($INBOX_DIR/description missing)
help / color / mirror / Atom feed[PATCH v17 4/4] Allow xlogreader to use different xlog blocksize.
26+ messages / 2 participants
[nested] [flat]
* [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize.
@ 2019-09-05 12:29 Kyotaro Horiguchi <[email protected]>
0 siblings, 0 replies; 26+ messages in thread
From: Kyotaro Horiguchi @ 2019-09-05 12:29 UTC (permalink / raw)
XLog reader facility and its callers currently relies on the same
constant XLOG_BLCKSZ. This patch moves the responsibility to determine
the block size to the callers of the facility.
---
src/backend/access/transam/twophase.c | 3 +-
src/backend/access/transam/xlog.c | 9 ++--
src/backend/access/transam/xlogreader.c | 57 ++++++++++++-----------
src/backend/replication/logical/logical.c | 5 +-
src/backend/replication/walsender.c | 4 +-
src/bin/pg_rewind/parsexlog.c | 6 +--
src/bin/pg_waldump/pg_waldump.c | 2 +-
src/include/access/xlogreader.h | 3 +-
8 files changed, 50 insertions(+), 39 deletions(-)
diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index 7d5a96a5f5..9f9e7de253 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -1325,7 +1325,8 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
XLogReaderState *xlogreader;
char *errormsg;
- xlogreader = XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+ xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+ NULL, wal_segment_close);
if (!xlogreader)
ereport(ERROR,
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index b517ca85ad..0822ac70cb 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -1222,7 +1222,10 @@ XLogInsertRecord(XLogRecData *rdata,
appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len);
if (!debug_reader)
- debug_reader = XLogReaderAllocate(wal_segment_size, NULL, NULL);
+ {
+ debug_reader =
+ XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, NULL);
+ }
if (!debug_reader)
{
@@ -6531,8 +6534,8 @@ StartupXLOG(void)
OwnLatch(&XLogCtl->recoveryWakeupLatch);
/* Set up XLOG reader facility */
- xlogreader =
- XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+ xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+ NULL, wal_segment_close);
if (!xlogreader)
ereport(ERROR,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index ba3e4868f7..e5baf13086 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -72,7 +72,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...)
* Returns NULL if the xlogreader couldn't be allocated.
*/
XLogReaderState *
-XLogReaderAllocate(int wal_segment_size, const char *waldir,
+XLogReaderAllocate(int wal_segment_size, int page_size, const char *waldir,
WALSegmentCleanupCB cleanup_cb)
{
XLogReaderState *state;
@@ -87,6 +87,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
state->cleanup_cb = cleanup_cb;
state->max_block_id = -1;
+ state->pageSize = page_size;
/*
* Permanently allocate readBuf. We do it this way, rather than just
@@ -95,7 +96,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
* isn't guaranteed to have any particular alignment, whereas
* palloc_extended() will provide MAXALIGN'd storage.
*/
- state->readBuf = (char *) palloc_extended(XLOG_BLCKSZ,
+ state->readBuf = (char *) palloc_extended(state->pageSize,
MCXT_ALLOC_NO_OOM);
if (!state->readBuf)
{
@@ -163,7 +164,7 @@ XLogReaderFree(XLogReaderState *state)
* readRecordBufSize is set to the new buffer size.
*
* To avoid useless small increases, round its size to a multiple of
- * XLOG_BLCKSZ, and make sure it's at least 5*Max(BLCKSZ, XLOG_BLCKSZ) to start
+ * page size, and make sure it's at least 5*Max(BLCKSZ, <page size>) to start
* with. (That is enough for all "normal" records, but very large commit or
* abort records might need more space.)
*/
@@ -172,8 +173,8 @@ allocate_recordbuf(XLogReaderState *state, uint32 reclength)
{
uint32 newSize = reclength;
- newSize += XLOG_BLCKSZ - (newSize % XLOG_BLCKSZ);
- newSize = Max(newSize, 5 * Max(BLCKSZ, XLOG_BLCKSZ));
+ newSize += state->pageSize - (newSize % state->pageSize);
+ newSize = Max(newSize, 5 * Max(BLCKSZ, state->pageSize));
#ifndef FRONTEND
@@ -238,6 +239,8 @@ void
XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
{
Assert(!XLogRecPtrIsInvalid(RecPtr));
+ Assert(state->readBuf != NULL &&
+ state->readBuf == (void *) MAXALIGN(state->readBuf));
ResetDecoder(state);
@@ -366,15 +369,15 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
XLogPageHeader pageHeader;
targetPagePtr =
- state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
- targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+ state->ReadRecPtr - (state->ReadRecPtr % state->pageSize);
+ targetRecOff = state->ReadRecPtr % state->pageSize;
/*
* Check if we have enough data. For the first record in the page,
* the requesting length doesn't contain page header.
*/
if (XLogNeedData(state, targetPagePtr,
- Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+ Min(targetRecOff + SizeOfXLogRecord, state->pageSize),
targetRecOff != 0))
return XLREAD_NEED_DATA;
@@ -421,7 +424,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
* verified that we got the whole header.
*/
prec = (XLogRecord *) (state->readBuf +
- state->ReadRecPtr % XLOG_BLCKSZ);
+ state->ReadRecPtr % state->pageSize);
total_len = prec->xl_tot_len;
/*
@@ -432,7 +435,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
* ensure that we enter the XLREAD_CONTINUATION state below;
* otherwise we might fail to apply ValidXLogRecordHeader at all.
*/
- if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
+ if (targetRecOff <= state->pageSize - SizeOfXLogRecord)
{
if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
state->PrevRecPtr, prec))
@@ -477,10 +480,10 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
* available
*/
targetPagePtr =
- state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
- targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+ state->ReadRecPtr - (state->ReadRecPtr % state->pageSize);
+ targetRecOff = state->ReadRecPtr % state->pageSize;
- request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ);
+ request_len = Min(targetRecOff + total_len, state->pageSize);
record_len = request_len - targetRecOff;
/* ReadRecPtr contains page header */
@@ -549,7 +552,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
/* Calculate pointer to beginning of next page */
state->recordContRecPtr = state->ReadRecPtr + record_len;
- Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0);
+ Assert(state->recordContRecPtr % state->pageSize == 0);
state->readRecordState = XLREAD_CONTINUATION;
}
@@ -576,7 +579,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
/* this request contains page header */
Assert (targetPagePtr != 0);
if (XLogNeedData(state, targetPagePtr,
- Min(state->recordRemainLen, XLOG_BLCKSZ),
+ Min(state->recordRemainLen, state->pageSize),
false))
return XLREAD_NEED_DATA;
@@ -628,7 +631,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
Assert(state->readLen >= pageHeaderSize);
contdata = (char *) state->readBuf + pageHeaderSize;
- record_len = XLOG_BLCKSZ - pageHeaderSize;
+ record_len = state->pageSize - pageHeaderSize;
if (pageHeader->xlp_rem_len < record_len)
record_len = pageHeader->xlp_rem_len;
@@ -655,7 +658,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
}
/* Calculate pointer to beginning of next page, and continue */
- state->recordContRecPtr += XLOG_BLCKSZ;
+ state->recordContRecPtr += state->pageSize;
}
/* targetPagePtr is pointing the last-read page here */
@@ -789,10 +792,10 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
XLogPageHeaderSize((XLogPageHeader) state->readBuf);
addLen = pageHeaderSize;
- if (reqLen + pageHeaderSize <= XLOG_BLCKSZ)
+ if (reqLen + pageHeaderSize <= state->pageSize)
addLen = pageHeaderSize;
else
- addLen = XLOG_BLCKSZ - reqLen;
+ addLen = state->pageSize - reqLen;
Assert(addLen >= 0);
}
@@ -805,7 +808,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
/* Data is not in our buffer, request the caller for it. */
XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize);
targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize);
- Assert((pageptr % XLOG_BLCKSZ) == 0);
+ Assert((pageptr % state->pageSize) == 0);
/*
* Every time we request to load new data of a page to the caller, even if
@@ -829,7 +832,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
* will not come back here, but will request the actual target page.
*/
state->readPagePtr = pageptr - targetPageOff;
- state->readLen = XLOG_BLCKSZ;
+ state->readLen = state->pageSize;
return true;
}
@@ -962,7 +965,7 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
int32 offset;
XLogPageHeader hdr = (XLogPageHeader) phdr;
- Assert((recptr % XLOG_BLCKSZ) == 0);
+ Assert((recptr % state->pageSize) == 0);
XLByteToSeg(recptr, segno, state->segcxt.ws_segsize);
offset = XLogSegmentOffset(recptr, state->segcxt.ws_segsize);
@@ -1016,10 +1019,10 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
"WAL file is from different database system: incorrect segment size in page header");
return false;
}
- else if (longhdr->xlp_xlog_blcksz != XLOG_BLCKSZ)
+ else if (longhdr->xlp_xlog_blcksz != state->pageSize)
{
report_invalid_record(state,
- "WAL file is from different database system: incorrect XLOG_BLCKSZ in page header");
+ "WAL file is from different database system: incorrect block size in page header");
return false;
}
}
@@ -1140,7 +1143,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
* XLogNeedData() is prepared to handle that and will read at
* least short page-header worth of data
*/
- targetRecOff = tmpRecPtr % XLOG_BLCKSZ;
+ targetRecOff = tmpRecPtr % state->pageSize;
/* scroll back to page boundary */
targetPagePtr = tmpRecPtr - targetRecOff;
@@ -1174,8 +1177,8 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
*
* Note that record headers are MAXALIGN'ed
*/
- if (MAXALIGN(header->xlp_rem_len) >= (XLOG_BLCKSZ - pageHeaderSize))
- tmpRecPtr = targetPagePtr + XLOG_BLCKSZ;
+ if (MAXALIGN(header->xlp_rem_len) >= (state->pageSize - pageHeaderSize))
+ tmpRecPtr = targetPagePtr + state->pageSize;
else
{
/*
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index 9cbd8afa49..f2dd495310 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -199,11 +199,14 @@ StartupDecodingContext(List *output_plugin_options,
ctx->slot = slot;
- ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, cleanup_cb);
+ ctx->reader =
+ XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, cleanup_cb);
+
if (!ctx->reader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
errmsg("out of memory")));
+ ctx->reader->readBuf = palloc(XLOG_BLCKSZ);
ctx->page_read = page_read;
ctx->reorder = ReorderBufferAllocate();
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index 062093727d..8e95600ef7 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -579,8 +579,8 @@ StartReplication(StartReplicationCmd *cmd)
errmsg("IDENTIFY_SYSTEM has not been run before START_REPLICATION")));
/* create xlogreader for physical replication */
- xlogreader =
- XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+ xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+ NULL, wal_segment_close);
if (!xlogreader)
ereport(ERROR,
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index 712c85281c..8a6209b061 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -61,7 +61,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
XLogReaderState *xlogreader;
char *errormsg;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+ xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
@@ -121,7 +121,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex,
char *errormsg;
XLogRecPtr endptr;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+ xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
@@ -181,7 +181,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
forkptr += SizeOfXLogShortPHD;
}
- xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+ xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index f514de6b13..b635537659 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -1041,7 +1041,7 @@ main(int argc, char **argv)
/* we have everything we need, start reading */
xlogreader_state =
- XLogReaderAllocate(WalSegSz, waldir, WALDumpCloseSegment);
+ XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, waldir, WALDumpCloseSegment);
if (!xlogreader_state)
fatal_error("out of memory");
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index b9dad7ee51..1853516333 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -151,6 +151,7 @@ struct XLogReaderState
* ----------------------------------------
*/
/* variables to communicate with page reader */
+ int pageSize; /* size of a page */
XLogRecPtr readPagePtr; /* page pointer to read */
int32 readLen; /* bytes requested to reader, or actual bytes
* read by reader, which must be larger than
@@ -243,7 +244,7 @@ struct XLogReaderState
/* Get a new XLogReader */
extern XLogReaderState *XLogReaderAllocate(int wal_segment_size,
- const char *waldir,
+ int page_size, const char *waldir,
WALSegmentCleanupCB cleanup_cb);
/* Free an XLogReader */
--
2.27.0
----Next_Part(Thu_Mar__4_11_28_53_2021_014)----
^ permalink raw reply [nested|flat] 26+ messages in thread
* [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize.
@ 2019-09-05 12:29 Kyotaro Horiguchi <[email protected]>
0 siblings, 0 replies; 26+ messages in thread
From: Kyotaro Horiguchi @ 2019-09-05 12:29 UTC (permalink / raw)
XLog reader facility and its callers currently relies on the same
constant XLOG_BLCKSZ. This patch moves the responsibility to determine
the block size to the callers of the facility.
---
src/backend/access/transam/twophase.c | 3 +-
src/backend/access/transam/xlog.c | 9 ++--
src/backend/access/transam/xlogreader.c | 57 ++++++++++++-----------
src/backend/replication/logical/logical.c | 5 +-
src/backend/replication/walsender.c | 4 +-
src/bin/pg_rewind/parsexlog.c | 6 +--
src/bin/pg_waldump/pg_waldump.c | 2 +-
src/include/access/xlogreader.h | 3 +-
8 files changed, 50 insertions(+), 39 deletions(-)
diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index 7d5a96a5f5..9f9e7de253 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -1325,7 +1325,8 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
XLogReaderState *xlogreader;
char *errormsg;
- xlogreader = XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+ xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+ NULL, wal_segment_close);
if (!xlogreader)
ereport(ERROR,
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index b517ca85ad..0822ac70cb 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -1222,7 +1222,10 @@ XLogInsertRecord(XLogRecData *rdata,
appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len);
if (!debug_reader)
- debug_reader = XLogReaderAllocate(wal_segment_size, NULL, NULL);
+ {
+ debug_reader =
+ XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, NULL);
+ }
if (!debug_reader)
{
@@ -6531,8 +6534,8 @@ StartupXLOG(void)
OwnLatch(&XLogCtl->recoveryWakeupLatch);
/* Set up XLOG reader facility */
- xlogreader =
- XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+ xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+ NULL, wal_segment_close);
if (!xlogreader)
ereport(ERROR,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index ba3e4868f7..e5baf13086 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -72,7 +72,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...)
* Returns NULL if the xlogreader couldn't be allocated.
*/
XLogReaderState *
-XLogReaderAllocate(int wal_segment_size, const char *waldir,
+XLogReaderAllocate(int wal_segment_size, int page_size, const char *waldir,
WALSegmentCleanupCB cleanup_cb)
{
XLogReaderState *state;
@@ -87,6 +87,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
state->cleanup_cb = cleanup_cb;
state->max_block_id = -1;
+ state->pageSize = page_size;
/*
* Permanently allocate readBuf. We do it this way, rather than just
@@ -95,7 +96,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
* isn't guaranteed to have any particular alignment, whereas
* palloc_extended() will provide MAXALIGN'd storage.
*/
- state->readBuf = (char *) palloc_extended(XLOG_BLCKSZ,
+ state->readBuf = (char *) palloc_extended(state->pageSize,
MCXT_ALLOC_NO_OOM);
if (!state->readBuf)
{
@@ -163,7 +164,7 @@ XLogReaderFree(XLogReaderState *state)
* readRecordBufSize is set to the new buffer size.
*
* To avoid useless small increases, round its size to a multiple of
- * XLOG_BLCKSZ, and make sure it's at least 5*Max(BLCKSZ, XLOG_BLCKSZ) to start
+ * page size, and make sure it's at least 5*Max(BLCKSZ, <page size>) to start
* with. (That is enough for all "normal" records, but very large commit or
* abort records might need more space.)
*/
@@ -172,8 +173,8 @@ allocate_recordbuf(XLogReaderState *state, uint32 reclength)
{
uint32 newSize = reclength;
- newSize += XLOG_BLCKSZ - (newSize % XLOG_BLCKSZ);
- newSize = Max(newSize, 5 * Max(BLCKSZ, XLOG_BLCKSZ));
+ newSize += state->pageSize - (newSize % state->pageSize);
+ newSize = Max(newSize, 5 * Max(BLCKSZ, state->pageSize));
#ifndef FRONTEND
@@ -238,6 +239,8 @@ void
XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
{
Assert(!XLogRecPtrIsInvalid(RecPtr));
+ Assert(state->readBuf != NULL &&
+ state->readBuf == (void *) MAXALIGN(state->readBuf));
ResetDecoder(state);
@@ -366,15 +369,15 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
XLogPageHeader pageHeader;
targetPagePtr =
- state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
- targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+ state->ReadRecPtr - (state->ReadRecPtr % state->pageSize);
+ targetRecOff = state->ReadRecPtr % state->pageSize;
/*
* Check if we have enough data. For the first record in the page,
* the requesting length doesn't contain page header.
*/
if (XLogNeedData(state, targetPagePtr,
- Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+ Min(targetRecOff + SizeOfXLogRecord, state->pageSize),
targetRecOff != 0))
return XLREAD_NEED_DATA;
@@ -421,7 +424,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
* verified that we got the whole header.
*/
prec = (XLogRecord *) (state->readBuf +
- state->ReadRecPtr % XLOG_BLCKSZ);
+ state->ReadRecPtr % state->pageSize);
total_len = prec->xl_tot_len;
/*
@@ -432,7 +435,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
* ensure that we enter the XLREAD_CONTINUATION state below;
* otherwise we might fail to apply ValidXLogRecordHeader at all.
*/
- if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
+ if (targetRecOff <= state->pageSize - SizeOfXLogRecord)
{
if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
state->PrevRecPtr, prec))
@@ -477,10 +480,10 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
* available
*/
targetPagePtr =
- state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
- targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+ state->ReadRecPtr - (state->ReadRecPtr % state->pageSize);
+ targetRecOff = state->ReadRecPtr % state->pageSize;
- request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ);
+ request_len = Min(targetRecOff + total_len, state->pageSize);
record_len = request_len - targetRecOff;
/* ReadRecPtr contains page header */
@@ -549,7 +552,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
/* Calculate pointer to beginning of next page */
state->recordContRecPtr = state->ReadRecPtr + record_len;
- Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0);
+ Assert(state->recordContRecPtr % state->pageSize == 0);
state->readRecordState = XLREAD_CONTINUATION;
}
@@ -576,7 +579,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
/* this request contains page header */
Assert (targetPagePtr != 0);
if (XLogNeedData(state, targetPagePtr,
- Min(state->recordRemainLen, XLOG_BLCKSZ),
+ Min(state->recordRemainLen, state->pageSize),
false))
return XLREAD_NEED_DATA;
@@ -628,7 +631,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
Assert(state->readLen >= pageHeaderSize);
contdata = (char *) state->readBuf + pageHeaderSize;
- record_len = XLOG_BLCKSZ - pageHeaderSize;
+ record_len = state->pageSize - pageHeaderSize;
if (pageHeader->xlp_rem_len < record_len)
record_len = pageHeader->xlp_rem_len;
@@ -655,7 +658,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
}
/* Calculate pointer to beginning of next page, and continue */
- state->recordContRecPtr += XLOG_BLCKSZ;
+ state->recordContRecPtr += state->pageSize;
}
/* targetPagePtr is pointing the last-read page here */
@@ -789,10 +792,10 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
XLogPageHeaderSize((XLogPageHeader) state->readBuf);
addLen = pageHeaderSize;
- if (reqLen + pageHeaderSize <= XLOG_BLCKSZ)
+ if (reqLen + pageHeaderSize <= state->pageSize)
addLen = pageHeaderSize;
else
- addLen = XLOG_BLCKSZ - reqLen;
+ addLen = state->pageSize - reqLen;
Assert(addLen >= 0);
}
@@ -805,7 +808,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
/* Data is not in our buffer, request the caller for it. */
XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize);
targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize);
- Assert((pageptr % XLOG_BLCKSZ) == 0);
+ Assert((pageptr % state->pageSize) == 0);
/*
* Every time we request to load new data of a page to the caller, even if
@@ -829,7 +832,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
* will not come back here, but will request the actual target page.
*/
state->readPagePtr = pageptr - targetPageOff;
- state->readLen = XLOG_BLCKSZ;
+ state->readLen = state->pageSize;
return true;
}
@@ -962,7 +965,7 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
int32 offset;
XLogPageHeader hdr = (XLogPageHeader) phdr;
- Assert((recptr % XLOG_BLCKSZ) == 0);
+ Assert((recptr % state->pageSize) == 0);
XLByteToSeg(recptr, segno, state->segcxt.ws_segsize);
offset = XLogSegmentOffset(recptr, state->segcxt.ws_segsize);
@@ -1016,10 +1019,10 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
"WAL file is from different database system: incorrect segment size in page header");
return false;
}
- else if (longhdr->xlp_xlog_blcksz != XLOG_BLCKSZ)
+ else if (longhdr->xlp_xlog_blcksz != state->pageSize)
{
report_invalid_record(state,
- "WAL file is from different database system: incorrect XLOG_BLCKSZ in page header");
+ "WAL file is from different database system: incorrect block size in page header");
return false;
}
}
@@ -1140,7 +1143,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
* XLogNeedData() is prepared to handle that and will read at
* least short page-header worth of data
*/
- targetRecOff = tmpRecPtr % XLOG_BLCKSZ;
+ targetRecOff = tmpRecPtr % state->pageSize;
/* scroll back to page boundary */
targetPagePtr = tmpRecPtr - targetRecOff;
@@ -1174,8 +1177,8 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
*
* Note that record headers are MAXALIGN'ed
*/
- if (MAXALIGN(header->xlp_rem_len) >= (XLOG_BLCKSZ - pageHeaderSize))
- tmpRecPtr = targetPagePtr + XLOG_BLCKSZ;
+ if (MAXALIGN(header->xlp_rem_len) >= (state->pageSize - pageHeaderSize))
+ tmpRecPtr = targetPagePtr + state->pageSize;
else
{
/*
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index 9cbd8afa49..f2dd495310 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -199,11 +199,14 @@ StartupDecodingContext(List *output_plugin_options,
ctx->slot = slot;
- ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, cleanup_cb);
+ ctx->reader =
+ XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, cleanup_cb);
+
if (!ctx->reader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
errmsg("out of memory")));
+ ctx->reader->readBuf = palloc(XLOG_BLCKSZ);
ctx->page_read = page_read;
ctx->reorder = ReorderBufferAllocate();
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index 062093727d..8e95600ef7 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -579,8 +579,8 @@ StartReplication(StartReplicationCmd *cmd)
errmsg("IDENTIFY_SYSTEM has not been run before START_REPLICATION")));
/* create xlogreader for physical replication */
- xlogreader =
- XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+ xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+ NULL, wal_segment_close);
if (!xlogreader)
ereport(ERROR,
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index 712c85281c..8a6209b061 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -61,7 +61,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
XLogReaderState *xlogreader;
char *errormsg;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+ xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
@@ -121,7 +121,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex,
char *errormsg;
XLogRecPtr endptr;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+ xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
@@ -181,7 +181,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
forkptr += SizeOfXLogShortPHD;
}
- xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+ xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index f514de6b13..b635537659 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -1041,7 +1041,7 @@ main(int argc, char **argv)
/* we have everything we need, start reading */
xlogreader_state =
- XLogReaderAllocate(WalSegSz, waldir, WALDumpCloseSegment);
+ XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, waldir, WALDumpCloseSegment);
if (!xlogreader_state)
fatal_error("out of memory");
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index b9dad7ee51..1853516333 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -151,6 +151,7 @@ struct XLogReaderState
* ----------------------------------------
*/
/* variables to communicate with page reader */
+ int pageSize; /* size of a page */
XLogRecPtr readPagePtr; /* page pointer to read */
int32 readLen; /* bytes requested to reader, or actual bytes
* read by reader, which must be larger than
@@ -243,7 +244,7 @@ struct XLogReaderState
/* Get a new XLogReader */
extern XLogReaderState *XLogReaderAllocate(int wal_segment_size,
- const char *waldir,
+ int page_size, const char *waldir,
WALSegmentCleanupCB cleanup_cb);
/* Free an XLogReader */
--
2.27.0
----Next_Part(Thu_Mar__4_11_28_53_2021_014)----
^ permalink raw reply [nested|flat] 26+ messages in thread
* [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize.
@ 2019-09-05 12:29 Kyotaro Horiguchi <[email protected]>
0 siblings, 0 replies; 26+ messages in thread
From: Kyotaro Horiguchi @ 2019-09-05 12:29 UTC (permalink / raw)
XLog reader facility and its callers currently relies on the same
constant XLOG_BLCKSZ. This patch moves the responsibility to determine
the block size to the callers of the facility.
---
src/backend/access/transam/twophase.c | 3 +-
src/backend/access/transam/xlog.c | 9 ++--
src/backend/access/transam/xlogreader.c | 57 ++++++++++++-----------
src/backend/replication/logical/logical.c | 5 +-
src/backend/replication/walsender.c | 4 +-
src/bin/pg_rewind/parsexlog.c | 6 +--
src/bin/pg_waldump/pg_waldump.c | 2 +-
src/include/access/xlogreader.h | 3 +-
8 files changed, 50 insertions(+), 39 deletions(-)
diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index 7d5a96a5f5..9f9e7de253 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -1325,7 +1325,8 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
XLogReaderState *xlogreader;
char *errormsg;
- xlogreader = XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+ xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+ NULL, wal_segment_close);
if (!xlogreader)
ereport(ERROR,
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index b517ca85ad..0822ac70cb 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -1222,7 +1222,10 @@ XLogInsertRecord(XLogRecData *rdata,
appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len);
if (!debug_reader)
- debug_reader = XLogReaderAllocate(wal_segment_size, NULL, NULL);
+ {
+ debug_reader =
+ XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, NULL);
+ }
if (!debug_reader)
{
@@ -6531,8 +6534,8 @@ StartupXLOG(void)
OwnLatch(&XLogCtl->recoveryWakeupLatch);
/* Set up XLOG reader facility */
- xlogreader =
- XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+ xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+ NULL, wal_segment_close);
if (!xlogreader)
ereport(ERROR,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index ba3e4868f7..e5baf13086 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -72,7 +72,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...)
* Returns NULL if the xlogreader couldn't be allocated.
*/
XLogReaderState *
-XLogReaderAllocate(int wal_segment_size, const char *waldir,
+XLogReaderAllocate(int wal_segment_size, int page_size, const char *waldir,
WALSegmentCleanupCB cleanup_cb)
{
XLogReaderState *state;
@@ -87,6 +87,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
state->cleanup_cb = cleanup_cb;
state->max_block_id = -1;
+ state->pageSize = page_size;
/*
* Permanently allocate readBuf. We do it this way, rather than just
@@ -95,7 +96,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
* isn't guaranteed to have any particular alignment, whereas
* palloc_extended() will provide MAXALIGN'd storage.
*/
- state->readBuf = (char *) palloc_extended(XLOG_BLCKSZ,
+ state->readBuf = (char *) palloc_extended(state->pageSize,
MCXT_ALLOC_NO_OOM);
if (!state->readBuf)
{
@@ -163,7 +164,7 @@ XLogReaderFree(XLogReaderState *state)
* readRecordBufSize is set to the new buffer size.
*
* To avoid useless small increases, round its size to a multiple of
- * XLOG_BLCKSZ, and make sure it's at least 5*Max(BLCKSZ, XLOG_BLCKSZ) to start
+ * page size, and make sure it's at least 5*Max(BLCKSZ, <page size>) to start
* with. (That is enough for all "normal" records, but very large commit or
* abort records might need more space.)
*/
@@ -172,8 +173,8 @@ allocate_recordbuf(XLogReaderState *state, uint32 reclength)
{
uint32 newSize = reclength;
- newSize += XLOG_BLCKSZ - (newSize % XLOG_BLCKSZ);
- newSize = Max(newSize, 5 * Max(BLCKSZ, XLOG_BLCKSZ));
+ newSize += state->pageSize - (newSize % state->pageSize);
+ newSize = Max(newSize, 5 * Max(BLCKSZ, state->pageSize));
#ifndef FRONTEND
@@ -238,6 +239,8 @@ void
XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
{
Assert(!XLogRecPtrIsInvalid(RecPtr));
+ Assert(state->readBuf != NULL &&
+ state->readBuf == (void *) MAXALIGN(state->readBuf));
ResetDecoder(state);
@@ -366,15 +369,15 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
XLogPageHeader pageHeader;
targetPagePtr =
- state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
- targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+ state->ReadRecPtr - (state->ReadRecPtr % state->pageSize);
+ targetRecOff = state->ReadRecPtr % state->pageSize;
/*
* Check if we have enough data. For the first record in the page,
* the requesting length doesn't contain page header.
*/
if (XLogNeedData(state, targetPagePtr,
- Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+ Min(targetRecOff + SizeOfXLogRecord, state->pageSize),
targetRecOff != 0))
return XLREAD_NEED_DATA;
@@ -421,7 +424,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
* verified that we got the whole header.
*/
prec = (XLogRecord *) (state->readBuf +
- state->ReadRecPtr % XLOG_BLCKSZ);
+ state->ReadRecPtr % state->pageSize);
total_len = prec->xl_tot_len;
/*
@@ -432,7 +435,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
* ensure that we enter the XLREAD_CONTINUATION state below;
* otherwise we might fail to apply ValidXLogRecordHeader at all.
*/
- if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
+ if (targetRecOff <= state->pageSize - SizeOfXLogRecord)
{
if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
state->PrevRecPtr, prec))
@@ -477,10 +480,10 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
* available
*/
targetPagePtr =
- state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
- targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+ state->ReadRecPtr - (state->ReadRecPtr % state->pageSize);
+ targetRecOff = state->ReadRecPtr % state->pageSize;
- request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ);
+ request_len = Min(targetRecOff + total_len, state->pageSize);
record_len = request_len - targetRecOff;
/* ReadRecPtr contains page header */
@@ -549,7 +552,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
/* Calculate pointer to beginning of next page */
state->recordContRecPtr = state->ReadRecPtr + record_len;
- Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0);
+ Assert(state->recordContRecPtr % state->pageSize == 0);
state->readRecordState = XLREAD_CONTINUATION;
}
@@ -576,7 +579,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
/* this request contains page header */
Assert (targetPagePtr != 0);
if (XLogNeedData(state, targetPagePtr,
- Min(state->recordRemainLen, XLOG_BLCKSZ),
+ Min(state->recordRemainLen, state->pageSize),
false))
return XLREAD_NEED_DATA;
@@ -628,7 +631,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
Assert(state->readLen >= pageHeaderSize);
contdata = (char *) state->readBuf + pageHeaderSize;
- record_len = XLOG_BLCKSZ - pageHeaderSize;
+ record_len = state->pageSize - pageHeaderSize;
if (pageHeader->xlp_rem_len < record_len)
record_len = pageHeader->xlp_rem_len;
@@ -655,7 +658,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
}
/* Calculate pointer to beginning of next page, and continue */
- state->recordContRecPtr += XLOG_BLCKSZ;
+ state->recordContRecPtr += state->pageSize;
}
/* targetPagePtr is pointing the last-read page here */
@@ -789,10 +792,10 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
XLogPageHeaderSize((XLogPageHeader) state->readBuf);
addLen = pageHeaderSize;
- if (reqLen + pageHeaderSize <= XLOG_BLCKSZ)
+ if (reqLen + pageHeaderSize <= state->pageSize)
addLen = pageHeaderSize;
else
- addLen = XLOG_BLCKSZ - reqLen;
+ addLen = state->pageSize - reqLen;
Assert(addLen >= 0);
}
@@ -805,7 +808,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
/* Data is not in our buffer, request the caller for it. */
XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize);
targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize);
- Assert((pageptr % XLOG_BLCKSZ) == 0);
+ Assert((pageptr % state->pageSize) == 0);
/*
* Every time we request to load new data of a page to the caller, even if
@@ -829,7 +832,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
* will not come back here, but will request the actual target page.
*/
state->readPagePtr = pageptr - targetPageOff;
- state->readLen = XLOG_BLCKSZ;
+ state->readLen = state->pageSize;
return true;
}
@@ -962,7 +965,7 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
int32 offset;
XLogPageHeader hdr = (XLogPageHeader) phdr;
- Assert((recptr % XLOG_BLCKSZ) == 0);
+ Assert((recptr % state->pageSize) == 0);
XLByteToSeg(recptr, segno, state->segcxt.ws_segsize);
offset = XLogSegmentOffset(recptr, state->segcxt.ws_segsize);
@@ -1016,10 +1019,10 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
"WAL file is from different database system: incorrect segment size in page header");
return false;
}
- else if (longhdr->xlp_xlog_blcksz != XLOG_BLCKSZ)
+ else if (longhdr->xlp_xlog_blcksz != state->pageSize)
{
report_invalid_record(state,
- "WAL file is from different database system: incorrect XLOG_BLCKSZ in page header");
+ "WAL file is from different database system: incorrect block size in page header");
return false;
}
}
@@ -1140,7 +1143,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
* XLogNeedData() is prepared to handle that and will read at
* least short page-header worth of data
*/
- targetRecOff = tmpRecPtr % XLOG_BLCKSZ;
+ targetRecOff = tmpRecPtr % state->pageSize;
/* scroll back to page boundary */
targetPagePtr = tmpRecPtr - targetRecOff;
@@ -1174,8 +1177,8 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
*
* Note that record headers are MAXALIGN'ed
*/
- if (MAXALIGN(header->xlp_rem_len) >= (XLOG_BLCKSZ - pageHeaderSize))
- tmpRecPtr = targetPagePtr + XLOG_BLCKSZ;
+ if (MAXALIGN(header->xlp_rem_len) >= (state->pageSize - pageHeaderSize))
+ tmpRecPtr = targetPagePtr + state->pageSize;
else
{
/*
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index 9cbd8afa49..f2dd495310 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -199,11 +199,14 @@ StartupDecodingContext(List *output_plugin_options,
ctx->slot = slot;
- ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, cleanup_cb);
+ ctx->reader =
+ XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, cleanup_cb);
+
if (!ctx->reader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
errmsg("out of memory")));
+ ctx->reader->readBuf = palloc(XLOG_BLCKSZ);
ctx->page_read = page_read;
ctx->reorder = ReorderBufferAllocate();
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index 062093727d..8e95600ef7 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -579,8 +579,8 @@ StartReplication(StartReplicationCmd *cmd)
errmsg("IDENTIFY_SYSTEM has not been run before START_REPLICATION")));
/* create xlogreader for physical replication */
- xlogreader =
- XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+ xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+ NULL, wal_segment_close);
if (!xlogreader)
ereport(ERROR,
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index 712c85281c..8a6209b061 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -61,7 +61,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
XLogReaderState *xlogreader;
char *errormsg;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+ xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
@@ -121,7 +121,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex,
char *errormsg;
XLogRecPtr endptr;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+ xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
@@ -181,7 +181,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
forkptr += SizeOfXLogShortPHD;
}
- xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+ xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index f514de6b13..b635537659 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -1041,7 +1041,7 @@ main(int argc, char **argv)
/* we have everything we need, start reading */
xlogreader_state =
- XLogReaderAllocate(WalSegSz, waldir, WALDumpCloseSegment);
+ XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, waldir, WALDumpCloseSegment);
if (!xlogreader_state)
fatal_error("out of memory");
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index b9dad7ee51..1853516333 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -151,6 +151,7 @@ struct XLogReaderState
* ----------------------------------------
*/
/* variables to communicate with page reader */
+ int pageSize; /* size of a page */
XLogRecPtr readPagePtr; /* page pointer to read */
int32 readLen; /* bytes requested to reader, or actual bytes
* read by reader, which must be larger than
@@ -243,7 +244,7 @@ struct XLogReaderState
/* Get a new XLogReader */
extern XLogReaderState *XLogReaderAllocate(int wal_segment_size,
- const char *waldir,
+ int page_size, const char *waldir,
WALSegmentCleanupCB cleanup_cb);
/* Free an XLogReader */
--
2.27.0
----Next_Part(Thu_Mar__4_11_28_53_2021_014)----
^ permalink raw reply [nested|flat] 26+ messages in thread
* [PATCH v16 4/4] Allow xlogreader to use different xlog blocksize.
@ 2019-09-05 12:29 Kyotaro Horiguchi <[email protected]>
0 siblings, 0 replies; 26+ messages in thread
From: Kyotaro Horiguchi @ 2019-09-05 12:29 UTC (permalink / raw)
XLog reader facility and its callers currently relies on the same
constant XLOG_BLCKSZ. This patch moves the responsibility to determine
the block size to the callers of the facility.
---
src/backend/access/transam/twophase.c | 3 +-
src/backend/access/transam/xlog.c | 9 ++--
src/backend/access/transam/xlogreader.c | 57 ++++++++++++-----------
src/backend/replication/logical/logical.c | 5 +-
src/backend/replication/walsender.c | 4 +-
src/bin/pg_rewind/parsexlog.c | 6 +--
src/bin/pg_waldump/pg_waldump.c | 2 +-
src/include/access/xlogreader.h | 3 +-
8 files changed, 50 insertions(+), 39 deletions(-)
diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index 9a56b3487e..b64c61f742 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -1324,7 +1324,8 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
XLogReaderState *xlogreader;
char *errormsg;
- xlogreader = XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+ xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+ NULL, wal_segment_close);
if (!xlogreader)
ereport(ERROR,
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index eccb88b611..ec78de5dd9 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -1213,7 +1213,10 @@ XLogInsertRecord(XLogRecData *rdata,
appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len);
if (!debug_reader)
- debug_reader = XLogReaderAllocate(wal_segment_size, NULL, NULL);
+ {
+ debug_reader =
+ XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, NULL);
+ }
if (!debug_reader)
{
@@ -6469,8 +6472,8 @@ StartupXLOG(void)
OwnLatch(&XLogCtl->recoveryWakeupLatch);
/* Set up XLOG reader facility */
- xlogreader =
- XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+ xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+ NULL, wal_segment_close);
if (!xlogreader)
ereport(ERROR,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index d829e57baa..582b1dd11b 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -72,7 +72,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...)
* Returns NULL if the xlogreader couldn't be allocated.
*/
XLogReaderState *
-XLogReaderAllocate(int wal_segment_size, const char *waldir,
+XLogReaderAllocate(int wal_segment_size, int page_size, const char *waldir,
WALSegmentCleanupCB cleanup_cb)
{
XLogReaderState *state;
@@ -87,6 +87,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
state->cleanup_cb = cleanup_cb;
state->max_block_id = -1;
+ state->pageSize = page_size;
/*
* Permanently allocate readBuf. We do it this way, rather than just
@@ -95,7 +96,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
* isn't guaranteed to have any particular alignment, whereas
* palloc_extended() will provide MAXALIGN'd storage.
*/
- state->readBuf = (char *) palloc_extended(XLOG_BLCKSZ,
+ state->readBuf = (char *) palloc_extended(state->pageSize,
MCXT_ALLOC_NO_OOM);
if (!state->readBuf)
{
@@ -163,7 +164,7 @@ XLogReaderFree(XLogReaderState *state)
* readRecordBufSize is set to the new buffer size.
*
* To avoid useless small increases, round its size to a multiple of
- * XLOG_BLCKSZ, and make sure it's at least 5*Max(BLCKSZ, XLOG_BLCKSZ) to start
+ * page size, and make sure it's at least 5*Max(BLCKSZ, <page size>) to start
* with. (That is enough for all "normal" records, but very large commit or
* abort records might need more space.)
*/
@@ -172,8 +173,8 @@ allocate_recordbuf(XLogReaderState *state, uint32 reclength)
{
uint32 newSize = reclength;
- newSize += XLOG_BLCKSZ - (newSize % XLOG_BLCKSZ);
- newSize = Max(newSize, 5 * Max(BLCKSZ, XLOG_BLCKSZ));
+ newSize += state->pageSize - (newSize % state->pageSize);
+ newSize = Max(newSize, 5 * Max(BLCKSZ, state->pageSize));
#ifndef FRONTEND
@@ -238,6 +239,8 @@ void
XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
{
Assert(!XLogRecPtrIsInvalid(RecPtr));
+ Assert(state->readBuf != NULL &&
+ state->readBuf == (void *) MAXALIGN(state->readBuf));
ResetDecoder(state);
@@ -366,15 +369,15 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
XLogPageHeader pageHeader;
targetPagePtr =
- state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
- targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+ state->ReadRecPtr - (state->ReadRecPtr % state->pageSize);
+ targetRecOff = state->ReadRecPtr % state->pageSize;
/*
* Check if we have enough data. For the first record in the page,
* the requesting length doesn't contain page header.
*/
if (XLogNeedData(state, targetPagePtr,
- Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+ Min(targetRecOff + SizeOfXLogRecord, state->pageSize),
targetRecOff != 0))
return XLREAD_NEED_DATA;
@@ -422,7 +425,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
* verified that we got the whole header.
*/
prec = (XLogRecord *) (state->readBuf +
- state->ReadRecPtr % XLOG_BLCKSZ);
+ state->ReadRecPtr % state->pageSize);
total_len = prec->xl_tot_len;
/*
@@ -433,7 +436,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
* ensure that we enter the XLREAD_CONTINUATION state below;
* otherwise we might fail to apply ValidXLogRecordHeader at all.
*/
- if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
+ if (targetRecOff <= state->pageSize - SizeOfXLogRecord)
{
if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
state->PrevRecPtr, prec))
@@ -479,10 +482,10 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
* available
*/
targetPagePtr =
- state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
- targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+ state->ReadRecPtr - (state->ReadRecPtr % state->pageSize);
+ targetRecOff = state->ReadRecPtr % state->pageSize;
- request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ);
+ request_len = Min(targetRecOff + total_len, state->pageSize);
record_len = request_len - targetRecOff;
/* ReadRecPtr contains page header */
@@ -552,7 +555,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
/* Calculate pointer to beginning of next page */
state->recordContRecPtr = state->ReadRecPtr + record_len;
- Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0);
+ Assert(state->recordContRecPtr % state->pageSize == 0);
state->readRecordState = XLREAD_CONTINUATION;
}
@@ -579,7 +582,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
/* this request contains page header */
Assert (targetPagePtr != 0);
if (XLogNeedData(state, targetPagePtr,
- Min(state->recordRemainLen, XLOG_BLCKSZ),
+ Min(state->recordRemainLen, state->pageSize),
false))
return XLREAD_NEED_DATA;
@@ -631,7 +634,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
Assert(state->readLen >= pageHeaderSize);
contdata = (char *) state->readBuf + pageHeaderSize;
- record_len = XLOG_BLCKSZ - pageHeaderSize;
+ record_len = state->pageSize - pageHeaderSize;
if (pageHeader->xlp_rem_len < record_len)
record_len = pageHeader->xlp_rem_len;
@@ -658,7 +661,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
}
/* Calculate pointer to beginning of next page, and continue */
- state->recordContRecPtr += XLOG_BLCKSZ;
+ state->recordContRecPtr += state->pageSize;
}
/* targetPagePtr is pointing the last-read page here */
@@ -792,10 +795,10 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
XLogPageHeaderSize((XLogPageHeader) state->readBuf);
addLen = pageHeaderSize;
- if (reqLen + pageHeaderSize <= XLOG_BLCKSZ)
+ if (reqLen + pageHeaderSize <= state->pageSize)
addLen = pageHeaderSize;
else
- addLen = XLOG_BLCKSZ - reqLen;
+ addLen = state->pageSize - reqLen;
Assert(addLen >= 0);
}
@@ -808,7 +811,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
/* Data is not in our buffer, request the caller for it. */
XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize);
targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize);
- Assert((pageptr % XLOG_BLCKSZ) == 0);
+ Assert((pageptr % state->pageSize) == 0);
/*
* Every time we request to load new data of a page to the caller, even if
@@ -832,7 +835,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
* will not come back here, but will request the actual target page.
*/
state->readPagePtr = pageptr - targetPageOff;
- state->readLen = XLOG_BLCKSZ;
+ state->readLen = state->pageSize;
return true;
}
@@ -968,7 +971,7 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
int32 offset;
XLogPageHeader hdr = (XLogPageHeader) phdr;
- Assert((recptr % XLOG_BLCKSZ) == 0);
+ Assert((recptr % state->pageSize) == 0);
XLByteToSeg(recptr, segno, state->segcxt.ws_segsize);
offset = XLogSegmentOffset(recptr, state->segcxt.ws_segsize);
@@ -1022,10 +1025,10 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
"WAL file is from different database system: incorrect segment size in page header");
return false;
}
- else if (longhdr->xlp_xlog_blcksz != XLOG_BLCKSZ)
+ else if (longhdr->xlp_xlog_blcksz != state->pageSize)
{
report_invalid_record(state,
- "WAL file is from different database system: incorrect XLOG_BLCKSZ in page header");
+ "WAL file is from different database system: incorrect block size in page header");
return false;
}
}
@@ -1146,7 +1149,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
* XLogNeedData() is prepared to handle that and will read at
* least short page-header worth of data
*/
- targetRecOff = tmpRecPtr % XLOG_BLCKSZ;
+ targetRecOff = tmpRecPtr % state->pageSize;
/* scroll back to page boundary */
targetPagePtr = tmpRecPtr - targetRecOff;
@@ -1180,8 +1183,8 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
*
* Note that record headers are MAXALIGN'ed
*/
- if (MAXALIGN(header->xlp_rem_len) >= (XLOG_BLCKSZ - pageHeaderSize))
- tmpRecPtr = targetPagePtr + XLOG_BLCKSZ;
+ if (MAXALIGN(header->xlp_rem_len) >= (state->pageSize - pageHeaderSize))
+ tmpRecPtr = targetPagePtr + state->pageSize;
else
{
/*
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index 97def6f781..ee322ba796 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -189,11 +189,14 @@ StartupDecodingContext(List *output_plugin_options,
ctx->slot = slot;
- ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, cleanup_cb);
+ ctx->reader =
+ XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, cleanup_cb);
+
if (!ctx->reader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
errmsg("out of memory")));
+ ctx->reader->readBuf = palloc(XLOG_BLCKSZ);
ctx->page_read = page_read;
ctx->reorder = ReorderBufferAllocate();
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index 33d952f9fd..a0914ddb25 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -578,8 +578,8 @@ StartReplication(StartReplicationCmd *cmd)
errmsg("IDENTIFY_SYSTEM has not been run before START_REPLICATION")));
/* create xlogreader for physical replication */
- xlogreader =
- XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+ xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+ NULL, wal_segment_close);
if (!xlogreader)
ereport(ERROR,
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index 7344a7c692..c5881594b7 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -58,7 +58,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
XLogReaderState *xlogreader;
char *errormsg;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+ xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
@@ -112,7 +112,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex,
char *errormsg;
XLogRecPtr endptr;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+ xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
@@ -172,7 +172,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
forkptr += SizeOfXLogShortPHD;
}
- xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+ xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index 9329a6a6d6..025479431d 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -1043,7 +1043,7 @@ main(int argc, char **argv)
/* we have everything we need, start reading */
xlogreader_state =
- XLogReaderAllocate(WalSegSz, waldir, WALDumpCloseSegment);
+ XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, waldir, WALDumpCloseSegment);
if (!xlogreader_state)
fatal_error("out of memory");
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index 732d463a49..e9d7d70db7 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -151,6 +151,7 @@ struct XLogReaderState
* ----------------------------------------
*/
/* variables to communicate with page reader */
+ int pageSize; /* size of a page */
XLogRecPtr readPagePtr; /* page pointer to read */
int32 readLen; /* bytes requested to reader, or actual bytes
* read by reader, which must be larger than
@@ -243,7 +244,7 @@ struct XLogReaderState
/* Get a new XLogReader */
extern XLogReaderState *XLogReaderAllocate(int wal_segment_size,
- const char *waldir,
+ int page_size, const char *waldir,
WALSegmentCleanupCB cleanup_cb);
/* Free an XLogReader */
--
2.18.4
----Next_Part(Tue_Sep__8_16_35_16_2020_701)----
^ permalink raw reply [nested|flat] 26+ messages in thread
* [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize.
@ 2019-09-05 12:29 Kyotaro Horiguchi <[email protected]>
0 siblings, 0 replies; 26+ messages in thread
From: Kyotaro Horiguchi @ 2019-09-05 12:29 UTC (permalink / raw)
XLog reader facility and its callers currently relies on the same
constant XLOG_BLCKSZ. This patch moves the responsibility to determine
the block size to the callers of the facility.
---
src/backend/access/transam/twophase.c | 3 +-
src/backend/access/transam/xlog.c | 9 ++--
src/backend/access/transam/xlogreader.c | 57 ++++++++++++-----------
src/backend/replication/logical/logical.c | 5 +-
src/backend/replication/walsender.c | 4 +-
src/bin/pg_rewind/parsexlog.c | 6 +--
src/bin/pg_waldump/pg_waldump.c | 2 +-
src/include/access/xlogreader.h | 3 +-
8 files changed, 50 insertions(+), 39 deletions(-)
diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index 7d5a96a5f5..9f9e7de253 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -1325,7 +1325,8 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
XLogReaderState *xlogreader;
char *errormsg;
- xlogreader = XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+ xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+ NULL, wal_segment_close);
if (!xlogreader)
ereport(ERROR,
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index b517ca85ad..0822ac70cb 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -1222,7 +1222,10 @@ XLogInsertRecord(XLogRecData *rdata,
appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len);
if (!debug_reader)
- debug_reader = XLogReaderAllocate(wal_segment_size, NULL, NULL);
+ {
+ debug_reader =
+ XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, NULL);
+ }
if (!debug_reader)
{
@@ -6531,8 +6534,8 @@ StartupXLOG(void)
OwnLatch(&XLogCtl->recoveryWakeupLatch);
/* Set up XLOG reader facility */
- xlogreader =
- XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+ xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+ NULL, wal_segment_close);
if (!xlogreader)
ereport(ERROR,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index ba3e4868f7..e5baf13086 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -72,7 +72,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...)
* Returns NULL if the xlogreader couldn't be allocated.
*/
XLogReaderState *
-XLogReaderAllocate(int wal_segment_size, const char *waldir,
+XLogReaderAllocate(int wal_segment_size, int page_size, const char *waldir,
WALSegmentCleanupCB cleanup_cb)
{
XLogReaderState *state;
@@ -87,6 +87,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
state->cleanup_cb = cleanup_cb;
state->max_block_id = -1;
+ state->pageSize = page_size;
/*
* Permanently allocate readBuf. We do it this way, rather than just
@@ -95,7 +96,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
* isn't guaranteed to have any particular alignment, whereas
* palloc_extended() will provide MAXALIGN'd storage.
*/
- state->readBuf = (char *) palloc_extended(XLOG_BLCKSZ,
+ state->readBuf = (char *) palloc_extended(state->pageSize,
MCXT_ALLOC_NO_OOM);
if (!state->readBuf)
{
@@ -163,7 +164,7 @@ XLogReaderFree(XLogReaderState *state)
* readRecordBufSize is set to the new buffer size.
*
* To avoid useless small increases, round its size to a multiple of
- * XLOG_BLCKSZ, and make sure it's at least 5*Max(BLCKSZ, XLOG_BLCKSZ) to start
+ * page size, and make sure it's at least 5*Max(BLCKSZ, <page size>) to start
* with. (That is enough for all "normal" records, but very large commit or
* abort records might need more space.)
*/
@@ -172,8 +173,8 @@ allocate_recordbuf(XLogReaderState *state, uint32 reclength)
{
uint32 newSize = reclength;
- newSize += XLOG_BLCKSZ - (newSize % XLOG_BLCKSZ);
- newSize = Max(newSize, 5 * Max(BLCKSZ, XLOG_BLCKSZ));
+ newSize += state->pageSize - (newSize % state->pageSize);
+ newSize = Max(newSize, 5 * Max(BLCKSZ, state->pageSize));
#ifndef FRONTEND
@@ -238,6 +239,8 @@ void
XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
{
Assert(!XLogRecPtrIsInvalid(RecPtr));
+ Assert(state->readBuf != NULL &&
+ state->readBuf == (void *) MAXALIGN(state->readBuf));
ResetDecoder(state);
@@ -366,15 +369,15 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
XLogPageHeader pageHeader;
targetPagePtr =
- state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
- targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+ state->ReadRecPtr - (state->ReadRecPtr % state->pageSize);
+ targetRecOff = state->ReadRecPtr % state->pageSize;
/*
* Check if we have enough data. For the first record in the page,
* the requesting length doesn't contain page header.
*/
if (XLogNeedData(state, targetPagePtr,
- Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+ Min(targetRecOff + SizeOfXLogRecord, state->pageSize),
targetRecOff != 0))
return XLREAD_NEED_DATA;
@@ -421,7 +424,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
* verified that we got the whole header.
*/
prec = (XLogRecord *) (state->readBuf +
- state->ReadRecPtr % XLOG_BLCKSZ);
+ state->ReadRecPtr % state->pageSize);
total_len = prec->xl_tot_len;
/*
@@ -432,7 +435,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
* ensure that we enter the XLREAD_CONTINUATION state below;
* otherwise we might fail to apply ValidXLogRecordHeader at all.
*/
- if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
+ if (targetRecOff <= state->pageSize - SizeOfXLogRecord)
{
if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
state->PrevRecPtr, prec))
@@ -477,10 +480,10 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
* available
*/
targetPagePtr =
- state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
- targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+ state->ReadRecPtr - (state->ReadRecPtr % state->pageSize);
+ targetRecOff = state->ReadRecPtr % state->pageSize;
- request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ);
+ request_len = Min(targetRecOff + total_len, state->pageSize);
record_len = request_len - targetRecOff;
/* ReadRecPtr contains page header */
@@ -549,7 +552,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
/* Calculate pointer to beginning of next page */
state->recordContRecPtr = state->ReadRecPtr + record_len;
- Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0);
+ Assert(state->recordContRecPtr % state->pageSize == 0);
state->readRecordState = XLREAD_CONTINUATION;
}
@@ -576,7 +579,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
/* this request contains page header */
Assert (targetPagePtr != 0);
if (XLogNeedData(state, targetPagePtr,
- Min(state->recordRemainLen, XLOG_BLCKSZ),
+ Min(state->recordRemainLen, state->pageSize),
false))
return XLREAD_NEED_DATA;
@@ -628,7 +631,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
Assert(state->readLen >= pageHeaderSize);
contdata = (char *) state->readBuf + pageHeaderSize;
- record_len = XLOG_BLCKSZ - pageHeaderSize;
+ record_len = state->pageSize - pageHeaderSize;
if (pageHeader->xlp_rem_len < record_len)
record_len = pageHeader->xlp_rem_len;
@@ -655,7 +658,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
}
/* Calculate pointer to beginning of next page, and continue */
- state->recordContRecPtr += XLOG_BLCKSZ;
+ state->recordContRecPtr += state->pageSize;
}
/* targetPagePtr is pointing the last-read page here */
@@ -789,10 +792,10 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
XLogPageHeaderSize((XLogPageHeader) state->readBuf);
addLen = pageHeaderSize;
- if (reqLen + pageHeaderSize <= XLOG_BLCKSZ)
+ if (reqLen + pageHeaderSize <= state->pageSize)
addLen = pageHeaderSize;
else
- addLen = XLOG_BLCKSZ - reqLen;
+ addLen = state->pageSize - reqLen;
Assert(addLen >= 0);
}
@@ -805,7 +808,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
/* Data is not in our buffer, request the caller for it. */
XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize);
targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize);
- Assert((pageptr % XLOG_BLCKSZ) == 0);
+ Assert((pageptr % state->pageSize) == 0);
/*
* Every time we request to load new data of a page to the caller, even if
@@ -829,7 +832,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
* will not come back here, but will request the actual target page.
*/
state->readPagePtr = pageptr - targetPageOff;
- state->readLen = XLOG_BLCKSZ;
+ state->readLen = state->pageSize;
return true;
}
@@ -962,7 +965,7 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
int32 offset;
XLogPageHeader hdr = (XLogPageHeader) phdr;
- Assert((recptr % XLOG_BLCKSZ) == 0);
+ Assert((recptr % state->pageSize) == 0);
XLByteToSeg(recptr, segno, state->segcxt.ws_segsize);
offset = XLogSegmentOffset(recptr, state->segcxt.ws_segsize);
@@ -1016,10 +1019,10 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
"WAL file is from different database system: incorrect segment size in page header");
return false;
}
- else if (longhdr->xlp_xlog_blcksz != XLOG_BLCKSZ)
+ else if (longhdr->xlp_xlog_blcksz != state->pageSize)
{
report_invalid_record(state,
- "WAL file is from different database system: incorrect XLOG_BLCKSZ in page header");
+ "WAL file is from different database system: incorrect block size in page header");
return false;
}
}
@@ -1140,7 +1143,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
* XLogNeedData() is prepared to handle that and will read at
* least short page-header worth of data
*/
- targetRecOff = tmpRecPtr % XLOG_BLCKSZ;
+ targetRecOff = tmpRecPtr % state->pageSize;
/* scroll back to page boundary */
targetPagePtr = tmpRecPtr - targetRecOff;
@@ -1174,8 +1177,8 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
*
* Note that record headers are MAXALIGN'ed
*/
- if (MAXALIGN(header->xlp_rem_len) >= (XLOG_BLCKSZ - pageHeaderSize))
- tmpRecPtr = targetPagePtr + XLOG_BLCKSZ;
+ if (MAXALIGN(header->xlp_rem_len) >= (state->pageSize - pageHeaderSize))
+ tmpRecPtr = targetPagePtr + state->pageSize;
else
{
/*
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index 9cbd8afa49..f2dd495310 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -199,11 +199,14 @@ StartupDecodingContext(List *output_plugin_options,
ctx->slot = slot;
- ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, cleanup_cb);
+ ctx->reader =
+ XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, cleanup_cb);
+
if (!ctx->reader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
errmsg("out of memory")));
+ ctx->reader->readBuf = palloc(XLOG_BLCKSZ);
ctx->page_read = page_read;
ctx->reorder = ReorderBufferAllocate();
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index 062093727d..8e95600ef7 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -579,8 +579,8 @@ StartReplication(StartReplicationCmd *cmd)
errmsg("IDENTIFY_SYSTEM has not been run before START_REPLICATION")));
/* create xlogreader for physical replication */
- xlogreader =
- XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+ xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+ NULL, wal_segment_close);
if (!xlogreader)
ereport(ERROR,
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index 712c85281c..8a6209b061 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -61,7 +61,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
XLogReaderState *xlogreader;
char *errormsg;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+ xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
@@ -121,7 +121,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex,
char *errormsg;
XLogRecPtr endptr;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+ xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
@@ -181,7 +181,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
forkptr += SizeOfXLogShortPHD;
}
- xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+ xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index f514de6b13..b635537659 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -1041,7 +1041,7 @@ main(int argc, char **argv)
/* we have everything we need, start reading */
xlogreader_state =
- XLogReaderAllocate(WalSegSz, waldir, WALDumpCloseSegment);
+ XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, waldir, WALDumpCloseSegment);
if (!xlogreader_state)
fatal_error("out of memory");
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index b9dad7ee51..1853516333 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -151,6 +151,7 @@ struct XLogReaderState
* ----------------------------------------
*/
/* variables to communicate with page reader */
+ int pageSize; /* size of a page */
XLogRecPtr readPagePtr; /* page pointer to read */
int32 readLen; /* bytes requested to reader, or actual bytes
* read by reader, which must be larger than
@@ -243,7 +244,7 @@ struct XLogReaderState
/* Get a new XLogReader */
extern XLogReaderState *XLogReaderAllocate(int wal_segment_size,
- const char *waldir,
+ int page_size, const char *waldir,
WALSegmentCleanupCB cleanup_cb);
/* Free an XLogReader */
--
2.27.0
----Next_Part(Thu_Mar__4_11_28_53_2021_014)----
^ permalink raw reply [nested|flat] 26+ messages in thread
* [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize.
@ 2019-09-05 12:29 Kyotaro Horiguchi <[email protected]>
0 siblings, 0 replies; 26+ messages in thread
From: Kyotaro Horiguchi @ 2019-09-05 12:29 UTC (permalink / raw)
XLog reader facility and its callers currently relies on the same
constant XLOG_BLCKSZ. This patch moves the responsibility to determine
the block size to the callers of the facility.
---
src/backend/access/transam/twophase.c | 3 +-
src/backend/access/transam/xlog.c | 9 ++--
src/backend/access/transam/xlogreader.c | 57 ++++++++++++-----------
src/backend/replication/logical/logical.c | 5 +-
src/backend/replication/walsender.c | 4 +-
src/bin/pg_rewind/parsexlog.c | 6 +--
src/bin/pg_waldump/pg_waldump.c | 2 +-
src/include/access/xlogreader.h | 3 +-
8 files changed, 50 insertions(+), 39 deletions(-)
diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index 7d5a96a5f5..9f9e7de253 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -1325,7 +1325,8 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
XLogReaderState *xlogreader;
char *errormsg;
- xlogreader = XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+ xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+ NULL, wal_segment_close);
if (!xlogreader)
ereport(ERROR,
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index b517ca85ad..0822ac70cb 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -1222,7 +1222,10 @@ XLogInsertRecord(XLogRecData *rdata,
appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len);
if (!debug_reader)
- debug_reader = XLogReaderAllocate(wal_segment_size, NULL, NULL);
+ {
+ debug_reader =
+ XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, NULL);
+ }
if (!debug_reader)
{
@@ -6531,8 +6534,8 @@ StartupXLOG(void)
OwnLatch(&XLogCtl->recoveryWakeupLatch);
/* Set up XLOG reader facility */
- xlogreader =
- XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+ xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+ NULL, wal_segment_close);
if (!xlogreader)
ereport(ERROR,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index ba3e4868f7..e5baf13086 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -72,7 +72,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...)
* Returns NULL if the xlogreader couldn't be allocated.
*/
XLogReaderState *
-XLogReaderAllocate(int wal_segment_size, const char *waldir,
+XLogReaderAllocate(int wal_segment_size, int page_size, const char *waldir,
WALSegmentCleanupCB cleanup_cb)
{
XLogReaderState *state;
@@ -87,6 +87,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
state->cleanup_cb = cleanup_cb;
state->max_block_id = -1;
+ state->pageSize = page_size;
/*
* Permanently allocate readBuf. We do it this way, rather than just
@@ -95,7 +96,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
* isn't guaranteed to have any particular alignment, whereas
* palloc_extended() will provide MAXALIGN'd storage.
*/
- state->readBuf = (char *) palloc_extended(XLOG_BLCKSZ,
+ state->readBuf = (char *) palloc_extended(state->pageSize,
MCXT_ALLOC_NO_OOM);
if (!state->readBuf)
{
@@ -163,7 +164,7 @@ XLogReaderFree(XLogReaderState *state)
* readRecordBufSize is set to the new buffer size.
*
* To avoid useless small increases, round its size to a multiple of
- * XLOG_BLCKSZ, and make sure it's at least 5*Max(BLCKSZ, XLOG_BLCKSZ) to start
+ * page size, and make sure it's at least 5*Max(BLCKSZ, <page size>) to start
* with. (That is enough for all "normal" records, but very large commit or
* abort records might need more space.)
*/
@@ -172,8 +173,8 @@ allocate_recordbuf(XLogReaderState *state, uint32 reclength)
{
uint32 newSize = reclength;
- newSize += XLOG_BLCKSZ - (newSize % XLOG_BLCKSZ);
- newSize = Max(newSize, 5 * Max(BLCKSZ, XLOG_BLCKSZ));
+ newSize += state->pageSize - (newSize % state->pageSize);
+ newSize = Max(newSize, 5 * Max(BLCKSZ, state->pageSize));
#ifndef FRONTEND
@@ -238,6 +239,8 @@ void
XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
{
Assert(!XLogRecPtrIsInvalid(RecPtr));
+ Assert(state->readBuf != NULL &&
+ state->readBuf == (void *) MAXALIGN(state->readBuf));
ResetDecoder(state);
@@ -366,15 +369,15 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
XLogPageHeader pageHeader;
targetPagePtr =
- state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
- targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+ state->ReadRecPtr - (state->ReadRecPtr % state->pageSize);
+ targetRecOff = state->ReadRecPtr % state->pageSize;
/*
* Check if we have enough data. For the first record in the page,
* the requesting length doesn't contain page header.
*/
if (XLogNeedData(state, targetPagePtr,
- Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+ Min(targetRecOff + SizeOfXLogRecord, state->pageSize),
targetRecOff != 0))
return XLREAD_NEED_DATA;
@@ -421,7 +424,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
* verified that we got the whole header.
*/
prec = (XLogRecord *) (state->readBuf +
- state->ReadRecPtr % XLOG_BLCKSZ);
+ state->ReadRecPtr % state->pageSize);
total_len = prec->xl_tot_len;
/*
@@ -432,7 +435,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
* ensure that we enter the XLREAD_CONTINUATION state below;
* otherwise we might fail to apply ValidXLogRecordHeader at all.
*/
- if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
+ if (targetRecOff <= state->pageSize - SizeOfXLogRecord)
{
if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
state->PrevRecPtr, prec))
@@ -477,10 +480,10 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
* available
*/
targetPagePtr =
- state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
- targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+ state->ReadRecPtr - (state->ReadRecPtr % state->pageSize);
+ targetRecOff = state->ReadRecPtr % state->pageSize;
- request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ);
+ request_len = Min(targetRecOff + total_len, state->pageSize);
record_len = request_len - targetRecOff;
/* ReadRecPtr contains page header */
@@ -549,7 +552,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
/* Calculate pointer to beginning of next page */
state->recordContRecPtr = state->ReadRecPtr + record_len;
- Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0);
+ Assert(state->recordContRecPtr % state->pageSize == 0);
state->readRecordState = XLREAD_CONTINUATION;
}
@@ -576,7 +579,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
/* this request contains page header */
Assert (targetPagePtr != 0);
if (XLogNeedData(state, targetPagePtr,
- Min(state->recordRemainLen, XLOG_BLCKSZ),
+ Min(state->recordRemainLen, state->pageSize),
false))
return XLREAD_NEED_DATA;
@@ -628,7 +631,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
Assert(state->readLen >= pageHeaderSize);
contdata = (char *) state->readBuf + pageHeaderSize;
- record_len = XLOG_BLCKSZ - pageHeaderSize;
+ record_len = state->pageSize - pageHeaderSize;
if (pageHeader->xlp_rem_len < record_len)
record_len = pageHeader->xlp_rem_len;
@@ -655,7 +658,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
}
/* Calculate pointer to beginning of next page, and continue */
- state->recordContRecPtr += XLOG_BLCKSZ;
+ state->recordContRecPtr += state->pageSize;
}
/* targetPagePtr is pointing the last-read page here */
@@ -789,10 +792,10 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
XLogPageHeaderSize((XLogPageHeader) state->readBuf);
addLen = pageHeaderSize;
- if (reqLen + pageHeaderSize <= XLOG_BLCKSZ)
+ if (reqLen + pageHeaderSize <= state->pageSize)
addLen = pageHeaderSize;
else
- addLen = XLOG_BLCKSZ - reqLen;
+ addLen = state->pageSize - reqLen;
Assert(addLen >= 0);
}
@@ -805,7 +808,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
/* Data is not in our buffer, request the caller for it. */
XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize);
targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize);
- Assert((pageptr % XLOG_BLCKSZ) == 0);
+ Assert((pageptr % state->pageSize) == 0);
/*
* Every time we request to load new data of a page to the caller, even if
@@ -829,7 +832,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
* will not come back here, but will request the actual target page.
*/
state->readPagePtr = pageptr - targetPageOff;
- state->readLen = XLOG_BLCKSZ;
+ state->readLen = state->pageSize;
return true;
}
@@ -962,7 +965,7 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
int32 offset;
XLogPageHeader hdr = (XLogPageHeader) phdr;
- Assert((recptr % XLOG_BLCKSZ) == 0);
+ Assert((recptr % state->pageSize) == 0);
XLByteToSeg(recptr, segno, state->segcxt.ws_segsize);
offset = XLogSegmentOffset(recptr, state->segcxt.ws_segsize);
@@ -1016,10 +1019,10 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
"WAL file is from different database system: incorrect segment size in page header");
return false;
}
- else if (longhdr->xlp_xlog_blcksz != XLOG_BLCKSZ)
+ else if (longhdr->xlp_xlog_blcksz != state->pageSize)
{
report_invalid_record(state,
- "WAL file is from different database system: incorrect XLOG_BLCKSZ in page header");
+ "WAL file is from different database system: incorrect block size in page header");
return false;
}
}
@@ -1140,7 +1143,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
* XLogNeedData() is prepared to handle that and will read at
* least short page-header worth of data
*/
- targetRecOff = tmpRecPtr % XLOG_BLCKSZ;
+ targetRecOff = tmpRecPtr % state->pageSize;
/* scroll back to page boundary */
targetPagePtr = tmpRecPtr - targetRecOff;
@@ -1174,8 +1177,8 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
*
* Note that record headers are MAXALIGN'ed
*/
- if (MAXALIGN(header->xlp_rem_len) >= (XLOG_BLCKSZ - pageHeaderSize))
- tmpRecPtr = targetPagePtr + XLOG_BLCKSZ;
+ if (MAXALIGN(header->xlp_rem_len) >= (state->pageSize - pageHeaderSize))
+ tmpRecPtr = targetPagePtr + state->pageSize;
else
{
/*
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index 9cbd8afa49..f2dd495310 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -199,11 +199,14 @@ StartupDecodingContext(List *output_plugin_options,
ctx->slot = slot;
- ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, cleanup_cb);
+ ctx->reader =
+ XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, cleanup_cb);
+
if (!ctx->reader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
errmsg("out of memory")));
+ ctx->reader->readBuf = palloc(XLOG_BLCKSZ);
ctx->page_read = page_read;
ctx->reorder = ReorderBufferAllocate();
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index 062093727d..8e95600ef7 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -579,8 +579,8 @@ StartReplication(StartReplicationCmd *cmd)
errmsg("IDENTIFY_SYSTEM has not been run before START_REPLICATION")));
/* create xlogreader for physical replication */
- xlogreader =
- XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+ xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+ NULL, wal_segment_close);
if (!xlogreader)
ereport(ERROR,
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index 712c85281c..8a6209b061 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -61,7 +61,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
XLogReaderState *xlogreader;
char *errormsg;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+ xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
@@ -121,7 +121,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex,
char *errormsg;
XLogRecPtr endptr;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+ xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
@@ -181,7 +181,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
forkptr += SizeOfXLogShortPHD;
}
- xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+ xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index f514de6b13..b635537659 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -1041,7 +1041,7 @@ main(int argc, char **argv)
/* we have everything we need, start reading */
xlogreader_state =
- XLogReaderAllocate(WalSegSz, waldir, WALDumpCloseSegment);
+ XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, waldir, WALDumpCloseSegment);
if (!xlogreader_state)
fatal_error("out of memory");
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index b9dad7ee51..1853516333 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -151,6 +151,7 @@ struct XLogReaderState
* ----------------------------------------
*/
/* variables to communicate with page reader */
+ int pageSize; /* size of a page */
XLogRecPtr readPagePtr; /* page pointer to read */
int32 readLen; /* bytes requested to reader, or actual bytes
* read by reader, which must be larger than
@@ -243,7 +244,7 @@ struct XLogReaderState
/* Get a new XLogReader */
extern XLogReaderState *XLogReaderAllocate(int wal_segment_size,
- const char *waldir,
+ int page_size, const char *waldir,
WALSegmentCleanupCB cleanup_cb);
/* Free an XLogReader */
--
2.27.0
----Next_Part(Thu_Mar__4_11_28_53_2021_014)----
^ permalink raw reply [nested|flat] 26+ messages in thread
* [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize.
@ 2019-09-05 12:29 Kyotaro Horiguchi <[email protected]>
0 siblings, 0 replies; 26+ messages in thread
From: Kyotaro Horiguchi @ 2019-09-05 12:29 UTC (permalink / raw)
XLog reader facility and its callers currently relies on the same
constant XLOG_BLCKSZ. This patch moves the responsibility to determine
the block size to the callers of the facility.
---
src/backend/access/transam/twophase.c | 3 +-
src/backend/access/transam/xlog.c | 9 ++--
src/backend/access/transam/xlogreader.c | 57 ++++++++++++-----------
src/backend/replication/logical/logical.c | 5 +-
src/backend/replication/walsender.c | 4 +-
src/bin/pg_rewind/parsexlog.c | 6 +--
src/bin/pg_waldump/pg_waldump.c | 2 +-
src/include/access/xlogreader.h | 3 +-
8 files changed, 50 insertions(+), 39 deletions(-)
diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index 7d5a96a5f5..9f9e7de253 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -1325,7 +1325,8 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
XLogReaderState *xlogreader;
char *errormsg;
- xlogreader = XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+ xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+ NULL, wal_segment_close);
if (!xlogreader)
ereport(ERROR,
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index b517ca85ad..0822ac70cb 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -1222,7 +1222,10 @@ XLogInsertRecord(XLogRecData *rdata,
appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len);
if (!debug_reader)
- debug_reader = XLogReaderAllocate(wal_segment_size, NULL, NULL);
+ {
+ debug_reader =
+ XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, NULL);
+ }
if (!debug_reader)
{
@@ -6531,8 +6534,8 @@ StartupXLOG(void)
OwnLatch(&XLogCtl->recoveryWakeupLatch);
/* Set up XLOG reader facility */
- xlogreader =
- XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+ xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+ NULL, wal_segment_close);
if (!xlogreader)
ereport(ERROR,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index ba3e4868f7..e5baf13086 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -72,7 +72,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...)
* Returns NULL if the xlogreader couldn't be allocated.
*/
XLogReaderState *
-XLogReaderAllocate(int wal_segment_size, const char *waldir,
+XLogReaderAllocate(int wal_segment_size, int page_size, const char *waldir,
WALSegmentCleanupCB cleanup_cb)
{
XLogReaderState *state;
@@ -87,6 +87,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
state->cleanup_cb = cleanup_cb;
state->max_block_id = -1;
+ state->pageSize = page_size;
/*
* Permanently allocate readBuf. We do it this way, rather than just
@@ -95,7 +96,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
* isn't guaranteed to have any particular alignment, whereas
* palloc_extended() will provide MAXALIGN'd storage.
*/
- state->readBuf = (char *) palloc_extended(XLOG_BLCKSZ,
+ state->readBuf = (char *) palloc_extended(state->pageSize,
MCXT_ALLOC_NO_OOM);
if (!state->readBuf)
{
@@ -163,7 +164,7 @@ XLogReaderFree(XLogReaderState *state)
* readRecordBufSize is set to the new buffer size.
*
* To avoid useless small increases, round its size to a multiple of
- * XLOG_BLCKSZ, and make sure it's at least 5*Max(BLCKSZ, XLOG_BLCKSZ) to start
+ * page size, and make sure it's at least 5*Max(BLCKSZ, <page size>) to start
* with. (That is enough for all "normal" records, but very large commit or
* abort records might need more space.)
*/
@@ -172,8 +173,8 @@ allocate_recordbuf(XLogReaderState *state, uint32 reclength)
{
uint32 newSize = reclength;
- newSize += XLOG_BLCKSZ - (newSize % XLOG_BLCKSZ);
- newSize = Max(newSize, 5 * Max(BLCKSZ, XLOG_BLCKSZ));
+ newSize += state->pageSize - (newSize % state->pageSize);
+ newSize = Max(newSize, 5 * Max(BLCKSZ, state->pageSize));
#ifndef FRONTEND
@@ -238,6 +239,8 @@ void
XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
{
Assert(!XLogRecPtrIsInvalid(RecPtr));
+ Assert(state->readBuf != NULL &&
+ state->readBuf == (void *) MAXALIGN(state->readBuf));
ResetDecoder(state);
@@ -366,15 +369,15 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
XLogPageHeader pageHeader;
targetPagePtr =
- state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
- targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+ state->ReadRecPtr - (state->ReadRecPtr % state->pageSize);
+ targetRecOff = state->ReadRecPtr % state->pageSize;
/*
* Check if we have enough data. For the first record in the page,
* the requesting length doesn't contain page header.
*/
if (XLogNeedData(state, targetPagePtr,
- Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+ Min(targetRecOff + SizeOfXLogRecord, state->pageSize),
targetRecOff != 0))
return XLREAD_NEED_DATA;
@@ -421,7 +424,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
* verified that we got the whole header.
*/
prec = (XLogRecord *) (state->readBuf +
- state->ReadRecPtr % XLOG_BLCKSZ);
+ state->ReadRecPtr % state->pageSize);
total_len = prec->xl_tot_len;
/*
@@ -432,7 +435,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
* ensure that we enter the XLREAD_CONTINUATION state below;
* otherwise we might fail to apply ValidXLogRecordHeader at all.
*/
- if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
+ if (targetRecOff <= state->pageSize - SizeOfXLogRecord)
{
if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
state->PrevRecPtr, prec))
@@ -477,10 +480,10 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
* available
*/
targetPagePtr =
- state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
- targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+ state->ReadRecPtr - (state->ReadRecPtr % state->pageSize);
+ targetRecOff = state->ReadRecPtr % state->pageSize;
- request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ);
+ request_len = Min(targetRecOff + total_len, state->pageSize);
record_len = request_len - targetRecOff;
/* ReadRecPtr contains page header */
@@ -549,7 +552,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
/* Calculate pointer to beginning of next page */
state->recordContRecPtr = state->ReadRecPtr + record_len;
- Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0);
+ Assert(state->recordContRecPtr % state->pageSize == 0);
state->readRecordState = XLREAD_CONTINUATION;
}
@@ -576,7 +579,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
/* this request contains page header */
Assert (targetPagePtr != 0);
if (XLogNeedData(state, targetPagePtr,
- Min(state->recordRemainLen, XLOG_BLCKSZ),
+ Min(state->recordRemainLen, state->pageSize),
false))
return XLREAD_NEED_DATA;
@@ -628,7 +631,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
Assert(state->readLen >= pageHeaderSize);
contdata = (char *) state->readBuf + pageHeaderSize;
- record_len = XLOG_BLCKSZ - pageHeaderSize;
+ record_len = state->pageSize - pageHeaderSize;
if (pageHeader->xlp_rem_len < record_len)
record_len = pageHeader->xlp_rem_len;
@@ -655,7 +658,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
}
/* Calculate pointer to beginning of next page, and continue */
- state->recordContRecPtr += XLOG_BLCKSZ;
+ state->recordContRecPtr += state->pageSize;
}
/* targetPagePtr is pointing the last-read page here */
@@ -789,10 +792,10 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
XLogPageHeaderSize((XLogPageHeader) state->readBuf);
addLen = pageHeaderSize;
- if (reqLen + pageHeaderSize <= XLOG_BLCKSZ)
+ if (reqLen + pageHeaderSize <= state->pageSize)
addLen = pageHeaderSize;
else
- addLen = XLOG_BLCKSZ - reqLen;
+ addLen = state->pageSize - reqLen;
Assert(addLen >= 0);
}
@@ -805,7 +808,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
/* Data is not in our buffer, request the caller for it. */
XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize);
targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize);
- Assert((pageptr % XLOG_BLCKSZ) == 0);
+ Assert((pageptr % state->pageSize) == 0);
/*
* Every time we request to load new data of a page to the caller, even if
@@ -829,7 +832,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
* will not come back here, but will request the actual target page.
*/
state->readPagePtr = pageptr - targetPageOff;
- state->readLen = XLOG_BLCKSZ;
+ state->readLen = state->pageSize;
return true;
}
@@ -962,7 +965,7 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
int32 offset;
XLogPageHeader hdr = (XLogPageHeader) phdr;
- Assert((recptr % XLOG_BLCKSZ) == 0);
+ Assert((recptr % state->pageSize) == 0);
XLByteToSeg(recptr, segno, state->segcxt.ws_segsize);
offset = XLogSegmentOffset(recptr, state->segcxt.ws_segsize);
@@ -1016,10 +1019,10 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
"WAL file is from different database system: incorrect segment size in page header");
return false;
}
- else if (longhdr->xlp_xlog_blcksz != XLOG_BLCKSZ)
+ else if (longhdr->xlp_xlog_blcksz != state->pageSize)
{
report_invalid_record(state,
- "WAL file is from different database system: incorrect XLOG_BLCKSZ in page header");
+ "WAL file is from different database system: incorrect block size in page header");
return false;
}
}
@@ -1140,7 +1143,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
* XLogNeedData() is prepared to handle that and will read at
* least short page-header worth of data
*/
- targetRecOff = tmpRecPtr % XLOG_BLCKSZ;
+ targetRecOff = tmpRecPtr % state->pageSize;
/* scroll back to page boundary */
targetPagePtr = tmpRecPtr - targetRecOff;
@@ -1174,8 +1177,8 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
*
* Note that record headers are MAXALIGN'ed
*/
- if (MAXALIGN(header->xlp_rem_len) >= (XLOG_BLCKSZ - pageHeaderSize))
- tmpRecPtr = targetPagePtr + XLOG_BLCKSZ;
+ if (MAXALIGN(header->xlp_rem_len) >= (state->pageSize - pageHeaderSize))
+ tmpRecPtr = targetPagePtr + state->pageSize;
else
{
/*
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index 9cbd8afa49..f2dd495310 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -199,11 +199,14 @@ StartupDecodingContext(List *output_plugin_options,
ctx->slot = slot;
- ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, cleanup_cb);
+ ctx->reader =
+ XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, cleanup_cb);
+
if (!ctx->reader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
errmsg("out of memory")));
+ ctx->reader->readBuf = palloc(XLOG_BLCKSZ);
ctx->page_read = page_read;
ctx->reorder = ReorderBufferAllocate();
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index 062093727d..8e95600ef7 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -579,8 +579,8 @@ StartReplication(StartReplicationCmd *cmd)
errmsg("IDENTIFY_SYSTEM has not been run before START_REPLICATION")));
/* create xlogreader for physical replication */
- xlogreader =
- XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+ xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+ NULL, wal_segment_close);
if (!xlogreader)
ereport(ERROR,
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index 712c85281c..8a6209b061 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -61,7 +61,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
XLogReaderState *xlogreader;
char *errormsg;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+ xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
@@ -121,7 +121,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex,
char *errormsg;
XLogRecPtr endptr;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+ xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
@@ -181,7 +181,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
forkptr += SizeOfXLogShortPHD;
}
- xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+ xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index f514de6b13..b635537659 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -1041,7 +1041,7 @@ main(int argc, char **argv)
/* we have everything we need, start reading */
xlogreader_state =
- XLogReaderAllocate(WalSegSz, waldir, WALDumpCloseSegment);
+ XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, waldir, WALDumpCloseSegment);
if (!xlogreader_state)
fatal_error("out of memory");
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index b9dad7ee51..1853516333 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -151,6 +151,7 @@ struct XLogReaderState
* ----------------------------------------
*/
/* variables to communicate with page reader */
+ int pageSize; /* size of a page */
XLogRecPtr readPagePtr; /* page pointer to read */
int32 readLen; /* bytes requested to reader, or actual bytes
* read by reader, which must be larger than
@@ -243,7 +244,7 @@ struct XLogReaderState
/* Get a new XLogReader */
extern XLogReaderState *XLogReaderAllocate(int wal_segment_size,
- const char *waldir,
+ int page_size, const char *waldir,
WALSegmentCleanupCB cleanup_cb);
/* Free an XLogReader */
--
2.27.0
----Next_Part(Thu_Mar__4_11_28_53_2021_014)----
^ permalink raw reply [nested|flat] 26+ messages in thread
* [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize.
@ 2019-09-05 12:29 Kyotaro Horiguchi <[email protected]>
0 siblings, 0 replies; 26+ messages in thread
From: Kyotaro Horiguchi @ 2019-09-05 12:29 UTC (permalink / raw)
XLog reader facility and its callers currently relies on the same
constant XLOG_BLCKSZ. This patch moves the responsibility to determine
the block size to the callers of the facility.
---
src/backend/access/transam/twophase.c | 3 +-
src/backend/access/transam/xlog.c | 9 ++--
src/backend/access/transam/xlogreader.c | 57 ++++++++++++-----------
src/backend/replication/logical/logical.c | 5 +-
src/backend/replication/walsender.c | 4 +-
src/bin/pg_rewind/parsexlog.c | 6 +--
src/bin/pg_waldump/pg_waldump.c | 2 +-
src/include/access/xlogreader.h | 3 +-
8 files changed, 50 insertions(+), 39 deletions(-)
diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index 7d5a96a5f5..9f9e7de253 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -1325,7 +1325,8 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
XLogReaderState *xlogreader;
char *errormsg;
- xlogreader = XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+ xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+ NULL, wal_segment_close);
if (!xlogreader)
ereport(ERROR,
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index b517ca85ad..0822ac70cb 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -1222,7 +1222,10 @@ XLogInsertRecord(XLogRecData *rdata,
appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len);
if (!debug_reader)
- debug_reader = XLogReaderAllocate(wal_segment_size, NULL, NULL);
+ {
+ debug_reader =
+ XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, NULL);
+ }
if (!debug_reader)
{
@@ -6531,8 +6534,8 @@ StartupXLOG(void)
OwnLatch(&XLogCtl->recoveryWakeupLatch);
/* Set up XLOG reader facility */
- xlogreader =
- XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+ xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+ NULL, wal_segment_close);
if (!xlogreader)
ereport(ERROR,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index ba3e4868f7..e5baf13086 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -72,7 +72,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...)
* Returns NULL if the xlogreader couldn't be allocated.
*/
XLogReaderState *
-XLogReaderAllocate(int wal_segment_size, const char *waldir,
+XLogReaderAllocate(int wal_segment_size, int page_size, const char *waldir,
WALSegmentCleanupCB cleanup_cb)
{
XLogReaderState *state;
@@ -87,6 +87,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
state->cleanup_cb = cleanup_cb;
state->max_block_id = -1;
+ state->pageSize = page_size;
/*
* Permanently allocate readBuf. We do it this way, rather than just
@@ -95,7 +96,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
* isn't guaranteed to have any particular alignment, whereas
* palloc_extended() will provide MAXALIGN'd storage.
*/
- state->readBuf = (char *) palloc_extended(XLOG_BLCKSZ,
+ state->readBuf = (char *) palloc_extended(state->pageSize,
MCXT_ALLOC_NO_OOM);
if (!state->readBuf)
{
@@ -163,7 +164,7 @@ XLogReaderFree(XLogReaderState *state)
* readRecordBufSize is set to the new buffer size.
*
* To avoid useless small increases, round its size to a multiple of
- * XLOG_BLCKSZ, and make sure it's at least 5*Max(BLCKSZ, XLOG_BLCKSZ) to start
+ * page size, and make sure it's at least 5*Max(BLCKSZ, <page size>) to start
* with. (That is enough for all "normal" records, but very large commit or
* abort records might need more space.)
*/
@@ -172,8 +173,8 @@ allocate_recordbuf(XLogReaderState *state, uint32 reclength)
{
uint32 newSize = reclength;
- newSize += XLOG_BLCKSZ - (newSize % XLOG_BLCKSZ);
- newSize = Max(newSize, 5 * Max(BLCKSZ, XLOG_BLCKSZ));
+ newSize += state->pageSize - (newSize % state->pageSize);
+ newSize = Max(newSize, 5 * Max(BLCKSZ, state->pageSize));
#ifndef FRONTEND
@@ -238,6 +239,8 @@ void
XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
{
Assert(!XLogRecPtrIsInvalid(RecPtr));
+ Assert(state->readBuf != NULL &&
+ state->readBuf == (void *) MAXALIGN(state->readBuf));
ResetDecoder(state);
@@ -366,15 +369,15 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
XLogPageHeader pageHeader;
targetPagePtr =
- state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
- targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+ state->ReadRecPtr - (state->ReadRecPtr % state->pageSize);
+ targetRecOff = state->ReadRecPtr % state->pageSize;
/*
* Check if we have enough data. For the first record in the page,
* the requesting length doesn't contain page header.
*/
if (XLogNeedData(state, targetPagePtr,
- Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+ Min(targetRecOff + SizeOfXLogRecord, state->pageSize),
targetRecOff != 0))
return XLREAD_NEED_DATA;
@@ -421,7 +424,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
* verified that we got the whole header.
*/
prec = (XLogRecord *) (state->readBuf +
- state->ReadRecPtr % XLOG_BLCKSZ);
+ state->ReadRecPtr % state->pageSize);
total_len = prec->xl_tot_len;
/*
@@ -432,7 +435,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
* ensure that we enter the XLREAD_CONTINUATION state below;
* otherwise we might fail to apply ValidXLogRecordHeader at all.
*/
- if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
+ if (targetRecOff <= state->pageSize - SizeOfXLogRecord)
{
if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
state->PrevRecPtr, prec))
@@ -477,10 +480,10 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
* available
*/
targetPagePtr =
- state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
- targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+ state->ReadRecPtr - (state->ReadRecPtr % state->pageSize);
+ targetRecOff = state->ReadRecPtr % state->pageSize;
- request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ);
+ request_len = Min(targetRecOff + total_len, state->pageSize);
record_len = request_len - targetRecOff;
/* ReadRecPtr contains page header */
@@ -549,7 +552,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
/* Calculate pointer to beginning of next page */
state->recordContRecPtr = state->ReadRecPtr + record_len;
- Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0);
+ Assert(state->recordContRecPtr % state->pageSize == 0);
state->readRecordState = XLREAD_CONTINUATION;
}
@@ -576,7 +579,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
/* this request contains page header */
Assert (targetPagePtr != 0);
if (XLogNeedData(state, targetPagePtr,
- Min(state->recordRemainLen, XLOG_BLCKSZ),
+ Min(state->recordRemainLen, state->pageSize),
false))
return XLREAD_NEED_DATA;
@@ -628,7 +631,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
Assert(state->readLen >= pageHeaderSize);
contdata = (char *) state->readBuf + pageHeaderSize;
- record_len = XLOG_BLCKSZ - pageHeaderSize;
+ record_len = state->pageSize - pageHeaderSize;
if (pageHeader->xlp_rem_len < record_len)
record_len = pageHeader->xlp_rem_len;
@@ -655,7 +658,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
}
/* Calculate pointer to beginning of next page, and continue */
- state->recordContRecPtr += XLOG_BLCKSZ;
+ state->recordContRecPtr += state->pageSize;
}
/* targetPagePtr is pointing the last-read page here */
@@ -789,10 +792,10 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
XLogPageHeaderSize((XLogPageHeader) state->readBuf);
addLen = pageHeaderSize;
- if (reqLen + pageHeaderSize <= XLOG_BLCKSZ)
+ if (reqLen + pageHeaderSize <= state->pageSize)
addLen = pageHeaderSize;
else
- addLen = XLOG_BLCKSZ - reqLen;
+ addLen = state->pageSize - reqLen;
Assert(addLen >= 0);
}
@@ -805,7 +808,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
/* Data is not in our buffer, request the caller for it. */
XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize);
targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize);
- Assert((pageptr % XLOG_BLCKSZ) == 0);
+ Assert((pageptr % state->pageSize) == 0);
/*
* Every time we request to load new data of a page to the caller, even if
@@ -829,7 +832,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
* will not come back here, but will request the actual target page.
*/
state->readPagePtr = pageptr - targetPageOff;
- state->readLen = XLOG_BLCKSZ;
+ state->readLen = state->pageSize;
return true;
}
@@ -962,7 +965,7 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
int32 offset;
XLogPageHeader hdr = (XLogPageHeader) phdr;
- Assert((recptr % XLOG_BLCKSZ) == 0);
+ Assert((recptr % state->pageSize) == 0);
XLByteToSeg(recptr, segno, state->segcxt.ws_segsize);
offset = XLogSegmentOffset(recptr, state->segcxt.ws_segsize);
@@ -1016,10 +1019,10 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
"WAL file is from different database system: incorrect segment size in page header");
return false;
}
- else if (longhdr->xlp_xlog_blcksz != XLOG_BLCKSZ)
+ else if (longhdr->xlp_xlog_blcksz != state->pageSize)
{
report_invalid_record(state,
- "WAL file is from different database system: incorrect XLOG_BLCKSZ in page header");
+ "WAL file is from different database system: incorrect block size in page header");
return false;
}
}
@@ -1140,7 +1143,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
* XLogNeedData() is prepared to handle that and will read at
* least short page-header worth of data
*/
- targetRecOff = tmpRecPtr % XLOG_BLCKSZ;
+ targetRecOff = tmpRecPtr % state->pageSize;
/* scroll back to page boundary */
targetPagePtr = tmpRecPtr - targetRecOff;
@@ -1174,8 +1177,8 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
*
* Note that record headers are MAXALIGN'ed
*/
- if (MAXALIGN(header->xlp_rem_len) >= (XLOG_BLCKSZ - pageHeaderSize))
- tmpRecPtr = targetPagePtr + XLOG_BLCKSZ;
+ if (MAXALIGN(header->xlp_rem_len) >= (state->pageSize - pageHeaderSize))
+ tmpRecPtr = targetPagePtr + state->pageSize;
else
{
/*
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index 9cbd8afa49..f2dd495310 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -199,11 +199,14 @@ StartupDecodingContext(List *output_plugin_options,
ctx->slot = slot;
- ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, cleanup_cb);
+ ctx->reader =
+ XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, cleanup_cb);
+
if (!ctx->reader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
errmsg("out of memory")));
+ ctx->reader->readBuf = palloc(XLOG_BLCKSZ);
ctx->page_read = page_read;
ctx->reorder = ReorderBufferAllocate();
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index 062093727d..8e95600ef7 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -579,8 +579,8 @@ StartReplication(StartReplicationCmd *cmd)
errmsg("IDENTIFY_SYSTEM has not been run before START_REPLICATION")));
/* create xlogreader for physical replication */
- xlogreader =
- XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+ xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+ NULL, wal_segment_close);
if (!xlogreader)
ereport(ERROR,
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index 712c85281c..8a6209b061 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -61,7 +61,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
XLogReaderState *xlogreader;
char *errormsg;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+ xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
@@ -121,7 +121,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex,
char *errormsg;
XLogRecPtr endptr;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+ xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
@@ -181,7 +181,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
forkptr += SizeOfXLogShortPHD;
}
- xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+ xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index f514de6b13..b635537659 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -1041,7 +1041,7 @@ main(int argc, char **argv)
/* we have everything we need, start reading */
xlogreader_state =
- XLogReaderAllocate(WalSegSz, waldir, WALDumpCloseSegment);
+ XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, waldir, WALDumpCloseSegment);
if (!xlogreader_state)
fatal_error("out of memory");
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index b9dad7ee51..1853516333 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -151,6 +151,7 @@ struct XLogReaderState
* ----------------------------------------
*/
/* variables to communicate with page reader */
+ int pageSize; /* size of a page */
XLogRecPtr readPagePtr; /* page pointer to read */
int32 readLen; /* bytes requested to reader, or actual bytes
* read by reader, which must be larger than
@@ -243,7 +244,7 @@ struct XLogReaderState
/* Get a new XLogReader */
extern XLogReaderState *XLogReaderAllocate(int wal_segment_size,
- const char *waldir,
+ int page_size, const char *waldir,
WALSegmentCleanupCB cleanup_cb);
/* Free an XLogReader */
--
2.27.0
----Next_Part(Thu_Mar__4_11_28_53_2021_014)----
^ permalink raw reply [nested|flat] 26+ messages in thread
* [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize.
@ 2019-09-05 12:29 Kyotaro Horiguchi <[email protected]>
0 siblings, 0 replies; 26+ messages in thread
From: Kyotaro Horiguchi @ 2019-09-05 12:29 UTC (permalink / raw)
XLog reader facility and its callers currently relies on the same
constant XLOG_BLCKSZ. This patch moves the responsibility to determine
the block size to the callers of the facility.
---
src/backend/access/transam/twophase.c | 3 +-
src/backend/access/transam/xlog.c | 9 ++--
src/backend/access/transam/xlogreader.c | 57 ++++++++++++-----------
src/backend/replication/logical/logical.c | 5 +-
src/backend/replication/walsender.c | 4 +-
src/bin/pg_rewind/parsexlog.c | 6 +--
src/bin/pg_waldump/pg_waldump.c | 2 +-
src/include/access/xlogreader.h | 3 +-
8 files changed, 50 insertions(+), 39 deletions(-)
diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index 7d5a96a5f5..9f9e7de253 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -1325,7 +1325,8 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
XLogReaderState *xlogreader;
char *errormsg;
- xlogreader = XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+ xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+ NULL, wal_segment_close);
if (!xlogreader)
ereport(ERROR,
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index b517ca85ad..0822ac70cb 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -1222,7 +1222,10 @@ XLogInsertRecord(XLogRecData *rdata,
appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len);
if (!debug_reader)
- debug_reader = XLogReaderAllocate(wal_segment_size, NULL, NULL);
+ {
+ debug_reader =
+ XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, NULL);
+ }
if (!debug_reader)
{
@@ -6531,8 +6534,8 @@ StartupXLOG(void)
OwnLatch(&XLogCtl->recoveryWakeupLatch);
/* Set up XLOG reader facility */
- xlogreader =
- XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+ xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+ NULL, wal_segment_close);
if (!xlogreader)
ereport(ERROR,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index ba3e4868f7..e5baf13086 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -72,7 +72,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...)
* Returns NULL if the xlogreader couldn't be allocated.
*/
XLogReaderState *
-XLogReaderAllocate(int wal_segment_size, const char *waldir,
+XLogReaderAllocate(int wal_segment_size, int page_size, const char *waldir,
WALSegmentCleanupCB cleanup_cb)
{
XLogReaderState *state;
@@ -87,6 +87,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
state->cleanup_cb = cleanup_cb;
state->max_block_id = -1;
+ state->pageSize = page_size;
/*
* Permanently allocate readBuf. We do it this way, rather than just
@@ -95,7 +96,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
* isn't guaranteed to have any particular alignment, whereas
* palloc_extended() will provide MAXALIGN'd storage.
*/
- state->readBuf = (char *) palloc_extended(XLOG_BLCKSZ,
+ state->readBuf = (char *) palloc_extended(state->pageSize,
MCXT_ALLOC_NO_OOM);
if (!state->readBuf)
{
@@ -163,7 +164,7 @@ XLogReaderFree(XLogReaderState *state)
* readRecordBufSize is set to the new buffer size.
*
* To avoid useless small increases, round its size to a multiple of
- * XLOG_BLCKSZ, and make sure it's at least 5*Max(BLCKSZ, XLOG_BLCKSZ) to start
+ * page size, and make sure it's at least 5*Max(BLCKSZ, <page size>) to start
* with. (That is enough for all "normal" records, but very large commit or
* abort records might need more space.)
*/
@@ -172,8 +173,8 @@ allocate_recordbuf(XLogReaderState *state, uint32 reclength)
{
uint32 newSize = reclength;
- newSize += XLOG_BLCKSZ - (newSize % XLOG_BLCKSZ);
- newSize = Max(newSize, 5 * Max(BLCKSZ, XLOG_BLCKSZ));
+ newSize += state->pageSize - (newSize % state->pageSize);
+ newSize = Max(newSize, 5 * Max(BLCKSZ, state->pageSize));
#ifndef FRONTEND
@@ -238,6 +239,8 @@ void
XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
{
Assert(!XLogRecPtrIsInvalid(RecPtr));
+ Assert(state->readBuf != NULL &&
+ state->readBuf == (void *) MAXALIGN(state->readBuf));
ResetDecoder(state);
@@ -366,15 +369,15 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
XLogPageHeader pageHeader;
targetPagePtr =
- state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
- targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+ state->ReadRecPtr - (state->ReadRecPtr % state->pageSize);
+ targetRecOff = state->ReadRecPtr % state->pageSize;
/*
* Check if we have enough data. For the first record in the page,
* the requesting length doesn't contain page header.
*/
if (XLogNeedData(state, targetPagePtr,
- Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+ Min(targetRecOff + SizeOfXLogRecord, state->pageSize),
targetRecOff != 0))
return XLREAD_NEED_DATA;
@@ -421,7 +424,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
* verified that we got the whole header.
*/
prec = (XLogRecord *) (state->readBuf +
- state->ReadRecPtr % XLOG_BLCKSZ);
+ state->ReadRecPtr % state->pageSize);
total_len = prec->xl_tot_len;
/*
@@ -432,7 +435,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
* ensure that we enter the XLREAD_CONTINUATION state below;
* otherwise we might fail to apply ValidXLogRecordHeader at all.
*/
- if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
+ if (targetRecOff <= state->pageSize - SizeOfXLogRecord)
{
if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
state->PrevRecPtr, prec))
@@ -477,10 +480,10 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
* available
*/
targetPagePtr =
- state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
- targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+ state->ReadRecPtr - (state->ReadRecPtr % state->pageSize);
+ targetRecOff = state->ReadRecPtr % state->pageSize;
- request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ);
+ request_len = Min(targetRecOff + total_len, state->pageSize);
record_len = request_len - targetRecOff;
/* ReadRecPtr contains page header */
@@ -549,7 +552,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
/* Calculate pointer to beginning of next page */
state->recordContRecPtr = state->ReadRecPtr + record_len;
- Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0);
+ Assert(state->recordContRecPtr % state->pageSize == 0);
state->readRecordState = XLREAD_CONTINUATION;
}
@@ -576,7 +579,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
/* this request contains page header */
Assert (targetPagePtr != 0);
if (XLogNeedData(state, targetPagePtr,
- Min(state->recordRemainLen, XLOG_BLCKSZ),
+ Min(state->recordRemainLen, state->pageSize),
false))
return XLREAD_NEED_DATA;
@@ -628,7 +631,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
Assert(state->readLen >= pageHeaderSize);
contdata = (char *) state->readBuf + pageHeaderSize;
- record_len = XLOG_BLCKSZ - pageHeaderSize;
+ record_len = state->pageSize - pageHeaderSize;
if (pageHeader->xlp_rem_len < record_len)
record_len = pageHeader->xlp_rem_len;
@@ -655,7 +658,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
}
/* Calculate pointer to beginning of next page, and continue */
- state->recordContRecPtr += XLOG_BLCKSZ;
+ state->recordContRecPtr += state->pageSize;
}
/* targetPagePtr is pointing the last-read page here */
@@ -789,10 +792,10 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
XLogPageHeaderSize((XLogPageHeader) state->readBuf);
addLen = pageHeaderSize;
- if (reqLen + pageHeaderSize <= XLOG_BLCKSZ)
+ if (reqLen + pageHeaderSize <= state->pageSize)
addLen = pageHeaderSize;
else
- addLen = XLOG_BLCKSZ - reqLen;
+ addLen = state->pageSize - reqLen;
Assert(addLen >= 0);
}
@@ -805,7 +808,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
/* Data is not in our buffer, request the caller for it. */
XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize);
targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize);
- Assert((pageptr % XLOG_BLCKSZ) == 0);
+ Assert((pageptr % state->pageSize) == 0);
/*
* Every time we request to load new data of a page to the caller, even if
@@ -829,7 +832,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
* will not come back here, but will request the actual target page.
*/
state->readPagePtr = pageptr - targetPageOff;
- state->readLen = XLOG_BLCKSZ;
+ state->readLen = state->pageSize;
return true;
}
@@ -962,7 +965,7 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
int32 offset;
XLogPageHeader hdr = (XLogPageHeader) phdr;
- Assert((recptr % XLOG_BLCKSZ) == 0);
+ Assert((recptr % state->pageSize) == 0);
XLByteToSeg(recptr, segno, state->segcxt.ws_segsize);
offset = XLogSegmentOffset(recptr, state->segcxt.ws_segsize);
@@ -1016,10 +1019,10 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
"WAL file is from different database system: incorrect segment size in page header");
return false;
}
- else if (longhdr->xlp_xlog_blcksz != XLOG_BLCKSZ)
+ else if (longhdr->xlp_xlog_blcksz != state->pageSize)
{
report_invalid_record(state,
- "WAL file is from different database system: incorrect XLOG_BLCKSZ in page header");
+ "WAL file is from different database system: incorrect block size in page header");
return false;
}
}
@@ -1140,7 +1143,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
* XLogNeedData() is prepared to handle that and will read at
* least short page-header worth of data
*/
- targetRecOff = tmpRecPtr % XLOG_BLCKSZ;
+ targetRecOff = tmpRecPtr % state->pageSize;
/* scroll back to page boundary */
targetPagePtr = tmpRecPtr - targetRecOff;
@@ -1174,8 +1177,8 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
*
* Note that record headers are MAXALIGN'ed
*/
- if (MAXALIGN(header->xlp_rem_len) >= (XLOG_BLCKSZ - pageHeaderSize))
- tmpRecPtr = targetPagePtr + XLOG_BLCKSZ;
+ if (MAXALIGN(header->xlp_rem_len) >= (state->pageSize - pageHeaderSize))
+ tmpRecPtr = targetPagePtr + state->pageSize;
else
{
/*
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index 9cbd8afa49..f2dd495310 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -199,11 +199,14 @@ StartupDecodingContext(List *output_plugin_options,
ctx->slot = slot;
- ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, cleanup_cb);
+ ctx->reader =
+ XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, cleanup_cb);
+
if (!ctx->reader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
errmsg("out of memory")));
+ ctx->reader->readBuf = palloc(XLOG_BLCKSZ);
ctx->page_read = page_read;
ctx->reorder = ReorderBufferAllocate();
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index 062093727d..8e95600ef7 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -579,8 +579,8 @@ StartReplication(StartReplicationCmd *cmd)
errmsg("IDENTIFY_SYSTEM has not been run before START_REPLICATION")));
/* create xlogreader for physical replication */
- xlogreader =
- XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+ xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+ NULL, wal_segment_close);
if (!xlogreader)
ereport(ERROR,
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index 712c85281c..8a6209b061 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -61,7 +61,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
XLogReaderState *xlogreader;
char *errormsg;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+ xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
@@ -121,7 +121,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex,
char *errormsg;
XLogRecPtr endptr;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+ xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
@@ -181,7 +181,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
forkptr += SizeOfXLogShortPHD;
}
- xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+ xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index f514de6b13..b635537659 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -1041,7 +1041,7 @@ main(int argc, char **argv)
/* we have everything we need, start reading */
xlogreader_state =
- XLogReaderAllocate(WalSegSz, waldir, WALDumpCloseSegment);
+ XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, waldir, WALDumpCloseSegment);
if (!xlogreader_state)
fatal_error("out of memory");
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index b9dad7ee51..1853516333 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -151,6 +151,7 @@ struct XLogReaderState
* ----------------------------------------
*/
/* variables to communicate with page reader */
+ int pageSize; /* size of a page */
XLogRecPtr readPagePtr; /* page pointer to read */
int32 readLen; /* bytes requested to reader, or actual bytes
* read by reader, which must be larger than
@@ -243,7 +244,7 @@ struct XLogReaderState
/* Get a new XLogReader */
extern XLogReaderState *XLogReaderAllocate(int wal_segment_size,
- const char *waldir,
+ int page_size, const char *waldir,
WALSegmentCleanupCB cleanup_cb);
/* Free an XLogReader */
--
2.27.0
----Next_Part(Thu_Mar__4_11_28_53_2021_014)----
^ permalink raw reply [nested|flat] 26+ messages in thread
* [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize.
@ 2019-09-05 12:29 Kyotaro Horiguchi <[email protected]>
0 siblings, 0 replies; 26+ messages in thread
From: Kyotaro Horiguchi @ 2019-09-05 12:29 UTC (permalink / raw)
XLog reader facility and its callers currently relies on the same
constant XLOG_BLCKSZ. This patch moves the responsibility to determine
the block size to the callers of the facility.
---
src/backend/access/transam/twophase.c | 3 +-
src/backend/access/transam/xlog.c | 9 ++--
src/backend/access/transam/xlogreader.c | 57 ++++++++++++-----------
src/backend/replication/logical/logical.c | 5 +-
src/backend/replication/walsender.c | 4 +-
src/bin/pg_rewind/parsexlog.c | 6 +--
src/bin/pg_waldump/pg_waldump.c | 2 +-
src/include/access/xlogreader.h | 3 +-
8 files changed, 50 insertions(+), 39 deletions(-)
diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index 7d5a96a5f5..9f9e7de253 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -1325,7 +1325,8 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
XLogReaderState *xlogreader;
char *errormsg;
- xlogreader = XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+ xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+ NULL, wal_segment_close);
if (!xlogreader)
ereport(ERROR,
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index b517ca85ad..0822ac70cb 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -1222,7 +1222,10 @@ XLogInsertRecord(XLogRecData *rdata,
appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len);
if (!debug_reader)
- debug_reader = XLogReaderAllocate(wal_segment_size, NULL, NULL);
+ {
+ debug_reader =
+ XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, NULL);
+ }
if (!debug_reader)
{
@@ -6531,8 +6534,8 @@ StartupXLOG(void)
OwnLatch(&XLogCtl->recoveryWakeupLatch);
/* Set up XLOG reader facility */
- xlogreader =
- XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+ xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+ NULL, wal_segment_close);
if (!xlogreader)
ereport(ERROR,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index ba3e4868f7..e5baf13086 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -72,7 +72,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...)
* Returns NULL if the xlogreader couldn't be allocated.
*/
XLogReaderState *
-XLogReaderAllocate(int wal_segment_size, const char *waldir,
+XLogReaderAllocate(int wal_segment_size, int page_size, const char *waldir,
WALSegmentCleanupCB cleanup_cb)
{
XLogReaderState *state;
@@ -87,6 +87,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
state->cleanup_cb = cleanup_cb;
state->max_block_id = -1;
+ state->pageSize = page_size;
/*
* Permanently allocate readBuf. We do it this way, rather than just
@@ -95,7 +96,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
* isn't guaranteed to have any particular alignment, whereas
* palloc_extended() will provide MAXALIGN'd storage.
*/
- state->readBuf = (char *) palloc_extended(XLOG_BLCKSZ,
+ state->readBuf = (char *) palloc_extended(state->pageSize,
MCXT_ALLOC_NO_OOM);
if (!state->readBuf)
{
@@ -163,7 +164,7 @@ XLogReaderFree(XLogReaderState *state)
* readRecordBufSize is set to the new buffer size.
*
* To avoid useless small increases, round its size to a multiple of
- * XLOG_BLCKSZ, and make sure it's at least 5*Max(BLCKSZ, XLOG_BLCKSZ) to start
+ * page size, and make sure it's at least 5*Max(BLCKSZ, <page size>) to start
* with. (That is enough for all "normal" records, but very large commit or
* abort records might need more space.)
*/
@@ -172,8 +173,8 @@ allocate_recordbuf(XLogReaderState *state, uint32 reclength)
{
uint32 newSize = reclength;
- newSize += XLOG_BLCKSZ - (newSize % XLOG_BLCKSZ);
- newSize = Max(newSize, 5 * Max(BLCKSZ, XLOG_BLCKSZ));
+ newSize += state->pageSize - (newSize % state->pageSize);
+ newSize = Max(newSize, 5 * Max(BLCKSZ, state->pageSize));
#ifndef FRONTEND
@@ -238,6 +239,8 @@ void
XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
{
Assert(!XLogRecPtrIsInvalid(RecPtr));
+ Assert(state->readBuf != NULL &&
+ state->readBuf == (void *) MAXALIGN(state->readBuf));
ResetDecoder(state);
@@ -366,15 +369,15 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
XLogPageHeader pageHeader;
targetPagePtr =
- state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
- targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+ state->ReadRecPtr - (state->ReadRecPtr % state->pageSize);
+ targetRecOff = state->ReadRecPtr % state->pageSize;
/*
* Check if we have enough data. For the first record in the page,
* the requesting length doesn't contain page header.
*/
if (XLogNeedData(state, targetPagePtr,
- Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+ Min(targetRecOff + SizeOfXLogRecord, state->pageSize),
targetRecOff != 0))
return XLREAD_NEED_DATA;
@@ -421,7 +424,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
* verified that we got the whole header.
*/
prec = (XLogRecord *) (state->readBuf +
- state->ReadRecPtr % XLOG_BLCKSZ);
+ state->ReadRecPtr % state->pageSize);
total_len = prec->xl_tot_len;
/*
@@ -432,7 +435,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
* ensure that we enter the XLREAD_CONTINUATION state below;
* otherwise we might fail to apply ValidXLogRecordHeader at all.
*/
- if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
+ if (targetRecOff <= state->pageSize - SizeOfXLogRecord)
{
if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
state->PrevRecPtr, prec))
@@ -477,10 +480,10 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
* available
*/
targetPagePtr =
- state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
- targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+ state->ReadRecPtr - (state->ReadRecPtr % state->pageSize);
+ targetRecOff = state->ReadRecPtr % state->pageSize;
- request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ);
+ request_len = Min(targetRecOff + total_len, state->pageSize);
record_len = request_len - targetRecOff;
/* ReadRecPtr contains page header */
@@ -549,7 +552,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
/* Calculate pointer to beginning of next page */
state->recordContRecPtr = state->ReadRecPtr + record_len;
- Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0);
+ Assert(state->recordContRecPtr % state->pageSize == 0);
state->readRecordState = XLREAD_CONTINUATION;
}
@@ -576,7 +579,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
/* this request contains page header */
Assert (targetPagePtr != 0);
if (XLogNeedData(state, targetPagePtr,
- Min(state->recordRemainLen, XLOG_BLCKSZ),
+ Min(state->recordRemainLen, state->pageSize),
false))
return XLREAD_NEED_DATA;
@@ -628,7 +631,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
Assert(state->readLen >= pageHeaderSize);
contdata = (char *) state->readBuf + pageHeaderSize;
- record_len = XLOG_BLCKSZ - pageHeaderSize;
+ record_len = state->pageSize - pageHeaderSize;
if (pageHeader->xlp_rem_len < record_len)
record_len = pageHeader->xlp_rem_len;
@@ -655,7 +658,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
}
/* Calculate pointer to beginning of next page, and continue */
- state->recordContRecPtr += XLOG_BLCKSZ;
+ state->recordContRecPtr += state->pageSize;
}
/* targetPagePtr is pointing the last-read page here */
@@ -789,10 +792,10 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
XLogPageHeaderSize((XLogPageHeader) state->readBuf);
addLen = pageHeaderSize;
- if (reqLen + pageHeaderSize <= XLOG_BLCKSZ)
+ if (reqLen + pageHeaderSize <= state->pageSize)
addLen = pageHeaderSize;
else
- addLen = XLOG_BLCKSZ - reqLen;
+ addLen = state->pageSize - reqLen;
Assert(addLen >= 0);
}
@@ -805,7 +808,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
/* Data is not in our buffer, request the caller for it. */
XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize);
targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize);
- Assert((pageptr % XLOG_BLCKSZ) == 0);
+ Assert((pageptr % state->pageSize) == 0);
/*
* Every time we request to load new data of a page to the caller, even if
@@ -829,7 +832,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
* will not come back here, but will request the actual target page.
*/
state->readPagePtr = pageptr - targetPageOff;
- state->readLen = XLOG_BLCKSZ;
+ state->readLen = state->pageSize;
return true;
}
@@ -962,7 +965,7 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
int32 offset;
XLogPageHeader hdr = (XLogPageHeader) phdr;
- Assert((recptr % XLOG_BLCKSZ) == 0);
+ Assert((recptr % state->pageSize) == 0);
XLByteToSeg(recptr, segno, state->segcxt.ws_segsize);
offset = XLogSegmentOffset(recptr, state->segcxt.ws_segsize);
@@ -1016,10 +1019,10 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
"WAL file is from different database system: incorrect segment size in page header");
return false;
}
- else if (longhdr->xlp_xlog_blcksz != XLOG_BLCKSZ)
+ else if (longhdr->xlp_xlog_blcksz != state->pageSize)
{
report_invalid_record(state,
- "WAL file is from different database system: incorrect XLOG_BLCKSZ in page header");
+ "WAL file is from different database system: incorrect block size in page header");
return false;
}
}
@@ -1140,7 +1143,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
* XLogNeedData() is prepared to handle that and will read at
* least short page-header worth of data
*/
- targetRecOff = tmpRecPtr % XLOG_BLCKSZ;
+ targetRecOff = tmpRecPtr % state->pageSize;
/* scroll back to page boundary */
targetPagePtr = tmpRecPtr - targetRecOff;
@@ -1174,8 +1177,8 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
*
* Note that record headers are MAXALIGN'ed
*/
- if (MAXALIGN(header->xlp_rem_len) >= (XLOG_BLCKSZ - pageHeaderSize))
- tmpRecPtr = targetPagePtr + XLOG_BLCKSZ;
+ if (MAXALIGN(header->xlp_rem_len) >= (state->pageSize - pageHeaderSize))
+ tmpRecPtr = targetPagePtr + state->pageSize;
else
{
/*
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index 9cbd8afa49..f2dd495310 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -199,11 +199,14 @@ StartupDecodingContext(List *output_plugin_options,
ctx->slot = slot;
- ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, cleanup_cb);
+ ctx->reader =
+ XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, cleanup_cb);
+
if (!ctx->reader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
errmsg("out of memory")));
+ ctx->reader->readBuf = palloc(XLOG_BLCKSZ);
ctx->page_read = page_read;
ctx->reorder = ReorderBufferAllocate();
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index 062093727d..8e95600ef7 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -579,8 +579,8 @@ StartReplication(StartReplicationCmd *cmd)
errmsg("IDENTIFY_SYSTEM has not been run before START_REPLICATION")));
/* create xlogreader for physical replication */
- xlogreader =
- XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+ xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+ NULL, wal_segment_close);
if (!xlogreader)
ereport(ERROR,
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index 712c85281c..8a6209b061 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -61,7 +61,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
XLogReaderState *xlogreader;
char *errormsg;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+ xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
@@ -121,7 +121,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex,
char *errormsg;
XLogRecPtr endptr;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+ xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
@@ -181,7 +181,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
forkptr += SizeOfXLogShortPHD;
}
- xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+ xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index f514de6b13..b635537659 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -1041,7 +1041,7 @@ main(int argc, char **argv)
/* we have everything we need, start reading */
xlogreader_state =
- XLogReaderAllocate(WalSegSz, waldir, WALDumpCloseSegment);
+ XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, waldir, WALDumpCloseSegment);
if (!xlogreader_state)
fatal_error("out of memory");
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index b9dad7ee51..1853516333 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -151,6 +151,7 @@ struct XLogReaderState
* ----------------------------------------
*/
/* variables to communicate with page reader */
+ int pageSize; /* size of a page */
XLogRecPtr readPagePtr; /* page pointer to read */
int32 readLen; /* bytes requested to reader, or actual bytes
* read by reader, which must be larger than
@@ -243,7 +244,7 @@ struct XLogReaderState
/* Get a new XLogReader */
extern XLogReaderState *XLogReaderAllocate(int wal_segment_size,
- const char *waldir,
+ int page_size, const char *waldir,
WALSegmentCleanupCB cleanup_cb);
/* Free an XLogReader */
--
2.27.0
----Next_Part(Thu_Mar__4_11_28_53_2021_014)----
^ permalink raw reply [nested|flat] 26+ messages in thread
* [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize.
@ 2019-09-05 12:29 Kyotaro Horiguchi <[email protected]>
0 siblings, 0 replies; 26+ messages in thread
From: Kyotaro Horiguchi @ 2019-09-05 12:29 UTC (permalink / raw)
XLog reader facility and its callers currently relies on the same
constant XLOG_BLCKSZ. This patch moves the responsibility to determine
the block size to the callers of the facility.
---
src/backend/access/transam/twophase.c | 3 +-
src/backend/access/transam/xlog.c | 9 ++--
src/backend/access/transam/xlogreader.c | 57 ++++++++++++-----------
src/backend/replication/logical/logical.c | 5 +-
src/backend/replication/walsender.c | 4 +-
src/bin/pg_rewind/parsexlog.c | 6 +--
src/bin/pg_waldump/pg_waldump.c | 2 +-
src/include/access/xlogreader.h | 3 +-
8 files changed, 50 insertions(+), 39 deletions(-)
diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index 7d5a96a5f5..9f9e7de253 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -1325,7 +1325,8 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
XLogReaderState *xlogreader;
char *errormsg;
- xlogreader = XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+ xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+ NULL, wal_segment_close);
if (!xlogreader)
ereport(ERROR,
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index b517ca85ad..0822ac70cb 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -1222,7 +1222,10 @@ XLogInsertRecord(XLogRecData *rdata,
appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len);
if (!debug_reader)
- debug_reader = XLogReaderAllocate(wal_segment_size, NULL, NULL);
+ {
+ debug_reader =
+ XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, NULL);
+ }
if (!debug_reader)
{
@@ -6531,8 +6534,8 @@ StartupXLOG(void)
OwnLatch(&XLogCtl->recoveryWakeupLatch);
/* Set up XLOG reader facility */
- xlogreader =
- XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+ xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+ NULL, wal_segment_close);
if (!xlogreader)
ereport(ERROR,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index ba3e4868f7..e5baf13086 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -72,7 +72,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...)
* Returns NULL if the xlogreader couldn't be allocated.
*/
XLogReaderState *
-XLogReaderAllocate(int wal_segment_size, const char *waldir,
+XLogReaderAllocate(int wal_segment_size, int page_size, const char *waldir,
WALSegmentCleanupCB cleanup_cb)
{
XLogReaderState *state;
@@ -87,6 +87,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
state->cleanup_cb = cleanup_cb;
state->max_block_id = -1;
+ state->pageSize = page_size;
/*
* Permanently allocate readBuf. We do it this way, rather than just
@@ -95,7 +96,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
* isn't guaranteed to have any particular alignment, whereas
* palloc_extended() will provide MAXALIGN'd storage.
*/
- state->readBuf = (char *) palloc_extended(XLOG_BLCKSZ,
+ state->readBuf = (char *) palloc_extended(state->pageSize,
MCXT_ALLOC_NO_OOM);
if (!state->readBuf)
{
@@ -163,7 +164,7 @@ XLogReaderFree(XLogReaderState *state)
* readRecordBufSize is set to the new buffer size.
*
* To avoid useless small increases, round its size to a multiple of
- * XLOG_BLCKSZ, and make sure it's at least 5*Max(BLCKSZ, XLOG_BLCKSZ) to start
+ * page size, and make sure it's at least 5*Max(BLCKSZ, <page size>) to start
* with. (That is enough for all "normal" records, but very large commit or
* abort records might need more space.)
*/
@@ -172,8 +173,8 @@ allocate_recordbuf(XLogReaderState *state, uint32 reclength)
{
uint32 newSize = reclength;
- newSize += XLOG_BLCKSZ - (newSize % XLOG_BLCKSZ);
- newSize = Max(newSize, 5 * Max(BLCKSZ, XLOG_BLCKSZ));
+ newSize += state->pageSize - (newSize % state->pageSize);
+ newSize = Max(newSize, 5 * Max(BLCKSZ, state->pageSize));
#ifndef FRONTEND
@@ -238,6 +239,8 @@ void
XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
{
Assert(!XLogRecPtrIsInvalid(RecPtr));
+ Assert(state->readBuf != NULL &&
+ state->readBuf == (void *) MAXALIGN(state->readBuf));
ResetDecoder(state);
@@ -366,15 +369,15 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
XLogPageHeader pageHeader;
targetPagePtr =
- state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
- targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+ state->ReadRecPtr - (state->ReadRecPtr % state->pageSize);
+ targetRecOff = state->ReadRecPtr % state->pageSize;
/*
* Check if we have enough data. For the first record in the page,
* the requesting length doesn't contain page header.
*/
if (XLogNeedData(state, targetPagePtr,
- Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+ Min(targetRecOff + SizeOfXLogRecord, state->pageSize),
targetRecOff != 0))
return XLREAD_NEED_DATA;
@@ -421,7 +424,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
* verified that we got the whole header.
*/
prec = (XLogRecord *) (state->readBuf +
- state->ReadRecPtr % XLOG_BLCKSZ);
+ state->ReadRecPtr % state->pageSize);
total_len = prec->xl_tot_len;
/*
@@ -432,7 +435,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
* ensure that we enter the XLREAD_CONTINUATION state below;
* otherwise we might fail to apply ValidXLogRecordHeader at all.
*/
- if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
+ if (targetRecOff <= state->pageSize - SizeOfXLogRecord)
{
if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
state->PrevRecPtr, prec))
@@ -477,10 +480,10 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
* available
*/
targetPagePtr =
- state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
- targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+ state->ReadRecPtr - (state->ReadRecPtr % state->pageSize);
+ targetRecOff = state->ReadRecPtr % state->pageSize;
- request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ);
+ request_len = Min(targetRecOff + total_len, state->pageSize);
record_len = request_len - targetRecOff;
/* ReadRecPtr contains page header */
@@ -549,7 +552,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
/* Calculate pointer to beginning of next page */
state->recordContRecPtr = state->ReadRecPtr + record_len;
- Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0);
+ Assert(state->recordContRecPtr % state->pageSize == 0);
state->readRecordState = XLREAD_CONTINUATION;
}
@@ -576,7 +579,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
/* this request contains page header */
Assert (targetPagePtr != 0);
if (XLogNeedData(state, targetPagePtr,
- Min(state->recordRemainLen, XLOG_BLCKSZ),
+ Min(state->recordRemainLen, state->pageSize),
false))
return XLREAD_NEED_DATA;
@@ -628,7 +631,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
Assert(state->readLen >= pageHeaderSize);
contdata = (char *) state->readBuf + pageHeaderSize;
- record_len = XLOG_BLCKSZ - pageHeaderSize;
+ record_len = state->pageSize - pageHeaderSize;
if (pageHeader->xlp_rem_len < record_len)
record_len = pageHeader->xlp_rem_len;
@@ -655,7 +658,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
}
/* Calculate pointer to beginning of next page, and continue */
- state->recordContRecPtr += XLOG_BLCKSZ;
+ state->recordContRecPtr += state->pageSize;
}
/* targetPagePtr is pointing the last-read page here */
@@ -789,10 +792,10 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
XLogPageHeaderSize((XLogPageHeader) state->readBuf);
addLen = pageHeaderSize;
- if (reqLen + pageHeaderSize <= XLOG_BLCKSZ)
+ if (reqLen + pageHeaderSize <= state->pageSize)
addLen = pageHeaderSize;
else
- addLen = XLOG_BLCKSZ - reqLen;
+ addLen = state->pageSize - reqLen;
Assert(addLen >= 0);
}
@@ -805,7 +808,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
/* Data is not in our buffer, request the caller for it. */
XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize);
targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize);
- Assert((pageptr % XLOG_BLCKSZ) == 0);
+ Assert((pageptr % state->pageSize) == 0);
/*
* Every time we request to load new data of a page to the caller, even if
@@ -829,7 +832,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
* will not come back here, but will request the actual target page.
*/
state->readPagePtr = pageptr - targetPageOff;
- state->readLen = XLOG_BLCKSZ;
+ state->readLen = state->pageSize;
return true;
}
@@ -962,7 +965,7 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
int32 offset;
XLogPageHeader hdr = (XLogPageHeader) phdr;
- Assert((recptr % XLOG_BLCKSZ) == 0);
+ Assert((recptr % state->pageSize) == 0);
XLByteToSeg(recptr, segno, state->segcxt.ws_segsize);
offset = XLogSegmentOffset(recptr, state->segcxt.ws_segsize);
@@ -1016,10 +1019,10 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
"WAL file is from different database system: incorrect segment size in page header");
return false;
}
- else if (longhdr->xlp_xlog_blcksz != XLOG_BLCKSZ)
+ else if (longhdr->xlp_xlog_blcksz != state->pageSize)
{
report_invalid_record(state,
- "WAL file is from different database system: incorrect XLOG_BLCKSZ in page header");
+ "WAL file is from different database system: incorrect block size in page header");
return false;
}
}
@@ -1140,7 +1143,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
* XLogNeedData() is prepared to handle that and will read at
* least short page-header worth of data
*/
- targetRecOff = tmpRecPtr % XLOG_BLCKSZ;
+ targetRecOff = tmpRecPtr % state->pageSize;
/* scroll back to page boundary */
targetPagePtr = tmpRecPtr - targetRecOff;
@@ -1174,8 +1177,8 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
*
* Note that record headers are MAXALIGN'ed
*/
- if (MAXALIGN(header->xlp_rem_len) >= (XLOG_BLCKSZ - pageHeaderSize))
- tmpRecPtr = targetPagePtr + XLOG_BLCKSZ;
+ if (MAXALIGN(header->xlp_rem_len) >= (state->pageSize - pageHeaderSize))
+ tmpRecPtr = targetPagePtr + state->pageSize;
else
{
/*
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index 9cbd8afa49..f2dd495310 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -199,11 +199,14 @@ StartupDecodingContext(List *output_plugin_options,
ctx->slot = slot;
- ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, cleanup_cb);
+ ctx->reader =
+ XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, cleanup_cb);
+
if (!ctx->reader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
errmsg("out of memory")));
+ ctx->reader->readBuf = palloc(XLOG_BLCKSZ);
ctx->page_read = page_read;
ctx->reorder = ReorderBufferAllocate();
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index 062093727d..8e95600ef7 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -579,8 +579,8 @@ StartReplication(StartReplicationCmd *cmd)
errmsg("IDENTIFY_SYSTEM has not been run before START_REPLICATION")));
/* create xlogreader for physical replication */
- xlogreader =
- XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+ xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+ NULL, wal_segment_close);
if (!xlogreader)
ereport(ERROR,
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index 712c85281c..8a6209b061 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -61,7 +61,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
XLogReaderState *xlogreader;
char *errormsg;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+ xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
@@ -121,7 +121,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex,
char *errormsg;
XLogRecPtr endptr;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+ xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
@@ -181,7 +181,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
forkptr += SizeOfXLogShortPHD;
}
- xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+ xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index f514de6b13..b635537659 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -1041,7 +1041,7 @@ main(int argc, char **argv)
/* we have everything we need, start reading */
xlogreader_state =
- XLogReaderAllocate(WalSegSz, waldir, WALDumpCloseSegment);
+ XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, waldir, WALDumpCloseSegment);
if (!xlogreader_state)
fatal_error("out of memory");
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index b9dad7ee51..1853516333 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -151,6 +151,7 @@ struct XLogReaderState
* ----------------------------------------
*/
/* variables to communicate with page reader */
+ int pageSize; /* size of a page */
XLogRecPtr readPagePtr; /* page pointer to read */
int32 readLen; /* bytes requested to reader, or actual bytes
* read by reader, which must be larger than
@@ -243,7 +244,7 @@ struct XLogReaderState
/* Get a new XLogReader */
extern XLogReaderState *XLogReaderAllocate(int wal_segment_size,
- const char *waldir,
+ int page_size, const char *waldir,
WALSegmentCleanupCB cleanup_cb);
/* Free an XLogReader */
--
2.27.0
----Next_Part(Thu_Mar__4_11_28_53_2021_014)----
^ permalink raw reply [nested|flat] 26+ messages in thread
* [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize.
@ 2019-09-05 12:29 Kyotaro Horiguchi <[email protected]>
0 siblings, 0 replies; 26+ messages in thread
From: Kyotaro Horiguchi @ 2019-09-05 12:29 UTC (permalink / raw)
XLog reader facility and its callers currently relies on the same
constant XLOG_BLCKSZ. This patch moves the responsibility to determine
the block size to the callers of the facility.
---
src/backend/access/transam/twophase.c | 3 +-
src/backend/access/transam/xlog.c | 9 ++--
src/backend/access/transam/xlogreader.c | 57 ++++++++++++-----------
src/backend/replication/logical/logical.c | 5 +-
src/backend/replication/walsender.c | 4 +-
src/bin/pg_rewind/parsexlog.c | 6 +--
src/bin/pg_waldump/pg_waldump.c | 2 +-
src/include/access/xlogreader.h | 3 +-
8 files changed, 50 insertions(+), 39 deletions(-)
diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index 7d5a96a5f5..9f9e7de253 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -1325,7 +1325,8 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
XLogReaderState *xlogreader;
char *errormsg;
- xlogreader = XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+ xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+ NULL, wal_segment_close);
if (!xlogreader)
ereport(ERROR,
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index b517ca85ad..0822ac70cb 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -1222,7 +1222,10 @@ XLogInsertRecord(XLogRecData *rdata,
appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len);
if (!debug_reader)
- debug_reader = XLogReaderAllocate(wal_segment_size, NULL, NULL);
+ {
+ debug_reader =
+ XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, NULL);
+ }
if (!debug_reader)
{
@@ -6531,8 +6534,8 @@ StartupXLOG(void)
OwnLatch(&XLogCtl->recoveryWakeupLatch);
/* Set up XLOG reader facility */
- xlogreader =
- XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+ xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+ NULL, wal_segment_close);
if (!xlogreader)
ereport(ERROR,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index ba3e4868f7..e5baf13086 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -72,7 +72,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...)
* Returns NULL if the xlogreader couldn't be allocated.
*/
XLogReaderState *
-XLogReaderAllocate(int wal_segment_size, const char *waldir,
+XLogReaderAllocate(int wal_segment_size, int page_size, const char *waldir,
WALSegmentCleanupCB cleanup_cb)
{
XLogReaderState *state;
@@ -87,6 +87,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
state->cleanup_cb = cleanup_cb;
state->max_block_id = -1;
+ state->pageSize = page_size;
/*
* Permanently allocate readBuf. We do it this way, rather than just
@@ -95,7 +96,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
* isn't guaranteed to have any particular alignment, whereas
* palloc_extended() will provide MAXALIGN'd storage.
*/
- state->readBuf = (char *) palloc_extended(XLOG_BLCKSZ,
+ state->readBuf = (char *) palloc_extended(state->pageSize,
MCXT_ALLOC_NO_OOM);
if (!state->readBuf)
{
@@ -163,7 +164,7 @@ XLogReaderFree(XLogReaderState *state)
* readRecordBufSize is set to the new buffer size.
*
* To avoid useless small increases, round its size to a multiple of
- * XLOG_BLCKSZ, and make sure it's at least 5*Max(BLCKSZ, XLOG_BLCKSZ) to start
+ * page size, and make sure it's at least 5*Max(BLCKSZ, <page size>) to start
* with. (That is enough for all "normal" records, but very large commit or
* abort records might need more space.)
*/
@@ -172,8 +173,8 @@ allocate_recordbuf(XLogReaderState *state, uint32 reclength)
{
uint32 newSize = reclength;
- newSize += XLOG_BLCKSZ - (newSize % XLOG_BLCKSZ);
- newSize = Max(newSize, 5 * Max(BLCKSZ, XLOG_BLCKSZ));
+ newSize += state->pageSize - (newSize % state->pageSize);
+ newSize = Max(newSize, 5 * Max(BLCKSZ, state->pageSize));
#ifndef FRONTEND
@@ -238,6 +239,8 @@ void
XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
{
Assert(!XLogRecPtrIsInvalid(RecPtr));
+ Assert(state->readBuf != NULL &&
+ state->readBuf == (void *) MAXALIGN(state->readBuf));
ResetDecoder(state);
@@ -366,15 +369,15 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
XLogPageHeader pageHeader;
targetPagePtr =
- state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
- targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+ state->ReadRecPtr - (state->ReadRecPtr % state->pageSize);
+ targetRecOff = state->ReadRecPtr % state->pageSize;
/*
* Check if we have enough data. For the first record in the page,
* the requesting length doesn't contain page header.
*/
if (XLogNeedData(state, targetPagePtr,
- Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+ Min(targetRecOff + SizeOfXLogRecord, state->pageSize),
targetRecOff != 0))
return XLREAD_NEED_DATA;
@@ -421,7 +424,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
* verified that we got the whole header.
*/
prec = (XLogRecord *) (state->readBuf +
- state->ReadRecPtr % XLOG_BLCKSZ);
+ state->ReadRecPtr % state->pageSize);
total_len = prec->xl_tot_len;
/*
@@ -432,7 +435,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
* ensure that we enter the XLREAD_CONTINUATION state below;
* otherwise we might fail to apply ValidXLogRecordHeader at all.
*/
- if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
+ if (targetRecOff <= state->pageSize - SizeOfXLogRecord)
{
if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
state->PrevRecPtr, prec))
@@ -477,10 +480,10 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
* available
*/
targetPagePtr =
- state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
- targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+ state->ReadRecPtr - (state->ReadRecPtr % state->pageSize);
+ targetRecOff = state->ReadRecPtr % state->pageSize;
- request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ);
+ request_len = Min(targetRecOff + total_len, state->pageSize);
record_len = request_len - targetRecOff;
/* ReadRecPtr contains page header */
@@ -549,7 +552,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
/* Calculate pointer to beginning of next page */
state->recordContRecPtr = state->ReadRecPtr + record_len;
- Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0);
+ Assert(state->recordContRecPtr % state->pageSize == 0);
state->readRecordState = XLREAD_CONTINUATION;
}
@@ -576,7 +579,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
/* this request contains page header */
Assert (targetPagePtr != 0);
if (XLogNeedData(state, targetPagePtr,
- Min(state->recordRemainLen, XLOG_BLCKSZ),
+ Min(state->recordRemainLen, state->pageSize),
false))
return XLREAD_NEED_DATA;
@@ -628,7 +631,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
Assert(state->readLen >= pageHeaderSize);
contdata = (char *) state->readBuf + pageHeaderSize;
- record_len = XLOG_BLCKSZ - pageHeaderSize;
+ record_len = state->pageSize - pageHeaderSize;
if (pageHeader->xlp_rem_len < record_len)
record_len = pageHeader->xlp_rem_len;
@@ -655,7 +658,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
}
/* Calculate pointer to beginning of next page, and continue */
- state->recordContRecPtr += XLOG_BLCKSZ;
+ state->recordContRecPtr += state->pageSize;
}
/* targetPagePtr is pointing the last-read page here */
@@ -789,10 +792,10 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
XLogPageHeaderSize((XLogPageHeader) state->readBuf);
addLen = pageHeaderSize;
- if (reqLen + pageHeaderSize <= XLOG_BLCKSZ)
+ if (reqLen + pageHeaderSize <= state->pageSize)
addLen = pageHeaderSize;
else
- addLen = XLOG_BLCKSZ - reqLen;
+ addLen = state->pageSize - reqLen;
Assert(addLen >= 0);
}
@@ -805,7 +808,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
/* Data is not in our buffer, request the caller for it. */
XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize);
targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize);
- Assert((pageptr % XLOG_BLCKSZ) == 0);
+ Assert((pageptr % state->pageSize) == 0);
/*
* Every time we request to load new data of a page to the caller, even if
@@ -829,7 +832,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
* will not come back here, but will request the actual target page.
*/
state->readPagePtr = pageptr - targetPageOff;
- state->readLen = XLOG_BLCKSZ;
+ state->readLen = state->pageSize;
return true;
}
@@ -962,7 +965,7 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
int32 offset;
XLogPageHeader hdr = (XLogPageHeader) phdr;
- Assert((recptr % XLOG_BLCKSZ) == 0);
+ Assert((recptr % state->pageSize) == 0);
XLByteToSeg(recptr, segno, state->segcxt.ws_segsize);
offset = XLogSegmentOffset(recptr, state->segcxt.ws_segsize);
@@ -1016,10 +1019,10 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
"WAL file is from different database system: incorrect segment size in page header");
return false;
}
- else if (longhdr->xlp_xlog_blcksz != XLOG_BLCKSZ)
+ else if (longhdr->xlp_xlog_blcksz != state->pageSize)
{
report_invalid_record(state,
- "WAL file is from different database system: incorrect XLOG_BLCKSZ in page header");
+ "WAL file is from different database system: incorrect block size in page header");
return false;
}
}
@@ -1140,7 +1143,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
* XLogNeedData() is prepared to handle that and will read at
* least short page-header worth of data
*/
- targetRecOff = tmpRecPtr % XLOG_BLCKSZ;
+ targetRecOff = tmpRecPtr % state->pageSize;
/* scroll back to page boundary */
targetPagePtr = tmpRecPtr - targetRecOff;
@@ -1174,8 +1177,8 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
*
* Note that record headers are MAXALIGN'ed
*/
- if (MAXALIGN(header->xlp_rem_len) >= (XLOG_BLCKSZ - pageHeaderSize))
- tmpRecPtr = targetPagePtr + XLOG_BLCKSZ;
+ if (MAXALIGN(header->xlp_rem_len) >= (state->pageSize - pageHeaderSize))
+ tmpRecPtr = targetPagePtr + state->pageSize;
else
{
/*
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index 9cbd8afa49..f2dd495310 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -199,11 +199,14 @@ StartupDecodingContext(List *output_plugin_options,
ctx->slot = slot;
- ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, cleanup_cb);
+ ctx->reader =
+ XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, cleanup_cb);
+
if (!ctx->reader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
errmsg("out of memory")));
+ ctx->reader->readBuf = palloc(XLOG_BLCKSZ);
ctx->page_read = page_read;
ctx->reorder = ReorderBufferAllocate();
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index 062093727d..8e95600ef7 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -579,8 +579,8 @@ StartReplication(StartReplicationCmd *cmd)
errmsg("IDENTIFY_SYSTEM has not been run before START_REPLICATION")));
/* create xlogreader for physical replication */
- xlogreader =
- XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+ xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+ NULL, wal_segment_close);
if (!xlogreader)
ereport(ERROR,
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index 712c85281c..8a6209b061 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -61,7 +61,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
XLogReaderState *xlogreader;
char *errormsg;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+ xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
@@ -121,7 +121,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex,
char *errormsg;
XLogRecPtr endptr;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+ xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
@@ -181,7 +181,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
forkptr += SizeOfXLogShortPHD;
}
- xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+ xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index f514de6b13..b635537659 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -1041,7 +1041,7 @@ main(int argc, char **argv)
/* we have everything we need, start reading */
xlogreader_state =
- XLogReaderAllocate(WalSegSz, waldir, WALDumpCloseSegment);
+ XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, waldir, WALDumpCloseSegment);
if (!xlogreader_state)
fatal_error("out of memory");
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index b9dad7ee51..1853516333 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -151,6 +151,7 @@ struct XLogReaderState
* ----------------------------------------
*/
/* variables to communicate with page reader */
+ int pageSize; /* size of a page */
XLogRecPtr readPagePtr; /* page pointer to read */
int32 readLen; /* bytes requested to reader, or actual bytes
* read by reader, which must be larger than
@@ -243,7 +244,7 @@ struct XLogReaderState
/* Get a new XLogReader */
extern XLogReaderState *XLogReaderAllocate(int wal_segment_size,
- const char *waldir,
+ int page_size, const char *waldir,
WALSegmentCleanupCB cleanup_cb);
/* Free an XLogReader */
--
2.27.0
----Next_Part(Thu_Mar__4_11_28_53_2021_014)----
^ permalink raw reply [nested|flat] 26+ messages in thread
* [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize.
@ 2019-09-05 12:29 Kyotaro Horiguchi <[email protected]>
0 siblings, 0 replies; 26+ messages in thread
From: Kyotaro Horiguchi @ 2019-09-05 12:29 UTC (permalink / raw)
XLog reader facility and its callers currently relies on the same
constant XLOG_BLCKSZ. This patch moves the responsibility to determine
the block size to the callers of the facility.
---
src/backend/access/transam/twophase.c | 3 +-
src/backend/access/transam/xlog.c | 9 ++--
src/backend/access/transam/xlogreader.c | 57 ++++++++++++-----------
src/backend/replication/logical/logical.c | 5 +-
src/backend/replication/walsender.c | 4 +-
src/bin/pg_rewind/parsexlog.c | 6 +--
src/bin/pg_waldump/pg_waldump.c | 2 +-
src/include/access/xlogreader.h | 3 +-
8 files changed, 50 insertions(+), 39 deletions(-)
diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index 7d5a96a5f5..9f9e7de253 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -1325,7 +1325,8 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
XLogReaderState *xlogreader;
char *errormsg;
- xlogreader = XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+ xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+ NULL, wal_segment_close);
if (!xlogreader)
ereport(ERROR,
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index b517ca85ad..0822ac70cb 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -1222,7 +1222,10 @@ XLogInsertRecord(XLogRecData *rdata,
appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len);
if (!debug_reader)
- debug_reader = XLogReaderAllocate(wal_segment_size, NULL, NULL);
+ {
+ debug_reader =
+ XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, NULL);
+ }
if (!debug_reader)
{
@@ -6531,8 +6534,8 @@ StartupXLOG(void)
OwnLatch(&XLogCtl->recoveryWakeupLatch);
/* Set up XLOG reader facility */
- xlogreader =
- XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+ xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+ NULL, wal_segment_close);
if (!xlogreader)
ereport(ERROR,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index ba3e4868f7..e5baf13086 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -72,7 +72,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...)
* Returns NULL if the xlogreader couldn't be allocated.
*/
XLogReaderState *
-XLogReaderAllocate(int wal_segment_size, const char *waldir,
+XLogReaderAllocate(int wal_segment_size, int page_size, const char *waldir,
WALSegmentCleanupCB cleanup_cb)
{
XLogReaderState *state;
@@ -87,6 +87,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
state->cleanup_cb = cleanup_cb;
state->max_block_id = -1;
+ state->pageSize = page_size;
/*
* Permanently allocate readBuf. We do it this way, rather than just
@@ -95,7 +96,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
* isn't guaranteed to have any particular alignment, whereas
* palloc_extended() will provide MAXALIGN'd storage.
*/
- state->readBuf = (char *) palloc_extended(XLOG_BLCKSZ,
+ state->readBuf = (char *) palloc_extended(state->pageSize,
MCXT_ALLOC_NO_OOM);
if (!state->readBuf)
{
@@ -163,7 +164,7 @@ XLogReaderFree(XLogReaderState *state)
* readRecordBufSize is set to the new buffer size.
*
* To avoid useless small increases, round its size to a multiple of
- * XLOG_BLCKSZ, and make sure it's at least 5*Max(BLCKSZ, XLOG_BLCKSZ) to start
+ * page size, and make sure it's at least 5*Max(BLCKSZ, <page size>) to start
* with. (That is enough for all "normal" records, but very large commit or
* abort records might need more space.)
*/
@@ -172,8 +173,8 @@ allocate_recordbuf(XLogReaderState *state, uint32 reclength)
{
uint32 newSize = reclength;
- newSize += XLOG_BLCKSZ - (newSize % XLOG_BLCKSZ);
- newSize = Max(newSize, 5 * Max(BLCKSZ, XLOG_BLCKSZ));
+ newSize += state->pageSize - (newSize % state->pageSize);
+ newSize = Max(newSize, 5 * Max(BLCKSZ, state->pageSize));
#ifndef FRONTEND
@@ -238,6 +239,8 @@ void
XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
{
Assert(!XLogRecPtrIsInvalid(RecPtr));
+ Assert(state->readBuf != NULL &&
+ state->readBuf == (void *) MAXALIGN(state->readBuf));
ResetDecoder(state);
@@ -366,15 +369,15 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
XLogPageHeader pageHeader;
targetPagePtr =
- state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
- targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+ state->ReadRecPtr - (state->ReadRecPtr % state->pageSize);
+ targetRecOff = state->ReadRecPtr % state->pageSize;
/*
* Check if we have enough data. For the first record in the page,
* the requesting length doesn't contain page header.
*/
if (XLogNeedData(state, targetPagePtr,
- Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+ Min(targetRecOff + SizeOfXLogRecord, state->pageSize),
targetRecOff != 0))
return XLREAD_NEED_DATA;
@@ -421,7 +424,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
* verified that we got the whole header.
*/
prec = (XLogRecord *) (state->readBuf +
- state->ReadRecPtr % XLOG_BLCKSZ);
+ state->ReadRecPtr % state->pageSize);
total_len = prec->xl_tot_len;
/*
@@ -432,7 +435,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
* ensure that we enter the XLREAD_CONTINUATION state below;
* otherwise we might fail to apply ValidXLogRecordHeader at all.
*/
- if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
+ if (targetRecOff <= state->pageSize - SizeOfXLogRecord)
{
if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
state->PrevRecPtr, prec))
@@ -477,10 +480,10 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
* available
*/
targetPagePtr =
- state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
- targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+ state->ReadRecPtr - (state->ReadRecPtr % state->pageSize);
+ targetRecOff = state->ReadRecPtr % state->pageSize;
- request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ);
+ request_len = Min(targetRecOff + total_len, state->pageSize);
record_len = request_len - targetRecOff;
/* ReadRecPtr contains page header */
@@ -549,7 +552,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
/* Calculate pointer to beginning of next page */
state->recordContRecPtr = state->ReadRecPtr + record_len;
- Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0);
+ Assert(state->recordContRecPtr % state->pageSize == 0);
state->readRecordState = XLREAD_CONTINUATION;
}
@@ -576,7 +579,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
/* this request contains page header */
Assert (targetPagePtr != 0);
if (XLogNeedData(state, targetPagePtr,
- Min(state->recordRemainLen, XLOG_BLCKSZ),
+ Min(state->recordRemainLen, state->pageSize),
false))
return XLREAD_NEED_DATA;
@@ -628,7 +631,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
Assert(state->readLen >= pageHeaderSize);
contdata = (char *) state->readBuf + pageHeaderSize;
- record_len = XLOG_BLCKSZ - pageHeaderSize;
+ record_len = state->pageSize - pageHeaderSize;
if (pageHeader->xlp_rem_len < record_len)
record_len = pageHeader->xlp_rem_len;
@@ -655,7 +658,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
}
/* Calculate pointer to beginning of next page, and continue */
- state->recordContRecPtr += XLOG_BLCKSZ;
+ state->recordContRecPtr += state->pageSize;
}
/* targetPagePtr is pointing the last-read page here */
@@ -789,10 +792,10 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
XLogPageHeaderSize((XLogPageHeader) state->readBuf);
addLen = pageHeaderSize;
- if (reqLen + pageHeaderSize <= XLOG_BLCKSZ)
+ if (reqLen + pageHeaderSize <= state->pageSize)
addLen = pageHeaderSize;
else
- addLen = XLOG_BLCKSZ - reqLen;
+ addLen = state->pageSize - reqLen;
Assert(addLen >= 0);
}
@@ -805,7 +808,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
/* Data is not in our buffer, request the caller for it. */
XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize);
targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize);
- Assert((pageptr % XLOG_BLCKSZ) == 0);
+ Assert((pageptr % state->pageSize) == 0);
/*
* Every time we request to load new data of a page to the caller, even if
@@ -829,7 +832,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
* will not come back here, but will request the actual target page.
*/
state->readPagePtr = pageptr - targetPageOff;
- state->readLen = XLOG_BLCKSZ;
+ state->readLen = state->pageSize;
return true;
}
@@ -962,7 +965,7 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
int32 offset;
XLogPageHeader hdr = (XLogPageHeader) phdr;
- Assert((recptr % XLOG_BLCKSZ) == 0);
+ Assert((recptr % state->pageSize) == 0);
XLByteToSeg(recptr, segno, state->segcxt.ws_segsize);
offset = XLogSegmentOffset(recptr, state->segcxt.ws_segsize);
@@ -1016,10 +1019,10 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
"WAL file is from different database system: incorrect segment size in page header");
return false;
}
- else if (longhdr->xlp_xlog_blcksz != XLOG_BLCKSZ)
+ else if (longhdr->xlp_xlog_blcksz != state->pageSize)
{
report_invalid_record(state,
- "WAL file is from different database system: incorrect XLOG_BLCKSZ in page header");
+ "WAL file is from different database system: incorrect block size in page header");
return false;
}
}
@@ -1140,7 +1143,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
* XLogNeedData() is prepared to handle that and will read at
* least short page-header worth of data
*/
- targetRecOff = tmpRecPtr % XLOG_BLCKSZ;
+ targetRecOff = tmpRecPtr % state->pageSize;
/* scroll back to page boundary */
targetPagePtr = tmpRecPtr - targetRecOff;
@@ -1174,8 +1177,8 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
*
* Note that record headers are MAXALIGN'ed
*/
- if (MAXALIGN(header->xlp_rem_len) >= (XLOG_BLCKSZ - pageHeaderSize))
- tmpRecPtr = targetPagePtr + XLOG_BLCKSZ;
+ if (MAXALIGN(header->xlp_rem_len) >= (state->pageSize - pageHeaderSize))
+ tmpRecPtr = targetPagePtr + state->pageSize;
else
{
/*
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index 9cbd8afa49..f2dd495310 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -199,11 +199,14 @@ StartupDecodingContext(List *output_plugin_options,
ctx->slot = slot;
- ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, cleanup_cb);
+ ctx->reader =
+ XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, cleanup_cb);
+
if (!ctx->reader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
errmsg("out of memory")));
+ ctx->reader->readBuf = palloc(XLOG_BLCKSZ);
ctx->page_read = page_read;
ctx->reorder = ReorderBufferAllocate();
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index 062093727d..8e95600ef7 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -579,8 +579,8 @@ StartReplication(StartReplicationCmd *cmd)
errmsg("IDENTIFY_SYSTEM has not been run before START_REPLICATION")));
/* create xlogreader for physical replication */
- xlogreader =
- XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+ xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+ NULL, wal_segment_close);
if (!xlogreader)
ereport(ERROR,
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index 712c85281c..8a6209b061 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -61,7 +61,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
XLogReaderState *xlogreader;
char *errormsg;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+ xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
@@ -121,7 +121,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex,
char *errormsg;
XLogRecPtr endptr;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+ xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
@@ -181,7 +181,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
forkptr += SizeOfXLogShortPHD;
}
- xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+ xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index f514de6b13..b635537659 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -1041,7 +1041,7 @@ main(int argc, char **argv)
/* we have everything we need, start reading */
xlogreader_state =
- XLogReaderAllocate(WalSegSz, waldir, WALDumpCloseSegment);
+ XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, waldir, WALDumpCloseSegment);
if (!xlogreader_state)
fatal_error("out of memory");
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index b9dad7ee51..1853516333 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -151,6 +151,7 @@ struct XLogReaderState
* ----------------------------------------
*/
/* variables to communicate with page reader */
+ int pageSize; /* size of a page */
XLogRecPtr readPagePtr; /* page pointer to read */
int32 readLen; /* bytes requested to reader, or actual bytes
* read by reader, which must be larger than
@@ -243,7 +244,7 @@ struct XLogReaderState
/* Get a new XLogReader */
extern XLogReaderState *XLogReaderAllocate(int wal_segment_size,
- const char *waldir,
+ int page_size, const char *waldir,
WALSegmentCleanupCB cleanup_cb);
/* Free an XLogReader */
--
2.27.0
----Next_Part(Thu_Mar__4_11_28_53_2021_014)----
^ permalink raw reply [nested|flat] 26+ messages in thread
* [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize.
@ 2019-09-05 12:29 Kyotaro Horiguchi <[email protected]>
0 siblings, 0 replies; 26+ messages in thread
From: Kyotaro Horiguchi @ 2019-09-05 12:29 UTC (permalink / raw)
XLog reader facility and its callers currently relies on the same
constant XLOG_BLCKSZ. This patch moves the responsibility to determine
the block size to the callers of the facility.
---
src/backend/access/transam/twophase.c | 3 +-
src/backend/access/transam/xlog.c | 9 ++--
src/backend/access/transam/xlogreader.c | 57 ++++++++++++-----------
src/backend/replication/logical/logical.c | 5 +-
src/backend/replication/walsender.c | 4 +-
src/bin/pg_rewind/parsexlog.c | 6 +--
src/bin/pg_waldump/pg_waldump.c | 2 +-
src/include/access/xlogreader.h | 3 +-
8 files changed, 50 insertions(+), 39 deletions(-)
diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index 7d5a96a5f5..9f9e7de253 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -1325,7 +1325,8 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
XLogReaderState *xlogreader;
char *errormsg;
- xlogreader = XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+ xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+ NULL, wal_segment_close);
if (!xlogreader)
ereport(ERROR,
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index b517ca85ad..0822ac70cb 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -1222,7 +1222,10 @@ XLogInsertRecord(XLogRecData *rdata,
appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len);
if (!debug_reader)
- debug_reader = XLogReaderAllocate(wal_segment_size, NULL, NULL);
+ {
+ debug_reader =
+ XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, NULL);
+ }
if (!debug_reader)
{
@@ -6531,8 +6534,8 @@ StartupXLOG(void)
OwnLatch(&XLogCtl->recoveryWakeupLatch);
/* Set up XLOG reader facility */
- xlogreader =
- XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+ xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+ NULL, wal_segment_close);
if (!xlogreader)
ereport(ERROR,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index ba3e4868f7..e5baf13086 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -72,7 +72,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...)
* Returns NULL if the xlogreader couldn't be allocated.
*/
XLogReaderState *
-XLogReaderAllocate(int wal_segment_size, const char *waldir,
+XLogReaderAllocate(int wal_segment_size, int page_size, const char *waldir,
WALSegmentCleanupCB cleanup_cb)
{
XLogReaderState *state;
@@ -87,6 +87,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
state->cleanup_cb = cleanup_cb;
state->max_block_id = -1;
+ state->pageSize = page_size;
/*
* Permanently allocate readBuf. We do it this way, rather than just
@@ -95,7 +96,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
* isn't guaranteed to have any particular alignment, whereas
* palloc_extended() will provide MAXALIGN'd storage.
*/
- state->readBuf = (char *) palloc_extended(XLOG_BLCKSZ,
+ state->readBuf = (char *) palloc_extended(state->pageSize,
MCXT_ALLOC_NO_OOM);
if (!state->readBuf)
{
@@ -163,7 +164,7 @@ XLogReaderFree(XLogReaderState *state)
* readRecordBufSize is set to the new buffer size.
*
* To avoid useless small increases, round its size to a multiple of
- * XLOG_BLCKSZ, and make sure it's at least 5*Max(BLCKSZ, XLOG_BLCKSZ) to start
+ * page size, and make sure it's at least 5*Max(BLCKSZ, <page size>) to start
* with. (That is enough for all "normal" records, but very large commit or
* abort records might need more space.)
*/
@@ -172,8 +173,8 @@ allocate_recordbuf(XLogReaderState *state, uint32 reclength)
{
uint32 newSize = reclength;
- newSize += XLOG_BLCKSZ - (newSize % XLOG_BLCKSZ);
- newSize = Max(newSize, 5 * Max(BLCKSZ, XLOG_BLCKSZ));
+ newSize += state->pageSize - (newSize % state->pageSize);
+ newSize = Max(newSize, 5 * Max(BLCKSZ, state->pageSize));
#ifndef FRONTEND
@@ -238,6 +239,8 @@ void
XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
{
Assert(!XLogRecPtrIsInvalid(RecPtr));
+ Assert(state->readBuf != NULL &&
+ state->readBuf == (void *) MAXALIGN(state->readBuf));
ResetDecoder(state);
@@ -366,15 +369,15 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
XLogPageHeader pageHeader;
targetPagePtr =
- state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
- targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+ state->ReadRecPtr - (state->ReadRecPtr % state->pageSize);
+ targetRecOff = state->ReadRecPtr % state->pageSize;
/*
* Check if we have enough data. For the first record in the page,
* the requesting length doesn't contain page header.
*/
if (XLogNeedData(state, targetPagePtr,
- Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+ Min(targetRecOff + SizeOfXLogRecord, state->pageSize),
targetRecOff != 0))
return XLREAD_NEED_DATA;
@@ -421,7 +424,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
* verified that we got the whole header.
*/
prec = (XLogRecord *) (state->readBuf +
- state->ReadRecPtr % XLOG_BLCKSZ);
+ state->ReadRecPtr % state->pageSize);
total_len = prec->xl_tot_len;
/*
@@ -432,7 +435,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
* ensure that we enter the XLREAD_CONTINUATION state below;
* otherwise we might fail to apply ValidXLogRecordHeader at all.
*/
- if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
+ if (targetRecOff <= state->pageSize - SizeOfXLogRecord)
{
if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
state->PrevRecPtr, prec))
@@ -477,10 +480,10 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
* available
*/
targetPagePtr =
- state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
- targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+ state->ReadRecPtr - (state->ReadRecPtr % state->pageSize);
+ targetRecOff = state->ReadRecPtr % state->pageSize;
- request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ);
+ request_len = Min(targetRecOff + total_len, state->pageSize);
record_len = request_len - targetRecOff;
/* ReadRecPtr contains page header */
@@ -549,7 +552,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
/* Calculate pointer to beginning of next page */
state->recordContRecPtr = state->ReadRecPtr + record_len;
- Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0);
+ Assert(state->recordContRecPtr % state->pageSize == 0);
state->readRecordState = XLREAD_CONTINUATION;
}
@@ -576,7 +579,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
/* this request contains page header */
Assert (targetPagePtr != 0);
if (XLogNeedData(state, targetPagePtr,
- Min(state->recordRemainLen, XLOG_BLCKSZ),
+ Min(state->recordRemainLen, state->pageSize),
false))
return XLREAD_NEED_DATA;
@@ -628,7 +631,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
Assert(state->readLen >= pageHeaderSize);
contdata = (char *) state->readBuf + pageHeaderSize;
- record_len = XLOG_BLCKSZ - pageHeaderSize;
+ record_len = state->pageSize - pageHeaderSize;
if (pageHeader->xlp_rem_len < record_len)
record_len = pageHeader->xlp_rem_len;
@@ -655,7 +658,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
}
/* Calculate pointer to beginning of next page, and continue */
- state->recordContRecPtr += XLOG_BLCKSZ;
+ state->recordContRecPtr += state->pageSize;
}
/* targetPagePtr is pointing the last-read page here */
@@ -789,10 +792,10 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
XLogPageHeaderSize((XLogPageHeader) state->readBuf);
addLen = pageHeaderSize;
- if (reqLen + pageHeaderSize <= XLOG_BLCKSZ)
+ if (reqLen + pageHeaderSize <= state->pageSize)
addLen = pageHeaderSize;
else
- addLen = XLOG_BLCKSZ - reqLen;
+ addLen = state->pageSize - reqLen;
Assert(addLen >= 0);
}
@@ -805,7 +808,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
/* Data is not in our buffer, request the caller for it. */
XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize);
targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize);
- Assert((pageptr % XLOG_BLCKSZ) == 0);
+ Assert((pageptr % state->pageSize) == 0);
/*
* Every time we request to load new data of a page to the caller, even if
@@ -829,7 +832,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
* will not come back here, but will request the actual target page.
*/
state->readPagePtr = pageptr - targetPageOff;
- state->readLen = XLOG_BLCKSZ;
+ state->readLen = state->pageSize;
return true;
}
@@ -962,7 +965,7 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
int32 offset;
XLogPageHeader hdr = (XLogPageHeader) phdr;
- Assert((recptr % XLOG_BLCKSZ) == 0);
+ Assert((recptr % state->pageSize) == 0);
XLByteToSeg(recptr, segno, state->segcxt.ws_segsize);
offset = XLogSegmentOffset(recptr, state->segcxt.ws_segsize);
@@ -1016,10 +1019,10 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
"WAL file is from different database system: incorrect segment size in page header");
return false;
}
- else if (longhdr->xlp_xlog_blcksz != XLOG_BLCKSZ)
+ else if (longhdr->xlp_xlog_blcksz != state->pageSize)
{
report_invalid_record(state,
- "WAL file is from different database system: incorrect XLOG_BLCKSZ in page header");
+ "WAL file is from different database system: incorrect block size in page header");
return false;
}
}
@@ -1140,7 +1143,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
* XLogNeedData() is prepared to handle that and will read at
* least short page-header worth of data
*/
- targetRecOff = tmpRecPtr % XLOG_BLCKSZ;
+ targetRecOff = tmpRecPtr % state->pageSize;
/* scroll back to page boundary */
targetPagePtr = tmpRecPtr - targetRecOff;
@@ -1174,8 +1177,8 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
*
* Note that record headers are MAXALIGN'ed
*/
- if (MAXALIGN(header->xlp_rem_len) >= (XLOG_BLCKSZ - pageHeaderSize))
- tmpRecPtr = targetPagePtr + XLOG_BLCKSZ;
+ if (MAXALIGN(header->xlp_rem_len) >= (state->pageSize - pageHeaderSize))
+ tmpRecPtr = targetPagePtr + state->pageSize;
else
{
/*
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index 9cbd8afa49..f2dd495310 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -199,11 +199,14 @@ StartupDecodingContext(List *output_plugin_options,
ctx->slot = slot;
- ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, cleanup_cb);
+ ctx->reader =
+ XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, cleanup_cb);
+
if (!ctx->reader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
errmsg("out of memory")));
+ ctx->reader->readBuf = palloc(XLOG_BLCKSZ);
ctx->page_read = page_read;
ctx->reorder = ReorderBufferAllocate();
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index 062093727d..8e95600ef7 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -579,8 +579,8 @@ StartReplication(StartReplicationCmd *cmd)
errmsg("IDENTIFY_SYSTEM has not been run before START_REPLICATION")));
/* create xlogreader for physical replication */
- xlogreader =
- XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+ xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+ NULL, wal_segment_close);
if (!xlogreader)
ereport(ERROR,
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index 712c85281c..8a6209b061 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -61,7 +61,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
XLogReaderState *xlogreader;
char *errormsg;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+ xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
@@ -121,7 +121,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex,
char *errormsg;
XLogRecPtr endptr;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+ xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
@@ -181,7 +181,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
forkptr += SizeOfXLogShortPHD;
}
- xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+ xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index f514de6b13..b635537659 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -1041,7 +1041,7 @@ main(int argc, char **argv)
/* we have everything we need, start reading */
xlogreader_state =
- XLogReaderAllocate(WalSegSz, waldir, WALDumpCloseSegment);
+ XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, waldir, WALDumpCloseSegment);
if (!xlogreader_state)
fatal_error("out of memory");
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index b9dad7ee51..1853516333 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -151,6 +151,7 @@ struct XLogReaderState
* ----------------------------------------
*/
/* variables to communicate with page reader */
+ int pageSize; /* size of a page */
XLogRecPtr readPagePtr; /* page pointer to read */
int32 readLen; /* bytes requested to reader, or actual bytes
* read by reader, which must be larger than
@@ -243,7 +244,7 @@ struct XLogReaderState
/* Get a new XLogReader */
extern XLogReaderState *XLogReaderAllocate(int wal_segment_size,
- const char *waldir,
+ int page_size, const char *waldir,
WALSegmentCleanupCB cleanup_cb);
/* Free an XLogReader */
--
2.27.0
----Next_Part(Thu_Mar__4_11_28_53_2021_014)----
^ permalink raw reply [nested|flat] 26+ messages in thread
* [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize.
@ 2019-09-05 12:29 Kyotaro Horiguchi <[email protected]>
0 siblings, 0 replies; 26+ messages in thread
From: Kyotaro Horiguchi @ 2019-09-05 12:29 UTC (permalink / raw)
XLog reader facility and its callers currently relies on the same
constant XLOG_BLCKSZ. This patch moves the responsibility to determine
the block size to the callers of the facility.
---
src/backend/access/transam/twophase.c | 3 +-
src/backend/access/transam/xlog.c | 9 ++--
src/backend/access/transam/xlogreader.c | 57 ++++++++++++-----------
src/backend/replication/logical/logical.c | 5 +-
src/backend/replication/walsender.c | 4 +-
src/bin/pg_rewind/parsexlog.c | 6 +--
src/bin/pg_waldump/pg_waldump.c | 2 +-
src/include/access/xlogreader.h | 3 +-
8 files changed, 50 insertions(+), 39 deletions(-)
diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index 7d5a96a5f5..9f9e7de253 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -1325,7 +1325,8 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
XLogReaderState *xlogreader;
char *errormsg;
- xlogreader = XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+ xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+ NULL, wal_segment_close);
if (!xlogreader)
ereport(ERROR,
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index b517ca85ad..0822ac70cb 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -1222,7 +1222,10 @@ XLogInsertRecord(XLogRecData *rdata,
appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len);
if (!debug_reader)
- debug_reader = XLogReaderAllocate(wal_segment_size, NULL, NULL);
+ {
+ debug_reader =
+ XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, NULL);
+ }
if (!debug_reader)
{
@@ -6531,8 +6534,8 @@ StartupXLOG(void)
OwnLatch(&XLogCtl->recoveryWakeupLatch);
/* Set up XLOG reader facility */
- xlogreader =
- XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+ xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+ NULL, wal_segment_close);
if (!xlogreader)
ereport(ERROR,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index ba3e4868f7..e5baf13086 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -72,7 +72,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...)
* Returns NULL if the xlogreader couldn't be allocated.
*/
XLogReaderState *
-XLogReaderAllocate(int wal_segment_size, const char *waldir,
+XLogReaderAllocate(int wal_segment_size, int page_size, const char *waldir,
WALSegmentCleanupCB cleanup_cb)
{
XLogReaderState *state;
@@ -87,6 +87,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
state->cleanup_cb = cleanup_cb;
state->max_block_id = -1;
+ state->pageSize = page_size;
/*
* Permanently allocate readBuf. We do it this way, rather than just
@@ -95,7 +96,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
* isn't guaranteed to have any particular alignment, whereas
* palloc_extended() will provide MAXALIGN'd storage.
*/
- state->readBuf = (char *) palloc_extended(XLOG_BLCKSZ,
+ state->readBuf = (char *) palloc_extended(state->pageSize,
MCXT_ALLOC_NO_OOM);
if (!state->readBuf)
{
@@ -163,7 +164,7 @@ XLogReaderFree(XLogReaderState *state)
* readRecordBufSize is set to the new buffer size.
*
* To avoid useless small increases, round its size to a multiple of
- * XLOG_BLCKSZ, and make sure it's at least 5*Max(BLCKSZ, XLOG_BLCKSZ) to start
+ * page size, and make sure it's at least 5*Max(BLCKSZ, <page size>) to start
* with. (That is enough for all "normal" records, but very large commit or
* abort records might need more space.)
*/
@@ -172,8 +173,8 @@ allocate_recordbuf(XLogReaderState *state, uint32 reclength)
{
uint32 newSize = reclength;
- newSize += XLOG_BLCKSZ - (newSize % XLOG_BLCKSZ);
- newSize = Max(newSize, 5 * Max(BLCKSZ, XLOG_BLCKSZ));
+ newSize += state->pageSize - (newSize % state->pageSize);
+ newSize = Max(newSize, 5 * Max(BLCKSZ, state->pageSize));
#ifndef FRONTEND
@@ -238,6 +239,8 @@ void
XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
{
Assert(!XLogRecPtrIsInvalid(RecPtr));
+ Assert(state->readBuf != NULL &&
+ state->readBuf == (void *) MAXALIGN(state->readBuf));
ResetDecoder(state);
@@ -366,15 +369,15 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
XLogPageHeader pageHeader;
targetPagePtr =
- state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
- targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+ state->ReadRecPtr - (state->ReadRecPtr % state->pageSize);
+ targetRecOff = state->ReadRecPtr % state->pageSize;
/*
* Check if we have enough data. For the first record in the page,
* the requesting length doesn't contain page header.
*/
if (XLogNeedData(state, targetPagePtr,
- Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+ Min(targetRecOff + SizeOfXLogRecord, state->pageSize),
targetRecOff != 0))
return XLREAD_NEED_DATA;
@@ -421,7 +424,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
* verified that we got the whole header.
*/
prec = (XLogRecord *) (state->readBuf +
- state->ReadRecPtr % XLOG_BLCKSZ);
+ state->ReadRecPtr % state->pageSize);
total_len = prec->xl_tot_len;
/*
@@ -432,7 +435,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
* ensure that we enter the XLREAD_CONTINUATION state below;
* otherwise we might fail to apply ValidXLogRecordHeader at all.
*/
- if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
+ if (targetRecOff <= state->pageSize - SizeOfXLogRecord)
{
if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
state->PrevRecPtr, prec))
@@ -477,10 +480,10 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
* available
*/
targetPagePtr =
- state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
- targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+ state->ReadRecPtr - (state->ReadRecPtr % state->pageSize);
+ targetRecOff = state->ReadRecPtr % state->pageSize;
- request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ);
+ request_len = Min(targetRecOff + total_len, state->pageSize);
record_len = request_len - targetRecOff;
/* ReadRecPtr contains page header */
@@ -549,7 +552,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
/* Calculate pointer to beginning of next page */
state->recordContRecPtr = state->ReadRecPtr + record_len;
- Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0);
+ Assert(state->recordContRecPtr % state->pageSize == 0);
state->readRecordState = XLREAD_CONTINUATION;
}
@@ -576,7 +579,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
/* this request contains page header */
Assert (targetPagePtr != 0);
if (XLogNeedData(state, targetPagePtr,
- Min(state->recordRemainLen, XLOG_BLCKSZ),
+ Min(state->recordRemainLen, state->pageSize),
false))
return XLREAD_NEED_DATA;
@@ -628,7 +631,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
Assert(state->readLen >= pageHeaderSize);
contdata = (char *) state->readBuf + pageHeaderSize;
- record_len = XLOG_BLCKSZ - pageHeaderSize;
+ record_len = state->pageSize - pageHeaderSize;
if (pageHeader->xlp_rem_len < record_len)
record_len = pageHeader->xlp_rem_len;
@@ -655,7 +658,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
}
/* Calculate pointer to beginning of next page, and continue */
- state->recordContRecPtr += XLOG_BLCKSZ;
+ state->recordContRecPtr += state->pageSize;
}
/* targetPagePtr is pointing the last-read page here */
@@ -789,10 +792,10 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
XLogPageHeaderSize((XLogPageHeader) state->readBuf);
addLen = pageHeaderSize;
- if (reqLen + pageHeaderSize <= XLOG_BLCKSZ)
+ if (reqLen + pageHeaderSize <= state->pageSize)
addLen = pageHeaderSize;
else
- addLen = XLOG_BLCKSZ - reqLen;
+ addLen = state->pageSize - reqLen;
Assert(addLen >= 0);
}
@@ -805,7 +808,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
/* Data is not in our buffer, request the caller for it. */
XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize);
targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize);
- Assert((pageptr % XLOG_BLCKSZ) == 0);
+ Assert((pageptr % state->pageSize) == 0);
/*
* Every time we request to load new data of a page to the caller, even if
@@ -829,7 +832,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
* will not come back here, but will request the actual target page.
*/
state->readPagePtr = pageptr - targetPageOff;
- state->readLen = XLOG_BLCKSZ;
+ state->readLen = state->pageSize;
return true;
}
@@ -962,7 +965,7 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
int32 offset;
XLogPageHeader hdr = (XLogPageHeader) phdr;
- Assert((recptr % XLOG_BLCKSZ) == 0);
+ Assert((recptr % state->pageSize) == 0);
XLByteToSeg(recptr, segno, state->segcxt.ws_segsize);
offset = XLogSegmentOffset(recptr, state->segcxt.ws_segsize);
@@ -1016,10 +1019,10 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
"WAL file is from different database system: incorrect segment size in page header");
return false;
}
- else if (longhdr->xlp_xlog_blcksz != XLOG_BLCKSZ)
+ else if (longhdr->xlp_xlog_blcksz != state->pageSize)
{
report_invalid_record(state,
- "WAL file is from different database system: incorrect XLOG_BLCKSZ in page header");
+ "WAL file is from different database system: incorrect block size in page header");
return false;
}
}
@@ -1140,7 +1143,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
* XLogNeedData() is prepared to handle that and will read at
* least short page-header worth of data
*/
- targetRecOff = tmpRecPtr % XLOG_BLCKSZ;
+ targetRecOff = tmpRecPtr % state->pageSize;
/* scroll back to page boundary */
targetPagePtr = tmpRecPtr - targetRecOff;
@@ -1174,8 +1177,8 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
*
* Note that record headers are MAXALIGN'ed
*/
- if (MAXALIGN(header->xlp_rem_len) >= (XLOG_BLCKSZ - pageHeaderSize))
- tmpRecPtr = targetPagePtr + XLOG_BLCKSZ;
+ if (MAXALIGN(header->xlp_rem_len) >= (state->pageSize - pageHeaderSize))
+ tmpRecPtr = targetPagePtr + state->pageSize;
else
{
/*
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index 9cbd8afa49..f2dd495310 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -199,11 +199,14 @@ StartupDecodingContext(List *output_plugin_options,
ctx->slot = slot;
- ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, cleanup_cb);
+ ctx->reader =
+ XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, cleanup_cb);
+
if (!ctx->reader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
errmsg("out of memory")));
+ ctx->reader->readBuf = palloc(XLOG_BLCKSZ);
ctx->page_read = page_read;
ctx->reorder = ReorderBufferAllocate();
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index 062093727d..8e95600ef7 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -579,8 +579,8 @@ StartReplication(StartReplicationCmd *cmd)
errmsg("IDENTIFY_SYSTEM has not been run before START_REPLICATION")));
/* create xlogreader for physical replication */
- xlogreader =
- XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+ xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+ NULL, wal_segment_close);
if (!xlogreader)
ereport(ERROR,
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index 712c85281c..8a6209b061 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -61,7 +61,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
XLogReaderState *xlogreader;
char *errormsg;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+ xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
@@ -121,7 +121,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex,
char *errormsg;
XLogRecPtr endptr;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+ xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
@@ -181,7 +181,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
forkptr += SizeOfXLogShortPHD;
}
- xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+ xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index f514de6b13..b635537659 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -1041,7 +1041,7 @@ main(int argc, char **argv)
/* we have everything we need, start reading */
xlogreader_state =
- XLogReaderAllocate(WalSegSz, waldir, WALDumpCloseSegment);
+ XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, waldir, WALDumpCloseSegment);
if (!xlogreader_state)
fatal_error("out of memory");
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index b9dad7ee51..1853516333 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -151,6 +151,7 @@ struct XLogReaderState
* ----------------------------------------
*/
/* variables to communicate with page reader */
+ int pageSize; /* size of a page */
XLogRecPtr readPagePtr; /* page pointer to read */
int32 readLen; /* bytes requested to reader, or actual bytes
* read by reader, which must be larger than
@@ -243,7 +244,7 @@ struct XLogReaderState
/* Get a new XLogReader */
extern XLogReaderState *XLogReaderAllocate(int wal_segment_size,
- const char *waldir,
+ int page_size, const char *waldir,
WALSegmentCleanupCB cleanup_cb);
/* Free an XLogReader */
--
2.27.0
----Next_Part(Thu_Mar__4_11_28_53_2021_014)----
^ permalink raw reply [nested|flat] 26+ messages in thread
* [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize.
@ 2019-09-05 12:29 Kyotaro Horiguchi <[email protected]>
0 siblings, 0 replies; 26+ messages in thread
From: Kyotaro Horiguchi @ 2019-09-05 12:29 UTC (permalink / raw)
XLog reader facility and its callers currently relies on the same
constant XLOG_BLCKSZ. This patch moves the responsibility to determine
the block size to the callers of the facility.
---
src/backend/access/transam/twophase.c | 3 +-
src/backend/access/transam/xlog.c | 9 ++--
src/backend/access/transam/xlogreader.c | 57 ++++++++++++-----------
src/backend/replication/logical/logical.c | 5 +-
src/backend/replication/walsender.c | 4 +-
src/bin/pg_rewind/parsexlog.c | 6 +--
src/bin/pg_waldump/pg_waldump.c | 2 +-
src/include/access/xlogreader.h | 3 +-
8 files changed, 50 insertions(+), 39 deletions(-)
diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index 7d5a96a5f5..9f9e7de253 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -1325,7 +1325,8 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
XLogReaderState *xlogreader;
char *errormsg;
- xlogreader = XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+ xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+ NULL, wal_segment_close);
if (!xlogreader)
ereport(ERROR,
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index b517ca85ad..0822ac70cb 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -1222,7 +1222,10 @@ XLogInsertRecord(XLogRecData *rdata,
appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len);
if (!debug_reader)
- debug_reader = XLogReaderAllocate(wal_segment_size, NULL, NULL);
+ {
+ debug_reader =
+ XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, NULL);
+ }
if (!debug_reader)
{
@@ -6531,8 +6534,8 @@ StartupXLOG(void)
OwnLatch(&XLogCtl->recoveryWakeupLatch);
/* Set up XLOG reader facility */
- xlogreader =
- XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+ xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+ NULL, wal_segment_close);
if (!xlogreader)
ereport(ERROR,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index ba3e4868f7..e5baf13086 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -72,7 +72,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...)
* Returns NULL if the xlogreader couldn't be allocated.
*/
XLogReaderState *
-XLogReaderAllocate(int wal_segment_size, const char *waldir,
+XLogReaderAllocate(int wal_segment_size, int page_size, const char *waldir,
WALSegmentCleanupCB cleanup_cb)
{
XLogReaderState *state;
@@ -87,6 +87,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
state->cleanup_cb = cleanup_cb;
state->max_block_id = -1;
+ state->pageSize = page_size;
/*
* Permanently allocate readBuf. We do it this way, rather than just
@@ -95,7 +96,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
* isn't guaranteed to have any particular alignment, whereas
* palloc_extended() will provide MAXALIGN'd storage.
*/
- state->readBuf = (char *) palloc_extended(XLOG_BLCKSZ,
+ state->readBuf = (char *) palloc_extended(state->pageSize,
MCXT_ALLOC_NO_OOM);
if (!state->readBuf)
{
@@ -163,7 +164,7 @@ XLogReaderFree(XLogReaderState *state)
* readRecordBufSize is set to the new buffer size.
*
* To avoid useless small increases, round its size to a multiple of
- * XLOG_BLCKSZ, and make sure it's at least 5*Max(BLCKSZ, XLOG_BLCKSZ) to start
+ * page size, and make sure it's at least 5*Max(BLCKSZ, <page size>) to start
* with. (That is enough for all "normal" records, but very large commit or
* abort records might need more space.)
*/
@@ -172,8 +173,8 @@ allocate_recordbuf(XLogReaderState *state, uint32 reclength)
{
uint32 newSize = reclength;
- newSize += XLOG_BLCKSZ - (newSize % XLOG_BLCKSZ);
- newSize = Max(newSize, 5 * Max(BLCKSZ, XLOG_BLCKSZ));
+ newSize += state->pageSize - (newSize % state->pageSize);
+ newSize = Max(newSize, 5 * Max(BLCKSZ, state->pageSize));
#ifndef FRONTEND
@@ -238,6 +239,8 @@ void
XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
{
Assert(!XLogRecPtrIsInvalid(RecPtr));
+ Assert(state->readBuf != NULL &&
+ state->readBuf == (void *) MAXALIGN(state->readBuf));
ResetDecoder(state);
@@ -366,15 +369,15 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
XLogPageHeader pageHeader;
targetPagePtr =
- state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
- targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+ state->ReadRecPtr - (state->ReadRecPtr % state->pageSize);
+ targetRecOff = state->ReadRecPtr % state->pageSize;
/*
* Check if we have enough data. For the first record in the page,
* the requesting length doesn't contain page header.
*/
if (XLogNeedData(state, targetPagePtr,
- Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+ Min(targetRecOff + SizeOfXLogRecord, state->pageSize),
targetRecOff != 0))
return XLREAD_NEED_DATA;
@@ -421,7 +424,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
* verified that we got the whole header.
*/
prec = (XLogRecord *) (state->readBuf +
- state->ReadRecPtr % XLOG_BLCKSZ);
+ state->ReadRecPtr % state->pageSize);
total_len = prec->xl_tot_len;
/*
@@ -432,7 +435,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
* ensure that we enter the XLREAD_CONTINUATION state below;
* otherwise we might fail to apply ValidXLogRecordHeader at all.
*/
- if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
+ if (targetRecOff <= state->pageSize - SizeOfXLogRecord)
{
if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
state->PrevRecPtr, prec))
@@ -477,10 +480,10 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
* available
*/
targetPagePtr =
- state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
- targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+ state->ReadRecPtr - (state->ReadRecPtr % state->pageSize);
+ targetRecOff = state->ReadRecPtr % state->pageSize;
- request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ);
+ request_len = Min(targetRecOff + total_len, state->pageSize);
record_len = request_len - targetRecOff;
/* ReadRecPtr contains page header */
@@ -549,7 +552,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
/* Calculate pointer to beginning of next page */
state->recordContRecPtr = state->ReadRecPtr + record_len;
- Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0);
+ Assert(state->recordContRecPtr % state->pageSize == 0);
state->readRecordState = XLREAD_CONTINUATION;
}
@@ -576,7 +579,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
/* this request contains page header */
Assert (targetPagePtr != 0);
if (XLogNeedData(state, targetPagePtr,
- Min(state->recordRemainLen, XLOG_BLCKSZ),
+ Min(state->recordRemainLen, state->pageSize),
false))
return XLREAD_NEED_DATA;
@@ -628,7 +631,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
Assert(state->readLen >= pageHeaderSize);
contdata = (char *) state->readBuf + pageHeaderSize;
- record_len = XLOG_BLCKSZ - pageHeaderSize;
+ record_len = state->pageSize - pageHeaderSize;
if (pageHeader->xlp_rem_len < record_len)
record_len = pageHeader->xlp_rem_len;
@@ -655,7 +658,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
}
/* Calculate pointer to beginning of next page, and continue */
- state->recordContRecPtr += XLOG_BLCKSZ;
+ state->recordContRecPtr += state->pageSize;
}
/* targetPagePtr is pointing the last-read page here */
@@ -789,10 +792,10 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
XLogPageHeaderSize((XLogPageHeader) state->readBuf);
addLen = pageHeaderSize;
- if (reqLen + pageHeaderSize <= XLOG_BLCKSZ)
+ if (reqLen + pageHeaderSize <= state->pageSize)
addLen = pageHeaderSize;
else
- addLen = XLOG_BLCKSZ - reqLen;
+ addLen = state->pageSize - reqLen;
Assert(addLen >= 0);
}
@@ -805,7 +808,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
/* Data is not in our buffer, request the caller for it. */
XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize);
targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize);
- Assert((pageptr % XLOG_BLCKSZ) == 0);
+ Assert((pageptr % state->pageSize) == 0);
/*
* Every time we request to load new data of a page to the caller, even if
@@ -829,7 +832,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
* will not come back here, but will request the actual target page.
*/
state->readPagePtr = pageptr - targetPageOff;
- state->readLen = XLOG_BLCKSZ;
+ state->readLen = state->pageSize;
return true;
}
@@ -962,7 +965,7 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
int32 offset;
XLogPageHeader hdr = (XLogPageHeader) phdr;
- Assert((recptr % XLOG_BLCKSZ) == 0);
+ Assert((recptr % state->pageSize) == 0);
XLByteToSeg(recptr, segno, state->segcxt.ws_segsize);
offset = XLogSegmentOffset(recptr, state->segcxt.ws_segsize);
@@ -1016,10 +1019,10 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
"WAL file is from different database system: incorrect segment size in page header");
return false;
}
- else if (longhdr->xlp_xlog_blcksz != XLOG_BLCKSZ)
+ else if (longhdr->xlp_xlog_blcksz != state->pageSize)
{
report_invalid_record(state,
- "WAL file is from different database system: incorrect XLOG_BLCKSZ in page header");
+ "WAL file is from different database system: incorrect block size in page header");
return false;
}
}
@@ -1140,7 +1143,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
* XLogNeedData() is prepared to handle that and will read at
* least short page-header worth of data
*/
- targetRecOff = tmpRecPtr % XLOG_BLCKSZ;
+ targetRecOff = tmpRecPtr % state->pageSize;
/* scroll back to page boundary */
targetPagePtr = tmpRecPtr - targetRecOff;
@@ -1174,8 +1177,8 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
*
* Note that record headers are MAXALIGN'ed
*/
- if (MAXALIGN(header->xlp_rem_len) >= (XLOG_BLCKSZ - pageHeaderSize))
- tmpRecPtr = targetPagePtr + XLOG_BLCKSZ;
+ if (MAXALIGN(header->xlp_rem_len) >= (state->pageSize - pageHeaderSize))
+ tmpRecPtr = targetPagePtr + state->pageSize;
else
{
/*
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index 9cbd8afa49..f2dd495310 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -199,11 +199,14 @@ StartupDecodingContext(List *output_plugin_options,
ctx->slot = slot;
- ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, cleanup_cb);
+ ctx->reader =
+ XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, cleanup_cb);
+
if (!ctx->reader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
errmsg("out of memory")));
+ ctx->reader->readBuf = palloc(XLOG_BLCKSZ);
ctx->page_read = page_read;
ctx->reorder = ReorderBufferAllocate();
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index 062093727d..8e95600ef7 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -579,8 +579,8 @@ StartReplication(StartReplicationCmd *cmd)
errmsg("IDENTIFY_SYSTEM has not been run before START_REPLICATION")));
/* create xlogreader for physical replication */
- xlogreader =
- XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+ xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+ NULL, wal_segment_close);
if (!xlogreader)
ereport(ERROR,
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index 712c85281c..8a6209b061 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -61,7 +61,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
XLogReaderState *xlogreader;
char *errormsg;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+ xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
@@ -121,7 +121,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex,
char *errormsg;
XLogRecPtr endptr;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+ xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
@@ -181,7 +181,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
forkptr += SizeOfXLogShortPHD;
}
- xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+ xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index f514de6b13..b635537659 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -1041,7 +1041,7 @@ main(int argc, char **argv)
/* we have everything we need, start reading */
xlogreader_state =
- XLogReaderAllocate(WalSegSz, waldir, WALDumpCloseSegment);
+ XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, waldir, WALDumpCloseSegment);
if (!xlogreader_state)
fatal_error("out of memory");
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index b9dad7ee51..1853516333 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -151,6 +151,7 @@ struct XLogReaderState
* ----------------------------------------
*/
/* variables to communicate with page reader */
+ int pageSize; /* size of a page */
XLogRecPtr readPagePtr; /* page pointer to read */
int32 readLen; /* bytes requested to reader, or actual bytes
* read by reader, which must be larger than
@@ -243,7 +244,7 @@ struct XLogReaderState
/* Get a new XLogReader */
extern XLogReaderState *XLogReaderAllocate(int wal_segment_size,
- const char *waldir,
+ int page_size, const char *waldir,
WALSegmentCleanupCB cleanup_cb);
/* Free an XLogReader */
--
2.27.0
----Next_Part(Thu_Mar__4_11_28_53_2021_014)----
^ permalink raw reply [nested|flat] 26+ messages in thread
* [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize.
@ 2019-09-05 12:29 Kyotaro Horiguchi <[email protected]>
0 siblings, 0 replies; 26+ messages in thread
From: Kyotaro Horiguchi @ 2019-09-05 12:29 UTC (permalink / raw)
XLog reader facility and its callers currently relies on the same
constant XLOG_BLCKSZ. This patch moves the responsibility to determine
the block size to the callers of the facility.
---
src/backend/access/transam/twophase.c | 3 +-
src/backend/access/transam/xlog.c | 9 ++--
src/backend/access/transam/xlogreader.c | 57 ++++++++++++-----------
src/backend/replication/logical/logical.c | 5 +-
src/backend/replication/walsender.c | 4 +-
src/bin/pg_rewind/parsexlog.c | 6 +--
src/bin/pg_waldump/pg_waldump.c | 2 +-
src/include/access/xlogreader.h | 3 +-
8 files changed, 50 insertions(+), 39 deletions(-)
diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index 7d5a96a5f5..9f9e7de253 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -1325,7 +1325,8 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
XLogReaderState *xlogreader;
char *errormsg;
- xlogreader = XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+ xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+ NULL, wal_segment_close);
if (!xlogreader)
ereport(ERROR,
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index b517ca85ad..0822ac70cb 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -1222,7 +1222,10 @@ XLogInsertRecord(XLogRecData *rdata,
appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len);
if (!debug_reader)
- debug_reader = XLogReaderAllocate(wal_segment_size, NULL, NULL);
+ {
+ debug_reader =
+ XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, NULL);
+ }
if (!debug_reader)
{
@@ -6531,8 +6534,8 @@ StartupXLOG(void)
OwnLatch(&XLogCtl->recoveryWakeupLatch);
/* Set up XLOG reader facility */
- xlogreader =
- XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+ xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+ NULL, wal_segment_close);
if (!xlogreader)
ereport(ERROR,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index ba3e4868f7..e5baf13086 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -72,7 +72,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...)
* Returns NULL if the xlogreader couldn't be allocated.
*/
XLogReaderState *
-XLogReaderAllocate(int wal_segment_size, const char *waldir,
+XLogReaderAllocate(int wal_segment_size, int page_size, const char *waldir,
WALSegmentCleanupCB cleanup_cb)
{
XLogReaderState *state;
@@ -87,6 +87,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
state->cleanup_cb = cleanup_cb;
state->max_block_id = -1;
+ state->pageSize = page_size;
/*
* Permanently allocate readBuf. We do it this way, rather than just
@@ -95,7 +96,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
* isn't guaranteed to have any particular alignment, whereas
* palloc_extended() will provide MAXALIGN'd storage.
*/
- state->readBuf = (char *) palloc_extended(XLOG_BLCKSZ,
+ state->readBuf = (char *) palloc_extended(state->pageSize,
MCXT_ALLOC_NO_OOM);
if (!state->readBuf)
{
@@ -163,7 +164,7 @@ XLogReaderFree(XLogReaderState *state)
* readRecordBufSize is set to the new buffer size.
*
* To avoid useless small increases, round its size to a multiple of
- * XLOG_BLCKSZ, and make sure it's at least 5*Max(BLCKSZ, XLOG_BLCKSZ) to start
+ * page size, and make sure it's at least 5*Max(BLCKSZ, <page size>) to start
* with. (That is enough for all "normal" records, but very large commit or
* abort records might need more space.)
*/
@@ -172,8 +173,8 @@ allocate_recordbuf(XLogReaderState *state, uint32 reclength)
{
uint32 newSize = reclength;
- newSize += XLOG_BLCKSZ - (newSize % XLOG_BLCKSZ);
- newSize = Max(newSize, 5 * Max(BLCKSZ, XLOG_BLCKSZ));
+ newSize += state->pageSize - (newSize % state->pageSize);
+ newSize = Max(newSize, 5 * Max(BLCKSZ, state->pageSize));
#ifndef FRONTEND
@@ -238,6 +239,8 @@ void
XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
{
Assert(!XLogRecPtrIsInvalid(RecPtr));
+ Assert(state->readBuf != NULL &&
+ state->readBuf == (void *) MAXALIGN(state->readBuf));
ResetDecoder(state);
@@ -366,15 +369,15 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
XLogPageHeader pageHeader;
targetPagePtr =
- state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
- targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+ state->ReadRecPtr - (state->ReadRecPtr % state->pageSize);
+ targetRecOff = state->ReadRecPtr % state->pageSize;
/*
* Check if we have enough data. For the first record in the page,
* the requesting length doesn't contain page header.
*/
if (XLogNeedData(state, targetPagePtr,
- Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+ Min(targetRecOff + SizeOfXLogRecord, state->pageSize),
targetRecOff != 0))
return XLREAD_NEED_DATA;
@@ -421,7 +424,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
* verified that we got the whole header.
*/
prec = (XLogRecord *) (state->readBuf +
- state->ReadRecPtr % XLOG_BLCKSZ);
+ state->ReadRecPtr % state->pageSize);
total_len = prec->xl_tot_len;
/*
@@ -432,7 +435,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
* ensure that we enter the XLREAD_CONTINUATION state below;
* otherwise we might fail to apply ValidXLogRecordHeader at all.
*/
- if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
+ if (targetRecOff <= state->pageSize - SizeOfXLogRecord)
{
if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
state->PrevRecPtr, prec))
@@ -477,10 +480,10 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
* available
*/
targetPagePtr =
- state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
- targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+ state->ReadRecPtr - (state->ReadRecPtr % state->pageSize);
+ targetRecOff = state->ReadRecPtr % state->pageSize;
- request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ);
+ request_len = Min(targetRecOff + total_len, state->pageSize);
record_len = request_len - targetRecOff;
/* ReadRecPtr contains page header */
@@ -549,7 +552,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
/* Calculate pointer to beginning of next page */
state->recordContRecPtr = state->ReadRecPtr + record_len;
- Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0);
+ Assert(state->recordContRecPtr % state->pageSize == 0);
state->readRecordState = XLREAD_CONTINUATION;
}
@@ -576,7 +579,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
/* this request contains page header */
Assert (targetPagePtr != 0);
if (XLogNeedData(state, targetPagePtr,
- Min(state->recordRemainLen, XLOG_BLCKSZ),
+ Min(state->recordRemainLen, state->pageSize),
false))
return XLREAD_NEED_DATA;
@@ -628,7 +631,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
Assert(state->readLen >= pageHeaderSize);
contdata = (char *) state->readBuf + pageHeaderSize;
- record_len = XLOG_BLCKSZ - pageHeaderSize;
+ record_len = state->pageSize - pageHeaderSize;
if (pageHeader->xlp_rem_len < record_len)
record_len = pageHeader->xlp_rem_len;
@@ -655,7 +658,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
}
/* Calculate pointer to beginning of next page, and continue */
- state->recordContRecPtr += XLOG_BLCKSZ;
+ state->recordContRecPtr += state->pageSize;
}
/* targetPagePtr is pointing the last-read page here */
@@ -789,10 +792,10 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
XLogPageHeaderSize((XLogPageHeader) state->readBuf);
addLen = pageHeaderSize;
- if (reqLen + pageHeaderSize <= XLOG_BLCKSZ)
+ if (reqLen + pageHeaderSize <= state->pageSize)
addLen = pageHeaderSize;
else
- addLen = XLOG_BLCKSZ - reqLen;
+ addLen = state->pageSize - reqLen;
Assert(addLen >= 0);
}
@@ -805,7 +808,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
/* Data is not in our buffer, request the caller for it. */
XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize);
targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize);
- Assert((pageptr % XLOG_BLCKSZ) == 0);
+ Assert((pageptr % state->pageSize) == 0);
/*
* Every time we request to load new data of a page to the caller, even if
@@ -829,7 +832,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
* will not come back here, but will request the actual target page.
*/
state->readPagePtr = pageptr - targetPageOff;
- state->readLen = XLOG_BLCKSZ;
+ state->readLen = state->pageSize;
return true;
}
@@ -962,7 +965,7 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
int32 offset;
XLogPageHeader hdr = (XLogPageHeader) phdr;
- Assert((recptr % XLOG_BLCKSZ) == 0);
+ Assert((recptr % state->pageSize) == 0);
XLByteToSeg(recptr, segno, state->segcxt.ws_segsize);
offset = XLogSegmentOffset(recptr, state->segcxt.ws_segsize);
@@ -1016,10 +1019,10 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
"WAL file is from different database system: incorrect segment size in page header");
return false;
}
- else if (longhdr->xlp_xlog_blcksz != XLOG_BLCKSZ)
+ else if (longhdr->xlp_xlog_blcksz != state->pageSize)
{
report_invalid_record(state,
- "WAL file is from different database system: incorrect XLOG_BLCKSZ in page header");
+ "WAL file is from different database system: incorrect block size in page header");
return false;
}
}
@@ -1140,7 +1143,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
* XLogNeedData() is prepared to handle that and will read at
* least short page-header worth of data
*/
- targetRecOff = tmpRecPtr % XLOG_BLCKSZ;
+ targetRecOff = tmpRecPtr % state->pageSize;
/* scroll back to page boundary */
targetPagePtr = tmpRecPtr - targetRecOff;
@@ -1174,8 +1177,8 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
*
* Note that record headers are MAXALIGN'ed
*/
- if (MAXALIGN(header->xlp_rem_len) >= (XLOG_BLCKSZ - pageHeaderSize))
- tmpRecPtr = targetPagePtr + XLOG_BLCKSZ;
+ if (MAXALIGN(header->xlp_rem_len) >= (state->pageSize - pageHeaderSize))
+ tmpRecPtr = targetPagePtr + state->pageSize;
else
{
/*
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index 9cbd8afa49..f2dd495310 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -199,11 +199,14 @@ StartupDecodingContext(List *output_plugin_options,
ctx->slot = slot;
- ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, cleanup_cb);
+ ctx->reader =
+ XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, cleanup_cb);
+
if (!ctx->reader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
errmsg("out of memory")));
+ ctx->reader->readBuf = palloc(XLOG_BLCKSZ);
ctx->page_read = page_read;
ctx->reorder = ReorderBufferAllocate();
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index 062093727d..8e95600ef7 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -579,8 +579,8 @@ StartReplication(StartReplicationCmd *cmd)
errmsg("IDENTIFY_SYSTEM has not been run before START_REPLICATION")));
/* create xlogreader for physical replication */
- xlogreader =
- XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+ xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+ NULL, wal_segment_close);
if (!xlogreader)
ereport(ERROR,
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index 712c85281c..8a6209b061 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -61,7 +61,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
XLogReaderState *xlogreader;
char *errormsg;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+ xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
@@ -121,7 +121,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex,
char *errormsg;
XLogRecPtr endptr;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+ xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
@@ -181,7 +181,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
forkptr += SizeOfXLogShortPHD;
}
- xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+ xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index f514de6b13..b635537659 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -1041,7 +1041,7 @@ main(int argc, char **argv)
/* we have everything we need, start reading */
xlogreader_state =
- XLogReaderAllocate(WalSegSz, waldir, WALDumpCloseSegment);
+ XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, waldir, WALDumpCloseSegment);
if (!xlogreader_state)
fatal_error("out of memory");
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index b9dad7ee51..1853516333 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -151,6 +151,7 @@ struct XLogReaderState
* ----------------------------------------
*/
/* variables to communicate with page reader */
+ int pageSize; /* size of a page */
XLogRecPtr readPagePtr; /* page pointer to read */
int32 readLen; /* bytes requested to reader, or actual bytes
* read by reader, which must be larger than
@@ -243,7 +244,7 @@ struct XLogReaderState
/* Get a new XLogReader */
extern XLogReaderState *XLogReaderAllocate(int wal_segment_size,
- const char *waldir,
+ int page_size, const char *waldir,
WALSegmentCleanupCB cleanup_cb);
/* Free an XLogReader */
--
2.27.0
----Next_Part(Thu_Mar__4_11_28_53_2021_014)----
^ permalink raw reply [nested|flat] 26+ messages in thread
* [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize.
@ 2019-09-05 12:29 Kyotaro Horiguchi <[email protected]>
0 siblings, 0 replies; 26+ messages in thread
From: Kyotaro Horiguchi @ 2019-09-05 12:29 UTC (permalink / raw)
XLog reader facility and its callers currently relies on the same
constant XLOG_BLCKSZ. This patch moves the responsibility to determine
the block size to the callers of the facility.
---
src/backend/access/transam/twophase.c | 3 +-
src/backend/access/transam/xlog.c | 9 ++--
src/backend/access/transam/xlogreader.c | 57 ++++++++++++-----------
src/backend/replication/logical/logical.c | 5 +-
src/backend/replication/walsender.c | 4 +-
src/bin/pg_rewind/parsexlog.c | 6 +--
src/bin/pg_waldump/pg_waldump.c | 2 +-
src/include/access/xlogreader.h | 3 +-
8 files changed, 50 insertions(+), 39 deletions(-)
diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index 7d5a96a5f5..9f9e7de253 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -1325,7 +1325,8 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
XLogReaderState *xlogreader;
char *errormsg;
- xlogreader = XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+ xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+ NULL, wal_segment_close);
if (!xlogreader)
ereport(ERROR,
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index b517ca85ad..0822ac70cb 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -1222,7 +1222,10 @@ XLogInsertRecord(XLogRecData *rdata,
appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len);
if (!debug_reader)
- debug_reader = XLogReaderAllocate(wal_segment_size, NULL, NULL);
+ {
+ debug_reader =
+ XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, NULL);
+ }
if (!debug_reader)
{
@@ -6531,8 +6534,8 @@ StartupXLOG(void)
OwnLatch(&XLogCtl->recoveryWakeupLatch);
/* Set up XLOG reader facility */
- xlogreader =
- XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+ xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+ NULL, wal_segment_close);
if (!xlogreader)
ereport(ERROR,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index ba3e4868f7..e5baf13086 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -72,7 +72,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...)
* Returns NULL if the xlogreader couldn't be allocated.
*/
XLogReaderState *
-XLogReaderAllocate(int wal_segment_size, const char *waldir,
+XLogReaderAllocate(int wal_segment_size, int page_size, const char *waldir,
WALSegmentCleanupCB cleanup_cb)
{
XLogReaderState *state;
@@ -87,6 +87,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
state->cleanup_cb = cleanup_cb;
state->max_block_id = -1;
+ state->pageSize = page_size;
/*
* Permanently allocate readBuf. We do it this way, rather than just
@@ -95,7 +96,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
* isn't guaranteed to have any particular alignment, whereas
* palloc_extended() will provide MAXALIGN'd storage.
*/
- state->readBuf = (char *) palloc_extended(XLOG_BLCKSZ,
+ state->readBuf = (char *) palloc_extended(state->pageSize,
MCXT_ALLOC_NO_OOM);
if (!state->readBuf)
{
@@ -163,7 +164,7 @@ XLogReaderFree(XLogReaderState *state)
* readRecordBufSize is set to the new buffer size.
*
* To avoid useless small increases, round its size to a multiple of
- * XLOG_BLCKSZ, and make sure it's at least 5*Max(BLCKSZ, XLOG_BLCKSZ) to start
+ * page size, and make sure it's at least 5*Max(BLCKSZ, <page size>) to start
* with. (That is enough for all "normal" records, but very large commit or
* abort records might need more space.)
*/
@@ -172,8 +173,8 @@ allocate_recordbuf(XLogReaderState *state, uint32 reclength)
{
uint32 newSize = reclength;
- newSize += XLOG_BLCKSZ - (newSize % XLOG_BLCKSZ);
- newSize = Max(newSize, 5 * Max(BLCKSZ, XLOG_BLCKSZ));
+ newSize += state->pageSize - (newSize % state->pageSize);
+ newSize = Max(newSize, 5 * Max(BLCKSZ, state->pageSize));
#ifndef FRONTEND
@@ -238,6 +239,8 @@ void
XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
{
Assert(!XLogRecPtrIsInvalid(RecPtr));
+ Assert(state->readBuf != NULL &&
+ state->readBuf == (void *) MAXALIGN(state->readBuf));
ResetDecoder(state);
@@ -366,15 +369,15 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
XLogPageHeader pageHeader;
targetPagePtr =
- state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
- targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+ state->ReadRecPtr - (state->ReadRecPtr % state->pageSize);
+ targetRecOff = state->ReadRecPtr % state->pageSize;
/*
* Check if we have enough data. For the first record in the page,
* the requesting length doesn't contain page header.
*/
if (XLogNeedData(state, targetPagePtr,
- Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+ Min(targetRecOff + SizeOfXLogRecord, state->pageSize),
targetRecOff != 0))
return XLREAD_NEED_DATA;
@@ -421,7 +424,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
* verified that we got the whole header.
*/
prec = (XLogRecord *) (state->readBuf +
- state->ReadRecPtr % XLOG_BLCKSZ);
+ state->ReadRecPtr % state->pageSize);
total_len = prec->xl_tot_len;
/*
@@ -432,7 +435,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
* ensure that we enter the XLREAD_CONTINUATION state below;
* otherwise we might fail to apply ValidXLogRecordHeader at all.
*/
- if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
+ if (targetRecOff <= state->pageSize - SizeOfXLogRecord)
{
if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
state->PrevRecPtr, prec))
@@ -477,10 +480,10 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
* available
*/
targetPagePtr =
- state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
- targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+ state->ReadRecPtr - (state->ReadRecPtr % state->pageSize);
+ targetRecOff = state->ReadRecPtr % state->pageSize;
- request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ);
+ request_len = Min(targetRecOff + total_len, state->pageSize);
record_len = request_len - targetRecOff;
/* ReadRecPtr contains page header */
@@ -549,7 +552,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
/* Calculate pointer to beginning of next page */
state->recordContRecPtr = state->ReadRecPtr + record_len;
- Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0);
+ Assert(state->recordContRecPtr % state->pageSize == 0);
state->readRecordState = XLREAD_CONTINUATION;
}
@@ -576,7 +579,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
/* this request contains page header */
Assert (targetPagePtr != 0);
if (XLogNeedData(state, targetPagePtr,
- Min(state->recordRemainLen, XLOG_BLCKSZ),
+ Min(state->recordRemainLen, state->pageSize),
false))
return XLREAD_NEED_DATA;
@@ -628,7 +631,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
Assert(state->readLen >= pageHeaderSize);
contdata = (char *) state->readBuf + pageHeaderSize;
- record_len = XLOG_BLCKSZ - pageHeaderSize;
+ record_len = state->pageSize - pageHeaderSize;
if (pageHeader->xlp_rem_len < record_len)
record_len = pageHeader->xlp_rem_len;
@@ -655,7 +658,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
}
/* Calculate pointer to beginning of next page, and continue */
- state->recordContRecPtr += XLOG_BLCKSZ;
+ state->recordContRecPtr += state->pageSize;
}
/* targetPagePtr is pointing the last-read page here */
@@ -789,10 +792,10 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
XLogPageHeaderSize((XLogPageHeader) state->readBuf);
addLen = pageHeaderSize;
- if (reqLen + pageHeaderSize <= XLOG_BLCKSZ)
+ if (reqLen + pageHeaderSize <= state->pageSize)
addLen = pageHeaderSize;
else
- addLen = XLOG_BLCKSZ - reqLen;
+ addLen = state->pageSize - reqLen;
Assert(addLen >= 0);
}
@@ -805,7 +808,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
/* Data is not in our buffer, request the caller for it. */
XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize);
targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize);
- Assert((pageptr % XLOG_BLCKSZ) == 0);
+ Assert((pageptr % state->pageSize) == 0);
/*
* Every time we request to load new data of a page to the caller, even if
@@ -829,7 +832,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
* will not come back here, but will request the actual target page.
*/
state->readPagePtr = pageptr - targetPageOff;
- state->readLen = XLOG_BLCKSZ;
+ state->readLen = state->pageSize;
return true;
}
@@ -962,7 +965,7 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
int32 offset;
XLogPageHeader hdr = (XLogPageHeader) phdr;
- Assert((recptr % XLOG_BLCKSZ) == 0);
+ Assert((recptr % state->pageSize) == 0);
XLByteToSeg(recptr, segno, state->segcxt.ws_segsize);
offset = XLogSegmentOffset(recptr, state->segcxt.ws_segsize);
@@ -1016,10 +1019,10 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
"WAL file is from different database system: incorrect segment size in page header");
return false;
}
- else if (longhdr->xlp_xlog_blcksz != XLOG_BLCKSZ)
+ else if (longhdr->xlp_xlog_blcksz != state->pageSize)
{
report_invalid_record(state,
- "WAL file is from different database system: incorrect XLOG_BLCKSZ in page header");
+ "WAL file is from different database system: incorrect block size in page header");
return false;
}
}
@@ -1140,7 +1143,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
* XLogNeedData() is prepared to handle that and will read at
* least short page-header worth of data
*/
- targetRecOff = tmpRecPtr % XLOG_BLCKSZ;
+ targetRecOff = tmpRecPtr % state->pageSize;
/* scroll back to page boundary */
targetPagePtr = tmpRecPtr - targetRecOff;
@@ -1174,8 +1177,8 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
*
* Note that record headers are MAXALIGN'ed
*/
- if (MAXALIGN(header->xlp_rem_len) >= (XLOG_BLCKSZ - pageHeaderSize))
- tmpRecPtr = targetPagePtr + XLOG_BLCKSZ;
+ if (MAXALIGN(header->xlp_rem_len) >= (state->pageSize - pageHeaderSize))
+ tmpRecPtr = targetPagePtr + state->pageSize;
else
{
/*
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index 9cbd8afa49..f2dd495310 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -199,11 +199,14 @@ StartupDecodingContext(List *output_plugin_options,
ctx->slot = slot;
- ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, cleanup_cb);
+ ctx->reader =
+ XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, cleanup_cb);
+
if (!ctx->reader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
errmsg("out of memory")));
+ ctx->reader->readBuf = palloc(XLOG_BLCKSZ);
ctx->page_read = page_read;
ctx->reorder = ReorderBufferAllocate();
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index 062093727d..8e95600ef7 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -579,8 +579,8 @@ StartReplication(StartReplicationCmd *cmd)
errmsg("IDENTIFY_SYSTEM has not been run before START_REPLICATION")));
/* create xlogreader for physical replication */
- xlogreader =
- XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+ xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+ NULL, wal_segment_close);
if (!xlogreader)
ereport(ERROR,
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index 712c85281c..8a6209b061 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -61,7 +61,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
XLogReaderState *xlogreader;
char *errormsg;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+ xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
@@ -121,7 +121,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex,
char *errormsg;
XLogRecPtr endptr;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+ xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
@@ -181,7 +181,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
forkptr += SizeOfXLogShortPHD;
}
- xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+ xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index f514de6b13..b635537659 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -1041,7 +1041,7 @@ main(int argc, char **argv)
/* we have everything we need, start reading */
xlogreader_state =
- XLogReaderAllocate(WalSegSz, waldir, WALDumpCloseSegment);
+ XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, waldir, WALDumpCloseSegment);
if (!xlogreader_state)
fatal_error("out of memory");
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index b9dad7ee51..1853516333 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -151,6 +151,7 @@ struct XLogReaderState
* ----------------------------------------
*/
/* variables to communicate with page reader */
+ int pageSize; /* size of a page */
XLogRecPtr readPagePtr; /* page pointer to read */
int32 readLen; /* bytes requested to reader, or actual bytes
* read by reader, which must be larger than
@@ -243,7 +244,7 @@ struct XLogReaderState
/* Get a new XLogReader */
extern XLogReaderState *XLogReaderAllocate(int wal_segment_size,
- const char *waldir,
+ int page_size, const char *waldir,
WALSegmentCleanupCB cleanup_cb);
/* Free an XLogReader */
--
2.27.0
----Next_Part(Thu_Mar__4_11_28_53_2021_014)----
^ permalink raw reply [nested|flat] 26+ messages in thread
* [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize.
@ 2019-09-05 12:29 Kyotaro Horiguchi <[email protected]>
0 siblings, 0 replies; 26+ messages in thread
From: Kyotaro Horiguchi @ 2019-09-05 12:29 UTC (permalink / raw)
XLog reader facility and its callers currently relies on the same
constant XLOG_BLCKSZ. This patch moves the responsibility to determine
the block size to the callers of the facility.
---
src/backend/access/transam/twophase.c | 3 +-
src/backend/access/transam/xlog.c | 9 ++--
src/backend/access/transam/xlogreader.c | 57 ++++++++++++-----------
src/backend/replication/logical/logical.c | 5 +-
src/backend/replication/walsender.c | 4 +-
src/bin/pg_rewind/parsexlog.c | 6 +--
src/bin/pg_waldump/pg_waldump.c | 2 +-
src/include/access/xlogreader.h | 3 +-
8 files changed, 50 insertions(+), 39 deletions(-)
diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index 7d5a96a5f5..9f9e7de253 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -1325,7 +1325,8 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
XLogReaderState *xlogreader;
char *errormsg;
- xlogreader = XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+ xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+ NULL, wal_segment_close);
if (!xlogreader)
ereport(ERROR,
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index b517ca85ad..0822ac70cb 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -1222,7 +1222,10 @@ XLogInsertRecord(XLogRecData *rdata,
appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len);
if (!debug_reader)
- debug_reader = XLogReaderAllocate(wal_segment_size, NULL, NULL);
+ {
+ debug_reader =
+ XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, NULL);
+ }
if (!debug_reader)
{
@@ -6531,8 +6534,8 @@ StartupXLOG(void)
OwnLatch(&XLogCtl->recoveryWakeupLatch);
/* Set up XLOG reader facility */
- xlogreader =
- XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+ xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+ NULL, wal_segment_close);
if (!xlogreader)
ereport(ERROR,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index ba3e4868f7..e5baf13086 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -72,7 +72,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...)
* Returns NULL if the xlogreader couldn't be allocated.
*/
XLogReaderState *
-XLogReaderAllocate(int wal_segment_size, const char *waldir,
+XLogReaderAllocate(int wal_segment_size, int page_size, const char *waldir,
WALSegmentCleanupCB cleanup_cb)
{
XLogReaderState *state;
@@ -87,6 +87,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
state->cleanup_cb = cleanup_cb;
state->max_block_id = -1;
+ state->pageSize = page_size;
/*
* Permanently allocate readBuf. We do it this way, rather than just
@@ -95,7 +96,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
* isn't guaranteed to have any particular alignment, whereas
* palloc_extended() will provide MAXALIGN'd storage.
*/
- state->readBuf = (char *) palloc_extended(XLOG_BLCKSZ,
+ state->readBuf = (char *) palloc_extended(state->pageSize,
MCXT_ALLOC_NO_OOM);
if (!state->readBuf)
{
@@ -163,7 +164,7 @@ XLogReaderFree(XLogReaderState *state)
* readRecordBufSize is set to the new buffer size.
*
* To avoid useless small increases, round its size to a multiple of
- * XLOG_BLCKSZ, and make sure it's at least 5*Max(BLCKSZ, XLOG_BLCKSZ) to start
+ * page size, and make sure it's at least 5*Max(BLCKSZ, <page size>) to start
* with. (That is enough for all "normal" records, but very large commit or
* abort records might need more space.)
*/
@@ -172,8 +173,8 @@ allocate_recordbuf(XLogReaderState *state, uint32 reclength)
{
uint32 newSize = reclength;
- newSize += XLOG_BLCKSZ - (newSize % XLOG_BLCKSZ);
- newSize = Max(newSize, 5 * Max(BLCKSZ, XLOG_BLCKSZ));
+ newSize += state->pageSize - (newSize % state->pageSize);
+ newSize = Max(newSize, 5 * Max(BLCKSZ, state->pageSize));
#ifndef FRONTEND
@@ -238,6 +239,8 @@ void
XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
{
Assert(!XLogRecPtrIsInvalid(RecPtr));
+ Assert(state->readBuf != NULL &&
+ state->readBuf == (void *) MAXALIGN(state->readBuf));
ResetDecoder(state);
@@ -366,15 +369,15 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
XLogPageHeader pageHeader;
targetPagePtr =
- state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
- targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+ state->ReadRecPtr - (state->ReadRecPtr % state->pageSize);
+ targetRecOff = state->ReadRecPtr % state->pageSize;
/*
* Check if we have enough data. For the first record in the page,
* the requesting length doesn't contain page header.
*/
if (XLogNeedData(state, targetPagePtr,
- Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+ Min(targetRecOff + SizeOfXLogRecord, state->pageSize),
targetRecOff != 0))
return XLREAD_NEED_DATA;
@@ -421,7 +424,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
* verified that we got the whole header.
*/
prec = (XLogRecord *) (state->readBuf +
- state->ReadRecPtr % XLOG_BLCKSZ);
+ state->ReadRecPtr % state->pageSize);
total_len = prec->xl_tot_len;
/*
@@ -432,7 +435,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
* ensure that we enter the XLREAD_CONTINUATION state below;
* otherwise we might fail to apply ValidXLogRecordHeader at all.
*/
- if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
+ if (targetRecOff <= state->pageSize - SizeOfXLogRecord)
{
if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
state->PrevRecPtr, prec))
@@ -477,10 +480,10 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
* available
*/
targetPagePtr =
- state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
- targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+ state->ReadRecPtr - (state->ReadRecPtr % state->pageSize);
+ targetRecOff = state->ReadRecPtr % state->pageSize;
- request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ);
+ request_len = Min(targetRecOff + total_len, state->pageSize);
record_len = request_len - targetRecOff;
/* ReadRecPtr contains page header */
@@ -549,7 +552,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
/* Calculate pointer to beginning of next page */
state->recordContRecPtr = state->ReadRecPtr + record_len;
- Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0);
+ Assert(state->recordContRecPtr % state->pageSize == 0);
state->readRecordState = XLREAD_CONTINUATION;
}
@@ -576,7 +579,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
/* this request contains page header */
Assert (targetPagePtr != 0);
if (XLogNeedData(state, targetPagePtr,
- Min(state->recordRemainLen, XLOG_BLCKSZ),
+ Min(state->recordRemainLen, state->pageSize),
false))
return XLREAD_NEED_DATA;
@@ -628,7 +631,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
Assert(state->readLen >= pageHeaderSize);
contdata = (char *) state->readBuf + pageHeaderSize;
- record_len = XLOG_BLCKSZ - pageHeaderSize;
+ record_len = state->pageSize - pageHeaderSize;
if (pageHeader->xlp_rem_len < record_len)
record_len = pageHeader->xlp_rem_len;
@@ -655,7 +658,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
}
/* Calculate pointer to beginning of next page, and continue */
- state->recordContRecPtr += XLOG_BLCKSZ;
+ state->recordContRecPtr += state->pageSize;
}
/* targetPagePtr is pointing the last-read page here */
@@ -789,10 +792,10 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
XLogPageHeaderSize((XLogPageHeader) state->readBuf);
addLen = pageHeaderSize;
- if (reqLen + pageHeaderSize <= XLOG_BLCKSZ)
+ if (reqLen + pageHeaderSize <= state->pageSize)
addLen = pageHeaderSize;
else
- addLen = XLOG_BLCKSZ - reqLen;
+ addLen = state->pageSize - reqLen;
Assert(addLen >= 0);
}
@@ -805,7 +808,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
/* Data is not in our buffer, request the caller for it. */
XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize);
targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize);
- Assert((pageptr % XLOG_BLCKSZ) == 0);
+ Assert((pageptr % state->pageSize) == 0);
/*
* Every time we request to load new data of a page to the caller, even if
@@ -829,7 +832,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
* will not come back here, but will request the actual target page.
*/
state->readPagePtr = pageptr - targetPageOff;
- state->readLen = XLOG_BLCKSZ;
+ state->readLen = state->pageSize;
return true;
}
@@ -962,7 +965,7 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
int32 offset;
XLogPageHeader hdr = (XLogPageHeader) phdr;
- Assert((recptr % XLOG_BLCKSZ) == 0);
+ Assert((recptr % state->pageSize) == 0);
XLByteToSeg(recptr, segno, state->segcxt.ws_segsize);
offset = XLogSegmentOffset(recptr, state->segcxt.ws_segsize);
@@ -1016,10 +1019,10 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
"WAL file is from different database system: incorrect segment size in page header");
return false;
}
- else if (longhdr->xlp_xlog_blcksz != XLOG_BLCKSZ)
+ else if (longhdr->xlp_xlog_blcksz != state->pageSize)
{
report_invalid_record(state,
- "WAL file is from different database system: incorrect XLOG_BLCKSZ in page header");
+ "WAL file is from different database system: incorrect block size in page header");
return false;
}
}
@@ -1140,7 +1143,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
* XLogNeedData() is prepared to handle that and will read at
* least short page-header worth of data
*/
- targetRecOff = tmpRecPtr % XLOG_BLCKSZ;
+ targetRecOff = tmpRecPtr % state->pageSize;
/* scroll back to page boundary */
targetPagePtr = tmpRecPtr - targetRecOff;
@@ -1174,8 +1177,8 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
*
* Note that record headers are MAXALIGN'ed
*/
- if (MAXALIGN(header->xlp_rem_len) >= (XLOG_BLCKSZ - pageHeaderSize))
- tmpRecPtr = targetPagePtr + XLOG_BLCKSZ;
+ if (MAXALIGN(header->xlp_rem_len) >= (state->pageSize - pageHeaderSize))
+ tmpRecPtr = targetPagePtr + state->pageSize;
else
{
/*
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index 9cbd8afa49..f2dd495310 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -199,11 +199,14 @@ StartupDecodingContext(List *output_plugin_options,
ctx->slot = slot;
- ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, cleanup_cb);
+ ctx->reader =
+ XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, cleanup_cb);
+
if (!ctx->reader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
errmsg("out of memory")));
+ ctx->reader->readBuf = palloc(XLOG_BLCKSZ);
ctx->page_read = page_read;
ctx->reorder = ReorderBufferAllocate();
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index 062093727d..8e95600ef7 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -579,8 +579,8 @@ StartReplication(StartReplicationCmd *cmd)
errmsg("IDENTIFY_SYSTEM has not been run before START_REPLICATION")));
/* create xlogreader for physical replication */
- xlogreader =
- XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+ xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+ NULL, wal_segment_close);
if (!xlogreader)
ereport(ERROR,
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index 712c85281c..8a6209b061 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -61,7 +61,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
XLogReaderState *xlogreader;
char *errormsg;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+ xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
@@ -121,7 +121,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex,
char *errormsg;
XLogRecPtr endptr;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+ xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
@@ -181,7 +181,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
forkptr += SizeOfXLogShortPHD;
}
- xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+ xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index f514de6b13..b635537659 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -1041,7 +1041,7 @@ main(int argc, char **argv)
/* we have everything we need, start reading */
xlogreader_state =
- XLogReaderAllocate(WalSegSz, waldir, WALDumpCloseSegment);
+ XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, waldir, WALDumpCloseSegment);
if (!xlogreader_state)
fatal_error("out of memory");
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index b9dad7ee51..1853516333 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -151,6 +151,7 @@ struct XLogReaderState
* ----------------------------------------
*/
/* variables to communicate with page reader */
+ int pageSize; /* size of a page */
XLogRecPtr readPagePtr; /* page pointer to read */
int32 readLen; /* bytes requested to reader, or actual bytes
* read by reader, which must be larger than
@@ -243,7 +244,7 @@ struct XLogReaderState
/* Get a new XLogReader */
extern XLogReaderState *XLogReaderAllocate(int wal_segment_size,
- const char *waldir,
+ int page_size, const char *waldir,
WALSegmentCleanupCB cleanup_cb);
/* Free an XLogReader */
--
2.27.0
----Next_Part(Thu_Mar__4_11_28_53_2021_014)----
^ permalink raw reply [nested|flat] 26+ messages in thread
* [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize.
@ 2019-09-05 12:29 Kyotaro Horiguchi <[email protected]>
0 siblings, 0 replies; 26+ messages in thread
From: Kyotaro Horiguchi @ 2019-09-05 12:29 UTC (permalink / raw)
XLog reader facility and its callers currently relies on the same
constant XLOG_BLCKSZ. This patch moves the responsibility to determine
the block size to the callers of the facility.
---
src/backend/access/transam/twophase.c | 3 +-
src/backend/access/transam/xlog.c | 9 ++--
src/backend/access/transam/xlogreader.c | 57 ++++++++++++-----------
src/backend/replication/logical/logical.c | 5 +-
src/backend/replication/walsender.c | 4 +-
src/bin/pg_rewind/parsexlog.c | 6 +--
src/bin/pg_waldump/pg_waldump.c | 2 +-
src/include/access/xlogreader.h | 3 +-
8 files changed, 50 insertions(+), 39 deletions(-)
diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index 7d5a96a5f5..9f9e7de253 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -1325,7 +1325,8 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
XLogReaderState *xlogreader;
char *errormsg;
- xlogreader = XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+ xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+ NULL, wal_segment_close);
if (!xlogreader)
ereport(ERROR,
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index b517ca85ad..0822ac70cb 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -1222,7 +1222,10 @@ XLogInsertRecord(XLogRecData *rdata,
appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len);
if (!debug_reader)
- debug_reader = XLogReaderAllocate(wal_segment_size, NULL, NULL);
+ {
+ debug_reader =
+ XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, NULL);
+ }
if (!debug_reader)
{
@@ -6531,8 +6534,8 @@ StartupXLOG(void)
OwnLatch(&XLogCtl->recoveryWakeupLatch);
/* Set up XLOG reader facility */
- xlogreader =
- XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+ xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+ NULL, wal_segment_close);
if (!xlogreader)
ereport(ERROR,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index ba3e4868f7..e5baf13086 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -72,7 +72,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...)
* Returns NULL if the xlogreader couldn't be allocated.
*/
XLogReaderState *
-XLogReaderAllocate(int wal_segment_size, const char *waldir,
+XLogReaderAllocate(int wal_segment_size, int page_size, const char *waldir,
WALSegmentCleanupCB cleanup_cb)
{
XLogReaderState *state;
@@ -87,6 +87,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
state->cleanup_cb = cleanup_cb;
state->max_block_id = -1;
+ state->pageSize = page_size;
/*
* Permanently allocate readBuf. We do it this way, rather than just
@@ -95,7 +96,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
* isn't guaranteed to have any particular alignment, whereas
* palloc_extended() will provide MAXALIGN'd storage.
*/
- state->readBuf = (char *) palloc_extended(XLOG_BLCKSZ,
+ state->readBuf = (char *) palloc_extended(state->pageSize,
MCXT_ALLOC_NO_OOM);
if (!state->readBuf)
{
@@ -163,7 +164,7 @@ XLogReaderFree(XLogReaderState *state)
* readRecordBufSize is set to the new buffer size.
*
* To avoid useless small increases, round its size to a multiple of
- * XLOG_BLCKSZ, and make sure it's at least 5*Max(BLCKSZ, XLOG_BLCKSZ) to start
+ * page size, and make sure it's at least 5*Max(BLCKSZ, <page size>) to start
* with. (That is enough for all "normal" records, but very large commit or
* abort records might need more space.)
*/
@@ -172,8 +173,8 @@ allocate_recordbuf(XLogReaderState *state, uint32 reclength)
{
uint32 newSize = reclength;
- newSize += XLOG_BLCKSZ - (newSize % XLOG_BLCKSZ);
- newSize = Max(newSize, 5 * Max(BLCKSZ, XLOG_BLCKSZ));
+ newSize += state->pageSize - (newSize % state->pageSize);
+ newSize = Max(newSize, 5 * Max(BLCKSZ, state->pageSize));
#ifndef FRONTEND
@@ -238,6 +239,8 @@ void
XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
{
Assert(!XLogRecPtrIsInvalid(RecPtr));
+ Assert(state->readBuf != NULL &&
+ state->readBuf == (void *) MAXALIGN(state->readBuf));
ResetDecoder(state);
@@ -366,15 +369,15 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
XLogPageHeader pageHeader;
targetPagePtr =
- state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
- targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+ state->ReadRecPtr - (state->ReadRecPtr % state->pageSize);
+ targetRecOff = state->ReadRecPtr % state->pageSize;
/*
* Check if we have enough data. For the first record in the page,
* the requesting length doesn't contain page header.
*/
if (XLogNeedData(state, targetPagePtr,
- Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+ Min(targetRecOff + SizeOfXLogRecord, state->pageSize),
targetRecOff != 0))
return XLREAD_NEED_DATA;
@@ -421,7 +424,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
* verified that we got the whole header.
*/
prec = (XLogRecord *) (state->readBuf +
- state->ReadRecPtr % XLOG_BLCKSZ);
+ state->ReadRecPtr % state->pageSize);
total_len = prec->xl_tot_len;
/*
@@ -432,7 +435,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
* ensure that we enter the XLREAD_CONTINUATION state below;
* otherwise we might fail to apply ValidXLogRecordHeader at all.
*/
- if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
+ if (targetRecOff <= state->pageSize - SizeOfXLogRecord)
{
if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
state->PrevRecPtr, prec))
@@ -477,10 +480,10 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
* available
*/
targetPagePtr =
- state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
- targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+ state->ReadRecPtr - (state->ReadRecPtr % state->pageSize);
+ targetRecOff = state->ReadRecPtr % state->pageSize;
- request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ);
+ request_len = Min(targetRecOff + total_len, state->pageSize);
record_len = request_len - targetRecOff;
/* ReadRecPtr contains page header */
@@ -549,7 +552,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
/* Calculate pointer to beginning of next page */
state->recordContRecPtr = state->ReadRecPtr + record_len;
- Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0);
+ Assert(state->recordContRecPtr % state->pageSize == 0);
state->readRecordState = XLREAD_CONTINUATION;
}
@@ -576,7 +579,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
/* this request contains page header */
Assert (targetPagePtr != 0);
if (XLogNeedData(state, targetPagePtr,
- Min(state->recordRemainLen, XLOG_BLCKSZ),
+ Min(state->recordRemainLen, state->pageSize),
false))
return XLREAD_NEED_DATA;
@@ -628,7 +631,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
Assert(state->readLen >= pageHeaderSize);
contdata = (char *) state->readBuf + pageHeaderSize;
- record_len = XLOG_BLCKSZ - pageHeaderSize;
+ record_len = state->pageSize - pageHeaderSize;
if (pageHeader->xlp_rem_len < record_len)
record_len = pageHeader->xlp_rem_len;
@@ -655,7 +658,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
}
/* Calculate pointer to beginning of next page, and continue */
- state->recordContRecPtr += XLOG_BLCKSZ;
+ state->recordContRecPtr += state->pageSize;
}
/* targetPagePtr is pointing the last-read page here */
@@ -789,10 +792,10 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
XLogPageHeaderSize((XLogPageHeader) state->readBuf);
addLen = pageHeaderSize;
- if (reqLen + pageHeaderSize <= XLOG_BLCKSZ)
+ if (reqLen + pageHeaderSize <= state->pageSize)
addLen = pageHeaderSize;
else
- addLen = XLOG_BLCKSZ - reqLen;
+ addLen = state->pageSize - reqLen;
Assert(addLen >= 0);
}
@@ -805,7 +808,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
/* Data is not in our buffer, request the caller for it. */
XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize);
targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize);
- Assert((pageptr % XLOG_BLCKSZ) == 0);
+ Assert((pageptr % state->pageSize) == 0);
/*
* Every time we request to load new data of a page to the caller, even if
@@ -829,7 +832,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
* will not come back here, but will request the actual target page.
*/
state->readPagePtr = pageptr - targetPageOff;
- state->readLen = XLOG_BLCKSZ;
+ state->readLen = state->pageSize;
return true;
}
@@ -962,7 +965,7 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
int32 offset;
XLogPageHeader hdr = (XLogPageHeader) phdr;
- Assert((recptr % XLOG_BLCKSZ) == 0);
+ Assert((recptr % state->pageSize) == 0);
XLByteToSeg(recptr, segno, state->segcxt.ws_segsize);
offset = XLogSegmentOffset(recptr, state->segcxt.ws_segsize);
@@ -1016,10 +1019,10 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
"WAL file is from different database system: incorrect segment size in page header");
return false;
}
- else if (longhdr->xlp_xlog_blcksz != XLOG_BLCKSZ)
+ else if (longhdr->xlp_xlog_blcksz != state->pageSize)
{
report_invalid_record(state,
- "WAL file is from different database system: incorrect XLOG_BLCKSZ in page header");
+ "WAL file is from different database system: incorrect block size in page header");
return false;
}
}
@@ -1140,7 +1143,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
* XLogNeedData() is prepared to handle that and will read at
* least short page-header worth of data
*/
- targetRecOff = tmpRecPtr % XLOG_BLCKSZ;
+ targetRecOff = tmpRecPtr % state->pageSize;
/* scroll back to page boundary */
targetPagePtr = tmpRecPtr - targetRecOff;
@@ -1174,8 +1177,8 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
*
* Note that record headers are MAXALIGN'ed
*/
- if (MAXALIGN(header->xlp_rem_len) >= (XLOG_BLCKSZ - pageHeaderSize))
- tmpRecPtr = targetPagePtr + XLOG_BLCKSZ;
+ if (MAXALIGN(header->xlp_rem_len) >= (state->pageSize - pageHeaderSize))
+ tmpRecPtr = targetPagePtr + state->pageSize;
else
{
/*
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index 9cbd8afa49..f2dd495310 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -199,11 +199,14 @@ StartupDecodingContext(List *output_plugin_options,
ctx->slot = slot;
- ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, cleanup_cb);
+ ctx->reader =
+ XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, cleanup_cb);
+
if (!ctx->reader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
errmsg("out of memory")));
+ ctx->reader->readBuf = palloc(XLOG_BLCKSZ);
ctx->page_read = page_read;
ctx->reorder = ReorderBufferAllocate();
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index 062093727d..8e95600ef7 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -579,8 +579,8 @@ StartReplication(StartReplicationCmd *cmd)
errmsg("IDENTIFY_SYSTEM has not been run before START_REPLICATION")));
/* create xlogreader for physical replication */
- xlogreader =
- XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+ xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+ NULL, wal_segment_close);
if (!xlogreader)
ereport(ERROR,
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index 712c85281c..8a6209b061 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -61,7 +61,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
XLogReaderState *xlogreader;
char *errormsg;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+ xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
@@ -121,7 +121,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex,
char *errormsg;
XLogRecPtr endptr;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+ xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
@@ -181,7 +181,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
forkptr += SizeOfXLogShortPHD;
}
- xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+ xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index f514de6b13..b635537659 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -1041,7 +1041,7 @@ main(int argc, char **argv)
/* we have everything we need, start reading */
xlogreader_state =
- XLogReaderAllocate(WalSegSz, waldir, WALDumpCloseSegment);
+ XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, waldir, WALDumpCloseSegment);
if (!xlogreader_state)
fatal_error("out of memory");
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index b9dad7ee51..1853516333 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -151,6 +151,7 @@ struct XLogReaderState
* ----------------------------------------
*/
/* variables to communicate with page reader */
+ int pageSize; /* size of a page */
XLogRecPtr readPagePtr; /* page pointer to read */
int32 readLen; /* bytes requested to reader, or actual bytes
* read by reader, which must be larger than
@@ -243,7 +244,7 @@ struct XLogReaderState
/* Get a new XLogReader */
extern XLogReaderState *XLogReaderAllocate(int wal_segment_size,
- const char *waldir,
+ int page_size, const char *waldir,
WALSegmentCleanupCB cleanup_cb);
/* Free an XLogReader */
--
2.27.0
----Next_Part(Thu_Mar__4_11_28_53_2021_014)----
^ permalink raw reply [nested|flat] 26+ messages in thread
* [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize.
@ 2019-09-05 12:29 Kyotaro Horiguchi <[email protected]>
0 siblings, 0 replies; 26+ messages in thread
From: Kyotaro Horiguchi @ 2019-09-05 12:29 UTC (permalink / raw)
XLog reader facility and its callers currently relies on the same
constant XLOG_BLCKSZ. This patch moves the responsibility to determine
the block size to the callers of the facility.
---
src/backend/access/transam/twophase.c | 3 +-
src/backend/access/transam/xlog.c | 9 ++--
src/backend/access/transam/xlogreader.c | 57 ++++++++++++-----------
src/backend/replication/logical/logical.c | 5 +-
src/backend/replication/walsender.c | 4 +-
src/bin/pg_rewind/parsexlog.c | 6 +--
src/bin/pg_waldump/pg_waldump.c | 2 +-
src/include/access/xlogreader.h | 3 +-
8 files changed, 50 insertions(+), 39 deletions(-)
diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index 7d5a96a5f5..9f9e7de253 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -1325,7 +1325,8 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
XLogReaderState *xlogreader;
char *errormsg;
- xlogreader = XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+ xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+ NULL, wal_segment_close);
if (!xlogreader)
ereport(ERROR,
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index b517ca85ad..0822ac70cb 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -1222,7 +1222,10 @@ XLogInsertRecord(XLogRecData *rdata,
appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len);
if (!debug_reader)
- debug_reader = XLogReaderAllocate(wal_segment_size, NULL, NULL);
+ {
+ debug_reader =
+ XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, NULL);
+ }
if (!debug_reader)
{
@@ -6531,8 +6534,8 @@ StartupXLOG(void)
OwnLatch(&XLogCtl->recoveryWakeupLatch);
/* Set up XLOG reader facility */
- xlogreader =
- XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+ xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+ NULL, wal_segment_close);
if (!xlogreader)
ereport(ERROR,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index ba3e4868f7..e5baf13086 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -72,7 +72,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...)
* Returns NULL if the xlogreader couldn't be allocated.
*/
XLogReaderState *
-XLogReaderAllocate(int wal_segment_size, const char *waldir,
+XLogReaderAllocate(int wal_segment_size, int page_size, const char *waldir,
WALSegmentCleanupCB cleanup_cb)
{
XLogReaderState *state;
@@ -87,6 +87,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
state->cleanup_cb = cleanup_cb;
state->max_block_id = -1;
+ state->pageSize = page_size;
/*
* Permanently allocate readBuf. We do it this way, rather than just
@@ -95,7 +96,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
* isn't guaranteed to have any particular alignment, whereas
* palloc_extended() will provide MAXALIGN'd storage.
*/
- state->readBuf = (char *) palloc_extended(XLOG_BLCKSZ,
+ state->readBuf = (char *) palloc_extended(state->pageSize,
MCXT_ALLOC_NO_OOM);
if (!state->readBuf)
{
@@ -163,7 +164,7 @@ XLogReaderFree(XLogReaderState *state)
* readRecordBufSize is set to the new buffer size.
*
* To avoid useless small increases, round its size to a multiple of
- * XLOG_BLCKSZ, and make sure it's at least 5*Max(BLCKSZ, XLOG_BLCKSZ) to start
+ * page size, and make sure it's at least 5*Max(BLCKSZ, <page size>) to start
* with. (That is enough for all "normal" records, but very large commit or
* abort records might need more space.)
*/
@@ -172,8 +173,8 @@ allocate_recordbuf(XLogReaderState *state, uint32 reclength)
{
uint32 newSize = reclength;
- newSize += XLOG_BLCKSZ - (newSize % XLOG_BLCKSZ);
- newSize = Max(newSize, 5 * Max(BLCKSZ, XLOG_BLCKSZ));
+ newSize += state->pageSize - (newSize % state->pageSize);
+ newSize = Max(newSize, 5 * Max(BLCKSZ, state->pageSize));
#ifndef FRONTEND
@@ -238,6 +239,8 @@ void
XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
{
Assert(!XLogRecPtrIsInvalid(RecPtr));
+ Assert(state->readBuf != NULL &&
+ state->readBuf == (void *) MAXALIGN(state->readBuf));
ResetDecoder(state);
@@ -366,15 +369,15 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
XLogPageHeader pageHeader;
targetPagePtr =
- state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
- targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+ state->ReadRecPtr - (state->ReadRecPtr % state->pageSize);
+ targetRecOff = state->ReadRecPtr % state->pageSize;
/*
* Check if we have enough data. For the first record in the page,
* the requesting length doesn't contain page header.
*/
if (XLogNeedData(state, targetPagePtr,
- Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+ Min(targetRecOff + SizeOfXLogRecord, state->pageSize),
targetRecOff != 0))
return XLREAD_NEED_DATA;
@@ -421,7 +424,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
* verified that we got the whole header.
*/
prec = (XLogRecord *) (state->readBuf +
- state->ReadRecPtr % XLOG_BLCKSZ);
+ state->ReadRecPtr % state->pageSize);
total_len = prec->xl_tot_len;
/*
@@ -432,7 +435,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
* ensure that we enter the XLREAD_CONTINUATION state below;
* otherwise we might fail to apply ValidXLogRecordHeader at all.
*/
- if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
+ if (targetRecOff <= state->pageSize - SizeOfXLogRecord)
{
if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
state->PrevRecPtr, prec))
@@ -477,10 +480,10 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
* available
*/
targetPagePtr =
- state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
- targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+ state->ReadRecPtr - (state->ReadRecPtr % state->pageSize);
+ targetRecOff = state->ReadRecPtr % state->pageSize;
- request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ);
+ request_len = Min(targetRecOff + total_len, state->pageSize);
record_len = request_len - targetRecOff;
/* ReadRecPtr contains page header */
@@ -549,7 +552,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
/* Calculate pointer to beginning of next page */
state->recordContRecPtr = state->ReadRecPtr + record_len;
- Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0);
+ Assert(state->recordContRecPtr % state->pageSize == 0);
state->readRecordState = XLREAD_CONTINUATION;
}
@@ -576,7 +579,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
/* this request contains page header */
Assert (targetPagePtr != 0);
if (XLogNeedData(state, targetPagePtr,
- Min(state->recordRemainLen, XLOG_BLCKSZ),
+ Min(state->recordRemainLen, state->pageSize),
false))
return XLREAD_NEED_DATA;
@@ -628,7 +631,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
Assert(state->readLen >= pageHeaderSize);
contdata = (char *) state->readBuf + pageHeaderSize;
- record_len = XLOG_BLCKSZ - pageHeaderSize;
+ record_len = state->pageSize - pageHeaderSize;
if (pageHeader->xlp_rem_len < record_len)
record_len = pageHeader->xlp_rem_len;
@@ -655,7 +658,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
}
/* Calculate pointer to beginning of next page, and continue */
- state->recordContRecPtr += XLOG_BLCKSZ;
+ state->recordContRecPtr += state->pageSize;
}
/* targetPagePtr is pointing the last-read page here */
@@ -789,10 +792,10 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
XLogPageHeaderSize((XLogPageHeader) state->readBuf);
addLen = pageHeaderSize;
- if (reqLen + pageHeaderSize <= XLOG_BLCKSZ)
+ if (reqLen + pageHeaderSize <= state->pageSize)
addLen = pageHeaderSize;
else
- addLen = XLOG_BLCKSZ - reqLen;
+ addLen = state->pageSize - reqLen;
Assert(addLen >= 0);
}
@@ -805,7 +808,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
/* Data is not in our buffer, request the caller for it. */
XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize);
targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize);
- Assert((pageptr % XLOG_BLCKSZ) == 0);
+ Assert((pageptr % state->pageSize) == 0);
/*
* Every time we request to load new data of a page to the caller, even if
@@ -829,7 +832,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
* will not come back here, but will request the actual target page.
*/
state->readPagePtr = pageptr - targetPageOff;
- state->readLen = XLOG_BLCKSZ;
+ state->readLen = state->pageSize;
return true;
}
@@ -962,7 +965,7 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
int32 offset;
XLogPageHeader hdr = (XLogPageHeader) phdr;
- Assert((recptr % XLOG_BLCKSZ) == 0);
+ Assert((recptr % state->pageSize) == 0);
XLByteToSeg(recptr, segno, state->segcxt.ws_segsize);
offset = XLogSegmentOffset(recptr, state->segcxt.ws_segsize);
@@ -1016,10 +1019,10 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
"WAL file is from different database system: incorrect segment size in page header");
return false;
}
- else if (longhdr->xlp_xlog_blcksz != XLOG_BLCKSZ)
+ else if (longhdr->xlp_xlog_blcksz != state->pageSize)
{
report_invalid_record(state,
- "WAL file is from different database system: incorrect XLOG_BLCKSZ in page header");
+ "WAL file is from different database system: incorrect block size in page header");
return false;
}
}
@@ -1140,7 +1143,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
* XLogNeedData() is prepared to handle that and will read at
* least short page-header worth of data
*/
- targetRecOff = tmpRecPtr % XLOG_BLCKSZ;
+ targetRecOff = tmpRecPtr % state->pageSize;
/* scroll back to page boundary */
targetPagePtr = tmpRecPtr - targetRecOff;
@@ -1174,8 +1177,8 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
*
* Note that record headers are MAXALIGN'ed
*/
- if (MAXALIGN(header->xlp_rem_len) >= (XLOG_BLCKSZ - pageHeaderSize))
- tmpRecPtr = targetPagePtr + XLOG_BLCKSZ;
+ if (MAXALIGN(header->xlp_rem_len) >= (state->pageSize - pageHeaderSize))
+ tmpRecPtr = targetPagePtr + state->pageSize;
else
{
/*
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index 9cbd8afa49..f2dd495310 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -199,11 +199,14 @@ StartupDecodingContext(List *output_plugin_options,
ctx->slot = slot;
- ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, cleanup_cb);
+ ctx->reader =
+ XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, cleanup_cb);
+
if (!ctx->reader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
errmsg("out of memory")));
+ ctx->reader->readBuf = palloc(XLOG_BLCKSZ);
ctx->page_read = page_read;
ctx->reorder = ReorderBufferAllocate();
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index 062093727d..8e95600ef7 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -579,8 +579,8 @@ StartReplication(StartReplicationCmd *cmd)
errmsg("IDENTIFY_SYSTEM has not been run before START_REPLICATION")));
/* create xlogreader for physical replication */
- xlogreader =
- XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+ xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+ NULL, wal_segment_close);
if (!xlogreader)
ereport(ERROR,
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index 712c85281c..8a6209b061 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -61,7 +61,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
XLogReaderState *xlogreader;
char *errormsg;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+ xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
@@ -121,7 +121,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex,
char *errormsg;
XLogRecPtr endptr;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+ xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
@@ -181,7 +181,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
forkptr += SizeOfXLogShortPHD;
}
- xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+ xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index f514de6b13..b635537659 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -1041,7 +1041,7 @@ main(int argc, char **argv)
/* we have everything we need, start reading */
xlogreader_state =
- XLogReaderAllocate(WalSegSz, waldir, WALDumpCloseSegment);
+ XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, waldir, WALDumpCloseSegment);
if (!xlogreader_state)
fatal_error("out of memory");
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index b9dad7ee51..1853516333 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -151,6 +151,7 @@ struct XLogReaderState
* ----------------------------------------
*/
/* variables to communicate with page reader */
+ int pageSize; /* size of a page */
XLogRecPtr readPagePtr; /* page pointer to read */
int32 readLen; /* bytes requested to reader, or actual bytes
* read by reader, which must be larger than
@@ -243,7 +244,7 @@ struct XLogReaderState
/* Get a new XLogReader */
extern XLogReaderState *XLogReaderAllocate(int wal_segment_size,
- const char *waldir,
+ int page_size, const char *waldir,
WALSegmentCleanupCB cleanup_cb);
/* Free an XLogReader */
--
2.27.0
----Next_Part(Thu_Mar__4_11_28_53_2021_014)----
^ permalink raw reply [nested|flat] 26+ messages in thread
* [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize.
@ 2019-09-05 12:29 Kyotaro Horiguchi <[email protected]>
0 siblings, 0 replies; 26+ messages in thread
From: Kyotaro Horiguchi @ 2019-09-05 12:29 UTC (permalink / raw)
XLog reader facility and its callers currently relies on the same
constant XLOG_BLCKSZ. This patch moves the responsibility to determine
the block size to the callers of the facility.
---
src/backend/access/transam/twophase.c | 3 +-
src/backend/access/transam/xlog.c | 9 ++--
src/backend/access/transam/xlogreader.c | 57 ++++++++++++-----------
src/backend/replication/logical/logical.c | 5 +-
src/backend/replication/walsender.c | 4 +-
src/bin/pg_rewind/parsexlog.c | 6 +--
src/bin/pg_waldump/pg_waldump.c | 2 +-
src/include/access/xlogreader.h | 3 +-
8 files changed, 50 insertions(+), 39 deletions(-)
diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index 7d5a96a5f5..9f9e7de253 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -1325,7 +1325,8 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
XLogReaderState *xlogreader;
char *errormsg;
- xlogreader = XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+ xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+ NULL, wal_segment_close);
if (!xlogreader)
ereport(ERROR,
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index b517ca85ad..0822ac70cb 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -1222,7 +1222,10 @@ XLogInsertRecord(XLogRecData *rdata,
appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len);
if (!debug_reader)
- debug_reader = XLogReaderAllocate(wal_segment_size, NULL, NULL);
+ {
+ debug_reader =
+ XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, NULL);
+ }
if (!debug_reader)
{
@@ -6531,8 +6534,8 @@ StartupXLOG(void)
OwnLatch(&XLogCtl->recoveryWakeupLatch);
/* Set up XLOG reader facility */
- xlogreader =
- XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+ xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+ NULL, wal_segment_close);
if (!xlogreader)
ereport(ERROR,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index ba3e4868f7..e5baf13086 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -72,7 +72,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...)
* Returns NULL if the xlogreader couldn't be allocated.
*/
XLogReaderState *
-XLogReaderAllocate(int wal_segment_size, const char *waldir,
+XLogReaderAllocate(int wal_segment_size, int page_size, const char *waldir,
WALSegmentCleanupCB cleanup_cb)
{
XLogReaderState *state;
@@ -87,6 +87,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
state->cleanup_cb = cleanup_cb;
state->max_block_id = -1;
+ state->pageSize = page_size;
/*
* Permanently allocate readBuf. We do it this way, rather than just
@@ -95,7 +96,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
* isn't guaranteed to have any particular alignment, whereas
* palloc_extended() will provide MAXALIGN'd storage.
*/
- state->readBuf = (char *) palloc_extended(XLOG_BLCKSZ,
+ state->readBuf = (char *) palloc_extended(state->pageSize,
MCXT_ALLOC_NO_OOM);
if (!state->readBuf)
{
@@ -163,7 +164,7 @@ XLogReaderFree(XLogReaderState *state)
* readRecordBufSize is set to the new buffer size.
*
* To avoid useless small increases, round its size to a multiple of
- * XLOG_BLCKSZ, and make sure it's at least 5*Max(BLCKSZ, XLOG_BLCKSZ) to start
+ * page size, and make sure it's at least 5*Max(BLCKSZ, <page size>) to start
* with. (That is enough for all "normal" records, but very large commit or
* abort records might need more space.)
*/
@@ -172,8 +173,8 @@ allocate_recordbuf(XLogReaderState *state, uint32 reclength)
{
uint32 newSize = reclength;
- newSize += XLOG_BLCKSZ - (newSize % XLOG_BLCKSZ);
- newSize = Max(newSize, 5 * Max(BLCKSZ, XLOG_BLCKSZ));
+ newSize += state->pageSize - (newSize % state->pageSize);
+ newSize = Max(newSize, 5 * Max(BLCKSZ, state->pageSize));
#ifndef FRONTEND
@@ -238,6 +239,8 @@ void
XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
{
Assert(!XLogRecPtrIsInvalid(RecPtr));
+ Assert(state->readBuf != NULL &&
+ state->readBuf == (void *) MAXALIGN(state->readBuf));
ResetDecoder(state);
@@ -366,15 +369,15 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
XLogPageHeader pageHeader;
targetPagePtr =
- state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
- targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+ state->ReadRecPtr - (state->ReadRecPtr % state->pageSize);
+ targetRecOff = state->ReadRecPtr % state->pageSize;
/*
* Check if we have enough data. For the first record in the page,
* the requesting length doesn't contain page header.
*/
if (XLogNeedData(state, targetPagePtr,
- Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+ Min(targetRecOff + SizeOfXLogRecord, state->pageSize),
targetRecOff != 0))
return XLREAD_NEED_DATA;
@@ -421,7 +424,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
* verified that we got the whole header.
*/
prec = (XLogRecord *) (state->readBuf +
- state->ReadRecPtr % XLOG_BLCKSZ);
+ state->ReadRecPtr % state->pageSize);
total_len = prec->xl_tot_len;
/*
@@ -432,7 +435,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
* ensure that we enter the XLREAD_CONTINUATION state below;
* otherwise we might fail to apply ValidXLogRecordHeader at all.
*/
- if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
+ if (targetRecOff <= state->pageSize - SizeOfXLogRecord)
{
if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
state->PrevRecPtr, prec))
@@ -477,10 +480,10 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
* available
*/
targetPagePtr =
- state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
- targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+ state->ReadRecPtr - (state->ReadRecPtr % state->pageSize);
+ targetRecOff = state->ReadRecPtr % state->pageSize;
- request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ);
+ request_len = Min(targetRecOff + total_len, state->pageSize);
record_len = request_len - targetRecOff;
/* ReadRecPtr contains page header */
@@ -549,7 +552,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
/* Calculate pointer to beginning of next page */
state->recordContRecPtr = state->ReadRecPtr + record_len;
- Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0);
+ Assert(state->recordContRecPtr % state->pageSize == 0);
state->readRecordState = XLREAD_CONTINUATION;
}
@@ -576,7 +579,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
/* this request contains page header */
Assert (targetPagePtr != 0);
if (XLogNeedData(state, targetPagePtr,
- Min(state->recordRemainLen, XLOG_BLCKSZ),
+ Min(state->recordRemainLen, state->pageSize),
false))
return XLREAD_NEED_DATA;
@@ -628,7 +631,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
Assert(state->readLen >= pageHeaderSize);
contdata = (char *) state->readBuf + pageHeaderSize;
- record_len = XLOG_BLCKSZ - pageHeaderSize;
+ record_len = state->pageSize - pageHeaderSize;
if (pageHeader->xlp_rem_len < record_len)
record_len = pageHeader->xlp_rem_len;
@@ -655,7 +658,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
}
/* Calculate pointer to beginning of next page, and continue */
- state->recordContRecPtr += XLOG_BLCKSZ;
+ state->recordContRecPtr += state->pageSize;
}
/* targetPagePtr is pointing the last-read page here */
@@ -789,10 +792,10 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
XLogPageHeaderSize((XLogPageHeader) state->readBuf);
addLen = pageHeaderSize;
- if (reqLen + pageHeaderSize <= XLOG_BLCKSZ)
+ if (reqLen + pageHeaderSize <= state->pageSize)
addLen = pageHeaderSize;
else
- addLen = XLOG_BLCKSZ - reqLen;
+ addLen = state->pageSize - reqLen;
Assert(addLen >= 0);
}
@@ -805,7 +808,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
/* Data is not in our buffer, request the caller for it. */
XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize);
targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize);
- Assert((pageptr % XLOG_BLCKSZ) == 0);
+ Assert((pageptr % state->pageSize) == 0);
/*
* Every time we request to load new data of a page to the caller, even if
@@ -829,7 +832,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
* will not come back here, but will request the actual target page.
*/
state->readPagePtr = pageptr - targetPageOff;
- state->readLen = XLOG_BLCKSZ;
+ state->readLen = state->pageSize;
return true;
}
@@ -962,7 +965,7 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
int32 offset;
XLogPageHeader hdr = (XLogPageHeader) phdr;
- Assert((recptr % XLOG_BLCKSZ) == 0);
+ Assert((recptr % state->pageSize) == 0);
XLByteToSeg(recptr, segno, state->segcxt.ws_segsize);
offset = XLogSegmentOffset(recptr, state->segcxt.ws_segsize);
@@ -1016,10 +1019,10 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
"WAL file is from different database system: incorrect segment size in page header");
return false;
}
- else if (longhdr->xlp_xlog_blcksz != XLOG_BLCKSZ)
+ else if (longhdr->xlp_xlog_blcksz != state->pageSize)
{
report_invalid_record(state,
- "WAL file is from different database system: incorrect XLOG_BLCKSZ in page header");
+ "WAL file is from different database system: incorrect block size in page header");
return false;
}
}
@@ -1140,7 +1143,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
* XLogNeedData() is prepared to handle that and will read at
* least short page-header worth of data
*/
- targetRecOff = tmpRecPtr % XLOG_BLCKSZ;
+ targetRecOff = tmpRecPtr % state->pageSize;
/* scroll back to page boundary */
targetPagePtr = tmpRecPtr - targetRecOff;
@@ -1174,8 +1177,8 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
*
* Note that record headers are MAXALIGN'ed
*/
- if (MAXALIGN(header->xlp_rem_len) >= (XLOG_BLCKSZ - pageHeaderSize))
- tmpRecPtr = targetPagePtr + XLOG_BLCKSZ;
+ if (MAXALIGN(header->xlp_rem_len) >= (state->pageSize - pageHeaderSize))
+ tmpRecPtr = targetPagePtr + state->pageSize;
else
{
/*
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index 9cbd8afa49..f2dd495310 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -199,11 +199,14 @@ StartupDecodingContext(List *output_plugin_options,
ctx->slot = slot;
- ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, cleanup_cb);
+ ctx->reader =
+ XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, cleanup_cb);
+
if (!ctx->reader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
errmsg("out of memory")));
+ ctx->reader->readBuf = palloc(XLOG_BLCKSZ);
ctx->page_read = page_read;
ctx->reorder = ReorderBufferAllocate();
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index 062093727d..8e95600ef7 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -579,8 +579,8 @@ StartReplication(StartReplicationCmd *cmd)
errmsg("IDENTIFY_SYSTEM has not been run before START_REPLICATION")));
/* create xlogreader for physical replication */
- xlogreader =
- XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+ xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+ NULL, wal_segment_close);
if (!xlogreader)
ereport(ERROR,
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index 712c85281c..8a6209b061 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -61,7 +61,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
XLogReaderState *xlogreader;
char *errormsg;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+ xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
@@ -121,7 +121,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex,
char *errormsg;
XLogRecPtr endptr;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+ xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
@@ -181,7 +181,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
forkptr += SizeOfXLogShortPHD;
}
- xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+ xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index f514de6b13..b635537659 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -1041,7 +1041,7 @@ main(int argc, char **argv)
/* we have everything we need, start reading */
xlogreader_state =
- XLogReaderAllocate(WalSegSz, waldir, WALDumpCloseSegment);
+ XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, waldir, WALDumpCloseSegment);
if (!xlogreader_state)
fatal_error("out of memory");
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index b9dad7ee51..1853516333 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -151,6 +151,7 @@ struct XLogReaderState
* ----------------------------------------
*/
/* variables to communicate with page reader */
+ int pageSize; /* size of a page */
XLogRecPtr readPagePtr; /* page pointer to read */
int32 readLen; /* bytes requested to reader, or actual bytes
* read by reader, which must be larger than
@@ -243,7 +244,7 @@ struct XLogReaderState
/* Get a new XLogReader */
extern XLogReaderState *XLogReaderAllocate(int wal_segment_size,
- const char *waldir,
+ int page_size, const char *waldir,
WALSegmentCleanupCB cleanup_cb);
/* Free an XLogReader */
--
2.27.0
----Next_Part(Thu_Mar__4_11_28_53_2021_014)----
^ permalink raw reply [nested|flat] 26+ messages in thread
* [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize.
@ 2019-09-05 12:29 Kyotaro Horiguchi <[email protected]>
0 siblings, 0 replies; 26+ messages in thread
From: Kyotaro Horiguchi @ 2019-09-05 12:29 UTC (permalink / raw)
XLog reader facility and its callers currently relies on the same
constant XLOG_BLCKSZ. This patch moves the responsibility to determine
the block size to the callers of the facility.
---
src/backend/access/transam/twophase.c | 3 +-
src/backend/access/transam/xlog.c | 9 ++--
src/backend/access/transam/xlogreader.c | 57 ++++++++++++-----------
src/backend/replication/logical/logical.c | 5 +-
src/backend/replication/walsender.c | 4 +-
src/bin/pg_rewind/parsexlog.c | 6 +--
src/bin/pg_waldump/pg_waldump.c | 2 +-
src/include/access/xlogreader.h | 3 +-
8 files changed, 50 insertions(+), 39 deletions(-)
diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index 7d5a96a5f5..9f9e7de253 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -1325,7 +1325,8 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
XLogReaderState *xlogreader;
char *errormsg;
- xlogreader = XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+ xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+ NULL, wal_segment_close);
if (!xlogreader)
ereport(ERROR,
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index b517ca85ad..0822ac70cb 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -1222,7 +1222,10 @@ XLogInsertRecord(XLogRecData *rdata,
appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len);
if (!debug_reader)
- debug_reader = XLogReaderAllocate(wal_segment_size, NULL, NULL);
+ {
+ debug_reader =
+ XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, NULL);
+ }
if (!debug_reader)
{
@@ -6531,8 +6534,8 @@ StartupXLOG(void)
OwnLatch(&XLogCtl->recoveryWakeupLatch);
/* Set up XLOG reader facility */
- xlogreader =
- XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+ xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+ NULL, wal_segment_close);
if (!xlogreader)
ereport(ERROR,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index ba3e4868f7..e5baf13086 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -72,7 +72,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...)
* Returns NULL if the xlogreader couldn't be allocated.
*/
XLogReaderState *
-XLogReaderAllocate(int wal_segment_size, const char *waldir,
+XLogReaderAllocate(int wal_segment_size, int page_size, const char *waldir,
WALSegmentCleanupCB cleanup_cb)
{
XLogReaderState *state;
@@ -87,6 +87,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
state->cleanup_cb = cleanup_cb;
state->max_block_id = -1;
+ state->pageSize = page_size;
/*
* Permanently allocate readBuf. We do it this way, rather than just
@@ -95,7 +96,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
* isn't guaranteed to have any particular alignment, whereas
* palloc_extended() will provide MAXALIGN'd storage.
*/
- state->readBuf = (char *) palloc_extended(XLOG_BLCKSZ,
+ state->readBuf = (char *) palloc_extended(state->pageSize,
MCXT_ALLOC_NO_OOM);
if (!state->readBuf)
{
@@ -163,7 +164,7 @@ XLogReaderFree(XLogReaderState *state)
* readRecordBufSize is set to the new buffer size.
*
* To avoid useless small increases, round its size to a multiple of
- * XLOG_BLCKSZ, and make sure it's at least 5*Max(BLCKSZ, XLOG_BLCKSZ) to start
+ * page size, and make sure it's at least 5*Max(BLCKSZ, <page size>) to start
* with. (That is enough for all "normal" records, but very large commit or
* abort records might need more space.)
*/
@@ -172,8 +173,8 @@ allocate_recordbuf(XLogReaderState *state, uint32 reclength)
{
uint32 newSize = reclength;
- newSize += XLOG_BLCKSZ - (newSize % XLOG_BLCKSZ);
- newSize = Max(newSize, 5 * Max(BLCKSZ, XLOG_BLCKSZ));
+ newSize += state->pageSize - (newSize % state->pageSize);
+ newSize = Max(newSize, 5 * Max(BLCKSZ, state->pageSize));
#ifndef FRONTEND
@@ -238,6 +239,8 @@ void
XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
{
Assert(!XLogRecPtrIsInvalid(RecPtr));
+ Assert(state->readBuf != NULL &&
+ state->readBuf == (void *) MAXALIGN(state->readBuf));
ResetDecoder(state);
@@ -366,15 +369,15 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
XLogPageHeader pageHeader;
targetPagePtr =
- state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
- targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+ state->ReadRecPtr - (state->ReadRecPtr % state->pageSize);
+ targetRecOff = state->ReadRecPtr % state->pageSize;
/*
* Check if we have enough data. For the first record in the page,
* the requesting length doesn't contain page header.
*/
if (XLogNeedData(state, targetPagePtr,
- Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+ Min(targetRecOff + SizeOfXLogRecord, state->pageSize),
targetRecOff != 0))
return XLREAD_NEED_DATA;
@@ -421,7 +424,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
* verified that we got the whole header.
*/
prec = (XLogRecord *) (state->readBuf +
- state->ReadRecPtr % XLOG_BLCKSZ);
+ state->ReadRecPtr % state->pageSize);
total_len = prec->xl_tot_len;
/*
@@ -432,7 +435,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
* ensure that we enter the XLREAD_CONTINUATION state below;
* otherwise we might fail to apply ValidXLogRecordHeader at all.
*/
- if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
+ if (targetRecOff <= state->pageSize - SizeOfXLogRecord)
{
if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
state->PrevRecPtr, prec))
@@ -477,10 +480,10 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
* available
*/
targetPagePtr =
- state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
- targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+ state->ReadRecPtr - (state->ReadRecPtr % state->pageSize);
+ targetRecOff = state->ReadRecPtr % state->pageSize;
- request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ);
+ request_len = Min(targetRecOff + total_len, state->pageSize);
record_len = request_len - targetRecOff;
/* ReadRecPtr contains page header */
@@ -549,7 +552,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
/* Calculate pointer to beginning of next page */
state->recordContRecPtr = state->ReadRecPtr + record_len;
- Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0);
+ Assert(state->recordContRecPtr % state->pageSize == 0);
state->readRecordState = XLREAD_CONTINUATION;
}
@@ -576,7 +579,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
/* this request contains page header */
Assert (targetPagePtr != 0);
if (XLogNeedData(state, targetPagePtr,
- Min(state->recordRemainLen, XLOG_BLCKSZ),
+ Min(state->recordRemainLen, state->pageSize),
false))
return XLREAD_NEED_DATA;
@@ -628,7 +631,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
Assert(state->readLen >= pageHeaderSize);
contdata = (char *) state->readBuf + pageHeaderSize;
- record_len = XLOG_BLCKSZ - pageHeaderSize;
+ record_len = state->pageSize - pageHeaderSize;
if (pageHeader->xlp_rem_len < record_len)
record_len = pageHeader->xlp_rem_len;
@@ -655,7 +658,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
}
/* Calculate pointer to beginning of next page, and continue */
- state->recordContRecPtr += XLOG_BLCKSZ;
+ state->recordContRecPtr += state->pageSize;
}
/* targetPagePtr is pointing the last-read page here */
@@ -789,10 +792,10 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
XLogPageHeaderSize((XLogPageHeader) state->readBuf);
addLen = pageHeaderSize;
- if (reqLen + pageHeaderSize <= XLOG_BLCKSZ)
+ if (reqLen + pageHeaderSize <= state->pageSize)
addLen = pageHeaderSize;
else
- addLen = XLOG_BLCKSZ - reqLen;
+ addLen = state->pageSize - reqLen;
Assert(addLen >= 0);
}
@@ -805,7 +808,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
/* Data is not in our buffer, request the caller for it. */
XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize);
targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize);
- Assert((pageptr % XLOG_BLCKSZ) == 0);
+ Assert((pageptr % state->pageSize) == 0);
/*
* Every time we request to load new data of a page to the caller, even if
@@ -829,7 +832,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
* will not come back here, but will request the actual target page.
*/
state->readPagePtr = pageptr - targetPageOff;
- state->readLen = XLOG_BLCKSZ;
+ state->readLen = state->pageSize;
return true;
}
@@ -962,7 +965,7 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
int32 offset;
XLogPageHeader hdr = (XLogPageHeader) phdr;
- Assert((recptr % XLOG_BLCKSZ) == 0);
+ Assert((recptr % state->pageSize) == 0);
XLByteToSeg(recptr, segno, state->segcxt.ws_segsize);
offset = XLogSegmentOffset(recptr, state->segcxt.ws_segsize);
@@ -1016,10 +1019,10 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
"WAL file is from different database system: incorrect segment size in page header");
return false;
}
- else if (longhdr->xlp_xlog_blcksz != XLOG_BLCKSZ)
+ else if (longhdr->xlp_xlog_blcksz != state->pageSize)
{
report_invalid_record(state,
- "WAL file is from different database system: incorrect XLOG_BLCKSZ in page header");
+ "WAL file is from different database system: incorrect block size in page header");
return false;
}
}
@@ -1140,7 +1143,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
* XLogNeedData() is prepared to handle that and will read at
* least short page-header worth of data
*/
- targetRecOff = tmpRecPtr % XLOG_BLCKSZ;
+ targetRecOff = tmpRecPtr % state->pageSize;
/* scroll back to page boundary */
targetPagePtr = tmpRecPtr - targetRecOff;
@@ -1174,8 +1177,8 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
*
* Note that record headers are MAXALIGN'ed
*/
- if (MAXALIGN(header->xlp_rem_len) >= (XLOG_BLCKSZ - pageHeaderSize))
- tmpRecPtr = targetPagePtr + XLOG_BLCKSZ;
+ if (MAXALIGN(header->xlp_rem_len) >= (state->pageSize - pageHeaderSize))
+ tmpRecPtr = targetPagePtr + state->pageSize;
else
{
/*
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index 9cbd8afa49..f2dd495310 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -199,11 +199,14 @@ StartupDecodingContext(List *output_plugin_options,
ctx->slot = slot;
- ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, cleanup_cb);
+ ctx->reader =
+ XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, cleanup_cb);
+
if (!ctx->reader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
errmsg("out of memory")));
+ ctx->reader->readBuf = palloc(XLOG_BLCKSZ);
ctx->page_read = page_read;
ctx->reorder = ReorderBufferAllocate();
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index 062093727d..8e95600ef7 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -579,8 +579,8 @@ StartReplication(StartReplicationCmd *cmd)
errmsg("IDENTIFY_SYSTEM has not been run before START_REPLICATION")));
/* create xlogreader for physical replication */
- xlogreader =
- XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+ xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+ NULL, wal_segment_close);
if (!xlogreader)
ereport(ERROR,
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index 712c85281c..8a6209b061 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -61,7 +61,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
XLogReaderState *xlogreader;
char *errormsg;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+ xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
@@ -121,7 +121,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex,
char *errormsg;
XLogRecPtr endptr;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+ xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
@@ -181,7 +181,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
forkptr += SizeOfXLogShortPHD;
}
- xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+ xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index f514de6b13..b635537659 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -1041,7 +1041,7 @@ main(int argc, char **argv)
/* we have everything we need, start reading */
xlogreader_state =
- XLogReaderAllocate(WalSegSz, waldir, WALDumpCloseSegment);
+ XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, waldir, WALDumpCloseSegment);
if (!xlogreader_state)
fatal_error("out of memory");
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index b9dad7ee51..1853516333 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -151,6 +151,7 @@ struct XLogReaderState
* ----------------------------------------
*/
/* variables to communicate with page reader */
+ int pageSize; /* size of a page */
XLogRecPtr readPagePtr; /* page pointer to read */
int32 readLen; /* bytes requested to reader, or actual bytes
* read by reader, which must be larger than
@@ -243,7 +244,7 @@ struct XLogReaderState
/* Get a new XLogReader */
extern XLogReaderState *XLogReaderAllocate(int wal_segment_size,
- const char *waldir,
+ int page_size, const char *waldir,
WALSegmentCleanupCB cleanup_cb);
/* Free an XLogReader */
--
2.27.0
----Next_Part(Thu_Mar__4_11_28_53_2021_014)----
^ permalink raw reply [nested|flat] 26+ messages in thread
* [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize.
@ 2019-09-05 12:29 Kyotaro Horiguchi <[email protected]>
0 siblings, 0 replies; 26+ messages in thread
From: Kyotaro Horiguchi @ 2019-09-05 12:29 UTC (permalink / raw)
XLog reader facility and its callers currently relies on the same
constant XLOG_BLCKSZ. This patch moves the responsibility to determine
the block size to the callers of the facility.
---
src/backend/access/transam/twophase.c | 3 +-
src/backend/access/transam/xlog.c | 9 ++--
src/backend/access/transam/xlogreader.c | 57 ++++++++++++-----------
src/backend/replication/logical/logical.c | 5 +-
src/backend/replication/walsender.c | 4 +-
src/bin/pg_rewind/parsexlog.c | 6 +--
src/bin/pg_waldump/pg_waldump.c | 2 +-
src/include/access/xlogreader.h | 3 +-
8 files changed, 50 insertions(+), 39 deletions(-)
diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index 7d5a96a5f5..9f9e7de253 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -1325,7 +1325,8 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
XLogReaderState *xlogreader;
char *errormsg;
- xlogreader = XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+ xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+ NULL, wal_segment_close);
if (!xlogreader)
ereport(ERROR,
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index b517ca85ad..0822ac70cb 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -1222,7 +1222,10 @@ XLogInsertRecord(XLogRecData *rdata,
appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len);
if (!debug_reader)
- debug_reader = XLogReaderAllocate(wal_segment_size, NULL, NULL);
+ {
+ debug_reader =
+ XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, NULL);
+ }
if (!debug_reader)
{
@@ -6531,8 +6534,8 @@ StartupXLOG(void)
OwnLatch(&XLogCtl->recoveryWakeupLatch);
/* Set up XLOG reader facility */
- xlogreader =
- XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+ xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+ NULL, wal_segment_close);
if (!xlogreader)
ereport(ERROR,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index ba3e4868f7..e5baf13086 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -72,7 +72,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...)
* Returns NULL if the xlogreader couldn't be allocated.
*/
XLogReaderState *
-XLogReaderAllocate(int wal_segment_size, const char *waldir,
+XLogReaderAllocate(int wal_segment_size, int page_size, const char *waldir,
WALSegmentCleanupCB cleanup_cb)
{
XLogReaderState *state;
@@ -87,6 +87,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
state->cleanup_cb = cleanup_cb;
state->max_block_id = -1;
+ state->pageSize = page_size;
/*
* Permanently allocate readBuf. We do it this way, rather than just
@@ -95,7 +96,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
* isn't guaranteed to have any particular alignment, whereas
* palloc_extended() will provide MAXALIGN'd storage.
*/
- state->readBuf = (char *) palloc_extended(XLOG_BLCKSZ,
+ state->readBuf = (char *) palloc_extended(state->pageSize,
MCXT_ALLOC_NO_OOM);
if (!state->readBuf)
{
@@ -163,7 +164,7 @@ XLogReaderFree(XLogReaderState *state)
* readRecordBufSize is set to the new buffer size.
*
* To avoid useless small increases, round its size to a multiple of
- * XLOG_BLCKSZ, and make sure it's at least 5*Max(BLCKSZ, XLOG_BLCKSZ) to start
+ * page size, and make sure it's at least 5*Max(BLCKSZ, <page size>) to start
* with. (That is enough for all "normal" records, but very large commit or
* abort records might need more space.)
*/
@@ -172,8 +173,8 @@ allocate_recordbuf(XLogReaderState *state, uint32 reclength)
{
uint32 newSize = reclength;
- newSize += XLOG_BLCKSZ - (newSize % XLOG_BLCKSZ);
- newSize = Max(newSize, 5 * Max(BLCKSZ, XLOG_BLCKSZ));
+ newSize += state->pageSize - (newSize % state->pageSize);
+ newSize = Max(newSize, 5 * Max(BLCKSZ, state->pageSize));
#ifndef FRONTEND
@@ -238,6 +239,8 @@ void
XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
{
Assert(!XLogRecPtrIsInvalid(RecPtr));
+ Assert(state->readBuf != NULL &&
+ state->readBuf == (void *) MAXALIGN(state->readBuf));
ResetDecoder(state);
@@ -366,15 +369,15 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
XLogPageHeader pageHeader;
targetPagePtr =
- state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
- targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+ state->ReadRecPtr - (state->ReadRecPtr % state->pageSize);
+ targetRecOff = state->ReadRecPtr % state->pageSize;
/*
* Check if we have enough data. For the first record in the page,
* the requesting length doesn't contain page header.
*/
if (XLogNeedData(state, targetPagePtr,
- Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+ Min(targetRecOff + SizeOfXLogRecord, state->pageSize),
targetRecOff != 0))
return XLREAD_NEED_DATA;
@@ -421,7 +424,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
* verified that we got the whole header.
*/
prec = (XLogRecord *) (state->readBuf +
- state->ReadRecPtr % XLOG_BLCKSZ);
+ state->ReadRecPtr % state->pageSize);
total_len = prec->xl_tot_len;
/*
@@ -432,7 +435,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
* ensure that we enter the XLREAD_CONTINUATION state below;
* otherwise we might fail to apply ValidXLogRecordHeader at all.
*/
- if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
+ if (targetRecOff <= state->pageSize - SizeOfXLogRecord)
{
if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
state->PrevRecPtr, prec))
@@ -477,10 +480,10 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
* available
*/
targetPagePtr =
- state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
- targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+ state->ReadRecPtr - (state->ReadRecPtr % state->pageSize);
+ targetRecOff = state->ReadRecPtr % state->pageSize;
- request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ);
+ request_len = Min(targetRecOff + total_len, state->pageSize);
record_len = request_len - targetRecOff;
/* ReadRecPtr contains page header */
@@ -549,7 +552,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
/* Calculate pointer to beginning of next page */
state->recordContRecPtr = state->ReadRecPtr + record_len;
- Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0);
+ Assert(state->recordContRecPtr % state->pageSize == 0);
state->readRecordState = XLREAD_CONTINUATION;
}
@@ -576,7 +579,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
/* this request contains page header */
Assert (targetPagePtr != 0);
if (XLogNeedData(state, targetPagePtr,
- Min(state->recordRemainLen, XLOG_BLCKSZ),
+ Min(state->recordRemainLen, state->pageSize),
false))
return XLREAD_NEED_DATA;
@@ -628,7 +631,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
Assert(state->readLen >= pageHeaderSize);
contdata = (char *) state->readBuf + pageHeaderSize;
- record_len = XLOG_BLCKSZ - pageHeaderSize;
+ record_len = state->pageSize - pageHeaderSize;
if (pageHeader->xlp_rem_len < record_len)
record_len = pageHeader->xlp_rem_len;
@@ -655,7 +658,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
}
/* Calculate pointer to beginning of next page, and continue */
- state->recordContRecPtr += XLOG_BLCKSZ;
+ state->recordContRecPtr += state->pageSize;
}
/* targetPagePtr is pointing the last-read page here */
@@ -789,10 +792,10 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
XLogPageHeaderSize((XLogPageHeader) state->readBuf);
addLen = pageHeaderSize;
- if (reqLen + pageHeaderSize <= XLOG_BLCKSZ)
+ if (reqLen + pageHeaderSize <= state->pageSize)
addLen = pageHeaderSize;
else
- addLen = XLOG_BLCKSZ - reqLen;
+ addLen = state->pageSize - reqLen;
Assert(addLen >= 0);
}
@@ -805,7 +808,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
/* Data is not in our buffer, request the caller for it. */
XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize);
targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize);
- Assert((pageptr % XLOG_BLCKSZ) == 0);
+ Assert((pageptr % state->pageSize) == 0);
/*
* Every time we request to load new data of a page to the caller, even if
@@ -829,7 +832,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
* will not come back here, but will request the actual target page.
*/
state->readPagePtr = pageptr - targetPageOff;
- state->readLen = XLOG_BLCKSZ;
+ state->readLen = state->pageSize;
return true;
}
@@ -962,7 +965,7 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
int32 offset;
XLogPageHeader hdr = (XLogPageHeader) phdr;
- Assert((recptr % XLOG_BLCKSZ) == 0);
+ Assert((recptr % state->pageSize) == 0);
XLByteToSeg(recptr, segno, state->segcxt.ws_segsize);
offset = XLogSegmentOffset(recptr, state->segcxt.ws_segsize);
@@ -1016,10 +1019,10 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
"WAL file is from different database system: incorrect segment size in page header");
return false;
}
- else if (longhdr->xlp_xlog_blcksz != XLOG_BLCKSZ)
+ else if (longhdr->xlp_xlog_blcksz != state->pageSize)
{
report_invalid_record(state,
- "WAL file is from different database system: incorrect XLOG_BLCKSZ in page header");
+ "WAL file is from different database system: incorrect block size in page header");
return false;
}
}
@@ -1140,7 +1143,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
* XLogNeedData() is prepared to handle that and will read at
* least short page-header worth of data
*/
- targetRecOff = tmpRecPtr % XLOG_BLCKSZ;
+ targetRecOff = tmpRecPtr % state->pageSize;
/* scroll back to page boundary */
targetPagePtr = tmpRecPtr - targetRecOff;
@@ -1174,8 +1177,8 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
*
* Note that record headers are MAXALIGN'ed
*/
- if (MAXALIGN(header->xlp_rem_len) >= (XLOG_BLCKSZ - pageHeaderSize))
- tmpRecPtr = targetPagePtr + XLOG_BLCKSZ;
+ if (MAXALIGN(header->xlp_rem_len) >= (state->pageSize - pageHeaderSize))
+ tmpRecPtr = targetPagePtr + state->pageSize;
else
{
/*
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index 9cbd8afa49..f2dd495310 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -199,11 +199,14 @@ StartupDecodingContext(List *output_plugin_options,
ctx->slot = slot;
- ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, cleanup_cb);
+ ctx->reader =
+ XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, cleanup_cb);
+
if (!ctx->reader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
errmsg("out of memory")));
+ ctx->reader->readBuf = palloc(XLOG_BLCKSZ);
ctx->page_read = page_read;
ctx->reorder = ReorderBufferAllocate();
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index 062093727d..8e95600ef7 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -579,8 +579,8 @@ StartReplication(StartReplicationCmd *cmd)
errmsg("IDENTIFY_SYSTEM has not been run before START_REPLICATION")));
/* create xlogreader for physical replication */
- xlogreader =
- XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+ xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+ NULL, wal_segment_close);
if (!xlogreader)
ereport(ERROR,
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index 712c85281c..8a6209b061 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -61,7 +61,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
XLogReaderState *xlogreader;
char *errormsg;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+ xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
@@ -121,7 +121,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex,
char *errormsg;
XLogRecPtr endptr;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+ xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
@@ -181,7 +181,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
forkptr += SizeOfXLogShortPHD;
}
- xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+ xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index f514de6b13..b635537659 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -1041,7 +1041,7 @@ main(int argc, char **argv)
/* we have everything we need, start reading */
xlogreader_state =
- XLogReaderAllocate(WalSegSz, waldir, WALDumpCloseSegment);
+ XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, waldir, WALDumpCloseSegment);
if (!xlogreader_state)
fatal_error("out of memory");
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index b9dad7ee51..1853516333 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -151,6 +151,7 @@ struct XLogReaderState
* ----------------------------------------
*/
/* variables to communicate with page reader */
+ int pageSize; /* size of a page */
XLogRecPtr readPagePtr; /* page pointer to read */
int32 readLen; /* bytes requested to reader, or actual bytes
* read by reader, which must be larger than
@@ -243,7 +244,7 @@ struct XLogReaderState
/* Get a new XLogReader */
extern XLogReaderState *XLogReaderAllocate(int wal_segment_size,
- const char *waldir,
+ int page_size, const char *waldir,
WALSegmentCleanupCB cleanup_cb);
/* Free an XLogReader */
--
2.27.0
----Next_Part(Thu_Mar__4_11_28_53_2021_014)----
^ permalink raw reply [nested|flat] 26+ messages in thread
* [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize.
@ 2019-09-05 12:29 Kyotaro Horiguchi <[email protected]>
0 siblings, 0 replies; 26+ messages in thread
From: Kyotaro Horiguchi @ 2019-09-05 12:29 UTC (permalink / raw)
XLog reader facility and its callers currently relies on the same
constant XLOG_BLCKSZ. This patch moves the responsibility to determine
the block size to the callers of the facility.
---
src/backend/access/transam/twophase.c | 3 +-
src/backend/access/transam/xlog.c | 9 ++--
src/backend/access/transam/xlogreader.c | 57 ++++++++++++-----------
src/backend/replication/logical/logical.c | 5 +-
src/backend/replication/walsender.c | 4 +-
src/bin/pg_rewind/parsexlog.c | 6 +--
src/bin/pg_waldump/pg_waldump.c | 2 +-
src/include/access/xlogreader.h | 3 +-
8 files changed, 50 insertions(+), 39 deletions(-)
diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index 7d5a96a5f5..9f9e7de253 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -1325,7 +1325,8 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
XLogReaderState *xlogreader;
char *errormsg;
- xlogreader = XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+ xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+ NULL, wal_segment_close);
if (!xlogreader)
ereport(ERROR,
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index b517ca85ad..0822ac70cb 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -1222,7 +1222,10 @@ XLogInsertRecord(XLogRecData *rdata,
appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len);
if (!debug_reader)
- debug_reader = XLogReaderAllocate(wal_segment_size, NULL, NULL);
+ {
+ debug_reader =
+ XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, NULL);
+ }
if (!debug_reader)
{
@@ -6531,8 +6534,8 @@ StartupXLOG(void)
OwnLatch(&XLogCtl->recoveryWakeupLatch);
/* Set up XLOG reader facility */
- xlogreader =
- XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+ xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+ NULL, wal_segment_close);
if (!xlogreader)
ereport(ERROR,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index ba3e4868f7..e5baf13086 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -72,7 +72,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...)
* Returns NULL if the xlogreader couldn't be allocated.
*/
XLogReaderState *
-XLogReaderAllocate(int wal_segment_size, const char *waldir,
+XLogReaderAllocate(int wal_segment_size, int page_size, const char *waldir,
WALSegmentCleanupCB cleanup_cb)
{
XLogReaderState *state;
@@ -87,6 +87,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
state->cleanup_cb = cleanup_cb;
state->max_block_id = -1;
+ state->pageSize = page_size;
/*
* Permanently allocate readBuf. We do it this way, rather than just
@@ -95,7 +96,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
* isn't guaranteed to have any particular alignment, whereas
* palloc_extended() will provide MAXALIGN'd storage.
*/
- state->readBuf = (char *) palloc_extended(XLOG_BLCKSZ,
+ state->readBuf = (char *) palloc_extended(state->pageSize,
MCXT_ALLOC_NO_OOM);
if (!state->readBuf)
{
@@ -163,7 +164,7 @@ XLogReaderFree(XLogReaderState *state)
* readRecordBufSize is set to the new buffer size.
*
* To avoid useless small increases, round its size to a multiple of
- * XLOG_BLCKSZ, and make sure it's at least 5*Max(BLCKSZ, XLOG_BLCKSZ) to start
+ * page size, and make sure it's at least 5*Max(BLCKSZ, <page size>) to start
* with. (That is enough for all "normal" records, but very large commit or
* abort records might need more space.)
*/
@@ -172,8 +173,8 @@ allocate_recordbuf(XLogReaderState *state, uint32 reclength)
{
uint32 newSize = reclength;
- newSize += XLOG_BLCKSZ - (newSize % XLOG_BLCKSZ);
- newSize = Max(newSize, 5 * Max(BLCKSZ, XLOG_BLCKSZ));
+ newSize += state->pageSize - (newSize % state->pageSize);
+ newSize = Max(newSize, 5 * Max(BLCKSZ, state->pageSize));
#ifndef FRONTEND
@@ -238,6 +239,8 @@ void
XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
{
Assert(!XLogRecPtrIsInvalid(RecPtr));
+ Assert(state->readBuf != NULL &&
+ state->readBuf == (void *) MAXALIGN(state->readBuf));
ResetDecoder(state);
@@ -366,15 +369,15 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
XLogPageHeader pageHeader;
targetPagePtr =
- state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
- targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+ state->ReadRecPtr - (state->ReadRecPtr % state->pageSize);
+ targetRecOff = state->ReadRecPtr % state->pageSize;
/*
* Check if we have enough data. For the first record in the page,
* the requesting length doesn't contain page header.
*/
if (XLogNeedData(state, targetPagePtr,
- Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+ Min(targetRecOff + SizeOfXLogRecord, state->pageSize),
targetRecOff != 0))
return XLREAD_NEED_DATA;
@@ -421,7 +424,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
* verified that we got the whole header.
*/
prec = (XLogRecord *) (state->readBuf +
- state->ReadRecPtr % XLOG_BLCKSZ);
+ state->ReadRecPtr % state->pageSize);
total_len = prec->xl_tot_len;
/*
@@ -432,7 +435,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
* ensure that we enter the XLREAD_CONTINUATION state below;
* otherwise we might fail to apply ValidXLogRecordHeader at all.
*/
- if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
+ if (targetRecOff <= state->pageSize - SizeOfXLogRecord)
{
if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
state->PrevRecPtr, prec))
@@ -477,10 +480,10 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
* available
*/
targetPagePtr =
- state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
- targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+ state->ReadRecPtr - (state->ReadRecPtr % state->pageSize);
+ targetRecOff = state->ReadRecPtr % state->pageSize;
- request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ);
+ request_len = Min(targetRecOff + total_len, state->pageSize);
record_len = request_len - targetRecOff;
/* ReadRecPtr contains page header */
@@ -549,7 +552,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
/* Calculate pointer to beginning of next page */
state->recordContRecPtr = state->ReadRecPtr + record_len;
- Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0);
+ Assert(state->recordContRecPtr % state->pageSize == 0);
state->readRecordState = XLREAD_CONTINUATION;
}
@@ -576,7 +579,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
/* this request contains page header */
Assert (targetPagePtr != 0);
if (XLogNeedData(state, targetPagePtr,
- Min(state->recordRemainLen, XLOG_BLCKSZ),
+ Min(state->recordRemainLen, state->pageSize),
false))
return XLREAD_NEED_DATA;
@@ -628,7 +631,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
Assert(state->readLen >= pageHeaderSize);
contdata = (char *) state->readBuf + pageHeaderSize;
- record_len = XLOG_BLCKSZ - pageHeaderSize;
+ record_len = state->pageSize - pageHeaderSize;
if (pageHeader->xlp_rem_len < record_len)
record_len = pageHeader->xlp_rem_len;
@@ -655,7 +658,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
}
/* Calculate pointer to beginning of next page, and continue */
- state->recordContRecPtr += XLOG_BLCKSZ;
+ state->recordContRecPtr += state->pageSize;
}
/* targetPagePtr is pointing the last-read page here */
@@ -789,10 +792,10 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
XLogPageHeaderSize((XLogPageHeader) state->readBuf);
addLen = pageHeaderSize;
- if (reqLen + pageHeaderSize <= XLOG_BLCKSZ)
+ if (reqLen + pageHeaderSize <= state->pageSize)
addLen = pageHeaderSize;
else
- addLen = XLOG_BLCKSZ - reqLen;
+ addLen = state->pageSize - reqLen;
Assert(addLen >= 0);
}
@@ -805,7 +808,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
/* Data is not in our buffer, request the caller for it. */
XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize);
targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize);
- Assert((pageptr % XLOG_BLCKSZ) == 0);
+ Assert((pageptr % state->pageSize) == 0);
/*
* Every time we request to load new data of a page to the caller, even if
@@ -829,7 +832,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
* will not come back here, but will request the actual target page.
*/
state->readPagePtr = pageptr - targetPageOff;
- state->readLen = XLOG_BLCKSZ;
+ state->readLen = state->pageSize;
return true;
}
@@ -962,7 +965,7 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
int32 offset;
XLogPageHeader hdr = (XLogPageHeader) phdr;
- Assert((recptr % XLOG_BLCKSZ) == 0);
+ Assert((recptr % state->pageSize) == 0);
XLByteToSeg(recptr, segno, state->segcxt.ws_segsize);
offset = XLogSegmentOffset(recptr, state->segcxt.ws_segsize);
@@ -1016,10 +1019,10 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
"WAL file is from different database system: incorrect segment size in page header");
return false;
}
- else if (longhdr->xlp_xlog_blcksz != XLOG_BLCKSZ)
+ else if (longhdr->xlp_xlog_blcksz != state->pageSize)
{
report_invalid_record(state,
- "WAL file is from different database system: incorrect XLOG_BLCKSZ in page header");
+ "WAL file is from different database system: incorrect block size in page header");
return false;
}
}
@@ -1140,7 +1143,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
* XLogNeedData() is prepared to handle that and will read at
* least short page-header worth of data
*/
- targetRecOff = tmpRecPtr % XLOG_BLCKSZ;
+ targetRecOff = tmpRecPtr % state->pageSize;
/* scroll back to page boundary */
targetPagePtr = tmpRecPtr - targetRecOff;
@@ -1174,8 +1177,8 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
*
* Note that record headers are MAXALIGN'ed
*/
- if (MAXALIGN(header->xlp_rem_len) >= (XLOG_BLCKSZ - pageHeaderSize))
- tmpRecPtr = targetPagePtr + XLOG_BLCKSZ;
+ if (MAXALIGN(header->xlp_rem_len) >= (state->pageSize - pageHeaderSize))
+ tmpRecPtr = targetPagePtr + state->pageSize;
else
{
/*
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index 9cbd8afa49..f2dd495310 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -199,11 +199,14 @@ StartupDecodingContext(List *output_plugin_options,
ctx->slot = slot;
- ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, cleanup_cb);
+ ctx->reader =
+ XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, cleanup_cb);
+
if (!ctx->reader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
errmsg("out of memory")));
+ ctx->reader->readBuf = palloc(XLOG_BLCKSZ);
ctx->page_read = page_read;
ctx->reorder = ReorderBufferAllocate();
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index 062093727d..8e95600ef7 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -579,8 +579,8 @@ StartReplication(StartReplicationCmd *cmd)
errmsg("IDENTIFY_SYSTEM has not been run before START_REPLICATION")));
/* create xlogreader for physical replication */
- xlogreader =
- XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+ xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+ NULL, wal_segment_close);
if (!xlogreader)
ereport(ERROR,
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index 712c85281c..8a6209b061 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -61,7 +61,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
XLogReaderState *xlogreader;
char *errormsg;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+ xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
@@ -121,7 +121,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex,
char *errormsg;
XLogRecPtr endptr;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+ xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
@@ -181,7 +181,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
forkptr += SizeOfXLogShortPHD;
}
- xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+ xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index f514de6b13..b635537659 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -1041,7 +1041,7 @@ main(int argc, char **argv)
/* we have everything we need, start reading */
xlogreader_state =
- XLogReaderAllocate(WalSegSz, waldir, WALDumpCloseSegment);
+ XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, waldir, WALDumpCloseSegment);
if (!xlogreader_state)
fatal_error("out of memory");
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index b9dad7ee51..1853516333 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -151,6 +151,7 @@ struct XLogReaderState
* ----------------------------------------
*/
/* variables to communicate with page reader */
+ int pageSize; /* size of a page */
XLogRecPtr readPagePtr; /* page pointer to read */
int32 readLen; /* bytes requested to reader, or actual bytes
* read by reader, which must be larger than
@@ -243,7 +244,7 @@ struct XLogReaderState
/* Get a new XLogReader */
extern XLogReaderState *XLogReaderAllocate(int wal_segment_size,
- const char *waldir,
+ int page_size, const char *waldir,
WALSegmentCleanupCB cleanup_cb);
/* Free an XLogReader */
--
2.27.0
----Next_Part(Thu_Mar__4_11_28_53_2021_014)----
^ permalink raw reply [nested|flat] 26+ messages in thread
* Re: json_object returning jsonb reuslt different from returning json, returning text
@ 2022-04-25 05:19 alias <[email protected]>
0 siblings, 0 replies; 26+ messages in thread
From: alias @ 2022-04-25 05:19 UTC (permalink / raw)
To: PostgreSQL Hackers <[email protected]>
seems it's a bug around value 0.
SELECT JSON_OBJECTAGG(k: v ABSENT ON NULL WITH UNIQUE KEYS RETURNING jsonb)
FROM (VALUES (1, 1), (10, NULL),(4, null), (5, null),(6, null),(2, 2))
foo(k, v);
return:
{"1": 1, "2": 2}
SELECT JSON_OBJECTAGG(k: v ABSENT ON NULL WITH UNIQUE KEYS RETURNING jsonb)
FROM (VALUES (1, 1), (0, NULL),(4, null), (5, null),(6, null),(2, 2))
foo(k, v);
return
{"0": null, "1": 1, "2": 2}
On Mon, Apr 25, 2022 at 10:41 AM alias <[email protected]> wrote:
> select json_objectagg(
> k:v absent on null with unique keys returning text )
> from (
> values(1,1),(0, null),(3, null),(2,2),(4,null)
> ) foo(k, v);
>
> return
>
> json_objectagg
> ----------------------
> { "1" : 1, "2" : 2 }
> --------------------
>
> select json_objectagg(k:v absent on null with unique keys)
> from (
> values(1,1),(0, null),(3, null),(2,2),(4,null)
> ) foo(k, v);
>
> return
>
> json_objectagg ---------------------- { "1" : 1, "2" : 2 }
>
> *But*
>
> select json_objectagg(
> k:v absent on null with unique keys returning jsonb )
> from (
> values(1,1),(0, null),(3, null),(2,2),(4,null)
> ) foo(k, v);
>
> return
> json_objectagg ----------------------------- {"0": null, "1": 1, "2": 2}
>
> the last query "returning jsonb" should be { "1" : 1, "2" : 2 } ?
>
> version:
>
>> PostgreSQL 15devel (Ubuntu
>> 15~~devel~20220407.0430-1~713.git79b716c.pgdg20.04+1) on
>> x86_64-pc-linux-gnu,
>> compiled by gcc (Ubuntu 9.4.0-1ubuntu1~20.04.1) 9.4.0, 64-bit
>
>
^ permalink raw reply [nested|flat] 26+ messages in thread
end of thread, other threads:[~2022-04-25 05:19 UTC | newest]
Thread overview: 26+ messages (download: mbox mbox.gz follow: Atom feed)
-- links below jump to the message on this page --
2019-09-05 12:29 [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize. Kyotaro Horiguchi <[email protected]>
2019-09-05 12:29 [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize. Kyotaro Horiguchi <[email protected]>
2019-09-05 12:29 [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize. Kyotaro Horiguchi <[email protected]>
2019-09-05 12:29 [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize. Kyotaro Horiguchi <[email protected]>
2019-09-05 12:29 [PATCH v16 4/4] Allow xlogreader to use different xlog blocksize. Kyotaro Horiguchi <[email protected]>
2019-09-05 12:29 [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize. Kyotaro Horiguchi <[email protected]>
2019-09-05 12:29 [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize. Kyotaro Horiguchi <[email protected]>
2019-09-05 12:29 [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize. Kyotaro Horiguchi <[email protected]>
2019-09-05 12:29 [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize. Kyotaro Horiguchi <[email protected]>
2019-09-05 12:29 [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize. Kyotaro Horiguchi <[email protected]>
2019-09-05 12:29 [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize. Kyotaro Horiguchi <[email protected]>
2019-09-05 12:29 [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize. Kyotaro Horiguchi <[email protected]>
2019-09-05 12:29 [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize. Kyotaro Horiguchi <[email protected]>
2019-09-05 12:29 [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize. Kyotaro Horiguchi <[email protected]>
2019-09-05 12:29 [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize. Kyotaro Horiguchi <[email protected]>
2019-09-05 12:29 [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize. Kyotaro Horiguchi <[email protected]>
2019-09-05 12:29 [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize. Kyotaro Horiguchi <[email protected]>
2019-09-05 12:29 [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize. Kyotaro Horiguchi <[email protected]>
2019-09-05 12:29 [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize. Kyotaro Horiguchi <[email protected]>
2019-09-05 12:29 [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize. Kyotaro Horiguchi <[email protected]>
2019-09-05 12:29 [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize. Kyotaro Horiguchi <[email protected]>
2019-09-05 12:29 [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize. Kyotaro Horiguchi <[email protected]>
2019-09-05 12:29 [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize. Kyotaro Horiguchi <[email protected]>
2019-09-05 12:29 [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize. Kyotaro Horiguchi <[email protected]>
2019-09-05 12:29 [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize. Kyotaro Horiguchi <[email protected]>
2022-04-25 05:19 Re: json_object returning jsonb reuslt different from returning json, returning text alias <[email protected]>
This inbox is served by agora; see mirroring instructions
for how to clone and mirror all data and code used for this inbox