public inbox for [email protected]
help / color / mirror / Atom feed[PATCH v17 4/4] Allow xlogreader to use different xlog blocksize.
30+ messages / 4 participants
[nested] [flat]
* [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize.
@ 2019-09-05 12:29 Kyotaro Horiguchi <[email protected]>
0 siblings, 0 replies; 30+ messages in thread
From: Kyotaro Horiguchi @ 2019-09-05 12:29 UTC (permalink / raw)
XLog reader facility and its callers currently relies on the same
constant XLOG_BLCKSZ. This patch moves the responsibility to determine
the block size to the callers of the facility.
---
src/backend/access/transam/twophase.c | 3 +-
src/backend/access/transam/xlog.c | 9 ++--
src/backend/access/transam/xlogreader.c | 57 ++++++++++++-----------
src/backend/replication/logical/logical.c | 5 +-
src/backend/replication/walsender.c | 4 +-
src/bin/pg_rewind/parsexlog.c | 6 +--
src/bin/pg_waldump/pg_waldump.c | 2 +-
src/include/access/xlogreader.h | 3 +-
8 files changed, 50 insertions(+), 39 deletions(-)
diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index 7d5a96a5f5..9f9e7de253 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -1325,7 +1325,8 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
XLogReaderState *xlogreader;
char *errormsg;
- xlogreader = XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+ xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+ NULL, wal_segment_close);
if (!xlogreader)
ereport(ERROR,
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index b517ca85ad..0822ac70cb 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -1222,7 +1222,10 @@ XLogInsertRecord(XLogRecData *rdata,
appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len);
if (!debug_reader)
- debug_reader = XLogReaderAllocate(wal_segment_size, NULL, NULL);
+ {
+ debug_reader =
+ XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, NULL);
+ }
if (!debug_reader)
{
@@ -6531,8 +6534,8 @@ StartupXLOG(void)
OwnLatch(&XLogCtl->recoveryWakeupLatch);
/* Set up XLOG reader facility */
- xlogreader =
- XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+ xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+ NULL, wal_segment_close);
if (!xlogreader)
ereport(ERROR,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index ba3e4868f7..e5baf13086 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -72,7 +72,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...)
* Returns NULL if the xlogreader couldn't be allocated.
*/
XLogReaderState *
-XLogReaderAllocate(int wal_segment_size, const char *waldir,
+XLogReaderAllocate(int wal_segment_size, int page_size, const char *waldir,
WALSegmentCleanupCB cleanup_cb)
{
XLogReaderState *state;
@@ -87,6 +87,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
state->cleanup_cb = cleanup_cb;
state->max_block_id = -1;
+ state->pageSize = page_size;
/*
* Permanently allocate readBuf. We do it this way, rather than just
@@ -95,7 +96,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
* isn't guaranteed to have any particular alignment, whereas
* palloc_extended() will provide MAXALIGN'd storage.
*/
- state->readBuf = (char *) palloc_extended(XLOG_BLCKSZ,
+ state->readBuf = (char *) palloc_extended(state->pageSize,
MCXT_ALLOC_NO_OOM);
if (!state->readBuf)
{
@@ -163,7 +164,7 @@ XLogReaderFree(XLogReaderState *state)
* readRecordBufSize is set to the new buffer size.
*
* To avoid useless small increases, round its size to a multiple of
- * XLOG_BLCKSZ, and make sure it's at least 5*Max(BLCKSZ, XLOG_BLCKSZ) to start
+ * page size, and make sure it's at least 5*Max(BLCKSZ, <page size>) to start
* with. (That is enough for all "normal" records, but very large commit or
* abort records might need more space.)
*/
@@ -172,8 +173,8 @@ allocate_recordbuf(XLogReaderState *state, uint32 reclength)
{
uint32 newSize = reclength;
- newSize += XLOG_BLCKSZ - (newSize % XLOG_BLCKSZ);
- newSize = Max(newSize, 5 * Max(BLCKSZ, XLOG_BLCKSZ));
+ newSize += state->pageSize - (newSize % state->pageSize);
+ newSize = Max(newSize, 5 * Max(BLCKSZ, state->pageSize));
#ifndef FRONTEND
@@ -238,6 +239,8 @@ void
XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
{
Assert(!XLogRecPtrIsInvalid(RecPtr));
+ Assert(state->readBuf != NULL &&
+ state->readBuf == (void *) MAXALIGN(state->readBuf));
ResetDecoder(state);
@@ -366,15 +369,15 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
XLogPageHeader pageHeader;
targetPagePtr =
- state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
- targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+ state->ReadRecPtr - (state->ReadRecPtr % state->pageSize);
+ targetRecOff = state->ReadRecPtr % state->pageSize;
/*
* Check if we have enough data. For the first record in the page,
* the requesting length doesn't contain page header.
*/
if (XLogNeedData(state, targetPagePtr,
- Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+ Min(targetRecOff + SizeOfXLogRecord, state->pageSize),
targetRecOff != 0))
return XLREAD_NEED_DATA;
@@ -421,7 +424,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
* verified that we got the whole header.
*/
prec = (XLogRecord *) (state->readBuf +
- state->ReadRecPtr % XLOG_BLCKSZ);
+ state->ReadRecPtr % state->pageSize);
total_len = prec->xl_tot_len;
/*
@@ -432,7 +435,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
* ensure that we enter the XLREAD_CONTINUATION state below;
* otherwise we might fail to apply ValidXLogRecordHeader at all.
*/
- if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
+ if (targetRecOff <= state->pageSize - SizeOfXLogRecord)
{
if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
state->PrevRecPtr, prec))
@@ -477,10 +480,10 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
* available
*/
targetPagePtr =
- state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
- targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+ state->ReadRecPtr - (state->ReadRecPtr % state->pageSize);
+ targetRecOff = state->ReadRecPtr % state->pageSize;
- request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ);
+ request_len = Min(targetRecOff + total_len, state->pageSize);
record_len = request_len - targetRecOff;
/* ReadRecPtr contains page header */
@@ -549,7 +552,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
/* Calculate pointer to beginning of next page */
state->recordContRecPtr = state->ReadRecPtr + record_len;
- Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0);
+ Assert(state->recordContRecPtr % state->pageSize == 0);
state->readRecordState = XLREAD_CONTINUATION;
}
@@ -576,7 +579,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
/* this request contains page header */
Assert (targetPagePtr != 0);
if (XLogNeedData(state, targetPagePtr,
- Min(state->recordRemainLen, XLOG_BLCKSZ),
+ Min(state->recordRemainLen, state->pageSize),
false))
return XLREAD_NEED_DATA;
@@ -628,7 +631,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
Assert(state->readLen >= pageHeaderSize);
contdata = (char *) state->readBuf + pageHeaderSize;
- record_len = XLOG_BLCKSZ - pageHeaderSize;
+ record_len = state->pageSize - pageHeaderSize;
if (pageHeader->xlp_rem_len < record_len)
record_len = pageHeader->xlp_rem_len;
@@ -655,7 +658,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
}
/* Calculate pointer to beginning of next page, and continue */
- state->recordContRecPtr += XLOG_BLCKSZ;
+ state->recordContRecPtr += state->pageSize;
}
/* targetPagePtr is pointing the last-read page here */
@@ -789,10 +792,10 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
XLogPageHeaderSize((XLogPageHeader) state->readBuf);
addLen = pageHeaderSize;
- if (reqLen + pageHeaderSize <= XLOG_BLCKSZ)
+ if (reqLen + pageHeaderSize <= state->pageSize)
addLen = pageHeaderSize;
else
- addLen = XLOG_BLCKSZ - reqLen;
+ addLen = state->pageSize - reqLen;
Assert(addLen >= 0);
}
@@ -805,7 +808,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
/* Data is not in our buffer, request the caller for it. */
XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize);
targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize);
- Assert((pageptr % XLOG_BLCKSZ) == 0);
+ Assert((pageptr % state->pageSize) == 0);
/*
* Every time we request to load new data of a page to the caller, even if
@@ -829,7 +832,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
* will not come back here, but will request the actual target page.
*/
state->readPagePtr = pageptr - targetPageOff;
- state->readLen = XLOG_BLCKSZ;
+ state->readLen = state->pageSize;
return true;
}
@@ -962,7 +965,7 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
int32 offset;
XLogPageHeader hdr = (XLogPageHeader) phdr;
- Assert((recptr % XLOG_BLCKSZ) == 0);
+ Assert((recptr % state->pageSize) == 0);
XLByteToSeg(recptr, segno, state->segcxt.ws_segsize);
offset = XLogSegmentOffset(recptr, state->segcxt.ws_segsize);
@@ -1016,10 +1019,10 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
"WAL file is from different database system: incorrect segment size in page header");
return false;
}
- else if (longhdr->xlp_xlog_blcksz != XLOG_BLCKSZ)
+ else if (longhdr->xlp_xlog_blcksz != state->pageSize)
{
report_invalid_record(state,
- "WAL file is from different database system: incorrect XLOG_BLCKSZ in page header");
+ "WAL file is from different database system: incorrect block size in page header");
return false;
}
}
@@ -1140,7 +1143,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
* XLogNeedData() is prepared to handle that and will read at
* least short page-header worth of data
*/
- targetRecOff = tmpRecPtr % XLOG_BLCKSZ;
+ targetRecOff = tmpRecPtr % state->pageSize;
/* scroll back to page boundary */
targetPagePtr = tmpRecPtr - targetRecOff;
@@ -1174,8 +1177,8 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
*
* Note that record headers are MAXALIGN'ed
*/
- if (MAXALIGN(header->xlp_rem_len) >= (XLOG_BLCKSZ - pageHeaderSize))
- tmpRecPtr = targetPagePtr + XLOG_BLCKSZ;
+ if (MAXALIGN(header->xlp_rem_len) >= (state->pageSize - pageHeaderSize))
+ tmpRecPtr = targetPagePtr + state->pageSize;
else
{
/*
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index 9cbd8afa49..f2dd495310 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -199,11 +199,14 @@ StartupDecodingContext(List *output_plugin_options,
ctx->slot = slot;
- ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, cleanup_cb);
+ ctx->reader =
+ XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, cleanup_cb);
+
if (!ctx->reader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
errmsg("out of memory")));
+ ctx->reader->readBuf = palloc(XLOG_BLCKSZ);
ctx->page_read = page_read;
ctx->reorder = ReorderBufferAllocate();
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index 062093727d..8e95600ef7 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -579,8 +579,8 @@ StartReplication(StartReplicationCmd *cmd)
errmsg("IDENTIFY_SYSTEM has not been run before START_REPLICATION")));
/* create xlogreader for physical replication */
- xlogreader =
- XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+ xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+ NULL, wal_segment_close);
if (!xlogreader)
ereport(ERROR,
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index 712c85281c..8a6209b061 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -61,7 +61,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
XLogReaderState *xlogreader;
char *errormsg;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+ xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
@@ -121,7 +121,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex,
char *errormsg;
XLogRecPtr endptr;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+ xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
@@ -181,7 +181,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
forkptr += SizeOfXLogShortPHD;
}
- xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+ xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index f514de6b13..b635537659 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -1041,7 +1041,7 @@ main(int argc, char **argv)
/* we have everything we need, start reading */
xlogreader_state =
- XLogReaderAllocate(WalSegSz, waldir, WALDumpCloseSegment);
+ XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, waldir, WALDumpCloseSegment);
if (!xlogreader_state)
fatal_error("out of memory");
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index b9dad7ee51..1853516333 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -151,6 +151,7 @@ struct XLogReaderState
* ----------------------------------------
*/
/* variables to communicate with page reader */
+ int pageSize; /* size of a page */
XLogRecPtr readPagePtr; /* page pointer to read */
int32 readLen; /* bytes requested to reader, or actual bytes
* read by reader, which must be larger than
@@ -243,7 +244,7 @@ struct XLogReaderState
/* Get a new XLogReader */
extern XLogReaderState *XLogReaderAllocate(int wal_segment_size,
- const char *waldir,
+ int page_size, const char *waldir,
WALSegmentCleanupCB cleanup_cb);
/* Free an XLogReader */
--
2.27.0
----Next_Part(Thu_Mar__4_11_28_53_2021_014)----
^ permalink raw reply [nested|flat] 30+ messages in thread
* [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize.
@ 2019-09-05 12:29 Kyotaro Horiguchi <[email protected]>
0 siblings, 0 replies; 30+ messages in thread
From: Kyotaro Horiguchi @ 2019-09-05 12:29 UTC (permalink / raw)
XLog reader facility and its callers currently relies on the same
constant XLOG_BLCKSZ. This patch moves the responsibility to determine
the block size to the callers of the facility.
---
src/backend/access/transam/twophase.c | 3 +-
src/backend/access/transam/xlog.c | 9 ++--
src/backend/access/transam/xlogreader.c | 57 ++++++++++++-----------
src/backend/replication/logical/logical.c | 5 +-
src/backend/replication/walsender.c | 4 +-
src/bin/pg_rewind/parsexlog.c | 6 +--
src/bin/pg_waldump/pg_waldump.c | 2 +-
src/include/access/xlogreader.h | 3 +-
8 files changed, 50 insertions(+), 39 deletions(-)
diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index 7d5a96a5f5..9f9e7de253 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -1325,7 +1325,8 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
XLogReaderState *xlogreader;
char *errormsg;
- xlogreader = XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+ xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+ NULL, wal_segment_close);
if (!xlogreader)
ereport(ERROR,
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index b517ca85ad..0822ac70cb 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -1222,7 +1222,10 @@ XLogInsertRecord(XLogRecData *rdata,
appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len);
if (!debug_reader)
- debug_reader = XLogReaderAllocate(wal_segment_size, NULL, NULL);
+ {
+ debug_reader =
+ XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, NULL);
+ }
if (!debug_reader)
{
@@ -6531,8 +6534,8 @@ StartupXLOG(void)
OwnLatch(&XLogCtl->recoveryWakeupLatch);
/* Set up XLOG reader facility */
- xlogreader =
- XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+ xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+ NULL, wal_segment_close);
if (!xlogreader)
ereport(ERROR,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index ba3e4868f7..e5baf13086 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -72,7 +72,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...)
* Returns NULL if the xlogreader couldn't be allocated.
*/
XLogReaderState *
-XLogReaderAllocate(int wal_segment_size, const char *waldir,
+XLogReaderAllocate(int wal_segment_size, int page_size, const char *waldir,
WALSegmentCleanupCB cleanup_cb)
{
XLogReaderState *state;
@@ -87,6 +87,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
state->cleanup_cb = cleanup_cb;
state->max_block_id = -1;
+ state->pageSize = page_size;
/*
* Permanently allocate readBuf. We do it this way, rather than just
@@ -95,7 +96,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
* isn't guaranteed to have any particular alignment, whereas
* palloc_extended() will provide MAXALIGN'd storage.
*/
- state->readBuf = (char *) palloc_extended(XLOG_BLCKSZ,
+ state->readBuf = (char *) palloc_extended(state->pageSize,
MCXT_ALLOC_NO_OOM);
if (!state->readBuf)
{
@@ -163,7 +164,7 @@ XLogReaderFree(XLogReaderState *state)
* readRecordBufSize is set to the new buffer size.
*
* To avoid useless small increases, round its size to a multiple of
- * XLOG_BLCKSZ, and make sure it's at least 5*Max(BLCKSZ, XLOG_BLCKSZ) to start
+ * page size, and make sure it's at least 5*Max(BLCKSZ, <page size>) to start
* with. (That is enough for all "normal" records, but very large commit or
* abort records might need more space.)
*/
@@ -172,8 +173,8 @@ allocate_recordbuf(XLogReaderState *state, uint32 reclength)
{
uint32 newSize = reclength;
- newSize += XLOG_BLCKSZ - (newSize % XLOG_BLCKSZ);
- newSize = Max(newSize, 5 * Max(BLCKSZ, XLOG_BLCKSZ));
+ newSize += state->pageSize - (newSize % state->pageSize);
+ newSize = Max(newSize, 5 * Max(BLCKSZ, state->pageSize));
#ifndef FRONTEND
@@ -238,6 +239,8 @@ void
XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
{
Assert(!XLogRecPtrIsInvalid(RecPtr));
+ Assert(state->readBuf != NULL &&
+ state->readBuf == (void *) MAXALIGN(state->readBuf));
ResetDecoder(state);
@@ -366,15 +369,15 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
XLogPageHeader pageHeader;
targetPagePtr =
- state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
- targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+ state->ReadRecPtr - (state->ReadRecPtr % state->pageSize);
+ targetRecOff = state->ReadRecPtr % state->pageSize;
/*
* Check if we have enough data. For the first record in the page,
* the requesting length doesn't contain page header.
*/
if (XLogNeedData(state, targetPagePtr,
- Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+ Min(targetRecOff + SizeOfXLogRecord, state->pageSize),
targetRecOff != 0))
return XLREAD_NEED_DATA;
@@ -421,7 +424,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
* verified that we got the whole header.
*/
prec = (XLogRecord *) (state->readBuf +
- state->ReadRecPtr % XLOG_BLCKSZ);
+ state->ReadRecPtr % state->pageSize);
total_len = prec->xl_tot_len;
/*
@@ -432,7 +435,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
* ensure that we enter the XLREAD_CONTINUATION state below;
* otherwise we might fail to apply ValidXLogRecordHeader at all.
*/
- if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
+ if (targetRecOff <= state->pageSize - SizeOfXLogRecord)
{
if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
state->PrevRecPtr, prec))
@@ -477,10 +480,10 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
* available
*/
targetPagePtr =
- state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
- targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+ state->ReadRecPtr - (state->ReadRecPtr % state->pageSize);
+ targetRecOff = state->ReadRecPtr % state->pageSize;
- request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ);
+ request_len = Min(targetRecOff + total_len, state->pageSize);
record_len = request_len - targetRecOff;
/* ReadRecPtr contains page header */
@@ -549,7 +552,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
/* Calculate pointer to beginning of next page */
state->recordContRecPtr = state->ReadRecPtr + record_len;
- Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0);
+ Assert(state->recordContRecPtr % state->pageSize == 0);
state->readRecordState = XLREAD_CONTINUATION;
}
@@ -576,7 +579,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
/* this request contains page header */
Assert (targetPagePtr != 0);
if (XLogNeedData(state, targetPagePtr,
- Min(state->recordRemainLen, XLOG_BLCKSZ),
+ Min(state->recordRemainLen, state->pageSize),
false))
return XLREAD_NEED_DATA;
@@ -628,7 +631,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
Assert(state->readLen >= pageHeaderSize);
contdata = (char *) state->readBuf + pageHeaderSize;
- record_len = XLOG_BLCKSZ - pageHeaderSize;
+ record_len = state->pageSize - pageHeaderSize;
if (pageHeader->xlp_rem_len < record_len)
record_len = pageHeader->xlp_rem_len;
@@ -655,7 +658,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
}
/* Calculate pointer to beginning of next page, and continue */
- state->recordContRecPtr += XLOG_BLCKSZ;
+ state->recordContRecPtr += state->pageSize;
}
/* targetPagePtr is pointing the last-read page here */
@@ -789,10 +792,10 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
XLogPageHeaderSize((XLogPageHeader) state->readBuf);
addLen = pageHeaderSize;
- if (reqLen + pageHeaderSize <= XLOG_BLCKSZ)
+ if (reqLen + pageHeaderSize <= state->pageSize)
addLen = pageHeaderSize;
else
- addLen = XLOG_BLCKSZ - reqLen;
+ addLen = state->pageSize - reqLen;
Assert(addLen >= 0);
}
@@ -805,7 +808,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
/* Data is not in our buffer, request the caller for it. */
XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize);
targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize);
- Assert((pageptr % XLOG_BLCKSZ) == 0);
+ Assert((pageptr % state->pageSize) == 0);
/*
* Every time we request to load new data of a page to the caller, even if
@@ -829,7 +832,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
* will not come back here, but will request the actual target page.
*/
state->readPagePtr = pageptr - targetPageOff;
- state->readLen = XLOG_BLCKSZ;
+ state->readLen = state->pageSize;
return true;
}
@@ -962,7 +965,7 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
int32 offset;
XLogPageHeader hdr = (XLogPageHeader) phdr;
- Assert((recptr % XLOG_BLCKSZ) == 0);
+ Assert((recptr % state->pageSize) == 0);
XLByteToSeg(recptr, segno, state->segcxt.ws_segsize);
offset = XLogSegmentOffset(recptr, state->segcxt.ws_segsize);
@@ -1016,10 +1019,10 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
"WAL file is from different database system: incorrect segment size in page header");
return false;
}
- else if (longhdr->xlp_xlog_blcksz != XLOG_BLCKSZ)
+ else if (longhdr->xlp_xlog_blcksz != state->pageSize)
{
report_invalid_record(state,
- "WAL file is from different database system: incorrect XLOG_BLCKSZ in page header");
+ "WAL file is from different database system: incorrect block size in page header");
return false;
}
}
@@ -1140,7 +1143,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
* XLogNeedData() is prepared to handle that and will read at
* least short page-header worth of data
*/
- targetRecOff = tmpRecPtr % XLOG_BLCKSZ;
+ targetRecOff = tmpRecPtr % state->pageSize;
/* scroll back to page boundary */
targetPagePtr = tmpRecPtr - targetRecOff;
@@ -1174,8 +1177,8 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
*
* Note that record headers are MAXALIGN'ed
*/
- if (MAXALIGN(header->xlp_rem_len) >= (XLOG_BLCKSZ - pageHeaderSize))
- tmpRecPtr = targetPagePtr + XLOG_BLCKSZ;
+ if (MAXALIGN(header->xlp_rem_len) >= (state->pageSize - pageHeaderSize))
+ tmpRecPtr = targetPagePtr + state->pageSize;
else
{
/*
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index 9cbd8afa49..f2dd495310 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -199,11 +199,14 @@ StartupDecodingContext(List *output_plugin_options,
ctx->slot = slot;
- ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, cleanup_cb);
+ ctx->reader =
+ XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, cleanup_cb);
+
if (!ctx->reader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
errmsg("out of memory")));
+ ctx->reader->readBuf = palloc(XLOG_BLCKSZ);
ctx->page_read = page_read;
ctx->reorder = ReorderBufferAllocate();
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index 062093727d..8e95600ef7 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -579,8 +579,8 @@ StartReplication(StartReplicationCmd *cmd)
errmsg("IDENTIFY_SYSTEM has not been run before START_REPLICATION")));
/* create xlogreader for physical replication */
- xlogreader =
- XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+ xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+ NULL, wal_segment_close);
if (!xlogreader)
ereport(ERROR,
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index 712c85281c..8a6209b061 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -61,7 +61,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
XLogReaderState *xlogreader;
char *errormsg;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+ xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
@@ -121,7 +121,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex,
char *errormsg;
XLogRecPtr endptr;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+ xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
@@ -181,7 +181,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
forkptr += SizeOfXLogShortPHD;
}
- xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+ xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index f514de6b13..b635537659 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -1041,7 +1041,7 @@ main(int argc, char **argv)
/* we have everything we need, start reading */
xlogreader_state =
- XLogReaderAllocate(WalSegSz, waldir, WALDumpCloseSegment);
+ XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, waldir, WALDumpCloseSegment);
if (!xlogreader_state)
fatal_error("out of memory");
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index b9dad7ee51..1853516333 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -151,6 +151,7 @@ struct XLogReaderState
* ----------------------------------------
*/
/* variables to communicate with page reader */
+ int pageSize; /* size of a page */
XLogRecPtr readPagePtr; /* page pointer to read */
int32 readLen; /* bytes requested to reader, or actual bytes
* read by reader, which must be larger than
@@ -243,7 +244,7 @@ struct XLogReaderState
/* Get a new XLogReader */
extern XLogReaderState *XLogReaderAllocate(int wal_segment_size,
- const char *waldir,
+ int page_size, const char *waldir,
WALSegmentCleanupCB cleanup_cb);
/* Free an XLogReader */
--
2.27.0
----Next_Part(Thu_Mar__4_11_28_53_2021_014)----
^ permalink raw reply [nested|flat] 30+ messages in thread
* [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize.
@ 2019-09-05 12:29 Kyotaro Horiguchi <[email protected]>
0 siblings, 0 replies; 30+ messages in thread
From: Kyotaro Horiguchi @ 2019-09-05 12:29 UTC (permalink / raw)
XLog reader facility and its callers currently relies on the same
constant XLOG_BLCKSZ. This patch moves the responsibility to determine
the block size to the callers of the facility.
---
src/backend/access/transam/twophase.c | 3 +-
src/backend/access/transam/xlog.c | 9 ++--
src/backend/access/transam/xlogreader.c | 57 ++++++++++++-----------
src/backend/replication/logical/logical.c | 5 +-
src/backend/replication/walsender.c | 4 +-
src/bin/pg_rewind/parsexlog.c | 6 +--
src/bin/pg_waldump/pg_waldump.c | 2 +-
src/include/access/xlogreader.h | 3 +-
8 files changed, 50 insertions(+), 39 deletions(-)
diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index 7d5a96a5f5..9f9e7de253 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -1325,7 +1325,8 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
XLogReaderState *xlogreader;
char *errormsg;
- xlogreader = XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+ xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+ NULL, wal_segment_close);
if (!xlogreader)
ereport(ERROR,
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index b517ca85ad..0822ac70cb 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -1222,7 +1222,10 @@ XLogInsertRecord(XLogRecData *rdata,
appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len);
if (!debug_reader)
- debug_reader = XLogReaderAllocate(wal_segment_size, NULL, NULL);
+ {
+ debug_reader =
+ XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, NULL);
+ }
if (!debug_reader)
{
@@ -6531,8 +6534,8 @@ StartupXLOG(void)
OwnLatch(&XLogCtl->recoveryWakeupLatch);
/* Set up XLOG reader facility */
- xlogreader =
- XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+ xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+ NULL, wal_segment_close);
if (!xlogreader)
ereport(ERROR,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index ba3e4868f7..e5baf13086 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -72,7 +72,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...)
* Returns NULL if the xlogreader couldn't be allocated.
*/
XLogReaderState *
-XLogReaderAllocate(int wal_segment_size, const char *waldir,
+XLogReaderAllocate(int wal_segment_size, int page_size, const char *waldir,
WALSegmentCleanupCB cleanup_cb)
{
XLogReaderState *state;
@@ -87,6 +87,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
state->cleanup_cb = cleanup_cb;
state->max_block_id = -1;
+ state->pageSize = page_size;
/*
* Permanently allocate readBuf. We do it this way, rather than just
@@ -95,7 +96,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
* isn't guaranteed to have any particular alignment, whereas
* palloc_extended() will provide MAXALIGN'd storage.
*/
- state->readBuf = (char *) palloc_extended(XLOG_BLCKSZ,
+ state->readBuf = (char *) palloc_extended(state->pageSize,
MCXT_ALLOC_NO_OOM);
if (!state->readBuf)
{
@@ -163,7 +164,7 @@ XLogReaderFree(XLogReaderState *state)
* readRecordBufSize is set to the new buffer size.
*
* To avoid useless small increases, round its size to a multiple of
- * XLOG_BLCKSZ, and make sure it's at least 5*Max(BLCKSZ, XLOG_BLCKSZ) to start
+ * page size, and make sure it's at least 5*Max(BLCKSZ, <page size>) to start
* with. (That is enough for all "normal" records, but very large commit or
* abort records might need more space.)
*/
@@ -172,8 +173,8 @@ allocate_recordbuf(XLogReaderState *state, uint32 reclength)
{
uint32 newSize = reclength;
- newSize += XLOG_BLCKSZ - (newSize % XLOG_BLCKSZ);
- newSize = Max(newSize, 5 * Max(BLCKSZ, XLOG_BLCKSZ));
+ newSize += state->pageSize - (newSize % state->pageSize);
+ newSize = Max(newSize, 5 * Max(BLCKSZ, state->pageSize));
#ifndef FRONTEND
@@ -238,6 +239,8 @@ void
XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
{
Assert(!XLogRecPtrIsInvalid(RecPtr));
+ Assert(state->readBuf != NULL &&
+ state->readBuf == (void *) MAXALIGN(state->readBuf));
ResetDecoder(state);
@@ -366,15 +369,15 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
XLogPageHeader pageHeader;
targetPagePtr =
- state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
- targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+ state->ReadRecPtr - (state->ReadRecPtr % state->pageSize);
+ targetRecOff = state->ReadRecPtr % state->pageSize;
/*
* Check if we have enough data. For the first record in the page,
* the requesting length doesn't contain page header.
*/
if (XLogNeedData(state, targetPagePtr,
- Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+ Min(targetRecOff + SizeOfXLogRecord, state->pageSize),
targetRecOff != 0))
return XLREAD_NEED_DATA;
@@ -421,7 +424,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
* verified that we got the whole header.
*/
prec = (XLogRecord *) (state->readBuf +
- state->ReadRecPtr % XLOG_BLCKSZ);
+ state->ReadRecPtr % state->pageSize);
total_len = prec->xl_tot_len;
/*
@@ -432,7 +435,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
* ensure that we enter the XLREAD_CONTINUATION state below;
* otherwise we might fail to apply ValidXLogRecordHeader at all.
*/
- if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
+ if (targetRecOff <= state->pageSize - SizeOfXLogRecord)
{
if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
state->PrevRecPtr, prec))
@@ -477,10 +480,10 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
* available
*/
targetPagePtr =
- state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
- targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+ state->ReadRecPtr - (state->ReadRecPtr % state->pageSize);
+ targetRecOff = state->ReadRecPtr % state->pageSize;
- request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ);
+ request_len = Min(targetRecOff + total_len, state->pageSize);
record_len = request_len - targetRecOff;
/* ReadRecPtr contains page header */
@@ -549,7 +552,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
/* Calculate pointer to beginning of next page */
state->recordContRecPtr = state->ReadRecPtr + record_len;
- Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0);
+ Assert(state->recordContRecPtr % state->pageSize == 0);
state->readRecordState = XLREAD_CONTINUATION;
}
@@ -576,7 +579,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
/* this request contains page header */
Assert (targetPagePtr != 0);
if (XLogNeedData(state, targetPagePtr,
- Min(state->recordRemainLen, XLOG_BLCKSZ),
+ Min(state->recordRemainLen, state->pageSize),
false))
return XLREAD_NEED_DATA;
@@ -628,7 +631,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
Assert(state->readLen >= pageHeaderSize);
contdata = (char *) state->readBuf + pageHeaderSize;
- record_len = XLOG_BLCKSZ - pageHeaderSize;
+ record_len = state->pageSize - pageHeaderSize;
if (pageHeader->xlp_rem_len < record_len)
record_len = pageHeader->xlp_rem_len;
@@ -655,7 +658,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
}
/* Calculate pointer to beginning of next page, and continue */
- state->recordContRecPtr += XLOG_BLCKSZ;
+ state->recordContRecPtr += state->pageSize;
}
/* targetPagePtr is pointing the last-read page here */
@@ -789,10 +792,10 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
XLogPageHeaderSize((XLogPageHeader) state->readBuf);
addLen = pageHeaderSize;
- if (reqLen + pageHeaderSize <= XLOG_BLCKSZ)
+ if (reqLen + pageHeaderSize <= state->pageSize)
addLen = pageHeaderSize;
else
- addLen = XLOG_BLCKSZ - reqLen;
+ addLen = state->pageSize - reqLen;
Assert(addLen >= 0);
}
@@ -805,7 +808,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
/* Data is not in our buffer, request the caller for it. */
XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize);
targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize);
- Assert((pageptr % XLOG_BLCKSZ) == 0);
+ Assert((pageptr % state->pageSize) == 0);
/*
* Every time we request to load new data of a page to the caller, even if
@@ -829,7 +832,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
* will not come back here, but will request the actual target page.
*/
state->readPagePtr = pageptr - targetPageOff;
- state->readLen = XLOG_BLCKSZ;
+ state->readLen = state->pageSize;
return true;
}
@@ -962,7 +965,7 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
int32 offset;
XLogPageHeader hdr = (XLogPageHeader) phdr;
- Assert((recptr % XLOG_BLCKSZ) == 0);
+ Assert((recptr % state->pageSize) == 0);
XLByteToSeg(recptr, segno, state->segcxt.ws_segsize);
offset = XLogSegmentOffset(recptr, state->segcxt.ws_segsize);
@@ -1016,10 +1019,10 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
"WAL file is from different database system: incorrect segment size in page header");
return false;
}
- else if (longhdr->xlp_xlog_blcksz != XLOG_BLCKSZ)
+ else if (longhdr->xlp_xlog_blcksz != state->pageSize)
{
report_invalid_record(state,
- "WAL file is from different database system: incorrect XLOG_BLCKSZ in page header");
+ "WAL file is from different database system: incorrect block size in page header");
return false;
}
}
@@ -1140,7 +1143,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
* XLogNeedData() is prepared to handle that and will read at
* least short page-header worth of data
*/
- targetRecOff = tmpRecPtr % XLOG_BLCKSZ;
+ targetRecOff = tmpRecPtr % state->pageSize;
/* scroll back to page boundary */
targetPagePtr = tmpRecPtr - targetRecOff;
@@ -1174,8 +1177,8 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
*
* Note that record headers are MAXALIGN'ed
*/
- if (MAXALIGN(header->xlp_rem_len) >= (XLOG_BLCKSZ - pageHeaderSize))
- tmpRecPtr = targetPagePtr + XLOG_BLCKSZ;
+ if (MAXALIGN(header->xlp_rem_len) >= (state->pageSize - pageHeaderSize))
+ tmpRecPtr = targetPagePtr + state->pageSize;
else
{
/*
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index 9cbd8afa49..f2dd495310 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -199,11 +199,14 @@ StartupDecodingContext(List *output_plugin_options,
ctx->slot = slot;
- ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, cleanup_cb);
+ ctx->reader =
+ XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, cleanup_cb);
+
if (!ctx->reader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
errmsg("out of memory")));
+ ctx->reader->readBuf = palloc(XLOG_BLCKSZ);
ctx->page_read = page_read;
ctx->reorder = ReorderBufferAllocate();
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index 062093727d..8e95600ef7 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -579,8 +579,8 @@ StartReplication(StartReplicationCmd *cmd)
errmsg("IDENTIFY_SYSTEM has not been run before START_REPLICATION")));
/* create xlogreader for physical replication */
- xlogreader =
- XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+ xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+ NULL, wal_segment_close);
if (!xlogreader)
ereport(ERROR,
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index 712c85281c..8a6209b061 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -61,7 +61,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
XLogReaderState *xlogreader;
char *errormsg;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+ xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
@@ -121,7 +121,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex,
char *errormsg;
XLogRecPtr endptr;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+ xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
@@ -181,7 +181,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
forkptr += SizeOfXLogShortPHD;
}
- xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+ xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index f514de6b13..b635537659 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -1041,7 +1041,7 @@ main(int argc, char **argv)
/* we have everything we need, start reading */
xlogreader_state =
- XLogReaderAllocate(WalSegSz, waldir, WALDumpCloseSegment);
+ XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, waldir, WALDumpCloseSegment);
if (!xlogreader_state)
fatal_error("out of memory");
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index b9dad7ee51..1853516333 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -151,6 +151,7 @@ struct XLogReaderState
* ----------------------------------------
*/
/* variables to communicate with page reader */
+ int pageSize; /* size of a page */
XLogRecPtr readPagePtr; /* page pointer to read */
int32 readLen; /* bytes requested to reader, or actual bytes
* read by reader, which must be larger than
@@ -243,7 +244,7 @@ struct XLogReaderState
/* Get a new XLogReader */
extern XLogReaderState *XLogReaderAllocate(int wal_segment_size,
- const char *waldir,
+ int page_size, const char *waldir,
WALSegmentCleanupCB cleanup_cb);
/* Free an XLogReader */
--
2.27.0
----Next_Part(Thu_Mar__4_11_28_53_2021_014)----
^ permalink raw reply [nested|flat] 30+ messages in thread
* [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize.
@ 2019-09-05 12:29 Kyotaro Horiguchi <[email protected]>
0 siblings, 0 replies; 30+ messages in thread
From: Kyotaro Horiguchi @ 2019-09-05 12:29 UTC (permalink / raw)
XLog reader facility and its callers currently relies on the same
constant XLOG_BLCKSZ. This patch moves the responsibility to determine
the block size to the callers of the facility.
---
src/backend/access/transam/twophase.c | 3 +-
src/backend/access/transam/xlog.c | 9 ++--
src/backend/access/transam/xlogreader.c | 57 ++++++++++++-----------
src/backend/replication/logical/logical.c | 5 +-
src/backend/replication/walsender.c | 4 +-
src/bin/pg_rewind/parsexlog.c | 6 +--
src/bin/pg_waldump/pg_waldump.c | 2 +-
src/include/access/xlogreader.h | 3 +-
8 files changed, 50 insertions(+), 39 deletions(-)
diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index 7d5a96a5f5..9f9e7de253 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -1325,7 +1325,8 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
XLogReaderState *xlogreader;
char *errormsg;
- xlogreader = XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+ xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+ NULL, wal_segment_close);
if (!xlogreader)
ereport(ERROR,
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index b517ca85ad..0822ac70cb 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -1222,7 +1222,10 @@ XLogInsertRecord(XLogRecData *rdata,
appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len);
if (!debug_reader)
- debug_reader = XLogReaderAllocate(wal_segment_size, NULL, NULL);
+ {
+ debug_reader =
+ XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, NULL);
+ }
if (!debug_reader)
{
@@ -6531,8 +6534,8 @@ StartupXLOG(void)
OwnLatch(&XLogCtl->recoveryWakeupLatch);
/* Set up XLOG reader facility */
- xlogreader =
- XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+ xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+ NULL, wal_segment_close);
if (!xlogreader)
ereport(ERROR,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index ba3e4868f7..e5baf13086 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -72,7 +72,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...)
* Returns NULL if the xlogreader couldn't be allocated.
*/
XLogReaderState *
-XLogReaderAllocate(int wal_segment_size, const char *waldir,
+XLogReaderAllocate(int wal_segment_size, int page_size, const char *waldir,
WALSegmentCleanupCB cleanup_cb)
{
XLogReaderState *state;
@@ -87,6 +87,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
state->cleanup_cb = cleanup_cb;
state->max_block_id = -1;
+ state->pageSize = page_size;
/*
* Permanently allocate readBuf. We do it this way, rather than just
@@ -95,7 +96,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
* isn't guaranteed to have any particular alignment, whereas
* palloc_extended() will provide MAXALIGN'd storage.
*/
- state->readBuf = (char *) palloc_extended(XLOG_BLCKSZ,
+ state->readBuf = (char *) palloc_extended(state->pageSize,
MCXT_ALLOC_NO_OOM);
if (!state->readBuf)
{
@@ -163,7 +164,7 @@ XLogReaderFree(XLogReaderState *state)
* readRecordBufSize is set to the new buffer size.
*
* To avoid useless small increases, round its size to a multiple of
- * XLOG_BLCKSZ, and make sure it's at least 5*Max(BLCKSZ, XLOG_BLCKSZ) to start
+ * page size, and make sure it's at least 5*Max(BLCKSZ, <page size>) to start
* with. (That is enough for all "normal" records, but very large commit or
* abort records might need more space.)
*/
@@ -172,8 +173,8 @@ allocate_recordbuf(XLogReaderState *state, uint32 reclength)
{
uint32 newSize = reclength;
- newSize += XLOG_BLCKSZ - (newSize % XLOG_BLCKSZ);
- newSize = Max(newSize, 5 * Max(BLCKSZ, XLOG_BLCKSZ));
+ newSize += state->pageSize - (newSize % state->pageSize);
+ newSize = Max(newSize, 5 * Max(BLCKSZ, state->pageSize));
#ifndef FRONTEND
@@ -238,6 +239,8 @@ void
XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
{
Assert(!XLogRecPtrIsInvalid(RecPtr));
+ Assert(state->readBuf != NULL &&
+ state->readBuf == (void *) MAXALIGN(state->readBuf));
ResetDecoder(state);
@@ -366,15 +369,15 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
XLogPageHeader pageHeader;
targetPagePtr =
- state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
- targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+ state->ReadRecPtr - (state->ReadRecPtr % state->pageSize);
+ targetRecOff = state->ReadRecPtr % state->pageSize;
/*
* Check if we have enough data. For the first record in the page,
* the requesting length doesn't contain page header.
*/
if (XLogNeedData(state, targetPagePtr,
- Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+ Min(targetRecOff + SizeOfXLogRecord, state->pageSize),
targetRecOff != 0))
return XLREAD_NEED_DATA;
@@ -421,7 +424,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
* verified that we got the whole header.
*/
prec = (XLogRecord *) (state->readBuf +
- state->ReadRecPtr % XLOG_BLCKSZ);
+ state->ReadRecPtr % state->pageSize);
total_len = prec->xl_tot_len;
/*
@@ -432,7 +435,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
* ensure that we enter the XLREAD_CONTINUATION state below;
* otherwise we might fail to apply ValidXLogRecordHeader at all.
*/
- if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
+ if (targetRecOff <= state->pageSize - SizeOfXLogRecord)
{
if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
state->PrevRecPtr, prec))
@@ -477,10 +480,10 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
* available
*/
targetPagePtr =
- state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
- targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+ state->ReadRecPtr - (state->ReadRecPtr % state->pageSize);
+ targetRecOff = state->ReadRecPtr % state->pageSize;
- request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ);
+ request_len = Min(targetRecOff + total_len, state->pageSize);
record_len = request_len - targetRecOff;
/* ReadRecPtr contains page header */
@@ -549,7 +552,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
/* Calculate pointer to beginning of next page */
state->recordContRecPtr = state->ReadRecPtr + record_len;
- Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0);
+ Assert(state->recordContRecPtr % state->pageSize == 0);
state->readRecordState = XLREAD_CONTINUATION;
}
@@ -576,7 +579,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
/* this request contains page header */
Assert (targetPagePtr != 0);
if (XLogNeedData(state, targetPagePtr,
- Min(state->recordRemainLen, XLOG_BLCKSZ),
+ Min(state->recordRemainLen, state->pageSize),
false))
return XLREAD_NEED_DATA;
@@ -628,7 +631,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
Assert(state->readLen >= pageHeaderSize);
contdata = (char *) state->readBuf + pageHeaderSize;
- record_len = XLOG_BLCKSZ - pageHeaderSize;
+ record_len = state->pageSize - pageHeaderSize;
if (pageHeader->xlp_rem_len < record_len)
record_len = pageHeader->xlp_rem_len;
@@ -655,7 +658,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
}
/* Calculate pointer to beginning of next page, and continue */
- state->recordContRecPtr += XLOG_BLCKSZ;
+ state->recordContRecPtr += state->pageSize;
}
/* targetPagePtr is pointing the last-read page here */
@@ -789,10 +792,10 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
XLogPageHeaderSize((XLogPageHeader) state->readBuf);
addLen = pageHeaderSize;
- if (reqLen + pageHeaderSize <= XLOG_BLCKSZ)
+ if (reqLen + pageHeaderSize <= state->pageSize)
addLen = pageHeaderSize;
else
- addLen = XLOG_BLCKSZ - reqLen;
+ addLen = state->pageSize - reqLen;
Assert(addLen >= 0);
}
@@ -805,7 +808,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
/* Data is not in our buffer, request the caller for it. */
XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize);
targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize);
- Assert((pageptr % XLOG_BLCKSZ) == 0);
+ Assert((pageptr % state->pageSize) == 0);
/*
* Every time we request to load new data of a page to the caller, even if
@@ -829,7 +832,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
* will not come back here, but will request the actual target page.
*/
state->readPagePtr = pageptr - targetPageOff;
- state->readLen = XLOG_BLCKSZ;
+ state->readLen = state->pageSize;
return true;
}
@@ -962,7 +965,7 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
int32 offset;
XLogPageHeader hdr = (XLogPageHeader) phdr;
- Assert((recptr % XLOG_BLCKSZ) == 0);
+ Assert((recptr % state->pageSize) == 0);
XLByteToSeg(recptr, segno, state->segcxt.ws_segsize);
offset = XLogSegmentOffset(recptr, state->segcxt.ws_segsize);
@@ -1016,10 +1019,10 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
"WAL file is from different database system: incorrect segment size in page header");
return false;
}
- else if (longhdr->xlp_xlog_blcksz != XLOG_BLCKSZ)
+ else if (longhdr->xlp_xlog_blcksz != state->pageSize)
{
report_invalid_record(state,
- "WAL file is from different database system: incorrect XLOG_BLCKSZ in page header");
+ "WAL file is from different database system: incorrect block size in page header");
return false;
}
}
@@ -1140,7 +1143,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
* XLogNeedData() is prepared to handle that and will read at
* least short page-header worth of data
*/
- targetRecOff = tmpRecPtr % XLOG_BLCKSZ;
+ targetRecOff = tmpRecPtr % state->pageSize;
/* scroll back to page boundary */
targetPagePtr = tmpRecPtr - targetRecOff;
@@ -1174,8 +1177,8 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
*
* Note that record headers are MAXALIGN'ed
*/
- if (MAXALIGN(header->xlp_rem_len) >= (XLOG_BLCKSZ - pageHeaderSize))
- tmpRecPtr = targetPagePtr + XLOG_BLCKSZ;
+ if (MAXALIGN(header->xlp_rem_len) >= (state->pageSize - pageHeaderSize))
+ tmpRecPtr = targetPagePtr + state->pageSize;
else
{
/*
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index 9cbd8afa49..f2dd495310 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -199,11 +199,14 @@ StartupDecodingContext(List *output_plugin_options,
ctx->slot = slot;
- ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, cleanup_cb);
+ ctx->reader =
+ XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, cleanup_cb);
+
if (!ctx->reader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
errmsg("out of memory")));
+ ctx->reader->readBuf = palloc(XLOG_BLCKSZ);
ctx->page_read = page_read;
ctx->reorder = ReorderBufferAllocate();
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index 062093727d..8e95600ef7 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -579,8 +579,8 @@ StartReplication(StartReplicationCmd *cmd)
errmsg("IDENTIFY_SYSTEM has not been run before START_REPLICATION")));
/* create xlogreader for physical replication */
- xlogreader =
- XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+ xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+ NULL, wal_segment_close);
if (!xlogreader)
ereport(ERROR,
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index 712c85281c..8a6209b061 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -61,7 +61,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
XLogReaderState *xlogreader;
char *errormsg;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+ xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
@@ -121,7 +121,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex,
char *errormsg;
XLogRecPtr endptr;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+ xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
@@ -181,7 +181,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
forkptr += SizeOfXLogShortPHD;
}
- xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+ xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index f514de6b13..b635537659 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -1041,7 +1041,7 @@ main(int argc, char **argv)
/* we have everything we need, start reading */
xlogreader_state =
- XLogReaderAllocate(WalSegSz, waldir, WALDumpCloseSegment);
+ XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, waldir, WALDumpCloseSegment);
if (!xlogreader_state)
fatal_error("out of memory");
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index b9dad7ee51..1853516333 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -151,6 +151,7 @@ struct XLogReaderState
* ----------------------------------------
*/
/* variables to communicate with page reader */
+ int pageSize; /* size of a page */
XLogRecPtr readPagePtr; /* page pointer to read */
int32 readLen; /* bytes requested to reader, or actual bytes
* read by reader, which must be larger than
@@ -243,7 +244,7 @@ struct XLogReaderState
/* Get a new XLogReader */
extern XLogReaderState *XLogReaderAllocate(int wal_segment_size,
- const char *waldir,
+ int page_size, const char *waldir,
WALSegmentCleanupCB cleanup_cb);
/* Free an XLogReader */
--
2.27.0
----Next_Part(Thu_Mar__4_11_28_53_2021_014)----
^ permalink raw reply [nested|flat] 30+ messages in thread
* [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize.
@ 2019-09-05 12:29 Kyotaro Horiguchi <[email protected]>
0 siblings, 0 replies; 30+ messages in thread
From: Kyotaro Horiguchi @ 2019-09-05 12:29 UTC (permalink / raw)
XLog reader facility and its callers currently relies on the same
constant XLOG_BLCKSZ. This patch moves the responsibility to determine
the block size to the callers of the facility.
---
src/backend/access/transam/twophase.c | 3 +-
src/backend/access/transam/xlog.c | 9 ++--
src/backend/access/transam/xlogreader.c | 57 ++++++++++++-----------
src/backend/replication/logical/logical.c | 5 +-
src/backend/replication/walsender.c | 4 +-
src/bin/pg_rewind/parsexlog.c | 6 +--
src/bin/pg_waldump/pg_waldump.c | 2 +-
src/include/access/xlogreader.h | 3 +-
8 files changed, 50 insertions(+), 39 deletions(-)
diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index 7d5a96a5f5..9f9e7de253 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -1325,7 +1325,8 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
XLogReaderState *xlogreader;
char *errormsg;
- xlogreader = XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+ xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+ NULL, wal_segment_close);
if (!xlogreader)
ereport(ERROR,
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index b517ca85ad..0822ac70cb 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -1222,7 +1222,10 @@ XLogInsertRecord(XLogRecData *rdata,
appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len);
if (!debug_reader)
- debug_reader = XLogReaderAllocate(wal_segment_size, NULL, NULL);
+ {
+ debug_reader =
+ XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, NULL);
+ }
if (!debug_reader)
{
@@ -6531,8 +6534,8 @@ StartupXLOG(void)
OwnLatch(&XLogCtl->recoveryWakeupLatch);
/* Set up XLOG reader facility */
- xlogreader =
- XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+ xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+ NULL, wal_segment_close);
if (!xlogreader)
ereport(ERROR,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index ba3e4868f7..e5baf13086 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -72,7 +72,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...)
* Returns NULL if the xlogreader couldn't be allocated.
*/
XLogReaderState *
-XLogReaderAllocate(int wal_segment_size, const char *waldir,
+XLogReaderAllocate(int wal_segment_size, int page_size, const char *waldir,
WALSegmentCleanupCB cleanup_cb)
{
XLogReaderState *state;
@@ -87,6 +87,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
state->cleanup_cb = cleanup_cb;
state->max_block_id = -1;
+ state->pageSize = page_size;
/*
* Permanently allocate readBuf. We do it this way, rather than just
@@ -95,7 +96,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
* isn't guaranteed to have any particular alignment, whereas
* palloc_extended() will provide MAXALIGN'd storage.
*/
- state->readBuf = (char *) palloc_extended(XLOG_BLCKSZ,
+ state->readBuf = (char *) palloc_extended(state->pageSize,
MCXT_ALLOC_NO_OOM);
if (!state->readBuf)
{
@@ -163,7 +164,7 @@ XLogReaderFree(XLogReaderState *state)
* readRecordBufSize is set to the new buffer size.
*
* To avoid useless small increases, round its size to a multiple of
- * XLOG_BLCKSZ, and make sure it's at least 5*Max(BLCKSZ, XLOG_BLCKSZ) to start
+ * page size, and make sure it's at least 5*Max(BLCKSZ, <page size>) to start
* with. (That is enough for all "normal" records, but very large commit or
* abort records might need more space.)
*/
@@ -172,8 +173,8 @@ allocate_recordbuf(XLogReaderState *state, uint32 reclength)
{
uint32 newSize = reclength;
- newSize += XLOG_BLCKSZ - (newSize % XLOG_BLCKSZ);
- newSize = Max(newSize, 5 * Max(BLCKSZ, XLOG_BLCKSZ));
+ newSize += state->pageSize - (newSize % state->pageSize);
+ newSize = Max(newSize, 5 * Max(BLCKSZ, state->pageSize));
#ifndef FRONTEND
@@ -238,6 +239,8 @@ void
XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
{
Assert(!XLogRecPtrIsInvalid(RecPtr));
+ Assert(state->readBuf != NULL &&
+ state->readBuf == (void *) MAXALIGN(state->readBuf));
ResetDecoder(state);
@@ -366,15 +369,15 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
XLogPageHeader pageHeader;
targetPagePtr =
- state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
- targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+ state->ReadRecPtr - (state->ReadRecPtr % state->pageSize);
+ targetRecOff = state->ReadRecPtr % state->pageSize;
/*
* Check if we have enough data. For the first record in the page,
* the requesting length doesn't contain page header.
*/
if (XLogNeedData(state, targetPagePtr,
- Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+ Min(targetRecOff + SizeOfXLogRecord, state->pageSize),
targetRecOff != 0))
return XLREAD_NEED_DATA;
@@ -421,7 +424,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
* verified that we got the whole header.
*/
prec = (XLogRecord *) (state->readBuf +
- state->ReadRecPtr % XLOG_BLCKSZ);
+ state->ReadRecPtr % state->pageSize);
total_len = prec->xl_tot_len;
/*
@@ -432,7 +435,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
* ensure that we enter the XLREAD_CONTINUATION state below;
* otherwise we might fail to apply ValidXLogRecordHeader at all.
*/
- if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
+ if (targetRecOff <= state->pageSize - SizeOfXLogRecord)
{
if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
state->PrevRecPtr, prec))
@@ -477,10 +480,10 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
* available
*/
targetPagePtr =
- state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
- targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+ state->ReadRecPtr - (state->ReadRecPtr % state->pageSize);
+ targetRecOff = state->ReadRecPtr % state->pageSize;
- request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ);
+ request_len = Min(targetRecOff + total_len, state->pageSize);
record_len = request_len - targetRecOff;
/* ReadRecPtr contains page header */
@@ -549,7 +552,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
/* Calculate pointer to beginning of next page */
state->recordContRecPtr = state->ReadRecPtr + record_len;
- Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0);
+ Assert(state->recordContRecPtr % state->pageSize == 0);
state->readRecordState = XLREAD_CONTINUATION;
}
@@ -576,7 +579,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
/* this request contains page header */
Assert (targetPagePtr != 0);
if (XLogNeedData(state, targetPagePtr,
- Min(state->recordRemainLen, XLOG_BLCKSZ),
+ Min(state->recordRemainLen, state->pageSize),
false))
return XLREAD_NEED_DATA;
@@ -628,7 +631,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
Assert(state->readLen >= pageHeaderSize);
contdata = (char *) state->readBuf + pageHeaderSize;
- record_len = XLOG_BLCKSZ - pageHeaderSize;
+ record_len = state->pageSize - pageHeaderSize;
if (pageHeader->xlp_rem_len < record_len)
record_len = pageHeader->xlp_rem_len;
@@ -655,7 +658,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
}
/* Calculate pointer to beginning of next page, and continue */
- state->recordContRecPtr += XLOG_BLCKSZ;
+ state->recordContRecPtr += state->pageSize;
}
/* targetPagePtr is pointing the last-read page here */
@@ -789,10 +792,10 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
XLogPageHeaderSize((XLogPageHeader) state->readBuf);
addLen = pageHeaderSize;
- if (reqLen + pageHeaderSize <= XLOG_BLCKSZ)
+ if (reqLen + pageHeaderSize <= state->pageSize)
addLen = pageHeaderSize;
else
- addLen = XLOG_BLCKSZ - reqLen;
+ addLen = state->pageSize - reqLen;
Assert(addLen >= 0);
}
@@ -805,7 +808,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
/* Data is not in our buffer, request the caller for it. */
XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize);
targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize);
- Assert((pageptr % XLOG_BLCKSZ) == 0);
+ Assert((pageptr % state->pageSize) == 0);
/*
* Every time we request to load new data of a page to the caller, even if
@@ -829,7 +832,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
* will not come back here, but will request the actual target page.
*/
state->readPagePtr = pageptr - targetPageOff;
- state->readLen = XLOG_BLCKSZ;
+ state->readLen = state->pageSize;
return true;
}
@@ -962,7 +965,7 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
int32 offset;
XLogPageHeader hdr = (XLogPageHeader) phdr;
- Assert((recptr % XLOG_BLCKSZ) == 0);
+ Assert((recptr % state->pageSize) == 0);
XLByteToSeg(recptr, segno, state->segcxt.ws_segsize);
offset = XLogSegmentOffset(recptr, state->segcxt.ws_segsize);
@@ -1016,10 +1019,10 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
"WAL file is from different database system: incorrect segment size in page header");
return false;
}
- else if (longhdr->xlp_xlog_blcksz != XLOG_BLCKSZ)
+ else if (longhdr->xlp_xlog_blcksz != state->pageSize)
{
report_invalid_record(state,
- "WAL file is from different database system: incorrect XLOG_BLCKSZ in page header");
+ "WAL file is from different database system: incorrect block size in page header");
return false;
}
}
@@ -1140,7 +1143,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
* XLogNeedData() is prepared to handle that and will read at
* least short page-header worth of data
*/
- targetRecOff = tmpRecPtr % XLOG_BLCKSZ;
+ targetRecOff = tmpRecPtr % state->pageSize;
/* scroll back to page boundary */
targetPagePtr = tmpRecPtr - targetRecOff;
@@ -1174,8 +1177,8 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
*
* Note that record headers are MAXALIGN'ed
*/
- if (MAXALIGN(header->xlp_rem_len) >= (XLOG_BLCKSZ - pageHeaderSize))
- tmpRecPtr = targetPagePtr + XLOG_BLCKSZ;
+ if (MAXALIGN(header->xlp_rem_len) >= (state->pageSize - pageHeaderSize))
+ tmpRecPtr = targetPagePtr + state->pageSize;
else
{
/*
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index 9cbd8afa49..f2dd495310 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -199,11 +199,14 @@ StartupDecodingContext(List *output_plugin_options,
ctx->slot = slot;
- ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, cleanup_cb);
+ ctx->reader =
+ XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, cleanup_cb);
+
if (!ctx->reader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
errmsg("out of memory")));
+ ctx->reader->readBuf = palloc(XLOG_BLCKSZ);
ctx->page_read = page_read;
ctx->reorder = ReorderBufferAllocate();
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index 062093727d..8e95600ef7 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -579,8 +579,8 @@ StartReplication(StartReplicationCmd *cmd)
errmsg("IDENTIFY_SYSTEM has not been run before START_REPLICATION")));
/* create xlogreader for physical replication */
- xlogreader =
- XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+ xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+ NULL, wal_segment_close);
if (!xlogreader)
ereport(ERROR,
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index 712c85281c..8a6209b061 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -61,7 +61,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
XLogReaderState *xlogreader;
char *errormsg;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+ xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
@@ -121,7 +121,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex,
char *errormsg;
XLogRecPtr endptr;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+ xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
@@ -181,7 +181,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
forkptr += SizeOfXLogShortPHD;
}
- xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+ xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index f514de6b13..b635537659 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -1041,7 +1041,7 @@ main(int argc, char **argv)
/* we have everything we need, start reading */
xlogreader_state =
- XLogReaderAllocate(WalSegSz, waldir, WALDumpCloseSegment);
+ XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, waldir, WALDumpCloseSegment);
if (!xlogreader_state)
fatal_error("out of memory");
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index b9dad7ee51..1853516333 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -151,6 +151,7 @@ struct XLogReaderState
* ----------------------------------------
*/
/* variables to communicate with page reader */
+ int pageSize; /* size of a page */
XLogRecPtr readPagePtr; /* page pointer to read */
int32 readLen; /* bytes requested to reader, or actual bytes
* read by reader, which must be larger than
@@ -243,7 +244,7 @@ struct XLogReaderState
/* Get a new XLogReader */
extern XLogReaderState *XLogReaderAllocate(int wal_segment_size,
- const char *waldir,
+ int page_size, const char *waldir,
WALSegmentCleanupCB cleanup_cb);
/* Free an XLogReader */
--
2.27.0
----Next_Part(Thu_Mar__4_11_28_53_2021_014)----
^ permalink raw reply [nested|flat] 30+ messages in thread
* [PATCH v16 4/4] Allow xlogreader to use different xlog blocksize.
@ 2019-09-05 12:29 Kyotaro Horiguchi <[email protected]>
0 siblings, 0 replies; 30+ messages in thread
From: Kyotaro Horiguchi @ 2019-09-05 12:29 UTC (permalink / raw)
XLog reader facility and its callers currently relies on the same
constant XLOG_BLCKSZ. This patch moves the responsibility to determine
the block size to the callers of the facility.
---
src/backend/access/transam/twophase.c | 3 +-
src/backend/access/transam/xlog.c | 9 ++--
src/backend/access/transam/xlogreader.c | 57 ++++++++++++-----------
src/backend/replication/logical/logical.c | 5 +-
src/backend/replication/walsender.c | 4 +-
src/bin/pg_rewind/parsexlog.c | 6 +--
src/bin/pg_waldump/pg_waldump.c | 2 +-
src/include/access/xlogreader.h | 3 +-
8 files changed, 50 insertions(+), 39 deletions(-)
diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index 9a56b3487e..b64c61f742 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -1324,7 +1324,8 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
XLogReaderState *xlogreader;
char *errormsg;
- xlogreader = XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+ xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+ NULL, wal_segment_close);
if (!xlogreader)
ereport(ERROR,
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index eccb88b611..ec78de5dd9 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -1213,7 +1213,10 @@ XLogInsertRecord(XLogRecData *rdata,
appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len);
if (!debug_reader)
- debug_reader = XLogReaderAllocate(wal_segment_size, NULL, NULL);
+ {
+ debug_reader =
+ XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, NULL);
+ }
if (!debug_reader)
{
@@ -6469,8 +6472,8 @@ StartupXLOG(void)
OwnLatch(&XLogCtl->recoveryWakeupLatch);
/* Set up XLOG reader facility */
- xlogreader =
- XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+ xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+ NULL, wal_segment_close);
if (!xlogreader)
ereport(ERROR,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index d829e57baa..582b1dd11b 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -72,7 +72,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...)
* Returns NULL if the xlogreader couldn't be allocated.
*/
XLogReaderState *
-XLogReaderAllocate(int wal_segment_size, const char *waldir,
+XLogReaderAllocate(int wal_segment_size, int page_size, const char *waldir,
WALSegmentCleanupCB cleanup_cb)
{
XLogReaderState *state;
@@ -87,6 +87,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
state->cleanup_cb = cleanup_cb;
state->max_block_id = -1;
+ state->pageSize = page_size;
/*
* Permanently allocate readBuf. We do it this way, rather than just
@@ -95,7 +96,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
* isn't guaranteed to have any particular alignment, whereas
* palloc_extended() will provide MAXALIGN'd storage.
*/
- state->readBuf = (char *) palloc_extended(XLOG_BLCKSZ,
+ state->readBuf = (char *) palloc_extended(state->pageSize,
MCXT_ALLOC_NO_OOM);
if (!state->readBuf)
{
@@ -163,7 +164,7 @@ XLogReaderFree(XLogReaderState *state)
* readRecordBufSize is set to the new buffer size.
*
* To avoid useless small increases, round its size to a multiple of
- * XLOG_BLCKSZ, and make sure it's at least 5*Max(BLCKSZ, XLOG_BLCKSZ) to start
+ * page size, and make sure it's at least 5*Max(BLCKSZ, <page size>) to start
* with. (That is enough for all "normal" records, but very large commit or
* abort records might need more space.)
*/
@@ -172,8 +173,8 @@ allocate_recordbuf(XLogReaderState *state, uint32 reclength)
{
uint32 newSize = reclength;
- newSize += XLOG_BLCKSZ - (newSize % XLOG_BLCKSZ);
- newSize = Max(newSize, 5 * Max(BLCKSZ, XLOG_BLCKSZ));
+ newSize += state->pageSize - (newSize % state->pageSize);
+ newSize = Max(newSize, 5 * Max(BLCKSZ, state->pageSize));
#ifndef FRONTEND
@@ -238,6 +239,8 @@ void
XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
{
Assert(!XLogRecPtrIsInvalid(RecPtr));
+ Assert(state->readBuf != NULL &&
+ state->readBuf == (void *) MAXALIGN(state->readBuf));
ResetDecoder(state);
@@ -366,15 +369,15 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
XLogPageHeader pageHeader;
targetPagePtr =
- state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
- targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+ state->ReadRecPtr - (state->ReadRecPtr % state->pageSize);
+ targetRecOff = state->ReadRecPtr % state->pageSize;
/*
* Check if we have enough data. For the first record in the page,
* the requesting length doesn't contain page header.
*/
if (XLogNeedData(state, targetPagePtr,
- Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+ Min(targetRecOff + SizeOfXLogRecord, state->pageSize),
targetRecOff != 0))
return XLREAD_NEED_DATA;
@@ -422,7 +425,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
* verified that we got the whole header.
*/
prec = (XLogRecord *) (state->readBuf +
- state->ReadRecPtr % XLOG_BLCKSZ);
+ state->ReadRecPtr % state->pageSize);
total_len = prec->xl_tot_len;
/*
@@ -433,7 +436,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
* ensure that we enter the XLREAD_CONTINUATION state below;
* otherwise we might fail to apply ValidXLogRecordHeader at all.
*/
- if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
+ if (targetRecOff <= state->pageSize - SizeOfXLogRecord)
{
if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
state->PrevRecPtr, prec))
@@ -479,10 +482,10 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
* available
*/
targetPagePtr =
- state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
- targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+ state->ReadRecPtr - (state->ReadRecPtr % state->pageSize);
+ targetRecOff = state->ReadRecPtr % state->pageSize;
- request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ);
+ request_len = Min(targetRecOff + total_len, state->pageSize);
record_len = request_len - targetRecOff;
/* ReadRecPtr contains page header */
@@ -552,7 +555,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
/* Calculate pointer to beginning of next page */
state->recordContRecPtr = state->ReadRecPtr + record_len;
- Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0);
+ Assert(state->recordContRecPtr % state->pageSize == 0);
state->readRecordState = XLREAD_CONTINUATION;
}
@@ -579,7 +582,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
/* this request contains page header */
Assert (targetPagePtr != 0);
if (XLogNeedData(state, targetPagePtr,
- Min(state->recordRemainLen, XLOG_BLCKSZ),
+ Min(state->recordRemainLen, state->pageSize),
false))
return XLREAD_NEED_DATA;
@@ -631,7 +634,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
Assert(state->readLen >= pageHeaderSize);
contdata = (char *) state->readBuf + pageHeaderSize;
- record_len = XLOG_BLCKSZ - pageHeaderSize;
+ record_len = state->pageSize - pageHeaderSize;
if (pageHeader->xlp_rem_len < record_len)
record_len = pageHeader->xlp_rem_len;
@@ -658,7 +661,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
}
/* Calculate pointer to beginning of next page, and continue */
- state->recordContRecPtr += XLOG_BLCKSZ;
+ state->recordContRecPtr += state->pageSize;
}
/* targetPagePtr is pointing the last-read page here */
@@ -792,10 +795,10 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
XLogPageHeaderSize((XLogPageHeader) state->readBuf);
addLen = pageHeaderSize;
- if (reqLen + pageHeaderSize <= XLOG_BLCKSZ)
+ if (reqLen + pageHeaderSize <= state->pageSize)
addLen = pageHeaderSize;
else
- addLen = XLOG_BLCKSZ - reqLen;
+ addLen = state->pageSize - reqLen;
Assert(addLen >= 0);
}
@@ -808,7 +811,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
/* Data is not in our buffer, request the caller for it. */
XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize);
targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize);
- Assert((pageptr % XLOG_BLCKSZ) == 0);
+ Assert((pageptr % state->pageSize) == 0);
/*
* Every time we request to load new data of a page to the caller, even if
@@ -832,7 +835,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
* will not come back here, but will request the actual target page.
*/
state->readPagePtr = pageptr - targetPageOff;
- state->readLen = XLOG_BLCKSZ;
+ state->readLen = state->pageSize;
return true;
}
@@ -968,7 +971,7 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
int32 offset;
XLogPageHeader hdr = (XLogPageHeader) phdr;
- Assert((recptr % XLOG_BLCKSZ) == 0);
+ Assert((recptr % state->pageSize) == 0);
XLByteToSeg(recptr, segno, state->segcxt.ws_segsize);
offset = XLogSegmentOffset(recptr, state->segcxt.ws_segsize);
@@ -1022,10 +1025,10 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
"WAL file is from different database system: incorrect segment size in page header");
return false;
}
- else if (longhdr->xlp_xlog_blcksz != XLOG_BLCKSZ)
+ else if (longhdr->xlp_xlog_blcksz != state->pageSize)
{
report_invalid_record(state,
- "WAL file is from different database system: incorrect XLOG_BLCKSZ in page header");
+ "WAL file is from different database system: incorrect block size in page header");
return false;
}
}
@@ -1146,7 +1149,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
* XLogNeedData() is prepared to handle that and will read at
* least short page-header worth of data
*/
- targetRecOff = tmpRecPtr % XLOG_BLCKSZ;
+ targetRecOff = tmpRecPtr % state->pageSize;
/* scroll back to page boundary */
targetPagePtr = tmpRecPtr - targetRecOff;
@@ -1180,8 +1183,8 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
*
* Note that record headers are MAXALIGN'ed
*/
- if (MAXALIGN(header->xlp_rem_len) >= (XLOG_BLCKSZ - pageHeaderSize))
- tmpRecPtr = targetPagePtr + XLOG_BLCKSZ;
+ if (MAXALIGN(header->xlp_rem_len) >= (state->pageSize - pageHeaderSize))
+ tmpRecPtr = targetPagePtr + state->pageSize;
else
{
/*
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index 97def6f781..ee322ba796 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -189,11 +189,14 @@ StartupDecodingContext(List *output_plugin_options,
ctx->slot = slot;
- ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, cleanup_cb);
+ ctx->reader =
+ XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, cleanup_cb);
+
if (!ctx->reader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
errmsg("out of memory")));
+ ctx->reader->readBuf = palloc(XLOG_BLCKSZ);
ctx->page_read = page_read;
ctx->reorder = ReorderBufferAllocate();
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index 33d952f9fd..a0914ddb25 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -578,8 +578,8 @@ StartReplication(StartReplicationCmd *cmd)
errmsg("IDENTIFY_SYSTEM has not been run before START_REPLICATION")));
/* create xlogreader for physical replication */
- xlogreader =
- XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+ xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+ NULL, wal_segment_close);
if (!xlogreader)
ereport(ERROR,
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index 7344a7c692..c5881594b7 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -58,7 +58,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
XLogReaderState *xlogreader;
char *errormsg;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+ xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
@@ -112,7 +112,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex,
char *errormsg;
XLogRecPtr endptr;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+ xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
@@ -172,7 +172,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
forkptr += SizeOfXLogShortPHD;
}
- xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+ xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index 9329a6a6d6..025479431d 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -1043,7 +1043,7 @@ main(int argc, char **argv)
/* we have everything we need, start reading */
xlogreader_state =
- XLogReaderAllocate(WalSegSz, waldir, WALDumpCloseSegment);
+ XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, waldir, WALDumpCloseSegment);
if (!xlogreader_state)
fatal_error("out of memory");
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index 732d463a49..e9d7d70db7 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -151,6 +151,7 @@ struct XLogReaderState
* ----------------------------------------
*/
/* variables to communicate with page reader */
+ int pageSize; /* size of a page */
XLogRecPtr readPagePtr; /* page pointer to read */
int32 readLen; /* bytes requested to reader, or actual bytes
* read by reader, which must be larger than
@@ -243,7 +244,7 @@ struct XLogReaderState
/* Get a new XLogReader */
extern XLogReaderState *XLogReaderAllocate(int wal_segment_size,
- const char *waldir,
+ int page_size, const char *waldir,
WALSegmentCleanupCB cleanup_cb);
/* Free an XLogReader */
--
2.18.4
----Next_Part(Tue_Sep__8_16_35_16_2020_701)----
^ permalink raw reply [nested|flat] 30+ messages in thread
* [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize.
@ 2019-09-05 12:29 Kyotaro Horiguchi <[email protected]>
0 siblings, 0 replies; 30+ messages in thread
From: Kyotaro Horiguchi @ 2019-09-05 12:29 UTC (permalink / raw)
XLog reader facility and its callers currently relies on the same
constant XLOG_BLCKSZ. This patch moves the responsibility to determine
the block size to the callers of the facility.
---
src/backend/access/transam/twophase.c | 3 +-
src/backend/access/transam/xlog.c | 9 ++--
src/backend/access/transam/xlogreader.c | 57 ++++++++++++-----------
src/backend/replication/logical/logical.c | 5 +-
src/backend/replication/walsender.c | 4 +-
src/bin/pg_rewind/parsexlog.c | 6 +--
src/bin/pg_waldump/pg_waldump.c | 2 +-
src/include/access/xlogreader.h | 3 +-
8 files changed, 50 insertions(+), 39 deletions(-)
diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index 7d5a96a5f5..9f9e7de253 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -1325,7 +1325,8 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
XLogReaderState *xlogreader;
char *errormsg;
- xlogreader = XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+ xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+ NULL, wal_segment_close);
if (!xlogreader)
ereport(ERROR,
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index b517ca85ad..0822ac70cb 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -1222,7 +1222,10 @@ XLogInsertRecord(XLogRecData *rdata,
appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len);
if (!debug_reader)
- debug_reader = XLogReaderAllocate(wal_segment_size, NULL, NULL);
+ {
+ debug_reader =
+ XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, NULL);
+ }
if (!debug_reader)
{
@@ -6531,8 +6534,8 @@ StartupXLOG(void)
OwnLatch(&XLogCtl->recoveryWakeupLatch);
/* Set up XLOG reader facility */
- xlogreader =
- XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+ xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+ NULL, wal_segment_close);
if (!xlogreader)
ereport(ERROR,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index ba3e4868f7..e5baf13086 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -72,7 +72,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...)
* Returns NULL if the xlogreader couldn't be allocated.
*/
XLogReaderState *
-XLogReaderAllocate(int wal_segment_size, const char *waldir,
+XLogReaderAllocate(int wal_segment_size, int page_size, const char *waldir,
WALSegmentCleanupCB cleanup_cb)
{
XLogReaderState *state;
@@ -87,6 +87,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
state->cleanup_cb = cleanup_cb;
state->max_block_id = -1;
+ state->pageSize = page_size;
/*
* Permanently allocate readBuf. We do it this way, rather than just
@@ -95,7 +96,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
* isn't guaranteed to have any particular alignment, whereas
* palloc_extended() will provide MAXALIGN'd storage.
*/
- state->readBuf = (char *) palloc_extended(XLOG_BLCKSZ,
+ state->readBuf = (char *) palloc_extended(state->pageSize,
MCXT_ALLOC_NO_OOM);
if (!state->readBuf)
{
@@ -163,7 +164,7 @@ XLogReaderFree(XLogReaderState *state)
* readRecordBufSize is set to the new buffer size.
*
* To avoid useless small increases, round its size to a multiple of
- * XLOG_BLCKSZ, and make sure it's at least 5*Max(BLCKSZ, XLOG_BLCKSZ) to start
+ * page size, and make sure it's at least 5*Max(BLCKSZ, <page size>) to start
* with. (That is enough for all "normal" records, but very large commit or
* abort records might need more space.)
*/
@@ -172,8 +173,8 @@ allocate_recordbuf(XLogReaderState *state, uint32 reclength)
{
uint32 newSize = reclength;
- newSize += XLOG_BLCKSZ - (newSize % XLOG_BLCKSZ);
- newSize = Max(newSize, 5 * Max(BLCKSZ, XLOG_BLCKSZ));
+ newSize += state->pageSize - (newSize % state->pageSize);
+ newSize = Max(newSize, 5 * Max(BLCKSZ, state->pageSize));
#ifndef FRONTEND
@@ -238,6 +239,8 @@ void
XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
{
Assert(!XLogRecPtrIsInvalid(RecPtr));
+ Assert(state->readBuf != NULL &&
+ state->readBuf == (void *) MAXALIGN(state->readBuf));
ResetDecoder(state);
@@ -366,15 +369,15 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
XLogPageHeader pageHeader;
targetPagePtr =
- state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
- targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+ state->ReadRecPtr - (state->ReadRecPtr % state->pageSize);
+ targetRecOff = state->ReadRecPtr % state->pageSize;
/*
* Check if we have enough data. For the first record in the page,
* the requesting length doesn't contain page header.
*/
if (XLogNeedData(state, targetPagePtr,
- Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+ Min(targetRecOff + SizeOfXLogRecord, state->pageSize),
targetRecOff != 0))
return XLREAD_NEED_DATA;
@@ -421,7 +424,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
* verified that we got the whole header.
*/
prec = (XLogRecord *) (state->readBuf +
- state->ReadRecPtr % XLOG_BLCKSZ);
+ state->ReadRecPtr % state->pageSize);
total_len = prec->xl_tot_len;
/*
@@ -432,7 +435,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
* ensure that we enter the XLREAD_CONTINUATION state below;
* otherwise we might fail to apply ValidXLogRecordHeader at all.
*/
- if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
+ if (targetRecOff <= state->pageSize - SizeOfXLogRecord)
{
if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
state->PrevRecPtr, prec))
@@ -477,10 +480,10 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
* available
*/
targetPagePtr =
- state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
- targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+ state->ReadRecPtr - (state->ReadRecPtr % state->pageSize);
+ targetRecOff = state->ReadRecPtr % state->pageSize;
- request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ);
+ request_len = Min(targetRecOff + total_len, state->pageSize);
record_len = request_len - targetRecOff;
/* ReadRecPtr contains page header */
@@ -549,7 +552,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
/* Calculate pointer to beginning of next page */
state->recordContRecPtr = state->ReadRecPtr + record_len;
- Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0);
+ Assert(state->recordContRecPtr % state->pageSize == 0);
state->readRecordState = XLREAD_CONTINUATION;
}
@@ -576,7 +579,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
/* this request contains page header */
Assert (targetPagePtr != 0);
if (XLogNeedData(state, targetPagePtr,
- Min(state->recordRemainLen, XLOG_BLCKSZ),
+ Min(state->recordRemainLen, state->pageSize),
false))
return XLREAD_NEED_DATA;
@@ -628,7 +631,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
Assert(state->readLen >= pageHeaderSize);
contdata = (char *) state->readBuf + pageHeaderSize;
- record_len = XLOG_BLCKSZ - pageHeaderSize;
+ record_len = state->pageSize - pageHeaderSize;
if (pageHeader->xlp_rem_len < record_len)
record_len = pageHeader->xlp_rem_len;
@@ -655,7 +658,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
}
/* Calculate pointer to beginning of next page, and continue */
- state->recordContRecPtr += XLOG_BLCKSZ;
+ state->recordContRecPtr += state->pageSize;
}
/* targetPagePtr is pointing the last-read page here */
@@ -789,10 +792,10 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
XLogPageHeaderSize((XLogPageHeader) state->readBuf);
addLen = pageHeaderSize;
- if (reqLen + pageHeaderSize <= XLOG_BLCKSZ)
+ if (reqLen + pageHeaderSize <= state->pageSize)
addLen = pageHeaderSize;
else
- addLen = XLOG_BLCKSZ - reqLen;
+ addLen = state->pageSize - reqLen;
Assert(addLen >= 0);
}
@@ -805,7 +808,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
/* Data is not in our buffer, request the caller for it. */
XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize);
targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize);
- Assert((pageptr % XLOG_BLCKSZ) == 0);
+ Assert((pageptr % state->pageSize) == 0);
/*
* Every time we request to load new data of a page to the caller, even if
@@ -829,7 +832,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
* will not come back here, but will request the actual target page.
*/
state->readPagePtr = pageptr - targetPageOff;
- state->readLen = XLOG_BLCKSZ;
+ state->readLen = state->pageSize;
return true;
}
@@ -962,7 +965,7 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
int32 offset;
XLogPageHeader hdr = (XLogPageHeader) phdr;
- Assert((recptr % XLOG_BLCKSZ) == 0);
+ Assert((recptr % state->pageSize) == 0);
XLByteToSeg(recptr, segno, state->segcxt.ws_segsize);
offset = XLogSegmentOffset(recptr, state->segcxt.ws_segsize);
@@ -1016,10 +1019,10 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
"WAL file is from different database system: incorrect segment size in page header");
return false;
}
- else if (longhdr->xlp_xlog_blcksz != XLOG_BLCKSZ)
+ else if (longhdr->xlp_xlog_blcksz != state->pageSize)
{
report_invalid_record(state,
- "WAL file is from different database system: incorrect XLOG_BLCKSZ in page header");
+ "WAL file is from different database system: incorrect block size in page header");
return false;
}
}
@@ -1140,7 +1143,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
* XLogNeedData() is prepared to handle that and will read at
* least short page-header worth of data
*/
- targetRecOff = tmpRecPtr % XLOG_BLCKSZ;
+ targetRecOff = tmpRecPtr % state->pageSize;
/* scroll back to page boundary */
targetPagePtr = tmpRecPtr - targetRecOff;
@@ -1174,8 +1177,8 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
*
* Note that record headers are MAXALIGN'ed
*/
- if (MAXALIGN(header->xlp_rem_len) >= (XLOG_BLCKSZ - pageHeaderSize))
- tmpRecPtr = targetPagePtr + XLOG_BLCKSZ;
+ if (MAXALIGN(header->xlp_rem_len) >= (state->pageSize - pageHeaderSize))
+ tmpRecPtr = targetPagePtr + state->pageSize;
else
{
/*
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index 9cbd8afa49..f2dd495310 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -199,11 +199,14 @@ StartupDecodingContext(List *output_plugin_options,
ctx->slot = slot;
- ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, cleanup_cb);
+ ctx->reader =
+ XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, cleanup_cb);
+
if (!ctx->reader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
errmsg("out of memory")));
+ ctx->reader->readBuf = palloc(XLOG_BLCKSZ);
ctx->page_read = page_read;
ctx->reorder = ReorderBufferAllocate();
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index 062093727d..8e95600ef7 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -579,8 +579,8 @@ StartReplication(StartReplicationCmd *cmd)
errmsg("IDENTIFY_SYSTEM has not been run before START_REPLICATION")));
/* create xlogreader for physical replication */
- xlogreader =
- XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+ xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+ NULL, wal_segment_close);
if (!xlogreader)
ereport(ERROR,
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index 712c85281c..8a6209b061 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -61,7 +61,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
XLogReaderState *xlogreader;
char *errormsg;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+ xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
@@ -121,7 +121,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex,
char *errormsg;
XLogRecPtr endptr;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+ xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
@@ -181,7 +181,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
forkptr += SizeOfXLogShortPHD;
}
- xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+ xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index f514de6b13..b635537659 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -1041,7 +1041,7 @@ main(int argc, char **argv)
/* we have everything we need, start reading */
xlogreader_state =
- XLogReaderAllocate(WalSegSz, waldir, WALDumpCloseSegment);
+ XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, waldir, WALDumpCloseSegment);
if (!xlogreader_state)
fatal_error("out of memory");
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index b9dad7ee51..1853516333 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -151,6 +151,7 @@ struct XLogReaderState
* ----------------------------------------
*/
/* variables to communicate with page reader */
+ int pageSize; /* size of a page */
XLogRecPtr readPagePtr; /* page pointer to read */
int32 readLen; /* bytes requested to reader, or actual bytes
* read by reader, which must be larger than
@@ -243,7 +244,7 @@ struct XLogReaderState
/* Get a new XLogReader */
extern XLogReaderState *XLogReaderAllocate(int wal_segment_size,
- const char *waldir,
+ int page_size, const char *waldir,
WALSegmentCleanupCB cleanup_cb);
/* Free an XLogReader */
--
2.27.0
----Next_Part(Thu_Mar__4_11_28_53_2021_014)----
^ permalink raw reply [nested|flat] 30+ messages in thread
* [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize.
@ 2019-09-05 12:29 Kyotaro Horiguchi <[email protected]>
0 siblings, 0 replies; 30+ messages in thread
From: Kyotaro Horiguchi @ 2019-09-05 12:29 UTC (permalink / raw)
XLog reader facility and its callers currently relies on the same
constant XLOG_BLCKSZ. This patch moves the responsibility to determine
the block size to the callers of the facility.
---
src/backend/access/transam/twophase.c | 3 +-
src/backend/access/transam/xlog.c | 9 ++--
src/backend/access/transam/xlogreader.c | 57 ++++++++++++-----------
src/backend/replication/logical/logical.c | 5 +-
src/backend/replication/walsender.c | 4 +-
src/bin/pg_rewind/parsexlog.c | 6 +--
src/bin/pg_waldump/pg_waldump.c | 2 +-
src/include/access/xlogreader.h | 3 +-
8 files changed, 50 insertions(+), 39 deletions(-)
diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index 7d5a96a5f5..9f9e7de253 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -1325,7 +1325,8 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
XLogReaderState *xlogreader;
char *errormsg;
- xlogreader = XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+ xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+ NULL, wal_segment_close);
if (!xlogreader)
ereport(ERROR,
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index b517ca85ad..0822ac70cb 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -1222,7 +1222,10 @@ XLogInsertRecord(XLogRecData *rdata,
appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len);
if (!debug_reader)
- debug_reader = XLogReaderAllocate(wal_segment_size, NULL, NULL);
+ {
+ debug_reader =
+ XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, NULL);
+ }
if (!debug_reader)
{
@@ -6531,8 +6534,8 @@ StartupXLOG(void)
OwnLatch(&XLogCtl->recoveryWakeupLatch);
/* Set up XLOG reader facility */
- xlogreader =
- XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+ xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+ NULL, wal_segment_close);
if (!xlogreader)
ereport(ERROR,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index ba3e4868f7..e5baf13086 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -72,7 +72,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...)
* Returns NULL if the xlogreader couldn't be allocated.
*/
XLogReaderState *
-XLogReaderAllocate(int wal_segment_size, const char *waldir,
+XLogReaderAllocate(int wal_segment_size, int page_size, const char *waldir,
WALSegmentCleanupCB cleanup_cb)
{
XLogReaderState *state;
@@ -87,6 +87,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
state->cleanup_cb = cleanup_cb;
state->max_block_id = -1;
+ state->pageSize = page_size;
/*
* Permanently allocate readBuf. We do it this way, rather than just
@@ -95,7 +96,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
* isn't guaranteed to have any particular alignment, whereas
* palloc_extended() will provide MAXALIGN'd storage.
*/
- state->readBuf = (char *) palloc_extended(XLOG_BLCKSZ,
+ state->readBuf = (char *) palloc_extended(state->pageSize,
MCXT_ALLOC_NO_OOM);
if (!state->readBuf)
{
@@ -163,7 +164,7 @@ XLogReaderFree(XLogReaderState *state)
* readRecordBufSize is set to the new buffer size.
*
* To avoid useless small increases, round its size to a multiple of
- * XLOG_BLCKSZ, and make sure it's at least 5*Max(BLCKSZ, XLOG_BLCKSZ) to start
+ * page size, and make sure it's at least 5*Max(BLCKSZ, <page size>) to start
* with. (That is enough for all "normal" records, but very large commit or
* abort records might need more space.)
*/
@@ -172,8 +173,8 @@ allocate_recordbuf(XLogReaderState *state, uint32 reclength)
{
uint32 newSize = reclength;
- newSize += XLOG_BLCKSZ - (newSize % XLOG_BLCKSZ);
- newSize = Max(newSize, 5 * Max(BLCKSZ, XLOG_BLCKSZ));
+ newSize += state->pageSize - (newSize % state->pageSize);
+ newSize = Max(newSize, 5 * Max(BLCKSZ, state->pageSize));
#ifndef FRONTEND
@@ -238,6 +239,8 @@ void
XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
{
Assert(!XLogRecPtrIsInvalid(RecPtr));
+ Assert(state->readBuf != NULL &&
+ state->readBuf == (void *) MAXALIGN(state->readBuf));
ResetDecoder(state);
@@ -366,15 +369,15 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
XLogPageHeader pageHeader;
targetPagePtr =
- state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
- targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+ state->ReadRecPtr - (state->ReadRecPtr % state->pageSize);
+ targetRecOff = state->ReadRecPtr % state->pageSize;
/*
* Check if we have enough data. For the first record in the page,
* the requesting length doesn't contain page header.
*/
if (XLogNeedData(state, targetPagePtr,
- Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+ Min(targetRecOff + SizeOfXLogRecord, state->pageSize),
targetRecOff != 0))
return XLREAD_NEED_DATA;
@@ -421,7 +424,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
* verified that we got the whole header.
*/
prec = (XLogRecord *) (state->readBuf +
- state->ReadRecPtr % XLOG_BLCKSZ);
+ state->ReadRecPtr % state->pageSize);
total_len = prec->xl_tot_len;
/*
@@ -432,7 +435,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
* ensure that we enter the XLREAD_CONTINUATION state below;
* otherwise we might fail to apply ValidXLogRecordHeader at all.
*/
- if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
+ if (targetRecOff <= state->pageSize - SizeOfXLogRecord)
{
if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
state->PrevRecPtr, prec))
@@ -477,10 +480,10 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
* available
*/
targetPagePtr =
- state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
- targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+ state->ReadRecPtr - (state->ReadRecPtr % state->pageSize);
+ targetRecOff = state->ReadRecPtr % state->pageSize;
- request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ);
+ request_len = Min(targetRecOff + total_len, state->pageSize);
record_len = request_len - targetRecOff;
/* ReadRecPtr contains page header */
@@ -549,7 +552,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
/* Calculate pointer to beginning of next page */
state->recordContRecPtr = state->ReadRecPtr + record_len;
- Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0);
+ Assert(state->recordContRecPtr % state->pageSize == 0);
state->readRecordState = XLREAD_CONTINUATION;
}
@@ -576,7 +579,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
/* this request contains page header */
Assert (targetPagePtr != 0);
if (XLogNeedData(state, targetPagePtr,
- Min(state->recordRemainLen, XLOG_BLCKSZ),
+ Min(state->recordRemainLen, state->pageSize),
false))
return XLREAD_NEED_DATA;
@@ -628,7 +631,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
Assert(state->readLen >= pageHeaderSize);
contdata = (char *) state->readBuf + pageHeaderSize;
- record_len = XLOG_BLCKSZ - pageHeaderSize;
+ record_len = state->pageSize - pageHeaderSize;
if (pageHeader->xlp_rem_len < record_len)
record_len = pageHeader->xlp_rem_len;
@@ -655,7 +658,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
}
/* Calculate pointer to beginning of next page, and continue */
- state->recordContRecPtr += XLOG_BLCKSZ;
+ state->recordContRecPtr += state->pageSize;
}
/* targetPagePtr is pointing the last-read page here */
@@ -789,10 +792,10 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
XLogPageHeaderSize((XLogPageHeader) state->readBuf);
addLen = pageHeaderSize;
- if (reqLen + pageHeaderSize <= XLOG_BLCKSZ)
+ if (reqLen + pageHeaderSize <= state->pageSize)
addLen = pageHeaderSize;
else
- addLen = XLOG_BLCKSZ - reqLen;
+ addLen = state->pageSize - reqLen;
Assert(addLen >= 0);
}
@@ -805,7 +808,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
/* Data is not in our buffer, request the caller for it. */
XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize);
targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize);
- Assert((pageptr % XLOG_BLCKSZ) == 0);
+ Assert((pageptr % state->pageSize) == 0);
/*
* Every time we request to load new data of a page to the caller, even if
@@ -829,7 +832,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
* will not come back here, but will request the actual target page.
*/
state->readPagePtr = pageptr - targetPageOff;
- state->readLen = XLOG_BLCKSZ;
+ state->readLen = state->pageSize;
return true;
}
@@ -962,7 +965,7 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
int32 offset;
XLogPageHeader hdr = (XLogPageHeader) phdr;
- Assert((recptr % XLOG_BLCKSZ) == 0);
+ Assert((recptr % state->pageSize) == 0);
XLByteToSeg(recptr, segno, state->segcxt.ws_segsize);
offset = XLogSegmentOffset(recptr, state->segcxt.ws_segsize);
@@ -1016,10 +1019,10 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
"WAL file is from different database system: incorrect segment size in page header");
return false;
}
- else if (longhdr->xlp_xlog_blcksz != XLOG_BLCKSZ)
+ else if (longhdr->xlp_xlog_blcksz != state->pageSize)
{
report_invalid_record(state,
- "WAL file is from different database system: incorrect XLOG_BLCKSZ in page header");
+ "WAL file is from different database system: incorrect block size in page header");
return false;
}
}
@@ -1140,7 +1143,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
* XLogNeedData() is prepared to handle that and will read at
* least short page-header worth of data
*/
- targetRecOff = tmpRecPtr % XLOG_BLCKSZ;
+ targetRecOff = tmpRecPtr % state->pageSize;
/* scroll back to page boundary */
targetPagePtr = tmpRecPtr - targetRecOff;
@@ -1174,8 +1177,8 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
*
* Note that record headers are MAXALIGN'ed
*/
- if (MAXALIGN(header->xlp_rem_len) >= (XLOG_BLCKSZ - pageHeaderSize))
- tmpRecPtr = targetPagePtr + XLOG_BLCKSZ;
+ if (MAXALIGN(header->xlp_rem_len) >= (state->pageSize - pageHeaderSize))
+ tmpRecPtr = targetPagePtr + state->pageSize;
else
{
/*
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index 9cbd8afa49..f2dd495310 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -199,11 +199,14 @@ StartupDecodingContext(List *output_plugin_options,
ctx->slot = slot;
- ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, cleanup_cb);
+ ctx->reader =
+ XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, cleanup_cb);
+
if (!ctx->reader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
errmsg("out of memory")));
+ ctx->reader->readBuf = palloc(XLOG_BLCKSZ);
ctx->page_read = page_read;
ctx->reorder = ReorderBufferAllocate();
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index 062093727d..8e95600ef7 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -579,8 +579,8 @@ StartReplication(StartReplicationCmd *cmd)
errmsg("IDENTIFY_SYSTEM has not been run before START_REPLICATION")));
/* create xlogreader for physical replication */
- xlogreader =
- XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+ xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+ NULL, wal_segment_close);
if (!xlogreader)
ereport(ERROR,
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index 712c85281c..8a6209b061 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -61,7 +61,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
XLogReaderState *xlogreader;
char *errormsg;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+ xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
@@ -121,7 +121,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex,
char *errormsg;
XLogRecPtr endptr;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+ xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
@@ -181,7 +181,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
forkptr += SizeOfXLogShortPHD;
}
- xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+ xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index f514de6b13..b635537659 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -1041,7 +1041,7 @@ main(int argc, char **argv)
/* we have everything we need, start reading */
xlogreader_state =
- XLogReaderAllocate(WalSegSz, waldir, WALDumpCloseSegment);
+ XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, waldir, WALDumpCloseSegment);
if (!xlogreader_state)
fatal_error("out of memory");
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index b9dad7ee51..1853516333 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -151,6 +151,7 @@ struct XLogReaderState
* ----------------------------------------
*/
/* variables to communicate with page reader */
+ int pageSize; /* size of a page */
XLogRecPtr readPagePtr; /* page pointer to read */
int32 readLen; /* bytes requested to reader, or actual bytes
* read by reader, which must be larger than
@@ -243,7 +244,7 @@ struct XLogReaderState
/* Get a new XLogReader */
extern XLogReaderState *XLogReaderAllocate(int wal_segment_size,
- const char *waldir,
+ int page_size, const char *waldir,
WALSegmentCleanupCB cleanup_cb);
/* Free an XLogReader */
--
2.27.0
----Next_Part(Thu_Mar__4_11_28_53_2021_014)----
^ permalink raw reply [nested|flat] 30+ messages in thread
* [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize.
@ 2019-09-05 12:29 Kyotaro Horiguchi <[email protected]>
0 siblings, 0 replies; 30+ messages in thread
From: Kyotaro Horiguchi @ 2019-09-05 12:29 UTC (permalink / raw)
XLog reader facility and its callers currently relies on the same
constant XLOG_BLCKSZ. This patch moves the responsibility to determine
the block size to the callers of the facility.
---
src/backend/access/transam/twophase.c | 3 +-
src/backend/access/transam/xlog.c | 9 ++--
src/backend/access/transam/xlogreader.c | 57 ++++++++++++-----------
src/backend/replication/logical/logical.c | 5 +-
src/backend/replication/walsender.c | 4 +-
src/bin/pg_rewind/parsexlog.c | 6 +--
src/bin/pg_waldump/pg_waldump.c | 2 +-
src/include/access/xlogreader.h | 3 +-
8 files changed, 50 insertions(+), 39 deletions(-)
diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index 7d5a96a5f5..9f9e7de253 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -1325,7 +1325,8 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
XLogReaderState *xlogreader;
char *errormsg;
- xlogreader = XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+ xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+ NULL, wal_segment_close);
if (!xlogreader)
ereport(ERROR,
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index b517ca85ad..0822ac70cb 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -1222,7 +1222,10 @@ XLogInsertRecord(XLogRecData *rdata,
appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len);
if (!debug_reader)
- debug_reader = XLogReaderAllocate(wal_segment_size, NULL, NULL);
+ {
+ debug_reader =
+ XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, NULL);
+ }
if (!debug_reader)
{
@@ -6531,8 +6534,8 @@ StartupXLOG(void)
OwnLatch(&XLogCtl->recoveryWakeupLatch);
/* Set up XLOG reader facility */
- xlogreader =
- XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+ xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+ NULL, wal_segment_close);
if (!xlogreader)
ereport(ERROR,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index ba3e4868f7..e5baf13086 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -72,7 +72,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...)
* Returns NULL if the xlogreader couldn't be allocated.
*/
XLogReaderState *
-XLogReaderAllocate(int wal_segment_size, const char *waldir,
+XLogReaderAllocate(int wal_segment_size, int page_size, const char *waldir,
WALSegmentCleanupCB cleanup_cb)
{
XLogReaderState *state;
@@ -87,6 +87,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
state->cleanup_cb = cleanup_cb;
state->max_block_id = -1;
+ state->pageSize = page_size;
/*
* Permanently allocate readBuf. We do it this way, rather than just
@@ -95,7 +96,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
* isn't guaranteed to have any particular alignment, whereas
* palloc_extended() will provide MAXALIGN'd storage.
*/
- state->readBuf = (char *) palloc_extended(XLOG_BLCKSZ,
+ state->readBuf = (char *) palloc_extended(state->pageSize,
MCXT_ALLOC_NO_OOM);
if (!state->readBuf)
{
@@ -163,7 +164,7 @@ XLogReaderFree(XLogReaderState *state)
* readRecordBufSize is set to the new buffer size.
*
* To avoid useless small increases, round its size to a multiple of
- * XLOG_BLCKSZ, and make sure it's at least 5*Max(BLCKSZ, XLOG_BLCKSZ) to start
+ * page size, and make sure it's at least 5*Max(BLCKSZ, <page size>) to start
* with. (That is enough for all "normal" records, but very large commit or
* abort records might need more space.)
*/
@@ -172,8 +173,8 @@ allocate_recordbuf(XLogReaderState *state, uint32 reclength)
{
uint32 newSize = reclength;
- newSize += XLOG_BLCKSZ - (newSize % XLOG_BLCKSZ);
- newSize = Max(newSize, 5 * Max(BLCKSZ, XLOG_BLCKSZ));
+ newSize += state->pageSize - (newSize % state->pageSize);
+ newSize = Max(newSize, 5 * Max(BLCKSZ, state->pageSize));
#ifndef FRONTEND
@@ -238,6 +239,8 @@ void
XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
{
Assert(!XLogRecPtrIsInvalid(RecPtr));
+ Assert(state->readBuf != NULL &&
+ state->readBuf == (void *) MAXALIGN(state->readBuf));
ResetDecoder(state);
@@ -366,15 +369,15 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
XLogPageHeader pageHeader;
targetPagePtr =
- state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
- targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+ state->ReadRecPtr - (state->ReadRecPtr % state->pageSize);
+ targetRecOff = state->ReadRecPtr % state->pageSize;
/*
* Check if we have enough data. For the first record in the page,
* the requesting length doesn't contain page header.
*/
if (XLogNeedData(state, targetPagePtr,
- Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+ Min(targetRecOff + SizeOfXLogRecord, state->pageSize),
targetRecOff != 0))
return XLREAD_NEED_DATA;
@@ -421,7 +424,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
* verified that we got the whole header.
*/
prec = (XLogRecord *) (state->readBuf +
- state->ReadRecPtr % XLOG_BLCKSZ);
+ state->ReadRecPtr % state->pageSize);
total_len = prec->xl_tot_len;
/*
@@ -432,7 +435,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
* ensure that we enter the XLREAD_CONTINUATION state below;
* otherwise we might fail to apply ValidXLogRecordHeader at all.
*/
- if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
+ if (targetRecOff <= state->pageSize - SizeOfXLogRecord)
{
if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
state->PrevRecPtr, prec))
@@ -477,10 +480,10 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
* available
*/
targetPagePtr =
- state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
- targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+ state->ReadRecPtr - (state->ReadRecPtr % state->pageSize);
+ targetRecOff = state->ReadRecPtr % state->pageSize;
- request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ);
+ request_len = Min(targetRecOff + total_len, state->pageSize);
record_len = request_len - targetRecOff;
/* ReadRecPtr contains page header */
@@ -549,7 +552,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
/* Calculate pointer to beginning of next page */
state->recordContRecPtr = state->ReadRecPtr + record_len;
- Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0);
+ Assert(state->recordContRecPtr % state->pageSize == 0);
state->readRecordState = XLREAD_CONTINUATION;
}
@@ -576,7 +579,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
/* this request contains page header */
Assert (targetPagePtr != 0);
if (XLogNeedData(state, targetPagePtr,
- Min(state->recordRemainLen, XLOG_BLCKSZ),
+ Min(state->recordRemainLen, state->pageSize),
false))
return XLREAD_NEED_DATA;
@@ -628,7 +631,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
Assert(state->readLen >= pageHeaderSize);
contdata = (char *) state->readBuf + pageHeaderSize;
- record_len = XLOG_BLCKSZ - pageHeaderSize;
+ record_len = state->pageSize - pageHeaderSize;
if (pageHeader->xlp_rem_len < record_len)
record_len = pageHeader->xlp_rem_len;
@@ -655,7 +658,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
}
/* Calculate pointer to beginning of next page, and continue */
- state->recordContRecPtr += XLOG_BLCKSZ;
+ state->recordContRecPtr += state->pageSize;
}
/* targetPagePtr is pointing the last-read page here */
@@ -789,10 +792,10 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
XLogPageHeaderSize((XLogPageHeader) state->readBuf);
addLen = pageHeaderSize;
- if (reqLen + pageHeaderSize <= XLOG_BLCKSZ)
+ if (reqLen + pageHeaderSize <= state->pageSize)
addLen = pageHeaderSize;
else
- addLen = XLOG_BLCKSZ - reqLen;
+ addLen = state->pageSize - reqLen;
Assert(addLen >= 0);
}
@@ -805,7 +808,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
/* Data is not in our buffer, request the caller for it. */
XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize);
targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize);
- Assert((pageptr % XLOG_BLCKSZ) == 0);
+ Assert((pageptr % state->pageSize) == 0);
/*
* Every time we request to load new data of a page to the caller, even if
@@ -829,7 +832,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
* will not come back here, but will request the actual target page.
*/
state->readPagePtr = pageptr - targetPageOff;
- state->readLen = XLOG_BLCKSZ;
+ state->readLen = state->pageSize;
return true;
}
@@ -962,7 +965,7 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
int32 offset;
XLogPageHeader hdr = (XLogPageHeader) phdr;
- Assert((recptr % XLOG_BLCKSZ) == 0);
+ Assert((recptr % state->pageSize) == 0);
XLByteToSeg(recptr, segno, state->segcxt.ws_segsize);
offset = XLogSegmentOffset(recptr, state->segcxt.ws_segsize);
@@ -1016,10 +1019,10 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
"WAL file is from different database system: incorrect segment size in page header");
return false;
}
- else if (longhdr->xlp_xlog_blcksz != XLOG_BLCKSZ)
+ else if (longhdr->xlp_xlog_blcksz != state->pageSize)
{
report_invalid_record(state,
- "WAL file is from different database system: incorrect XLOG_BLCKSZ in page header");
+ "WAL file is from different database system: incorrect block size in page header");
return false;
}
}
@@ -1140,7 +1143,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
* XLogNeedData() is prepared to handle that and will read at
* least short page-header worth of data
*/
- targetRecOff = tmpRecPtr % XLOG_BLCKSZ;
+ targetRecOff = tmpRecPtr % state->pageSize;
/* scroll back to page boundary */
targetPagePtr = tmpRecPtr - targetRecOff;
@@ -1174,8 +1177,8 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
*
* Note that record headers are MAXALIGN'ed
*/
- if (MAXALIGN(header->xlp_rem_len) >= (XLOG_BLCKSZ - pageHeaderSize))
- tmpRecPtr = targetPagePtr + XLOG_BLCKSZ;
+ if (MAXALIGN(header->xlp_rem_len) >= (state->pageSize - pageHeaderSize))
+ tmpRecPtr = targetPagePtr + state->pageSize;
else
{
/*
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index 9cbd8afa49..f2dd495310 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -199,11 +199,14 @@ StartupDecodingContext(List *output_plugin_options,
ctx->slot = slot;
- ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, cleanup_cb);
+ ctx->reader =
+ XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, cleanup_cb);
+
if (!ctx->reader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
errmsg("out of memory")));
+ ctx->reader->readBuf = palloc(XLOG_BLCKSZ);
ctx->page_read = page_read;
ctx->reorder = ReorderBufferAllocate();
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index 062093727d..8e95600ef7 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -579,8 +579,8 @@ StartReplication(StartReplicationCmd *cmd)
errmsg("IDENTIFY_SYSTEM has not been run before START_REPLICATION")));
/* create xlogreader for physical replication */
- xlogreader =
- XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+ xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+ NULL, wal_segment_close);
if (!xlogreader)
ereport(ERROR,
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index 712c85281c..8a6209b061 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -61,7 +61,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
XLogReaderState *xlogreader;
char *errormsg;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+ xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
@@ -121,7 +121,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex,
char *errormsg;
XLogRecPtr endptr;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+ xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
@@ -181,7 +181,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
forkptr += SizeOfXLogShortPHD;
}
- xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+ xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index f514de6b13..b635537659 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -1041,7 +1041,7 @@ main(int argc, char **argv)
/* we have everything we need, start reading */
xlogreader_state =
- XLogReaderAllocate(WalSegSz, waldir, WALDumpCloseSegment);
+ XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, waldir, WALDumpCloseSegment);
if (!xlogreader_state)
fatal_error("out of memory");
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index b9dad7ee51..1853516333 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -151,6 +151,7 @@ struct XLogReaderState
* ----------------------------------------
*/
/* variables to communicate with page reader */
+ int pageSize; /* size of a page */
XLogRecPtr readPagePtr; /* page pointer to read */
int32 readLen; /* bytes requested to reader, or actual bytes
* read by reader, which must be larger than
@@ -243,7 +244,7 @@ struct XLogReaderState
/* Get a new XLogReader */
extern XLogReaderState *XLogReaderAllocate(int wal_segment_size,
- const char *waldir,
+ int page_size, const char *waldir,
WALSegmentCleanupCB cleanup_cb);
/* Free an XLogReader */
--
2.27.0
----Next_Part(Thu_Mar__4_11_28_53_2021_014)----
^ permalink raw reply [nested|flat] 30+ messages in thread
* [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize.
@ 2019-09-05 12:29 Kyotaro Horiguchi <[email protected]>
0 siblings, 0 replies; 30+ messages in thread
From: Kyotaro Horiguchi @ 2019-09-05 12:29 UTC (permalink / raw)
XLog reader facility and its callers currently relies on the same
constant XLOG_BLCKSZ. This patch moves the responsibility to determine
the block size to the callers of the facility.
---
src/backend/access/transam/twophase.c | 3 +-
src/backend/access/transam/xlog.c | 9 ++--
src/backend/access/transam/xlogreader.c | 57 ++++++++++++-----------
src/backend/replication/logical/logical.c | 5 +-
src/backend/replication/walsender.c | 4 +-
src/bin/pg_rewind/parsexlog.c | 6 +--
src/bin/pg_waldump/pg_waldump.c | 2 +-
src/include/access/xlogreader.h | 3 +-
8 files changed, 50 insertions(+), 39 deletions(-)
diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index 7d5a96a5f5..9f9e7de253 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -1325,7 +1325,8 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
XLogReaderState *xlogreader;
char *errormsg;
- xlogreader = XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+ xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+ NULL, wal_segment_close);
if (!xlogreader)
ereport(ERROR,
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index b517ca85ad..0822ac70cb 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -1222,7 +1222,10 @@ XLogInsertRecord(XLogRecData *rdata,
appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len);
if (!debug_reader)
- debug_reader = XLogReaderAllocate(wal_segment_size, NULL, NULL);
+ {
+ debug_reader =
+ XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, NULL);
+ }
if (!debug_reader)
{
@@ -6531,8 +6534,8 @@ StartupXLOG(void)
OwnLatch(&XLogCtl->recoveryWakeupLatch);
/* Set up XLOG reader facility */
- xlogreader =
- XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+ xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+ NULL, wal_segment_close);
if (!xlogreader)
ereport(ERROR,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index ba3e4868f7..e5baf13086 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -72,7 +72,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...)
* Returns NULL if the xlogreader couldn't be allocated.
*/
XLogReaderState *
-XLogReaderAllocate(int wal_segment_size, const char *waldir,
+XLogReaderAllocate(int wal_segment_size, int page_size, const char *waldir,
WALSegmentCleanupCB cleanup_cb)
{
XLogReaderState *state;
@@ -87,6 +87,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
state->cleanup_cb = cleanup_cb;
state->max_block_id = -1;
+ state->pageSize = page_size;
/*
* Permanently allocate readBuf. We do it this way, rather than just
@@ -95,7 +96,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
* isn't guaranteed to have any particular alignment, whereas
* palloc_extended() will provide MAXALIGN'd storage.
*/
- state->readBuf = (char *) palloc_extended(XLOG_BLCKSZ,
+ state->readBuf = (char *) palloc_extended(state->pageSize,
MCXT_ALLOC_NO_OOM);
if (!state->readBuf)
{
@@ -163,7 +164,7 @@ XLogReaderFree(XLogReaderState *state)
* readRecordBufSize is set to the new buffer size.
*
* To avoid useless small increases, round its size to a multiple of
- * XLOG_BLCKSZ, and make sure it's at least 5*Max(BLCKSZ, XLOG_BLCKSZ) to start
+ * page size, and make sure it's at least 5*Max(BLCKSZ, <page size>) to start
* with. (That is enough for all "normal" records, but very large commit or
* abort records might need more space.)
*/
@@ -172,8 +173,8 @@ allocate_recordbuf(XLogReaderState *state, uint32 reclength)
{
uint32 newSize = reclength;
- newSize += XLOG_BLCKSZ - (newSize % XLOG_BLCKSZ);
- newSize = Max(newSize, 5 * Max(BLCKSZ, XLOG_BLCKSZ));
+ newSize += state->pageSize - (newSize % state->pageSize);
+ newSize = Max(newSize, 5 * Max(BLCKSZ, state->pageSize));
#ifndef FRONTEND
@@ -238,6 +239,8 @@ void
XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
{
Assert(!XLogRecPtrIsInvalid(RecPtr));
+ Assert(state->readBuf != NULL &&
+ state->readBuf == (void *) MAXALIGN(state->readBuf));
ResetDecoder(state);
@@ -366,15 +369,15 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
XLogPageHeader pageHeader;
targetPagePtr =
- state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
- targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+ state->ReadRecPtr - (state->ReadRecPtr % state->pageSize);
+ targetRecOff = state->ReadRecPtr % state->pageSize;
/*
* Check if we have enough data. For the first record in the page,
* the requesting length doesn't contain page header.
*/
if (XLogNeedData(state, targetPagePtr,
- Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+ Min(targetRecOff + SizeOfXLogRecord, state->pageSize),
targetRecOff != 0))
return XLREAD_NEED_DATA;
@@ -421,7 +424,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
* verified that we got the whole header.
*/
prec = (XLogRecord *) (state->readBuf +
- state->ReadRecPtr % XLOG_BLCKSZ);
+ state->ReadRecPtr % state->pageSize);
total_len = prec->xl_tot_len;
/*
@@ -432,7 +435,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
* ensure that we enter the XLREAD_CONTINUATION state below;
* otherwise we might fail to apply ValidXLogRecordHeader at all.
*/
- if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
+ if (targetRecOff <= state->pageSize - SizeOfXLogRecord)
{
if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
state->PrevRecPtr, prec))
@@ -477,10 +480,10 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
* available
*/
targetPagePtr =
- state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
- targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+ state->ReadRecPtr - (state->ReadRecPtr % state->pageSize);
+ targetRecOff = state->ReadRecPtr % state->pageSize;
- request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ);
+ request_len = Min(targetRecOff + total_len, state->pageSize);
record_len = request_len - targetRecOff;
/* ReadRecPtr contains page header */
@@ -549,7 +552,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
/* Calculate pointer to beginning of next page */
state->recordContRecPtr = state->ReadRecPtr + record_len;
- Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0);
+ Assert(state->recordContRecPtr % state->pageSize == 0);
state->readRecordState = XLREAD_CONTINUATION;
}
@@ -576,7 +579,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
/* this request contains page header */
Assert (targetPagePtr != 0);
if (XLogNeedData(state, targetPagePtr,
- Min(state->recordRemainLen, XLOG_BLCKSZ),
+ Min(state->recordRemainLen, state->pageSize),
false))
return XLREAD_NEED_DATA;
@@ -628,7 +631,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
Assert(state->readLen >= pageHeaderSize);
contdata = (char *) state->readBuf + pageHeaderSize;
- record_len = XLOG_BLCKSZ - pageHeaderSize;
+ record_len = state->pageSize - pageHeaderSize;
if (pageHeader->xlp_rem_len < record_len)
record_len = pageHeader->xlp_rem_len;
@@ -655,7 +658,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
}
/* Calculate pointer to beginning of next page, and continue */
- state->recordContRecPtr += XLOG_BLCKSZ;
+ state->recordContRecPtr += state->pageSize;
}
/* targetPagePtr is pointing the last-read page here */
@@ -789,10 +792,10 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
XLogPageHeaderSize((XLogPageHeader) state->readBuf);
addLen = pageHeaderSize;
- if (reqLen + pageHeaderSize <= XLOG_BLCKSZ)
+ if (reqLen + pageHeaderSize <= state->pageSize)
addLen = pageHeaderSize;
else
- addLen = XLOG_BLCKSZ - reqLen;
+ addLen = state->pageSize - reqLen;
Assert(addLen >= 0);
}
@@ -805,7 +808,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
/* Data is not in our buffer, request the caller for it. */
XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize);
targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize);
- Assert((pageptr % XLOG_BLCKSZ) == 0);
+ Assert((pageptr % state->pageSize) == 0);
/*
* Every time we request to load new data of a page to the caller, even if
@@ -829,7 +832,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
* will not come back here, but will request the actual target page.
*/
state->readPagePtr = pageptr - targetPageOff;
- state->readLen = XLOG_BLCKSZ;
+ state->readLen = state->pageSize;
return true;
}
@@ -962,7 +965,7 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
int32 offset;
XLogPageHeader hdr = (XLogPageHeader) phdr;
- Assert((recptr % XLOG_BLCKSZ) == 0);
+ Assert((recptr % state->pageSize) == 0);
XLByteToSeg(recptr, segno, state->segcxt.ws_segsize);
offset = XLogSegmentOffset(recptr, state->segcxt.ws_segsize);
@@ -1016,10 +1019,10 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
"WAL file is from different database system: incorrect segment size in page header");
return false;
}
- else if (longhdr->xlp_xlog_blcksz != XLOG_BLCKSZ)
+ else if (longhdr->xlp_xlog_blcksz != state->pageSize)
{
report_invalid_record(state,
- "WAL file is from different database system: incorrect XLOG_BLCKSZ in page header");
+ "WAL file is from different database system: incorrect block size in page header");
return false;
}
}
@@ -1140,7 +1143,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
* XLogNeedData() is prepared to handle that and will read at
* least short page-header worth of data
*/
- targetRecOff = tmpRecPtr % XLOG_BLCKSZ;
+ targetRecOff = tmpRecPtr % state->pageSize;
/* scroll back to page boundary */
targetPagePtr = tmpRecPtr - targetRecOff;
@@ -1174,8 +1177,8 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
*
* Note that record headers are MAXALIGN'ed
*/
- if (MAXALIGN(header->xlp_rem_len) >= (XLOG_BLCKSZ - pageHeaderSize))
- tmpRecPtr = targetPagePtr + XLOG_BLCKSZ;
+ if (MAXALIGN(header->xlp_rem_len) >= (state->pageSize - pageHeaderSize))
+ tmpRecPtr = targetPagePtr + state->pageSize;
else
{
/*
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index 9cbd8afa49..f2dd495310 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -199,11 +199,14 @@ StartupDecodingContext(List *output_plugin_options,
ctx->slot = slot;
- ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, cleanup_cb);
+ ctx->reader =
+ XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, cleanup_cb);
+
if (!ctx->reader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
errmsg("out of memory")));
+ ctx->reader->readBuf = palloc(XLOG_BLCKSZ);
ctx->page_read = page_read;
ctx->reorder = ReorderBufferAllocate();
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index 062093727d..8e95600ef7 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -579,8 +579,8 @@ StartReplication(StartReplicationCmd *cmd)
errmsg("IDENTIFY_SYSTEM has not been run before START_REPLICATION")));
/* create xlogreader for physical replication */
- xlogreader =
- XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+ xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+ NULL, wal_segment_close);
if (!xlogreader)
ereport(ERROR,
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index 712c85281c..8a6209b061 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -61,7 +61,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
XLogReaderState *xlogreader;
char *errormsg;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+ xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
@@ -121,7 +121,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex,
char *errormsg;
XLogRecPtr endptr;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+ xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
@@ -181,7 +181,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
forkptr += SizeOfXLogShortPHD;
}
- xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+ xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index f514de6b13..b635537659 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -1041,7 +1041,7 @@ main(int argc, char **argv)
/* we have everything we need, start reading */
xlogreader_state =
- XLogReaderAllocate(WalSegSz, waldir, WALDumpCloseSegment);
+ XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, waldir, WALDumpCloseSegment);
if (!xlogreader_state)
fatal_error("out of memory");
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index b9dad7ee51..1853516333 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -151,6 +151,7 @@ struct XLogReaderState
* ----------------------------------------
*/
/* variables to communicate with page reader */
+ int pageSize; /* size of a page */
XLogRecPtr readPagePtr; /* page pointer to read */
int32 readLen; /* bytes requested to reader, or actual bytes
* read by reader, which must be larger than
@@ -243,7 +244,7 @@ struct XLogReaderState
/* Get a new XLogReader */
extern XLogReaderState *XLogReaderAllocate(int wal_segment_size,
- const char *waldir,
+ int page_size, const char *waldir,
WALSegmentCleanupCB cleanup_cb);
/* Free an XLogReader */
--
2.27.0
----Next_Part(Thu_Mar__4_11_28_53_2021_014)----
^ permalink raw reply [nested|flat] 30+ messages in thread
* [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize.
@ 2019-09-05 12:29 Kyotaro Horiguchi <[email protected]>
0 siblings, 0 replies; 30+ messages in thread
From: Kyotaro Horiguchi @ 2019-09-05 12:29 UTC (permalink / raw)
XLog reader facility and its callers currently relies on the same
constant XLOG_BLCKSZ. This patch moves the responsibility to determine
the block size to the callers of the facility.
---
src/backend/access/transam/twophase.c | 3 +-
src/backend/access/transam/xlog.c | 9 ++--
src/backend/access/transam/xlogreader.c | 57 ++++++++++++-----------
src/backend/replication/logical/logical.c | 5 +-
src/backend/replication/walsender.c | 4 +-
src/bin/pg_rewind/parsexlog.c | 6 +--
src/bin/pg_waldump/pg_waldump.c | 2 +-
src/include/access/xlogreader.h | 3 +-
8 files changed, 50 insertions(+), 39 deletions(-)
diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index 7d5a96a5f5..9f9e7de253 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -1325,7 +1325,8 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
XLogReaderState *xlogreader;
char *errormsg;
- xlogreader = XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+ xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+ NULL, wal_segment_close);
if (!xlogreader)
ereport(ERROR,
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index b517ca85ad..0822ac70cb 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -1222,7 +1222,10 @@ XLogInsertRecord(XLogRecData *rdata,
appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len);
if (!debug_reader)
- debug_reader = XLogReaderAllocate(wal_segment_size, NULL, NULL);
+ {
+ debug_reader =
+ XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, NULL);
+ }
if (!debug_reader)
{
@@ -6531,8 +6534,8 @@ StartupXLOG(void)
OwnLatch(&XLogCtl->recoveryWakeupLatch);
/* Set up XLOG reader facility */
- xlogreader =
- XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+ xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+ NULL, wal_segment_close);
if (!xlogreader)
ereport(ERROR,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index ba3e4868f7..e5baf13086 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -72,7 +72,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...)
* Returns NULL if the xlogreader couldn't be allocated.
*/
XLogReaderState *
-XLogReaderAllocate(int wal_segment_size, const char *waldir,
+XLogReaderAllocate(int wal_segment_size, int page_size, const char *waldir,
WALSegmentCleanupCB cleanup_cb)
{
XLogReaderState *state;
@@ -87,6 +87,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
state->cleanup_cb = cleanup_cb;
state->max_block_id = -1;
+ state->pageSize = page_size;
/*
* Permanently allocate readBuf. We do it this way, rather than just
@@ -95,7 +96,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
* isn't guaranteed to have any particular alignment, whereas
* palloc_extended() will provide MAXALIGN'd storage.
*/
- state->readBuf = (char *) palloc_extended(XLOG_BLCKSZ,
+ state->readBuf = (char *) palloc_extended(state->pageSize,
MCXT_ALLOC_NO_OOM);
if (!state->readBuf)
{
@@ -163,7 +164,7 @@ XLogReaderFree(XLogReaderState *state)
* readRecordBufSize is set to the new buffer size.
*
* To avoid useless small increases, round its size to a multiple of
- * XLOG_BLCKSZ, and make sure it's at least 5*Max(BLCKSZ, XLOG_BLCKSZ) to start
+ * page size, and make sure it's at least 5*Max(BLCKSZ, <page size>) to start
* with. (That is enough for all "normal" records, but very large commit or
* abort records might need more space.)
*/
@@ -172,8 +173,8 @@ allocate_recordbuf(XLogReaderState *state, uint32 reclength)
{
uint32 newSize = reclength;
- newSize += XLOG_BLCKSZ - (newSize % XLOG_BLCKSZ);
- newSize = Max(newSize, 5 * Max(BLCKSZ, XLOG_BLCKSZ));
+ newSize += state->pageSize - (newSize % state->pageSize);
+ newSize = Max(newSize, 5 * Max(BLCKSZ, state->pageSize));
#ifndef FRONTEND
@@ -238,6 +239,8 @@ void
XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
{
Assert(!XLogRecPtrIsInvalid(RecPtr));
+ Assert(state->readBuf != NULL &&
+ state->readBuf == (void *) MAXALIGN(state->readBuf));
ResetDecoder(state);
@@ -366,15 +369,15 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
XLogPageHeader pageHeader;
targetPagePtr =
- state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
- targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+ state->ReadRecPtr - (state->ReadRecPtr % state->pageSize);
+ targetRecOff = state->ReadRecPtr % state->pageSize;
/*
* Check if we have enough data. For the first record in the page,
* the requesting length doesn't contain page header.
*/
if (XLogNeedData(state, targetPagePtr,
- Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+ Min(targetRecOff + SizeOfXLogRecord, state->pageSize),
targetRecOff != 0))
return XLREAD_NEED_DATA;
@@ -421,7 +424,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
* verified that we got the whole header.
*/
prec = (XLogRecord *) (state->readBuf +
- state->ReadRecPtr % XLOG_BLCKSZ);
+ state->ReadRecPtr % state->pageSize);
total_len = prec->xl_tot_len;
/*
@@ -432,7 +435,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
* ensure that we enter the XLREAD_CONTINUATION state below;
* otherwise we might fail to apply ValidXLogRecordHeader at all.
*/
- if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
+ if (targetRecOff <= state->pageSize - SizeOfXLogRecord)
{
if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
state->PrevRecPtr, prec))
@@ -477,10 +480,10 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
* available
*/
targetPagePtr =
- state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
- targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+ state->ReadRecPtr - (state->ReadRecPtr % state->pageSize);
+ targetRecOff = state->ReadRecPtr % state->pageSize;
- request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ);
+ request_len = Min(targetRecOff + total_len, state->pageSize);
record_len = request_len - targetRecOff;
/* ReadRecPtr contains page header */
@@ -549,7 +552,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
/* Calculate pointer to beginning of next page */
state->recordContRecPtr = state->ReadRecPtr + record_len;
- Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0);
+ Assert(state->recordContRecPtr % state->pageSize == 0);
state->readRecordState = XLREAD_CONTINUATION;
}
@@ -576,7 +579,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
/* this request contains page header */
Assert (targetPagePtr != 0);
if (XLogNeedData(state, targetPagePtr,
- Min(state->recordRemainLen, XLOG_BLCKSZ),
+ Min(state->recordRemainLen, state->pageSize),
false))
return XLREAD_NEED_DATA;
@@ -628,7 +631,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
Assert(state->readLen >= pageHeaderSize);
contdata = (char *) state->readBuf + pageHeaderSize;
- record_len = XLOG_BLCKSZ - pageHeaderSize;
+ record_len = state->pageSize - pageHeaderSize;
if (pageHeader->xlp_rem_len < record_len)
record_len = pageHeader->xlp_rem_len;
@@ -655,7 +658,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
}
/* Calculate pointer to beginning of next page, and continue */
- state->recordContRecPtr += XLOG_BLCKSZ;
+ state->recordContRecPtr += state->pageSize;
}
/* targetPagePtr is pointing the last-read page here */
@@ -789,10 +792,10 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
XLogPageHeaderSize((XLogPageHeader) state->readBuf);
addLen = pageHeaderSize;
- if (reqLen + pageHeaderSize <= XLOG_BLCKSZ)
+ if (reqLen + pageHeaderSize <= state->pageSize)
addLen = pageHeaderSize;
else
- addLen = XLOG_BLCKSZ - reqLen;
+ addLen = state->pageSize - reqLen;
Assert(addLen >= 0);
}
@@ -805,7 +808,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
/* Data is not in our buffer, request the caller for it. */
XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize);
targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize);
- Assert((pageptr % XLOG_BLCKSZ) == 0);
+ Assert((pageptr % state->pageSize) == 0);
/*
* Every time we request to load new data of a page to the caller, even if
@@ -829,7 +832,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
* will not come back here, but will request the actual target page.
*/
state->readPagePtr = pageptr - targetPageOff;
- state->readLen = XLOG_BLCKSZ;
+ state->readLen = state->pageSize;
return true;
}
@@ -962,7 +965,7 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
int32 offset;
XLogPageHeader hdr = (XLogPageHeader) phdr;
- Assert((recptr % XLOG_BLCKSZ) == 0);
+ Assert((recptr % state->pageSize) == 0);
XLByteToSeg(recptr, segno, state->segcxt.ws_segsize);
offset = XLogSegmentOffset(recptr, state->segcxt.ws_segsize);
@@ -1016,10 +1019,10 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
"WAL file is from different database system: incorrect segment size in page header");
return false;
}
- else if (longhdr->xlp_xlog_blcksz != XLOG_BLCKSZ)
+ else if (longhdr->xlp_xlog_blcksz != state->pageSize)
{
report_invalid_record(state,
- "WAL file is from different database system: incorrect XLOG_BLCKSZ in page header");
+ "WAL file is from different database system: incorrect block size in page header");
return false;
}
}
@@ -1140,7 +1143,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
* XLogNeedData() is prepared to handle that and will read at
* least short page-header worth of data
*/
- targetRecOff = tmpRecPtr % XLOG_BLCKSZ;
+ targetRecOff = tmpRecPtr % state->pageSize;
/* scroll back to page boundary */
targetPagePtr = tmpRecPtr - targetRecOff;
@@ -1174,8 +1177,8 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
*
* Note that record headers are MAXALIGN'ed
*/
- if (MAXALIGN(header->xlp_rem_len) >= (XLOG_BLCKSZ - pageHeaderSize))
- tmpRecPtr = targetPagePtr + XLOG_BLCKSZ;
+ if (MAXALIGN(header->xlp_rem_len) >= (state->pageSize - pageHeaderSize))
+ tmpRecPtr = targetPagePtr + state->pageSize;
else
{
/*
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index 9cbd8afa49..f2dd495310 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -199,11 +199,14 @@ StartupDecodingContext(List *output_plugin_options,
ctx->slot = slot;
- ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, cleanup_cb);
+ ctx->reader =
+ XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, cleanup_cb);
+
if (!ctx->reader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
errmsg("out of memory")));
+ ctx->reader->readBuf = palloc(XLOG_BLCKSZ);
ctx->page_read = page_read;
ctx->reorder = ReorderBufferAllocate();
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index 062093727d..8e95600ef7 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -579,8 +579,8 @@ StartReplication(StartReplicationCmd *cmd)
errmsg("IDENTIFY_SYSTEM has not been run before START_REPLICATION")));
/* create xlogreader for physical replication */
- xlogreader =
- XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+ xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+ NULL, wal_segment_close);
if (!xlogreader)
ereport(ERROR,
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index 712c85281c..8a6209b061 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -61,7 +61,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
XLogReaderState *xlogreader;
char *errormsg;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+ xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
@@ -121,7 +121,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex,
char *errormsg;
XLogRecPtr endptr;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+ xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
@@ -181,7 +181,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
forkptr += SizeOfXLogShortPHD;
}
- xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+ xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index f514de6b13..b635537659 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -1041,7 +1041,7 @@ main(int argc, char **argv)
/* we have everything we need, start reading */
xlogreader_state =
- XLogReaderAllocate(WalSegSz, waldir, WALDumpCloseSegment);
+ XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, waldir, WALDumpCloseSegment);
if (!xlogreader_state)
fatal_error("out of memory");
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index b9dad7ee51..1853516333 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -151,6 +151,7 @@ struct XLogReaderState
* ----------------------------------------
*/
/* variables to communicate with page reader */
+ int pageSize; /* size of a page */
XLogRecPtr readPagePtr; /* page pointer to read */
int32 readLen; /* bytes requested to reader, or actual bytes
* read by reader, which must be larger than
@@ -243,7 +244,7 @@ struct XLogReaderState
/* Get a new XLogReader */
extern XLogReaderState *XLogReaderAllocate(int wal_segment_size,
- const char *waldir,
+ int page_size, const char *waldir,
WALSegmentCleanupCB cleanup_cb);
/* Free an XLogReader */
--
2.27.0
----Next_Part(Thu_Mar__4_11_28_53_2021_014)----
^ permalink raw reply [nested|flat] 30+ messages in thread
* [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize.
@ 2019-09-05 12:29 Kyotaro Horiguchi <[email protected]>
0 siblings, 0 replies; 30+ messages in thread
From: Kyotaro Horiguchi @ 2019-09-05 12:29 UTC (permalink / raw)
XLog reader facility and its callers currently relies on the same
constant XLOG_BLCKSZ. This patch moves the responsibility to determine
the block size to the callers of the facility.
---
src/backend/access/transam/twophase.c | 3 +-
src/backend/access/transam/xlog.c | 9 ++--
src/backend/access/transam/xlogreader.c | 57 ++++++++++++-----------
src/backend/replication/logical/logical.c | 5 +-
src/backend/replication/walsender.c | 4 +-
src/bin/pg_rewind/parsexlog.c | 6 +--
src/bin/pg_waldump/pg_waldump.c | 2 +-
src/include/access/xlogreader.h | 3 +-
8 files changed, 50 insertions(+), 39 deletions(-)
diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index 7d5a96a5f5..9f9e7de253 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -1325,7 +1325,8 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
XLogReaderState *xlogreader;
char *errormsg;
- xlogreader = XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+ xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+ NULL, wal_segment_close);
if (!xlogreader)
ereport(ERROR,
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index b517ca85ad..0822ac70cb 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -1222,7 +1222,10 @@ XLogInsertRecord(XLogRecData *rdata,
appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len);
if (!debug_reader)
- debug_reader = XLogReaderAllocate(wal_segment_size, NULL, NULL);
+ {
+ debug_reader =
+ XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, NULL);
+ }
if (!debug_reader)
{
@@ -6531,8 +6534,8 @@ StartupXLOG(void)
OwnLatch(&XLogCtl->recoveryWakeupLatch);
/* Set up XLOG reader facility */
- xlogreader =
- XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+ xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+ NULL, wal_segment_close);
if (!xlogreader)
ereport(ERROR,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index ba3e4868f7..e5baf13086 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -72,7 +72,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...)
* Returns NULL if the xlogreader couldn't be allocated.
*/
XLogReaderState *
-XLogReaderAllocate(int wal_segment_size, const char *waldir,
+XLogReaderAllocate(int wal_segment_size, int page_size, const char *waldir,
WALSegmentCleanupCB cleanup_cb)
{
XLogReaderState *state;
@@ -87,6 +87,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
state->cleanup_cb = cleanup_cb;
state->max_block_id = -1;
+ state->pageSize = page_size;
/*
* Permanently allocate readBuf. We do it this way, rather than just
@@ -95,7 +96,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
* isn't guaranteed to have any particular alignment, whereas
* palloc_extended() will provide MAXALIGN'd storage.
*/
- state->readBuf = (char *) palloc_extended(XLOG_BLCKSZ,
+ state->readBuf = (char *) palloc_extended(state->pageSize,
MCXT_ALLOC_NO_OOM);
if (!state->readBuf)
{
@@ -163,7 +164,7 @@ XLogReaderFree(XLogReaderState *state)
* readRecordBufSize is set to the new buffer size.
*
* To avoid useless small increases, round its size to a multiple of
- * XLOG_BLCKSZ, and make sure it's at least 5*Max(BLCKSZ, XLOG_BLCKSZ) to start
+ * page size, and make sure it's at least 5*Max(BLCKSZ, <page size>) to start
* with. (That is enough for all "normal" records, but very large commit or
* abort records might need more space.)
*/
@@ -172,8 +173,8 @@ allocate_recordbuf(XLogReaderState *state, uint32 reclength)
{
uint32 newSize = reclength;
- newSize += XLOG_BLCKSZ - (newSize % XLOG_BLCKSZ);
- newSize = Max(newSize, 5 * Max(BLCKSZ, XLOG_BLCKSZ));
+ newSize += state->pageSize - (newSize % state->pageSize);
+ newSize = Max(newSize, 5 * Max(BLCKSZ, state->pageSize));
#ifndef FRONTEND
@@ -238,6 +239,8 @@ void
XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
{
Assert(!XLogRecPtrIsInvalid(RecPtr));
+ Assert(state->readBuf != NULL &&
+ state->readBuf == (void *) MAXALIGN(state->readBuf));
ResetDecoder(state);
@@ -366,15 +369,15 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
XLogPageHeader pageHeader;
targetPagePtr =
- state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
- targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+ state->ReadRecPtr - (state->ReadRecPtr % state->pageSize);
+ targetRecOff = state->ReadRecPtr % state->pageSize;
/*
* Check if we have enough data. For the first record in the page,
* the requesting length doesn't contain page header.
*/
if (XLogNeedData(state, targetPagePtr,
- Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+ Min(targetRecOff + SizeOfXLogRecord, state->pageSize),
targetRecOff != 0))
return XLREAD_NEED_DATA;
@@ -421,7 +424,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
* verified that we got the whole header.
*/
prec = (XLogRecord *) (state->readBuf +
- state->ReadRecPtr % XLOG_BLCKSZ);
+ state->ReadRecPtr % state->pageSize);
total_len = prec->xl_tot_len;
/*
@@ -432,7 +435,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
* ensure that we enter the XLREAD_CONTINUATION state below;
* otherwise we might fail to apply ValidXLogRecordHeader at all.
*/
- if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
+ if (targetRecOff <= state->pageSize - SizeOfXLogRecord)
{
if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
state->PrevRecPtr, prec))
@@ -477,10 +480,10 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
* available
*/
targetPagePtr =
- state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
- targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+ state->ReadRecPtr - (state->ReadRecPtr % state->pageSize);
+ targetRecOff = state->ReadRecPtr % state->pageSize;
- request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ);
+ request_len = Min(targetRecOff + total_len, state->pageSize);
record_len = request_len - targetRecOff;
/* ReadRecPtr contains page header */
@@ -549,7 +552,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
/* Calculate pointer to beginning of next page */
state->recordContRecPtr = state->ReadRecPtr + record_len;
- Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0);
+ Assert(state->recordContRecPtr % state->pageSize == 0);
state->readRecordState = XLREAD_CONTINUATION;
}
@@ -576,7 +579,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
/* this request contains page header */
Assert (targetPagePtr != 0);
if (XLogNeedData(state, targetPagePtr,
- Min(state->recordRemainLen, XLOG_BLCKSZ),
+ Min(state->recordRemainLen, state->pageSize),
false))
return XLREAD_NEED_DATA;
@@ -628,7 +631,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
Assert(state->readLen >= pageHeaderSize);
contdata = (char *) state->readBuf + pageHeaderSize;
- record_len = XLOG_BLCKSZ - pageHeaderSize;
+ record_len = state->pageSize - pageHeaderSize;
if (pageHeader->xlp_rem_len < record_len)
record_len = pageHeader->xlp_rem_len;
@@ -655,7 +658,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
}
/* Calculate pointer to beginning of next page, and continue */
- state->recordContRecPtr += XLOG_BLCKSZ;
+ state->recordContRecPtr += state->pageSize;
}
/* targetPagePtr is pointing the last-read page here */
@@ -789,10 +792,10 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
XLogPageHeaderSize((XLogPageHeader) state->readBuf);
addLen = pageHeaderSize;
- if (reqLen + pageHeaderSize <= XLOG_BLCKSZ)
+ if (reqLen + pageHeaderSize <= state->pageSize)
addLen = pageHeaderSize;
else
- addLen = XLOG_BLCKSZ - reqLen;
+ addLen = state->pageSize - reqLen;
Assert(addLen >= 0);
}
@@ -805,7 +808,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
/* Data is not in our buffer, request the caller for it. */
XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize);
targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize);
- Assert((pageptr % XLOG_BLCKSZ) == 0);
+ Assert((pageptr % state->pageSize) == 0);
/*
* Every time we request to load new data of a page to the caller, even if
@@ -829,7 +832,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
* will not come back here, but will request the actual target page.
*/
state->readPagePtr = pageptr - targetPageOff;
- state->readLen = XLOG_BLCKSZ;
+ state->readLen = state->pageSize;
return true;
}
@@ -962,7 +965,7 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
int32 offset;
XLogPageHeader hdr = (XLogPageHeader) phdr;
- Assert((recptr % XLOG_BLCKSZ) == 0);
+ Assert((recptr % state->pageSize) == 0);
XLByteToSeg(recptr, segno, state->segcxt.ws_segsize);
offset = XLogSegmentOffset(recptr, state->segcxt.ws_segsize);
@@ -1016,10 +1019,10 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
"WAL file is from different database system: incorrect segment size in page header");
return false;
}
- else if (longhdr->xlp_xlog_blcksz != XLOG_BLCKSZ)
+ else if (longhdr->xlp_xlog_blcksz != state->pageSize)
{
report_invalid_record(state,
- "WAL file is from different database system: incorrect XLOG_BLCKSZ in page header");
+ "WAL file is from different database system: incorrect block size in page header");
return false;
}
}
@@ -1140,7 +1143,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
* XLogNeedData() is prepared to handle that and will read at
* least short page-header worth of data
*/
- targetRecOff = tmpRecPtr % XLOG_BLCKSZ;
+ targetRecOff = tmpRecPtr % state->pageSize;
/* scroll back to page boundary */
targetPagePtr = tmpRecPtr - targetRecOff;
@@ -1174,8 +1177,8 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
*
* Note that record headers are MAXALIGN'ed
*/
- if (MAXALIGN(header->xlp_rem_len) >= (XLOG_BLCKSZ - pageHeaderSize))
- tmpRecPtr = targetPagePtr + XLOG_BLCKSZ;
+ if (MAXALIGN(header->xlp_rem_len) >= (state->pageSize - pageHeaderSize))
+ tmpRecPtr = targetPagePtr + state->pageSize;
else
{
/*
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index 9cbd8afa49..f2dd495310 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -199,11 +199,14 @@ StartupDecodingContext(List *output_plugin_options,
ctx->slot = slot;
- ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, cleanup_cb);
+ ctx->reader =
+ XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, cleanup_cb);
+
if (!ctx->reader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
errmsg("out of memory")));
+ ctx->reader->readBuf = palloc(XLOG_BLCKSZ);
ctx->page_read = page_read;
ctx->reorder = ReorderBufferAllocate();
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index 062093727d..8e95600ef7 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -579,8 +579,8 @@ StartReplication(StartReplicationCmd *cmd)
errmsg("IDENTIFY_SYSTEM has not been run before START_REPLICATION")));
/* create xlogreader for physical replication */
- xlogreader =
- XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+ xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+ NULL, wal_segment_close);
if (!xlogreader)
ereport(ERROR,
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index 712c85281c..8a6209b061 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -61,7 +61,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
XLogReaderState *xlogreader;
char *errormsg;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+ xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
@@ -121,7 +121,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex,
char *errormsg;
XLogRecPtr endptr;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+ xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
@@ -181,7 +181,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
forkptr += SizeOfXLogShortPHD;
}
- xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+ xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index f514de6b13..b635537659 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -1041,7 +1041,7 @@ main(int argc, char **argv)
/* we have everything we need, start reading */
xlogreader_state =
- XLogReaderAllocate(WalSegSz, waldir, WALDumpCloseSegment);
+ XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, waldir, WALDumpCloseSegment);
if (!xlogreader_state)
fatal_error("out of memory");
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index b9dad7ee51..1853516333 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -151,6 +151,7 @@ struct XLogReaderState
* ----------------------------------------
*/
/* variables to communicate with page reader */
+ int pageSize; /* size of a page */
XLogRecPtr readPagePtr; /* page pointer to read */
int32 readLen; /* bytes requested to reader, or actual bytes
* read by reader, which must be larger than
@@ -243,7 +244,7 @@ struct XLogReaderState
/* Get a new XLogReader */
extern XLogReaderState *XLogReaderAllocate(int wal_segment_size,
- const char *waldir,
+ int page_size, const char *waldir,
WALSegmentCleanupCB cleanup_cb);
/* Free an XLogReader */
--
2.27.0
----Next_Part(Thu_Mar__4_11_28_53_2021_014)----
^ permalink raw reply [nested|flat] 30+ messages in thread
* [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize.
@ 2019-09-05 12:29 Kyotaro Horiguchi <[email protected]>
0 siblings, 0 replies; 30+ messages in thread
From: Kyotaro Horiguchi @ 2019-09-05 12:29 UTC (permalink / raw)
XLog reader facility and its callers currently relies on the same
constant XLOG_BLCKSZ. This patch moves the responsibility to determine
the block size to the callers of the facility.
---
src/backend/access/transam/twophase.c | 3 +-
src/backend/access/transam/xlog.c | 9 ++--
src/backend/access/transam/xlogreader.c | 57 ++++++++++++-----------
src/backend/replication/logical/logical.c | 5 +-
src/backend/replication/walsender.c | 4 +-
src/bin/pg_rewind/parsexlog.c | 6 +--
src/bin/pg_waldump/pg_waldump.c | 2 +-
src/include/access/xlogreader.h | 3 +-
8 files changed, 50 insertions(+), 39 deletions(-)
diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index 7d5a96a5f5..9f9e7de253 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -1325,7 +1325,8 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
XLogReaderState *xlogreader;
char *errormsg;
- xlogreader = XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+ xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+ NULL, wal_segment_close);
if (!xlogreader)
ereport(ERROR,
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index b517ca85ad..0822ac70cb 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -1222,7 +1222,10 @@ XLogInsertRecord(XLogRecData *rdata,
appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len);
if (!debug_reader)
- debug_reader = XLogReaderAllocate(wal_segment_size, NULL, NULL);
+ {
+ debug_reader =
+ XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, NULL);
+ }
if (!debug_reader)
{
@@ -6531,8 +6534,8 @@ StartupXLOG(void)
OwnLatch(&XLogCtl->recoveryWakeupLatch);
/* Set up XLOG reader facility */
- xlogreader =
- XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+ xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+ NULL, wal_segment_close);
if (!xlogreader)
ereport(ERROR,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index ba3e4868f7..e5baf13086 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -72,7 +72,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...)
* Returns NULL if the xlogreader couldn't be allocated.
*/
XLogReaderState *
-XLogReaderAllocate(int wal_segment_size, const char *waldir,
+XLogReaderAllocate(int wal_segment_size, int page_size, const char *waldir,
WALSegmentCleanupCB cleanup_cb)
{
XLogReaderState *state;
@@ -87,6 +87,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
state->cleanup_cb = cleanup_cb;
state->max_block_id = -1;
+ state->pageSize = page_size;
/*
* Permanently allocate readBuf. We do it this way, rather than just
@@ -95,7 +96,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
* isn't guaranteed to have any particular alignment, whereas
* palloc_extended() will provide MAXALIGN'd storage.
*/
- state->readBuf = (char *) palloc_extended(XLOG_BLCKSZ,
+ state->readBuf = (char *) palloc_extended(state->pageSize,
MCXT_ALLOC_NO_OOM);
if (!state->readBuf)
{
@@ -163,7 +164,7 @@ XLogReaderFree(XLogReaderState *state)
* readRecordBufSize is set to the new buffer size.
*
* To avoid useless small increases, round its size to a multiple of
- * XLOG_BLCKSZ, and make sure it's at least 5*Max(BLCKSZ, XLOG_BLCKSZ) to start
+ * page size, and make sure it's at least 5*Max(BLCKSZ, <page size>) to start
* with. (That is enough for all "normal" records, but very large commit or
* abort records might need more space.)
*/
@@ -172,8 +173,8 @@ allocate_recordbuf(XLogReaderState *state, uint32 reclength)
{
uint32 newSize = reclength;
- newSize += XLOG_BLCKSZ - (newSize % XLOG_BLCKSZ);
- newSize = Max(newSize, 5 * Max(BLCKSZ, XLOG_BLCKSZ));
+ newSize += state->pageSize - (newSize % state->pageSize);
+ newSize = Max(newSize, 5 * Max(BLCKSZ, state->pageSize));
#ifndef FRONTEND
@@ -238,6 +239,8 @@ void
XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
{
Assert(!XLogRecPtrIsInvalid(RecPtr));
+ Assert(state->readBuf != NULL &&
+ state->readBuf == (void *) MAXALIGN(state->readBuf));
ResetDecoder(state);
@@ -366,15 +369,15 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
XLogPageHeader pageHeader;
targetPagePtr =
- state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
- targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+ state->ReadRecPtr - (state->ReadRecPtr % state->pageSize);
+ targetRecOff = state->ReadRecPtr % state->pageSize;
/*
* Check if we have enough data. For the first record in the page,
* the requesting length doesn't contain page header.
*/
if (XLogNeedData(state, targetPagePtr,
- Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+ Min(targetRecOff + SizeOfXLogRecord, state->pageSize),
targetRecOff != 0))
return XLREAD_NEED_DATA;
@@ -421,7 +424,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
* verified that we got the whole header.
*/
prec = (XLogRecord *) (state->readBuf +
- state->ReadRecPtr % XLOG_BLCKSZ);
+ state->ReadRecPtr % state->pageSize);
total_len = prec->xl_tot_len;
/*
@@ -432,7 +435,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
* ensure that we enter the XLREAD_CONTINUATION state below;
* otherwise we might fail to apply ValidXLogRecordHeader at all.
*/
- if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
+ if (targetRecOff <= state->pageSize - SizeOfXLogRecord)
{
if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
state->PrevRecPtr, prec))
@@ -477,10 +480,10 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
* available
*/
targetPagePtr =
- state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
- targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+ state->ReadRecPtr - (state->ReadRecPtr % state->pageSize);
+ targetRecOff = state->ReadRecPtr % state->pageSize;
- request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ);
+ request_len = Min(targetRecOff + total_len, state->pageSize);
record_len = request_len - targetRecOff;
/* ReadRecPtr contains page header */
@@ -549,7 +552,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
/* Calculate pointer to beginning of next page */
state->recordContRecPtr = state->ReadRecPtr + record_len;
- Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0);
+ Assert(state->recordContRecPtr % state->pageSize == 0);
state->readRecordState = XLREAD_CONTINUATION;
}
@@ -576,7 +579,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
/* this request contains page header */
Assert (targetPagePtr != 0);
if (XLogNeedData(state, targetPagePtr,
- Min(state->recordRemainLen, XLOG_BLCKSZ),
+ Min(state->recordRemainLen, state->pageSize),
false))
return XLREAD_NEED_DATA;
@@ -628,7 +631,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
Assert(state->readLen >= pageHeaderSize);
contdata = (char *) state->readBuf + pageHeaderSize;
- record_len = XLOG_BLCKSZ - pageHeaderSize;
+ record_len = state->pageSize - pageHeaderSize;
if (pageHeader->xlp_rem_len < record_len)
record_len = pageHeader->xlp_rem_len;
@@ -655,7 +658,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
}
/* Calculate pointer to beginning of next page, and continue */
- state->recordContRecPtr += XLOG_BLCKSZ;
+ state->recordContRecPtr += state->pageSize;
}
/* targetPagePtr is pointing the last-read page here */
@@ -789,10 +792,10 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
XLogPageHeaderSize((XLogPageHeader) state->readBuf);
addLen = pageHeaderSize;
- if (reqLen + pageHeaderSize <= XLOG_BLCKSZ)
+ if (reqLen + pageHeaderSize <= state->pageSize)
addLen = pageHeaderSize;
else
- addLen = XLOG_BLCKSZ - reqLen;
+ addLen = state->pageSize - reqLen;
Assert(addLen >= 0);
}
@@ -805,7 +808,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
/* Data is not in our buffer, request the caller for it. */
XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize);
targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize);
- Assert((pageptr % XLOG_BLCKSZ) == 0);
+ Assert((pageptr % state->pageSize) == 0);
/*
* Every time we request to load new data of a page to the caller, even if
@@ -829,7 +832,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
* will not come back here, but will request the actual target page.
*/
state->readPagePtr = pageptr - targetPageOff;
- state->readLen = XLOG_BLCKSZ;
+ state->readLen = state->pageSize;
return true;
}
@@ -962,7 +965,7 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
int32 offset;
XLogPageHeader hdr = (XLogPageHeader) phdr;
- Assert((recptr % XLOG_BLCKSZ) == 0);
+ Assert((recptr % state->pageSize) == 0);
XLByteToSeg(recptr, segno, state->segcxt.ws_segsize);
offset = XLogSegmentOffset(recptr, state->segcxt.ws_segsize);
@@ -1016,10 +1019,10 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
"WAL file is from different database system: incorrect segment size in page header");
return false;
}
- else if (longhdr->xlp_xlog_blcksz != XLOG_BLCKSZ)
+ else if (longhdr->xlp_xlog_blcksz != state->pageSize)
{
report_invalid_record(state,
- "WAL file is from different database system: incorrect XLOG_BLCKSZ in page header");
+ "WAL file is from different database system: incorrect block size in page header");
return false;
}
}
@@ -1140,7 +1143,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
* XLogNeedData() is prepared to handle that and will read at
* least short page-header worth of data
*/
- targetRecOff = tmpRecPtr % XLOG_BLCKSZ;
+ targetRecOff = tmpRecPtr % state->pageSize;
/* scroll back to page boundary */
targetPagePtr = tmpRecPtr - targetRecOff;
@@ -1174,8 +1177,8 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
*
* Note that record headers are MAXALIGN'ed
*/
- if (MAXALIGN(header->xlp_rem_len) >= (XLOG_BLCKSZ - pageHeaderSize))
- tmpRecPtr = targetPagePtr + XLOG_BLCKSZ;
+ if (MAXALIGN(header->xlp_rem_len) >= (state->pageSize - pageHeaderSize))
+ tmpRecPtr = targetPagePtr + state->pageSize;
else
{
/*
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index 9cbd8afa49..f2dd495310 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -199,11 +199,14 @@ StartupDecodingContext(List *output_plugin_options,
ctx->slot = slot;
- ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, cleanup_cb);
+ ctx->reader =
+ XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, cleanup_cb);
+
if (!ctx->reader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
errmsg("out of memory")));
+ ctx->reader->readBuf = palloc(XLOG_BLCKSZ);
ctx->page_read = page_read;
ctx->reorder = ReorderBufferAllocate();
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index 062093727d..8e95600ef7 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -579,8 +579,8 @@ StartReplication(StartReplicationCmd *cmd)
errmsg("IDENTIFY_SYSTEM has not been run before START_REPLICATION")));
/* create xlogreader for physical replication */
- xlogreader =
- XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+ xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+ NULL, wal_segment_close);
if (!xlogreader)
ereport(ERROR,
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index 712c85281c..8a6209b061 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -61,7 +61,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
XLogReaderState *xlogreader;
char *errormsg;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+ xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
@@ -121,7 +121,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex,
char *errormsg;
XLogRecPtr endptr;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+ xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
@@ -181,7 +181,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
forkptr += SizeOfXLogShortPHD;
}
- xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+ xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index f514de6b13..b635537659 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -1041,7 +1041,7 @@ main(int argc, char **argv)
/* we have everything we need, start reading */
xlogreader_state =
- XLogReaderAllocate(WalSegSz, waldir, WALDumpCloseSegment);
+ XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, waldir, WALDumpCloseSegment);
if (!xlogreader_state)
fatal_error("out of memory");
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index b9dad7ee51..1853516333 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -151,6 +151,7 @@ struct XLogReaderState
* ----------------------------------------
*/
/* variables to communicate with page reader */
+ int pageSize; /* size of a page */
XLogRecPtr readPagePtr; /* page pointer to read */
int32 readLen; /* bytes requested to reader, or actual bytes
* read by reader, which must be larger than
@@ -243,7 +244,7 @@ struct XLogReaderState
/* Get a new XLogReader */
extern XLogReaderState *XLogReaderAllocate(int wal_segment_size,
- const char *waldir,
+ int page_size, const char *waldir,
WALSegmentCleanupCB cleanup_cb);
/* Free an XLogReader */
--
2.27.0
----Next_Part(Thu_Mar__4_11_28_53_2021_014)----
^ permalink raw reply [nested|flat] 30+ messages in thread
* [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize.
@ 2019-09-05 12:29 Kyotaro Horiguchi <[email protected]>
0 siblings, 0 replies; 30+ messages in thread
From: Kyotaro Horiguchi @ 2019-09-05 12:29 UTC (permalink / raw)
XLog reader facility and its callers currently relies on the same
constant XLOG_BLCKSZ. This patch moves the responsibility to determine
the block size to the callers of the facility.
---
src/backend/access/transam/twophase.c | 3 +-
src/backend/access/transam/xlog.c | 9 ++--
src/backend/access/transam/xlogreader.c | 57 ++++++++++++-----------
src/backend/replication/logical/logical.c | 5 +-
src/backend/replication/walsender.c | 4 +-
src/bin/pg_rewind/parsexlog.c | 6 +--
src/bin/pg_waldump/pg_waldump.c | 2 +-
src/include/access/xlogreader.h | 3 +-
8 files changed, 50 insertions(+), 39 deletions(-)
diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index 7d5a96a5f5..9f9e7de253 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -1325,7 +1325,8 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
XLogReaderState *xlogreader;
char *errormsg;
- xlogreader = XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+ xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+ NULL, wal_segment_close);
if (!xlogreader)
ereport(ERROR,
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index b517ca85ad..0822ac70cb 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -1222,7 +1222,10 @@ XLogInsertRecord(XLogRecData *rdata,
appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len);
if (!debug_reader)
- debug_reader = XLogReaderAllocate(wal_segment_size, NULL, NULL);
+ {
+ debug_reader =
+ XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, NULL);
+ }
if (!debug_reader)
{
@@ -6531,8 +6534,8 @@ StartupXLOG(void)
OwnLatch(&XLogCtl->recoveryWakeupLatch);
/* Set up XLOG reader facility */
- xlogreader =
- XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+ xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+ NULL, wal_segment_close);
if (!xlogreader)
ereport(ERROR,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index ba3e4868f7..e5baf13086 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -72,7 +72,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...)
* Returns NULL if the xlogreader couldn't be allocated.
*/
XLogReaderState *
-XLogReaderAllocate(int wal_segment_size, const char *waldir,
+XLogReaderAllocate(int wal_segment_size, int page_size, const char *waldir,
WALSegmentCleanupCB cleanup_cb)
{
XLogReaderState *state;
@@ -87,6 +87,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
state->cleanup_cb = cleanup_cb;
state->max_block_id = -1;
+ state->pageSize = page_size;
/*
* Permanently allocate readBuf. We do it this way, rather than just
@@ -95,7 +96,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
* isn't guaranteed to have any particular alignment, whereas
* palloc_extended() will provide MAXALIGN'd storage.
*/
- state->readBuf = (char *) palloc_extended(XLOG_BLCKSZ,
+ state->readBuf = (char *) palloc_extended(state->pageSize,
MCXT_ALLOC_NO_OOM);
if (!state->readBuf)
{
@@ -163,7 +164,7 @@ XLogReaderFree(XLogReaderState *state)
* readRecordBufSize is set to the new buffer size.
*
* To avoid useless small increases, round its size to a multiple of
- * XLOG_BLCKSZ, and make sure it's at least 5*Max(BLCKSZ, XLOG_BLCKSZ) to start
+ * page size, and make sure it's at least 5*Max(BLCKSZ, <page size>) to start
* with. (That is enough for all "normal" records, but very large commit or
* abort records might need more space.)
*/
@@ -172,8 +173,8 @@ allocate_recordbuf(XLogReaderState *state, uint32 reclength)
{
uint32 newSize = reclength;
- newSize += XLOG_BLCKSZ - (newSize % XLOG_BLCKSZ);
- newSize = Max(newSize, 5 * Max(BLCKSZ, XLOG_BLCKSZ));
+ newSize += state->pageSize - (newSize % state->pageSize);
+ newSize = Max(newSize, 5 * Max(BLCKSZ, state->pageSize));
#ifndef FRONTEND
@@ -238,6 +239,8 @@ void
XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
{
Assert(!XLogRecPtrIsInvalid(RecPtr));
+ Assert(state->readBuf != NULL &&
+ state->readBuf == (void *) MAXALIGN(state->readBuf));
ResetDecoder(state);
@@ -366,15 +369,15 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
XLogPageHeader pageHeader;
targetPagePtr =
- state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
- targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+ state->ReadRecPtr - (state->ReadRecPtr % state->pageSize);
+ targetRecOff = state->ReadRecPtr % state->pageSize;
/*
* Check if we have enough data. For the first record in the page,
* the requesting length doesn't contain page header.
*/
if (XLogNeedData(state, targetPagePtr,
- Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+ Min(targetRecOff + SizeOfXLogRecord, state->pageSize),
targetRecOff != 0))
return XLREAD_NEED_DATA;
@@ -421,7 +424,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
* verified that we got the whole header.
*/
prec = (XLogRecord *) (state->readBuf +
- state->ReadRecPtr % XLOG_BLCKSZ);
+ state->ReadRecPtr % state->pageSize);
total_len = prec->xl_tot_len;
/*
@@ -432,7 +435,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
* ensure that we enter the XLREAD_CONTINUATION state below;
* otherwise we might fail to apply ValidXLogRecordHeader at all.
*/
- if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
+ if (targetRecOff <= state->pageSize - SizeOfXLogRecord)
{
if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
state->PrevRecPtr, prec))
@@ -477,10 +480,10 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
* available
*/
targetPagePtr =
- state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
- targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+ state->ReadRecPtr - (state->ReadRecPtr % state->pageSize);
+ targetRecOff = state->ReadRecPtr % state->pageSize;
- request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ);
+ request_len = Min(targetRecOff + total_len, state->pageSize);
record_len = request_len - targetRecOff;
/* ReadRecPtr contains page header */
@@ -549,7 +552,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
/* Calculate pointer to beginning of next page */
state->recordContRecPtr = state->ReadRecPtr + record_len;
- Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0);
+ Assert(state->recordContRecPtr % state->pageSize == 0);
state->readRecordState = XLREAD_CONTINUATION;
}
@@ -576,7 +579,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
/* this request contains page header */
Assert (targetPagePtr != 0);
if (XLogNeedData(state, targetPagePtr,
- Min(state->recordRemainLen, XLOG_BLCKSZ),
+ Min(state->recordRemainLen, state->pageSize),
false))
return XLREAD_NEED_DATA;
@@ -628,7 +631,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
Assert(state->readLen >= pageHeaderSize);
contdata = (char *) state->readBuf + pageHeaderSize;
- record_len = XLOG_BLCKSZ - pageHeaderSize;
+ record_len = state->pageSize - pageHeaderSize;
if (pageHeader->xlp_rem_len < record_len)
record_len = pageHeader->xlp_rem_len;
@@ -655,7 +658,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
}
/* Calculate pointer to beginning of next page, and continue */
- state->recordContRecPtr += XLOG_BLCKSZ;
+ state->recordContRecPtr += state->pageSize;
}
/* targetPagePtr is pointing the last-read page here */
@@ -789,10 +792,10 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
XLogPageHeaderSize((XLogPageHeader) state->readBuf);
addLen = pageHeaderSize;
- if (reqLen + pageHeaderSize <= XLOG_BLCKSZ)
+ if (reqLen + pageHeaderSize <= state->pageSize)
addLen = pageHeaderSize;
else
- addLen = XLOG_BLCKSZ - reqLen;
+ addLen = state->pageSize - reqLen;
Assert(addLen >= 0);
}
@@ -805,7 +808,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
/* Data is not in our buffer, request the caller for it. */
XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize);
targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize);
- Assert((pageptr % XLOG_BLCKSZ) == 0);
+ Assert((pageptr % state->pageSize) == 0);
/*
* Every time we request to load new data of a page to the caller, even if
@@ -829,7 +832,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
* will not come back here, but will request the actual target page.
*/
state->readPagePtr = pageptr - targetPageOff;
- state->readLen = XLOG_BLCKSZ;
+ state->readLen = state->pageSize;
return true;
}
@@ -962,7 +965,7 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
int32 offset;
XLogPageHeader hdr = (XLogPageHeader) phdr;
- Assert((recptr % XLOG_BLCKSZ) == 0);
+ Assert((recptr % state->pageSize) == 0);
XLByteToSeg(recptr, segno, state->segcxt.ws_segsize);
offset = XLogSegmentOffset(recptr, state->segcxt.ws_segsize);
@@ -1016,10 +1019,10 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
"WAL file is from different database system: incorrect segment size in page header");
return false;
}
- else if (longhdr->xlp_xlog_blcksz != XLOG_BLCKSZ)
+ else if (longhdr->xlp_xlog_blcksz != state->pageSize)
{
report_invalid_record(state,
- "WAL file is from different database system: incorrect XLOG_BLCKSZ in page header");
+ "WAL file is from different database system: incorrect block size in page header");
return false;
}
}
@@ -1140,7 +1143,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
* XLogNeedData() is prepared to handle that and will read at
* least short page-header worth of data
*/
- targetRecOff = tmpRecPtr % XLOG_BLCKSZ;
+ targetRecOff = tmpRecPtr % state->pageSize;
/* scroll back to page boundary */
targetPagePtr = tmpRecPtr - targetRecOff;
@@ -1174,8 +1177,8 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
*
* Note that record headers are MAXALIGN'ed
*/
- if (MAXALIGN(header->xlp_rem_len) >= (XLOG_BLCKSZ - pageHeaderSize))
- tmpRecPtr = targetPagePtr + XLOG_BLCKSZ;
+ if (MAXALIGN(header->xlp_rem_len) >= (state->pageSize - pageHeaderSize))
+ tmpRecPtr = targetPagePtr + state->pageSize;
else
{
/*
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index 9cbd8afa49..f2dd495310 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -199,11 +199,14 @@ StartupDecodingContext(List *output_plugin_options,
ctx->slot = slot;
- ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, cleanup_cb);
+ ctx->reader =
+ XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, cleanup_cb);
+
if (!ctx->reader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
errmsg("out of memory")));
+ ctx->reader->readBuf = palloc(XLOG_BLCKSZ);
ctx->page_read = page_read;
ctx->reorder = ReorderBufferAllocate();
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index 062093727d..8e95600ef7 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -579,8 +579,8 @@ StartReplication(StartReplicationCmd *cmd)
errmsg("IDENTIFY_SYSTEM has not been run before START_REPLICATION")));
/* create xlogreader for physical replication */
- xlogreader =
- XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+ xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+ NULL, wal_segment_close);
if (!xlogreader)
ereport(ERROR,
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index 712c85281c..8a6209b061 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -61,7 +61,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
XLogReaderState *xlogreader;
char *errormsg;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+ xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
@@ -121,7 +121,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex,
char *errormsg;
XLogRecPtr endptr;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+ xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
@@ -181,7 +181,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
forkptr += SizeOfXLogShortPHD;
}
- xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+ xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index f514de6b13..b635537659 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -1041,7 +1041,7 @@ main(int argc, char **argv)
/* we have everything we need, start reading */
xlogreader_state =
- XLogReaderAllocate(WalSegSz, waldir, WALDumpCloseSegment);
+ XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, waldir, WALDumpCloseSegment);
if (!xlogreader_state)
fatal_error("out of memory");
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index b9dad7ee51..1853516333 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -151,6 +151,7 @@ struct XLogReaderState
* ----------------------------------------
*/
/* variables to communicate with page reader */
+ int pageSize; /* size of a page */
XLogRecPtr readPagePtr; /* page pointer to read */
int32 readLen; /* bytes requested to reader, or actual bytes
* read by reader, which must be larger than
@@ -243,7 +244,7 @@ struct XLogReaderState
/* Get a new XLogReader */
extern XLogReaderState *XLogReaderAllocate(int wal_segment_size,
- const char *waldir,
+ int page_size, const char *waldir,
WALSegmentCleanupCB cleanup_cb);
/* Free an XLogReader */
--
2.27.0
----Next_Part(Thu_Mar__4_11_28_53_2021_014)----
^ permalink raw reply [nested|flat] 30+ messages in thread
* [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize.
@ 2019-09-05 12:29 Kyotaro Horiguchi <[email protected]>
0 siblings, 0 replies; 30+ messages in thread
From: Kyotaro Horiguchi @ 2019-09-05 12:29 UTC (permalink / raw)
XLog reader facility and its callers currently relies on the same
constant XLOG_BLCKSZ. This patch moves the responsibility to determine
the block size to the callers of the facility.
---
src/backend/access/transam/twophase.c | 3 +-
src/backend/access/transam/xlog.c | 9 ++--
src/backend/access/transam/xlogreader.c | 57 ++++++++++++-----------
src/backend/replication/logical/logical.c | 5 +-
src/backend/replication/walsender.c | 4 +-
src/bin/pg_rewind/parsexlog.c | 6 +--
src/bin/pg_waldump/pg_waldump.c | 2 +-
src/include/access/xlogreader.h | 3 +-
8 files changed, 50 insertions(+), 39 deletions(-)
diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index 7d5a96a5f5..9f9e7de253 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -1325,7 +1325,8 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
XLogReaderState *xlogreader;
char *errormsg;
- xlogreader = XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+ xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+ NULL, wal_segment_close);
if (!xlogreader)
ereport(ERROR,
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index b517ca85ad..0822ac70cb 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -1222,7 +1222,10 @@ XLogInsertRecord(XLogRecData *rdata,
appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len);
if (!debug_reader)
- debug_reader = XLogReaderAllocate(wal_segment_size, NULL, NULL);
+ {
+ debug_reader =
+ XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, NULL);
+ }
if (!debug_reader)
{
@@ -6531,8 +6534,8 @@ StartupXLOG(void)
OwnLatch(&XLogCtl->recoveryWakeupLatch);
/* Set up XLOG reader facility */
- xlogreader =
- XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+ xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+ NULL, wal_segment_close);
if (!xlogreader)
ereport(ERROR,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index ba3e4868f7..e5baf13086 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -72,7 +72,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...)
* Returns NULL if the xlogreader couldn't be allocated.
*/
XLogReaderState *
-XLogReaderAllocate(int wal_segment_size, const char *waldir,
+XLogReaderAllocate(int wal_segment_size, int page_size, const char *waldir,
WALSegmentCleanupCB cleanup_cb)
{
XLogReaderState *state;
@@ -87,6 +87,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
state->cleanup_cb = cleanup_cb;
state->max_block_id = -1;
+ state->pageSize = page_size;
/*
* Permanently allocate readBuf. We do it this way, rather than just
@@ -95,7 +96,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
* isn't guaranteed to have any particular alignment, whereas
* palloc_extended() will provide MAXALIGN'd storage.
*/
- state->readBuf = (char *) palloc_extended(XLOG_BLCKSZ,
+ state->readBuf = (char *) palloc_extended(state->pageSize,
MCXT_ALLOC_NO_OOM);
if (!state->readBuf)
{
@@ -163,7 +164,7 @@ XLogReaderFree(XLogReaderState *state)
* readRecordBufSize is set to the new buffer size.
*
* To avoid useless small increases, round its size to a multiple of
- * XLOG_BLCKSZ, and make sure it's at least 5*Max(BLCKSZ, XLOG_BLCKSZ) to start
+ * page size, and make sure it's at least 5*Max(BLCKSZ, <page size>) to start
* with. (That is enough for all "normal" records, but very large commit or
* abort records might need more space.)
*/
@@ -172,8 +173,8 @@ allocate_recordbuf(XLogReaderState *state, uint32 reclength)
{
uint32 newSize = reclength;
- newSize += XLOG_BLCKSZ - (newSize % XLOG_BLCKSZ);
- newSize = Max(newSize, 5 * Max(BLCKSZ, XLOG_BLCKSZ));
+ newSize += state->pageSize - (newSize % state->pageSize);
+ newSize = Max(newSize, 5 * Max(BLCKSZ, state->pageSize));
#ifndef FRONTEND
@@ -238,6 +239,8 @@ void
XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
{
Assert(!XLogRecPtrIsInvalid(RecPtr));
+ Assert(state->readBuf != NULL &&
+ state->readBuf == (void *) MAXALIGN(state->readBuf));
ResetDecoder(state);
@@ -366,15 +369,15 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
XLogPageHeader pageHeader;
targetPagePtr =
- state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
- targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+ state->ReadRecPtr - (state->ReadRecPtr % state->pageSize);
+ targetRecOff = state->ReadRecPtr % state->pageSize;
/*
* Check if we have enough data. For the first record in the page,
* the requesting length doesn't contain page header.
*/
if (XLogNeedData(state, targetPagePtr,
- Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+ Min(targetRecOff + SizeOfXLogRecord, state->pageSize),
targetRecOff != 0))
return XLREAD_NEED_DATA;
@@ -421,7 +424,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
* verified that we got the whole header.
*/
prec = (XLogRecord *) (state->readBuf +
- state->ReadRecPtr % XLOG_BLCKSZ);
+ state->ReadRecPtr % state->pageSize);
total_len = prec->xl_tot_len;
/*
@@ -432,7 +435,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
* ensure that we enter the XLREAD_CONTINUATION state below;
* otherwise we might fail to apply ValidXLogRecordHeader at all.
*/
- if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
+ if (targetRecOff <= state->pageSize - SizeOfXLogRecord)
{
if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
state->PrevRecPtr, prec))
@@ -477,10 +480,10 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
* available
*/
targetPagePtr =
- state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
- targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+ state->ReadRecPtr - (state->ReadRecPtr % state->pageSize);
+ targetRecOff = state->ReadRecPtr % state->pageSize;
- request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ);
+ request_len = Min(targetRecOff + total_len, state->pageSize);
record_len = request_len - targetRecOff;
/* ReadRecPtr contains page header */
@@ -549,7 +552,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
/* Calculate pointer to beginning of next page */
state->recordContRecPtr = state->ReadRecPtr + record_len;
- Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0);
+ Assert(state->recordContRecPtr % state->pageSize == 0);
state->readRecordState = XLREAD_CONTINUATION;
}
@@ -576,7 +579,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
/* this request contains page header */
Assert (targetPagePtr != 0);
if (XLogNeedData(state, targetPagePtr,
- Min(state->recordRemainLen, XLOG_BLCKSZ),
+ Min(state->recordRemainLen, state->pageSize),
false))
return XLREAD_NEED_DATA;
@@ -628,7 +631,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
Assert(state->readLen >= pageHeaderSize);
contdata = (char *) state->readBuf + pageHeaderSize;
- record_len = XLOG_BLCKSZ - pageHeaderSize;
+ record_len = state->pageSize - pageHeaderSize;
if (pageHeader->xlp_rem_len < record_len)
record_len = pageHeader->xlp_rem_len;
@@ -655,7 +658,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
}
/* Calculate pointer to beginning of next page, and continue */
- state->recordContRecPtr += XLOG_BLCKSZ;
+ state->recordContRecPtr += state->pageSize;
}
/* targetPagePtr is pointing the last-read page here */
@@ -789,10 +792,10 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
XLogPageHeaderSize((XLogPageHeader) state->readBuf);
addLen = pageHeaderSize;
- if (reqLen + pageHeaderSize <= XLOG_BLCKSZ)
+ if (reqLen + pageHeaderSize <= state->pageSize)
addLen = pageHeaderSize;
else
- addLen = XLOG_BLCKSZ - reqLen;
+ addLen = state->pageSize - reqLen;
Assert(addLen >= 0);
}
@@ -805,7 +808,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
/* Data is not in our buffer, request the caller for it. */
XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize);
targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize);
- Assert((pageptr % XLOG_BLCKSZ) == 0);
+ Assert((pageptr % state->pageSize) == 0);
/*
* Every time we request to load new data of a page to the caller, even if
@@ -829,7 +832,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
* will not come back here, but will request the actual target page.
*/
state->readPagePtr = pageptr - targetPageOff;
- state->readLen = XLOG_BLCKSZ;
+ state->readLen = state->pageSize;
return true;
}
@@ -962,7 +965,7 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
int32 offset;
XLogPageHeader hdr = (XLogPageHeader) phdr;
- Assert((recptr % XLOG_BLCKSZ) == 0);
+ Assert((recptr % state->pageSize) == 0);
XLByteToSeg(recptr, segno, state->segcxt.ws_segsize);
offset = XLogSegmentOffset(recptr, state->segcxt.ws_segsize);
@@ -1016,10 +1019,10 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
"WAL file is from different database system: incorrect segment size in page header");
return false;
}
- else if (longhdr->xlp_xlog_blcksz != XLOG_BLCKSZ)
+ else if (longhdr->xlp_xlog_blcksz != state->pageSize)
{
report_invalid_record(state,
- "WAL file is from different database system: incorrect XLOG_BLCKSZ in page header");
+ "WAL file is from different database system: incorrect block size in page header");
return false;
}
}
@@ -1140,7 +1143,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
* XLogNeedData() is prepared to handle that and will read at
* least short page-header worth of data
*/
- targetRecOff = tmpRecPtr % XLOG_BLCKSZ;
+ targetRecOff = tmpRecPtr % state->pageSize;
/* scroll back to page boundary */
targetPagePtr = tmpRecPtr - targetRecOff;
@@ -1174,8 +1177,8 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
*
* Note that record headers are MAXALIGN'ed
*/
- if (MAXALIGN(header->xlp_rem_len) >= (XLOG_BLCKSZ - pageHeaderSize))
- tmpRecPtr = targetPagePtr + XLOG_BLCKSZ;
+ if (MAXALIGN(header->xlp_rem_len) >= (state->pageSize - pageHeaderSize))
+ tmpRecPtr = targetPagePtr + state->pageSize;
else
{
/*
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index 9cbd8afa49..f2dd495310 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -199,11 +199,14 @@ StartupDecodingContext(List *output_plugin_options,
ctx->slot = slot;
- ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, cleanup_cb);
+ ctx->reader =
+ XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, cleanup_cb);
+
if (!ctx->reader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
errmsg("out of memory")));
+ ctx->reader->readBuf = palloc(XLOG_BLCKSZ);
ctx->page_read = page_read;
ctx->reorder = ReorderBufferAllocate();
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index 062093727d..8e95600ef7 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -579,8 +579,8 @@ StartReplication(StartReplicationCmd *cmd)
errmsg("IDENTIFY_SYSTEM has not been run before START_REPLICATION")));
/* create xlogreader for physical replication */
- xlogreader =
- XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+ xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+ NULL, wal_segment_close);
if (!xlogreader)
ereport(ERROR,
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index 712c85281c..8a6209b061 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -61,7 +61,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
XLogReaderState *xlogreader;
char *errormsg;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+ xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
@@ -121,7 +121,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex,
char *errormsg;
XLogRecPtr endptr;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+ xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
@@ -181,7 +181,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
forkptr += SizeOfXLogShortPHD;
}
- xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+ xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index f514de6b13..b635537659 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -1041,7 +1041,7 @@ main(int argc, char **argv)
/* we have everything we need, start reading */
xlogreader_state =
- XLogReaderAllocate(WalSegSz, waldir, WALDumpCloseSegment);
+ XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, waldir, WALDumpCloseSegment);
if (!xlogreader_state)
fatal_error("out of memory");
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index b9dad7ee51..1853516333 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -151,6 +151,7 @@ struct XLogReaderState
* ----------------------------------------
*/
/* variables to communicate with page reader */
+ int pageSize; /* size of a page */
XLogRecPtr readPagePtr; /* page pointer to read */
int32 readLen; /* bytes requested to reader, or actual bytes
* read by reader, which must be larger than
@@ -243,7 +244,7 @@ struct XLogReaderState
/* Get a new XLogReader */
extern XLogReaderState *XLogReaderAllocate(int wal_segment_size,
- const char *waldir,
+ int page_size, const char *waldir,
WALSegmentCleanupCB cleanup_cb);
/* Free an XLogReader */
--
2.27.0
----Next_Part(Thu_Mar__4_11_28_53_2021_014)----
^ permalink raw reply [nested|flat] 30+ messages in thread
* [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize.
@ 2019-09-05 12:29 Kyotaro Horiguchi <[email protected]>
0 siblings, 0 replies; 30+ messages in thread
From: Kyotaro Horiguchi @ 2019-09-05 12:29 UTC (permalink / raw)
XLog reader facility and its callers currently relies on the same
constant XLOG_BLCKSZ. This patch moves the responsibility to determine
the block size to the callers of the facility.
---
src/backend/access/transam/twophase.c | 3 +-
src/backend/access/transam/xlog.c | 9 ++--
src/backend/access/transam/xlogreader.c | 57 ++++++++++++-----------
src/backend/replication/logical/logical.c | 5 +-
src/backend/replication/walsender.c | 4 +-
src/bin/pg_rewind/parsexlog.c | 6 +--
src/bin/pg_waldump/pg_waldump.c | 2 +-
src/include/access/xlogreader.h | 3 +-
8 files changed, 50 insertions(+), 39 deletions(-)
diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index 7d5a96a5f5..9f9e7de253 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -1325,7 +1325,8 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
XLogReaderState *xlogreader;
char *errormsg;
- xlogreader = XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+ xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+ NULL, wal_segment_close);
if (!xlogreader)
ereport(ERROR,
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index b517ca85ad..0822ac70cb 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -1222,7 +1222,10 @@ XLogInsertRecord(XLogRecData *rdata,
appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len);
if (!debug_reader)
- debug_reader = XLogReaderAllocate(wal_segment_size, NULL, NULL);
+ {
+ debug_reader =
+ XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, NULL);
+ }
if (!debug_reader)
{
@@ -6531,8 +6534,8 @@ StartupXLOG(void)
OwnLatch(&XLogCtl->recoveryWakeupLatch);
/* Set up XLOG reader facility */
- xlogreader =
- XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+ xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+ NULL, wal_segment_close);
if (!xlogreader)
ereport(ERROR,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index ba3e4868f7..e5baf13086 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -72,7 +72,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...)
* Returns NULL if the xlogreader couldn't be allocated.
*/
XLogReaderState *
-XLogReaderAllocate(int wal_segment_size, const char *waldir,
+XLogReaderAllocate(int wal_segment_size, int page_size, const char *waldir,
WALSegmentCleanupCB cleanup_cb)
{
XLogReaderState *state;
@@ -87,6 +87,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
state->cleanup_cb = cleanup_cb;
state->max_block_id = -1;
+ state->pageSize = page_size;
/*
* Permanently allocate readBuf. We do it this way, rather than just
@@ -95,7 +96,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
* isn't guaranteed to have any particular alignment, whereas
* palloc_extended() will provide MAXALIGN'd storage.
*/
- state->readBuf = (char *) palloc_extended(XLOG_BLCKSZ,
+ state->readBuf = (char *) palloc_extended(state->pageSize,
MCXT_ALLOC_NO_OOM);
if (!state->readBuf)
{
@@ -163,7 +164,7 @@ XLogReaderFree(XLogReaderState *state)
* readRecordBufSize is set to the new buffer size.
*
* To avoid useless small increases, round its size to a multiple of
- * XLOG_BLCKSZ, and make sure it's at least 5*Max(BLCKSZ, XLOG_BLCKSZ) to start
+ * page size, and make sure it's at least 5*Max(BLCKSZ, <page size>) to start
* with. (That is enough for all "normal" records, but very large commit or
* abort records might need more space.)
*/
@@ -172,8 +173,8 @@ allocate_recordbuf(XLogReaderState *state, uint32 reclength)
{
uint32 newSize = reclength;
- newSize += XLOG_BLCKSZ - (newSize % XLOG_BLCKSZ);
- newSize = Max(newSize, 5 * Max(BLCKSZ, XLOG_BLCKSZ));
+ newSize += state->pageSize - (newSize % state->pageSize);
+ newSize = Max(newSize, 5 * Max(BLCKSZ, state->pageSize));
#ifndef FRONTEND
@@ -238,6 +239,8 @@ void
XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
{
Assert(!XLogRecPtrIsInvalid(RecPtr));
+ Assert(state->readBuf != NULL &&
+ state->readBuf == (void *) MAXALIGN(state->readBuf));
ResetDecoder(state);
@@ -366,15 +369,15 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
XLogPageHeader pageHeader;
targetPagePtr =
- state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
- targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+ state->ReadRecPtr - (state->ReadRecPtr % state->pageSize);
+ targetRecOff = state->ReadRecPtr % state->pageSize;
/*
* Check if we have enough data. For the first record in the page,
* the requesting length doesn't contain page header.
*/
if (XLogNeedData(state, targetPagePtr,
- Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+ Min(targetRecOff + SizeOfXLogRecord, state->pageSize),
targetRecOff != 0))
return XLREAD_NEED_DATA;
@@ -421,7 +424,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
* verified that we got the whole header.
*/
prec = (XLogRecord *) (state->readBuf +
- state->ReadRecPtr % XLOG_BLCKSZ);
+ state->ReadRecPtr % state->pageSize);
total_len = prec->xl_tot_len;
/*
@@ -432,7 +435,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
* ensure that we enter the XLREAD_CONTINUATION state below;
* otherwise we might fail to apply ValidXLogRecordHeader at all.
*/
- if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
+ if (targetRecOff <= state->pageSize - SizeOfXLogRecord)
{
if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
state->PrevRecPtr, prec))
@@ -477,10 +480,10 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
* available
*/
targetPagePtr =
- state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
- targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+ state->ReadRecPtr - (state->ReadRecPtr % state->pageSize);
+ targetRecOff = state->ReadRecPtr % state->pageSize;
- request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ);
+ request_len = Min(targetRecOff + total_len, state->pageSize);
record_len = request_len - targetRecOff;
/* ReadRecPtr contains page header */
@@ -549,7 +552,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
/* Calculate pointer to beginning of next page */
state->recordContRecPtr = state->ReadRecPtr + record_len;
- Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0);
+ Assert(state->recordContRecPtr % state->pageSize == 0);
state->readRecordState = XLREAD_CONTINUATION;
}
@@ -576,7 +579,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
/* this request contains page header */
Assert (targetPagePtr != 0);
if (XLogNeedData(state, targetPagePtr,
- Min(state->recordRemainLen, XLOG_BLCKSZ),
+ Min(state->recordRemainLen, state->pageSize),
false))
return XLREAD_NEED_DATA;
@@ -628,7 +631,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
Assert(state->readLen >= pageHeaderSize);
contdata = (char *) state->readBuf + pageHeaderSize;
- record_len = XLOG_BLCKSZ - pageHeaderSize;
+ record_len = state->pageSize - pageHeaderSize;
if (pageHeader->xlp_rem_len < record_len)
record_len = pageHeader->xlp_rem_len;
@@ -655,7 +658,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
}
/* Calculate pointer to beginning of next page, and continue */
- state->recordContRecPtr += XLOG_BLCKSZ;
+ state->recordContRecPtr += state->pageSize;
}
/* targetPagePtr is pointing the last-read page here */
@@ -789,10 +792,10 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
XLogPageHeaderSize((XLogPageHeader) state->readBuf);
addLen = pageHeaderSize;
- if (reqLen + pageHeaderSize <= XLOG_BLCKSZ)
+ if (reqLen + pageHeaderSize <= state->pageSize)
addLen = pageHeaderSize;
else
- addLen = XLOG_BLCKSZ - reqLen;
+ addLen = state->pageSize - reqLen;
Assert(addLen >= 0);
}
@@ -805,7 +808,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
/* Data is not in our buffer, request the caller for it. */
XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize);
targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize);
- Assert((pageptr % XLOG_BLCKSZ) == 0);
+ Assert((pageptr % state->pageSize) == 0);
/*
* Every time we request to load new data of a page to the caller, even if
@@ -829,7 +832,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
* will not come back here, but will request the actual target page.
*/
state->readPagePtr = pageptr - targetPageOff;
- state->readLen = XLOG_BLCKSZ;
+ state->readLen = state->pageSize;
return true;
}
@@ -962,7 +965,7 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
int32 offset;
XLogPageHeader hdr = (XLogPageHeader) phdr;
- Assert((recptr % XLOG_BLCKSZ) == 0);
+ Assert((recptr % state->pageSize) == 0);
XLByteToSeg(recptr, segno, state->segcxt.ws_segsize);
offset = XLogSegmentOffset(recptr, state->segcxt.ws_segsize);
@@ -1016,10 +1019,10 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
"WAL file is from different database system: incorrect segment size in page header");
return false;
}
- else if (longhdr->xlp_xlog_blcksz != XLOG_BLCKSZ)
+ else if (longhdr->xlp_xlog_blcksz != state->pageSize)
{
report_invalid_record(state,
- "WAL file is from different database system: incorrect XLOG_BLCKSZ in page header");
+ "WAL file is from different database system: incorrect block size in page header");
return false;
}
}
@@ -1140,7 +1143,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
* XLogNeedData() is prepared to handle that and will read at
* least short page-header worth of data
*/
- targetRecOff = tmpRecPtr % XLOG_BLCKSZ;
+ targetRecOff = tmpRecPtr % state->pageSize;
/* scroll back to page boundary */
targetPagePtr = tmpRecPtr - targetRecOff;
@@ -1174,8 +1177,8 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
*
* Note that record headers are MAXALIGN'ed
*/
- if (MAXALIGN(header->xlp_rem_len) >= (XLOG_BLCKSZ - pageHeaderSize))
- tmpRecPtr = targetPagePtr + XLOG_BLCKSZ;
+ if (MAXALIGN(header->xlp_rem_len) >= (state->pageSize - pageHeaderSize))
+ tmpRecPtr = targetPagePtr + state->pageSize;
else
{
/*
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index 9cbd8afa49..f2dd495310 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -199,11 +199,14 @@ StartupDecodingContext(List *output_plugin_options,
ctx->slot = slot;
- ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, cleanup_cb);
+ ctx->reader =
+ XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, cleanup_cb);
+
if (!ctx->reader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
errmsg("out of memory")));
+ ctx->reader->readBuf = palloc(XLOG_BLCKSZ);
ctx->page_read = page_read;
ctx->reorder = ReorderBufferAllocate();
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index 062093727d..8e95600ef7 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -579,8 +579,8 @@ StartReplication(StartReplicationCmd *cmd)
errmsg("IDENTIFY_SYSTEM has not been run before START_REPLICATION")));
/* create xlogreader for physical replication */
- xlogreader =
- XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+ xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+ NULL, wal_segment_close);
if (!xlogreader)
ereport(ERROR,
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index 712c85281c..8a6209b061 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -61,7 +61,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
XLogReaderState *xlogreader;
char *errormsg;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+ xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
@@ -121,7 +121,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex,
char *errormsg;
XLogRecPtr endptr;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+ xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
@@ -181,7 +181,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
forkptr += SizeOfXLogShortPHD;
}
- xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+ xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index f514de6b13..b635537659 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -1041,7 +1041,7 @@ main(int argc, char **argv)
/* we have everything we need, start reading */
xlogreader_state =
- XLogReaderAllocate(WalSegSz, waldir, WALDumpCloseSegment);
+ XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, waldir, WALDumpCloseSegment);
if (!xlogreader_state)
fatal_error("out of memory");
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index b9dad7ee51..1853516333 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -151,6 +151,7 @@ struct XLogReaderState
* ----------------------------------------
*/
/* variables to communicate with page reader */
+ int pageSize; /* size of a page */
XLogRecPtr readPagePtr; /* page pointer to read */
int32 readLen; /* bytes requested to reader, or actual bytes
* read by reader, which must be larger than
@@ -243,7 +244,7 @@ struct XLogReaderState
/* Get a new XLogReader */
extern XLogReaderState *XLogReaderAllocate(int wal_segment_size,
- const char *waldir,
+ int page_size, const char *waldir,
WALSegmentCleanupCB cleanup_cb);
/* Free an XLogReader */
--
2.27.0
----Next_Part(Thu_Mar__4_11_28_53_2021_014)----
^ permalink raw reply [nested|flat] 30+ messages in thread
* [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize.
@ 2019-09-05 12:29 Kyotaro Horiguchi <[email protected]>
0 siblings, 0 replies; 30+ messages in thread
From: Kyotaro Horiguchi @ 2019-09-05 12:29 UTC (permalink / raw)
XLog reader facility and its callers currently relies on the same
constant XLOG_BLCKSZ. This patch moves the responsibility to determine
the block size to the callers of the facility.
---
src/backend/access/transam/twophase.c | 3 +-
src/backend/access/transam/xlog.c | 9 ++--
src/backend/access/transam/xlogreader.c | 57 ++++++++++++-----------
src/backend/replication/logical/logical.c | 5 +-
src/backend/replication/walsender.c | 4 +-
src/bin/pg_rewind/parsexlog.c | 6 +--
src/bin/pg_waldump/pg_waldump.c | 2 +-
src/include/access/xlogreader.h | 3 +-
8 files changed, 50 insertions(+), 39 deletions(-)
diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index 7d5a96a5f5..9f9e7de253 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -1325,7 +1325,8 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
XLogReaderState *xlogreader;
char *errormsg;
- xlogreader = XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+ xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+ NULL, wal_segment_close);
if (!xlogreader)
ereport(ERROR,
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index b517ca85ad..0822ac70cb 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -1222,7 +1222,10 @@ XLogInsertRecord(XLogRecData *rdata,
appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len);
if (!debug_reader)
- debug_reader = XLogReaderAllocate(wal_segment_size, NULL, NULL);
+ {
+ debug_reader =
+ XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, NULL);
+ }
if (!debug_reader)
{
@@ -6531,8 +6534,8 @@ StartupXLOG(void)
OwnLatch(&XLogCtl->recoveryWakeupLatch);
/* Set up XLOG reader facility */
- xlogreader =
- XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+ xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+ NULL, wal_segment_close);
if (!xlogreader)
ereport(ERROR,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index ba3e4868f7..e5baf13086 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -72,7 +72,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...)
* Returns NULL if the xlogreader couldn't be allocated.
*/
XLogReaderState *
-XLogReaderAllocate(int wal_segment_size, const char *waldir,
+XLogReaderAllocate(int wal_segment_size, int page_size, const char *waldir,
WALSegmentCleanupCB cleanup_cb)
{
XLogReaderState *state;
@@ -87,6 +87,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
state->cleanup_cb = cleanup_cb;
state->max_block_id = -1;
+ state->pageSize = page_size;
/*
* Permanently allocate readBuf. We do it this way, rather than just
@@ -95,7 +96,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
* isn't guaranteed to have any particular alignment, whereas
* palloc_extended() will provide MAXALIGN'd storage.
*/
- state->readBuf = (char *) palloc_extended(XLOG_BLCKSZ,
+ state->readBuf = (char *) palloc_extended(state->pageSize,
MCXT_ALLOC_NO_OOM);
if (!state->readBuf)
{
@@ -163,7 +164,7 @@ XLogReaderFree(XLogReaderState *state)
* readRecordBufSize is set to the new buffer size.
*
* To avoid useless small increases, round its size to a multiple of
- * XLOG_BLCKSZ, and make sure it's at least 5*Max(BLCKSZ, XLOG_BLCKSZ) to start
+ * page size, and make sure it's at least 5*Max(BLCKSZ, <page size>) to start
* with. (That is enough for all "normal" records, but very large commit or
* abort records might need more space.)
*/
@@ -172,8 +173,8 @@ allocate_recordbuf(XLogReaderState *state, uint32 reclength)
{
uint32 newSize = reclength;
- newSize += XLOG_BLCKSZ - (newSize % XLOG_BLCKSZ);
- newSize = Max(newSize, 5 * Max(BLCKSZ, XLOG_BLCKSZ));
+ newSize += state->pageSize - (newSize % state->pageSize);
+ newSize = Max(newSize, 5 * Max(BLCKSZ, state->pageSize));
#ifndef FRONTEND
@@ -238,6 +239,8 @@ void
XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
{
Assert(!XLogRecPtrIsInvalid(RecPtr));
+ Assert(state->readBuf != NULL &&
+ state->readBuf == (void *) MAXALIGN(state->readBuf));
ResetDecoder(state);
@@ -366,15 +369,15 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
XLogPageHeader pageHeader;
targetPagePtr =
- state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
- targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+ state->ReadRecPtr - (state->ReadRecPtr % state->pageSize);
+ targetRecOff = state->ReadRecPtr % state->pageSize;
/*
* Check if we have enough data. For the first record in the page,
* the requesting length doesn't contain page header.
*/
if (XLogNeedData(state, targetPagePtr,
- Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+ Min(targetRecOff + SizeOfXLogRecord, state->pageSize),
targetRecOff != 0))
return XLREAD_NEED_DATA;
@@ -421,7 +424,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
* verified that we got the whole header.
*/
prec = (XLogRecord *) (state->readBuf +
- state->ReadRecPtr % XLOG_BLCKSZ);
+ state->ReadRecPtr % state->pageSize);
total_len = prec->xl_tot_len;
/*
@@ -432,7 +435,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
* ensure that we enter the XLREAD_CONTINUATION state below;
* otherwise we might fail to apply ValidXLogRecordHeader at all.
*/
- if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
+ if (targetRecOff <= state->pageSize - SizeOfXLogRecord)
{
if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
state->PrevRecPtr, prec))
@@ -477,10 +480,10 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
* available
*/
targetPagePtr =
- state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
- targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+ state->ReadRecPtr - (state->ReadRecPtr % state->pageSize);
+ targetRecOff = state->ReadRecPtr % state->pageSize;
- request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ);
+ request_len = Min(targetRecOff + total_len, state->pageSize);
record_len = request_len - targetRecOff;
/* ReadRecPtr contains page header */
@@ -549,7 +552,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
/* Calculate pointer to beginning of next page */
state->recordContRecPtr = state->ReadRecPtr + record_len;
- Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0);
+ Assert(state->recordContRecPtr % state->pageSize == 0);
state->readRecordState = XLREAD_CONTINUATION;
}
@@ -576,7 +579,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
/* this request contains page header */
Assert (targetPagePtr != 0);
if (XLogNeedData(state, targetPagePtr,
- Min(state->recordRemainLen, XLOG_BLCKSZ),
+ Min(state->recordRemainLen, state->pageSize),
false))
return XLREAD_NEED_DATA;
@@ -628,7 +631,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
Assert(state->readLen >= pageHeaderSize);
contdata = (char *) state->readBuf + pageHeaderSize;
- record_len = XLOG_BLCKSZ - pageHeaderSize;
+ record_len = state->pageSize - pageHeaderSize;
if (pageHeader->xlp_rem_len < record_len)
record_len = pageHeader->xlp_rem_len;
@@ -655,7 +658,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
}
/* Calculate pointer to beginning of next page, and continue */
- state->recordContRecPtr += XLOG_BLCKSZ;
+ state->recordContRecPtr += state->pageSize;
}
/* targetPagePtr is pointing the last-read page here */
@@ -789,10 +792,10 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
XLogPageHeaderSize((XLogPageHeader) state->readBuf);
addLen = pageHeaderSize;
- if (reqLen + pageHeaderSize <= XLOG_BLCKSZ)
+ if (reqLen + pageHeaderSize <= state->pageSize)
addLen = pageHeaderSize;
else
- addLen = XLOG_BLCKSZ - reqLen;
+ addLen = state->pageSize - reqLen;
Assert(addLen >= 0);
}
@@ -805,7 +808,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
/* Data is not in our buffer, request the caller for it. */
XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize);
targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize);
- Assert((pageptr % XLOG_BLCKSZ) == 0);
+ Assert((pageptr % state->pageSize) == 0);
/*
* Every time we request to load new data of a page to the caller, even if
@@ -829,7 +832,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
* will not come back here, but will request the actual target page.
*/
state->readPagePtr = pageptr - targetPageOff;
- state->readLen = XLOG_BLCKSZ;
+ state->readLen = state->pageSize;
return true;
}
@@ -962,7 +965,7 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
int32 offset;
XLogPageHeader hdr = (XLogPageHeader) phdr;
- Assert((recptr % XLOG_BLCKSZ) == 0);
+ Assert((recptr % state->pageSize) == 0);
XLByteToSeg(recptr, segno, state->segcxt.ws_segsize);
offset = XLogSegmentOffset(recptr, state->segcxt.ws_segsize);
@@ -1016,10 +1019,10 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
"WAL file is from different database system: incorrect segment size in page header");
return false;
}
- else if (longhdr->xlp_xlog_blcksz != XLOG_BLCKSZ)
+ else if (longhdr->xlp_xlog_blcksz != state->pageSize)
{
report_invalid_record(state,
- "WAL file is from different database system: incorrect XLOG_BLCKSZ in page header");
+ "WAL file is from different database system: incorrect block size in page header");
return false;
}
}
@@ -1140,7 +1143,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
* XLogNeedData() is prepared to handle that and will read at
* least short page-header worth of data
*/
- targetRecOff = tmpRecPtr % XLOG_BLCKSZ;
+ targetRecOff = tmpRecPtr % state->pageSize;
/* scroll back to page boundary */
targetPagePtr = tmpRecPtr - targetRecOff;
@@ -1174,8 +1177,8 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
*
* Note that record headers are MAXALIGN'ed
*/
- if (MAXALIGN(header->xlp_rem_len) >= (XLOG_BLCKSZ - pageHeaderSize))
- tmpRecPtr = targetPagePtr + XLOG_BLCKSZ;
+ if (MAXALIGN(header->xlp_rem_len) >= (state->pageSize - pageHeaderSize))
+ tmpRecPtr = targetPagePtr + state->pageSize;
else
{
/*
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index 9cbd8afa49..f2dd495310 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -199,11 +199,14 @@ StartupDecodingContext(List *output_plugin_options,
ctx->slot = slot;
- ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, cleanup_cb);
+ ctx->reader =
+ XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, cleanup_cb);
+
if (!ctx->reader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
errmsg("out of memory")));
+ ctx->reader->readBuf = palloc(XLOG_BLCKSZ);
ctx->page_read = page_read;
ctx->reorder = ReorderBufferAllocate();
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index 062093727d..8e95600ef7 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -579,8 +579,8 @@ StartReplication(StartReplicationCmd *cmd)
errmsg("IDENTIFY_SYSTEM has not been run before START_REPLICATION")));
/* create xlogreader for physical replication */
- xlogreader =
- XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+ xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+ NULL, wal_segment_close);
if (!xlogreader)
ereport(ERROR,
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index 712c85281c..8a6209b061 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -61,7 +61,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
XLogReaderState *xlogreader;
char *errormsg;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+ xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
@@ -121,7 +121,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex,
char *errormsg;
XLogRecPtr endptr;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+ xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
@@ -181,7 +181,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
forkptr += SizeOfXLogShortPHD;
}
- xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+ xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index f514de6b13..b635537659 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -1041,7 +1041,7 @@ main(int argc, char **argv)
/* we have everything we need, start reading */
xlogreader_state =
- XLogReaderAllocate(WalSegSz, waldir, WALDumpCloseSegment);
+ XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, waldir, WALDumpCloseSegment);
if (!xlogreader_state)
fatal_error("out of memory");
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index b9dad7ee51..1853516333 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -151,6 +151,7 @@ struct XLogReaderState
* ----------------------------------------
*/
/* variables to communicate with page reader */
+ int pageSize; /* size of a page */
XLogRecPtr readPagePtr; /* page pointer to read */
int32 readLen; /* bytes requested to reader, or actual bytes
* read by reader, which must be larger than
@@ -243,7 +244,7 @@ struct XLogReaderState
/* Get a new XLogReader */
extern XLogReaderState *XLogReaderAllocate(int wal_segment_size,
- const char *waldir,
+ int page_size, const char *waldir,
WALSegmentCleanupCB cleanup_cb);
/* Free an XLogReader */
--
2.27.0
----Next_Part(Thu_Mar__4_11_28_53_2021_014)----
^ permalink raw reply [nested|flat] 30+ messages in thread
* [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize.
@ 2019-09-05 12:29 Kyotaro Horiguchi <[email protected]>
0 siblings, 0 replies; 30+ messages in thread
From: Kyotaro Horiguchi @ 2019-09-05 12:29 UTC (permalink / raw)
XLog reader facility and its callers currently relies on the same
constant XLOG_BLCKSZ. This patch moves the responsibility to determine
the block size to the callers of the facility.
---
src/backend/access/transam/twophase.c | 3 +-
src/backend/access/transam/xlog.c | 9 ++--
src/backend/access/transam/xlogreader.c | 57 ++++++++++++-----------
src/backend/replication/logical/logical.c | 5 +-
src/backend/replication/walsender.c | 4 +-
src/bin/pg_rewind/parsexlog.c | 6 +--
src/bin/pg_waldump/pg_waldump.c | 2 +-
src/include/access/xlogreader.h | 3 +-
8 files changed, 50 insertions(+), 39 deletions(-)
diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index 7d5a96a5f5..9f9e7de253 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -1325,7 +1325,8 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
XLogReaderState *xlogreader;
char *errormsg;
- xlogreader = XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+ xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+ NULL, wal_segment_close);
if (!xlogreader)
ereport(ERROR,
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index b517ca85ad..0822ac70cb 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -1222,7 +1222,10 @@ XLogInsertRecord(XLogRecData *rdata,
appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len);
if (!debug_reader)
- debug_reader = XLogReaderAllocate(wal_segment_size, NULL, NULL);
+ {
+ debug_reader =
+ XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, NULL);
+ }
if (!debug_reader)
{
@@ -6531,8 +6534,8 @@ StartupXLOG(void)
OwnLatch(&XLogCtl->recoveryWakeupLatch);
/* Set up XLOG reader facility */
- xlogreader =
- XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+ xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+ NULL, wal_segment_close);
if (!xlogreader)
ereport(ERROR,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index ba3e4868f7..e5baf13086 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -72,7 +72,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...)
* Returns NULL if the xlogreader couldn't be allocated.
*/
XLogReaderState *
-XLogReaderAllocate(int wal_segment_size, const char *waldir,
+XLogReaderAllocate(int wal_segment_size, int page_size, const char *waldir,
WALSegmentCleanupCB cleanup_cb)
{
XLogReaderState *state;
@@ -87,6 +87,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
state->cleanup_cb = cleanup_cb;
state->max_block_id = -1;
+ state->pageSize = page_size;
/*
* Permanently allocate readBuf. We do it this way, rather than just
@@ -95,7 +96,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
* isn't guaranteed to have any particular alignment, whereas
* palloc_extended() will provide MAXALIGN'd storage.
*/
- state->readBuf = (char *) palloc_extended(XLOG_BLCKSZ,
+ state->readBuf = (char *) palloc_extended(state->pageSize,
MCXT_ALLOC_NO_OOM);
if (!state->readBuf)
{
@@ -163,7 +164,7 @@ XLogReaderFree(XLogReaderState *state)
* readRecordBufSize is set to the new buffer size.
*
* To avoid useless small increases, round its size to a multiple of
- * XLOG_BLCKSZ, and make sure it's at least 5*Max(BLCKSZ, XLOG_BLCKSZ) to start
+ * page size, and make sure it's at least 5*Max(BLCKSZ, <page size>) to start
* with. (That is enough for all "normal" records, but very large commit or
* abort records might need more space.)
*/
@@ -172,8 +173,8 @@ allocate_recordbuf(XLogReaderState *state, uint32 reclength)
{
uint32 newSize = reclength;
- newSize += XLOG_BLCKSZ - (newSize % XLOG_BLCKSZ);
- newSize = Max(newSize, 5 * Max(BLCKSZ, XLOG_BLCKSZ));
+ newSize += state->pageSize - (newSize % state->pageSize);
+ newSize = Max(newSize, 5 * Max(BLCKSZ, state->pageSize));
#ifndef FRONTEND
@@ -238,6 +239,8 @@ void
XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
{
Assert(!XLogRecPtrIsInvalid(RecPtr));
+ Assert(state->readBuf != NULL &&
+ state->readBuf == (void *) MAXALIGN(state->readBuf));
ResetDecoder(state);
@@ -366,15 +369,15 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
XLogPageHeader pageHeader;
targetPagePtr =
- state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
- targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+ state->ReadRecPtr - (state->ReadRecPtr % state->pageSize);
+ targetRecOff = state->ReadRecPtr % state->pageSize;
/*
* Check if we have enough data. For the first record in the page,
* the requesting length doesn't contain page header.
*/
if (XLogNeedData(state, targetPagePtr,
- Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+ Min(targetRecOff + SizeOfXLogRecord, state->pageSize),
targetRecOff != 0))
return XLREAD_NEED_DATA;
@@ -421,7 +424,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
* verified that we got the whole header.
*/
prec = (XLogRecord *) (state->readBuf +
- state->ReadRecPtr % XLOG_BLCKSZ);
+ state->ReadRecPtr % state->pageSize);
total_len = prec->xl_tot_len;
/*
@@ -432,7 +435,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
* ensure that we enter the XLREAD_CONTINUATION state below;
* otherwise we might fail to apply ValidXLogRecordHeader at all.
*/
- if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
+ if (targetRecOff <= state->pageSize - SizeOfXLogRecord)
{
if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
state->PrevRecPtr, prec))
@@ -477,10 +480,10 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
* available
*/
targetPagePtr =
- state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
- targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+ state->ReadRecPtr - (state->ReadRecPtr % state->pageSize);
+ targetRecOff = state->ReadRecPtr % state->pageSize;
- request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ);
+ request_len = Min(targetRecOff + total_len, state->pageSize);
record_len = request_len - targetRecOff;
/* ReadRecPtr contains page header */
@@ -549,7 +552,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
/* Calculate pointer to beginning of next page */
state->recordContRecPtr = state->ReadRecPtr + record_len;
- Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0);
+ Assert(state->recordContRecPtr % state->pageSize == 0);
state->readRecordState = XLREAD_CONTINUATION;
}
@@ -576,7 +579,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
/* this request contains page header */
Assert (targetPagePtr != 0);
if (XLogNeedData(state, targetPagePtr,
- Min(state->recordRemainLen, XLOG_BLCKSZ),
+ Min(state->recordRemainLen, state->pageSize),
false))
return XLREAD_NEED_DATA;
@@ -628,7 +631,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
Assert(state->readLen >= pageHeaderSize);
contdata = (char *) state->readBuf + pageHeaderSize;
- record_len = XLOG_BLCKSZ - pageHeaderSize;
+ record_len = state->pageSize - pageHeaderSize;
if (pageHeader->xlp_rem_len < record_len)
record_len = pageHeader->xlp_rem_len;
@@ -655,7 +658,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
}
/* Calculate pointer to beginning of next page, and continue */
- state->recordContRecPtr += XLOG_BLCKSZ;
+ state->recordContRecPtr += state->pageSize;
}
/* targetPagePtr is pointing the last-read page here */
@@ -789,10 +792,10 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
XLogPageHeaderSize((XLogPageHeader) state->readBuf);
addLen = pageHeaderSize;
- if (reqLen + pageHeaderSize <= XLOG_BLCKSZ)
+ if (reqLen + pageHeaderSize <= state->pageSize)
addLen = pageHeaderSize;
else
- addLen = XLOG_BLCKSZ - reqLen;
+ addLen = state->pageSize - reqLen;
Assert(addLen >= 0);
}
@@ -805,7 +808,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
/* Data is not in our buffer, request the caller for it. */
XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize);
targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize);
- Assert((pageptr % XLOG_BLCKSZ) == 0);
+ Assert((pageptr % state->pageSize) == 0);
/*
* Every time we request to load new data of a page to the caller, even if
@@ -829,7 +832,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
* will not come back here, but will request the actual target page.
*/
state->readPagePtr = pageptr - targetPageOff;
- state->readLen = XLOG_BLCKSZ;
+ state->readLen = state->pageSize;
return true;
}
@@ -962,7 +965,7 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
int32 offset;
XLogPageHeader hdr = (XLogPageHeader) phdr;
- Assert((recptr % XLOG_BLCKSZ) == 0);
+ Assert((recptr % state->pageSize) == 0);
XLByteToSeg(recptr, segno, state->segcxt.ws_segsize);
offset = XLogSegmentOffset(recptr, state->segcxt.ws_segsize);
@@ -1016,10 +1019,10 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
"WAL file is from different database system: incorrect segment size in page header");
return false;
}
- else if (longhdr->xlp_xlog_blcksz != XLOG_BLCKSZ)
+ else if (longhdr->xlp_xlog_blcksz != state->pageSize)
{
report_invalid_record(state,
- "WAL file is from different database system: incorrect XLOG_BLCKSZ in page header");
+ "WAL file is from different database system: incorrect block size in page header");
return false;
}
}
@@ -1140,7 +1143,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
* XLogNeedData() is prepared to handle that and will read at
* least short page-header worth of data
*/
- targetRecOff = tmpRecPtr % XLOG_BLCKSZ;
+ targetRecOff = tmpRecPtr % state->pageSize;
/* scroll back to page boundary */
targetPagePtr = tmpRecPtr - targetRecOff;
@@ -1174,8 +1177,8 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
*
* Note that record headers are MAXALIGN'ed
*/
- if (MAXALIGN(header->xlp_rem_len) >= (XLOG_BLCKSZ - pageHeaderSize))
- tmpRecPtr = targetPagePtr + XLOG_BLCKSZ;
+ if (MAXALIGN(header->xlp_rem_len) >= (state->pageSize - pageHeaderSize))
+ tmpRecPtr = targetPagePtr + state->pageSize;
else
{
/*
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index 9cbd8afa49..f2dd495310 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -199,11 +199,14 @@ StartupDecodingContext(List *output_plugin_options,
ctx->slot = slot;
- ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, cleanup_cb);
+ ctx->reader =
+ XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, cleanup_cb);
+
if (!ctx->reader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
errmsg("out of memory")));
+ ctx->reader->readBuf = palloc(XLOG_BLCKSZ);
ctx->page_read = page_read;
ctx->reorder = ReorderBufferAllocate();
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index 062093727d..8e95600ef7 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -579,8 +579,8 @@ StartReplication(StartReplicationCmd *cmd)
errmsg("IDENTIFY_SYSTEM has not been run before START_REPLICATION")));
/* create xlogreader for physical replication */
- xlogreader =
- XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+ xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+ NULL, wal_segment_close);
if (!xlogreader)
ereport(ERROR,
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index 712c85281c..8a6209b061 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -61,7 +61,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
XLogReaderState *xlogreader;
char *errormsg;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+ xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
@@ -121,7 +121,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex,
char *errormsg;
XLogRecPtr endptr;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+ xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
@@ -181,7 +181,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
forkptr += SizeOfXLogShortPHD;
}
- xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+ xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index f514de6b13..b635537659 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -1041,7 +1041,7 @@ main(int argc, char **argv)
/* we have everything we need, start reading */
xlogreader_state =
- XLogReaderAllocate(WalSegSz, waldir, WALDumpCloseSegment);
+ XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, waldir, WALDumpCloseSegment);
if (!xlogreader_state)
fatal_error("out of memory");
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index b9dad7ee51..1853516333 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -151,6 +151,7 @@ struct XLogReaderState
* ----------------------------------------
*/
/* variables to communicate with page reader */
+ int pageSize; /* size of a page */
XLogRecPtr readPagePtr; /* page pointer to read */
int32 readLen; /* bytes requested to reader, or actual bytes
* read by reader, which must be larger than
@@ -243,7 +244,7 @@ struct XLogReaderState
/* Get a new XLogReader */
extern XLogReaderState *XLogReaderAllocate(int wal_segment_size,
- const char *waldir,
+ int page_size, const char *waldir,
WALSegmentCleanupCB cleanup_cb);
/* Free an XLogReader */
--
2.27.0
----Next_Part(Thu_Mar__4_11_28_53_2021_014)----
^ permalink raw reply [nested|flat] 30+ messages in thread
* [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize.
@ 2019-09-05 12:29 Kyotaro Horiguchi <[email protected]>
0 siblings, 0 replies; 30+ messages in thread
From: Kyotaro Horiguchi @ 2019-09-05 12:29 UTC (permalink / raw)
XLog reader facility and its callers currently relies on the same
constant XLOG_BLCKSZ. This patch moves the responsibility to determine
the block size to the callers of the facility.
---
src/backend/access/transam/twophase.c | 3 +-
src/backend/access/transam/xlog.c | 9 ++--
src/backend/access/transam/xlogreader.c | 57 ++++++++++++-----------
src/backend/replication/logical/logical.c | 5 +-
src/backend/replication/walsender.c | 4 +-
src/bin/pg_rewind/parsexlog.c | 6 +--
src/bin/pg_waldump/pg_waldump.c | 2 +-
src/include/access/xlogreader.h | 3 +-
8 files changed, 50 insertions(+), 39 deletions(-)
diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index 7d5a96a5f5..9f9e7de253 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -1325,7 +1325,8 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
XLogReaderState *xlogreader;
char *errormsg;
- xlogreader = XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+ xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+ NULL, wal_segment_close);
if (!xlogreader)
ereport(ERROR,
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index b517ca85ad..0822ac70cb 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -1222,7 +1222,10 @@ XLogInsertRecord(XLogRecData *rdata,
appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len);
if (!debug_reader)
- debug_reader = XLogReaderAllocate(wal_segment_size, NULL, NULL);
+ {
+ debug_reader =
+ XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, NULL);
+ }
if (!debug_reader)
{
@@ -6531,8 +6534,8 @@ StartupXLOG(void)
OwnLatch(&XLogCtl->recoveryWakeupLatch);
/* Set up XLOG reader facility */
- xlogreader =
- XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+ xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+ NULL, wal_segment_close);
if (!xlogreader)
ereport(ERROR,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index ba3e4868f7..e5baf13086 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -72,7 +72,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...)
* Returns NULL if the xlogreader couldn't be allocated.
*/
XLogReaderState *
-XLogReaderAllocate(int wal_segment_size, const char *waldir,
+XLogReaderAllocate(int wal_segment_size, int page_size, const char *waldir,
WALSegmentCleanupCB cleanup_cb)
{
XLogReaderState *state;
@@ -87,6 +87,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
state->cleanup_cb = cleanup_cb;
state->max_block_id = -1;
+ state->pageSize = page_size;
/*
* Permanently allocate readBuf. We do it this way, rather than just
@@ -95,7 +96,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
* isn't guaranteed to have any particular alignment, whereas
* palloc_extended() will provide MAXALIGN'd storage.
*/
- state->readBuf = (char *) palloc_extended(XLOG_BLCKSZ,
+ state->readBuf = (char *) palloc_extended(state->pageSize,
MCXT_ALLOC_NO_OOM);
if (!state->readBuf)
{
@@ -163,7 +164,7 @@ XLogReaderFree(XLogReaderState *state)
* readRecordBufSize is set to the new buffer size.
*
* To avoid useless small increases, round its size to a multiple of
- * XLOG_BLCKSZ, and make sure it's at least 5*Max(BLCKSZ, XLOG_BLCKSZ) to start
+ * page size, and make sure it's at least 5*Max(BLCKSZ, <page size>) to start
* with. (That is enough for all "normal" records, but very large commit or
* abort records might need more space.)
*/
@@ -172,8 +173,8 @@ allocate_recordbuf(XLogReaderState *state, uint32 reclength)
{
uint32 newSize = reclength;
- newSize += XLOG_BLCKSZ - (newSize % XLOG_BLCKSZ);
- newSize = Max(newSize, 5 * Max(BLCKSZ, XLOG_BLCKSZ));
+ newSize += state->pageSize - (newSize % state->pageSize);
+ newSize = Max(newSize, 5 * Max(BLCKSZ, state->pageSize));
#ifndef FRONTEND
@@ -238,6 +239,8 @@ void
XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
{
Assert(!XLogRecPtrIsInvalid(RecPtr));
+ Assert(state->readBuf != NULL &&
+ state->readBuf == (void *) MAXALIGN(state->readBuf));
ResetDecoder(state);
@@ -366,15 +369,15 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
XLogPageHeader pageHeader;
targetPagePtr =
- state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
- targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+ state->ReadRecPtr - (state->ReadRecPtr % state->pageSize);
+ targetRecOff = state->ReadRecPtr % state->pageSize;
/*
* Check if we have enough data. For the first record in the page,
* the requesting length doesn't contain page header.
*/
if (XLogNeedData(state, targetPagePtr,
- Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+ Min(targetRecOff + SizeOfXLogRecord, state->pageSize),
targetRecOff != 0))
return XLREAD_NEED_DATA;
@@ -421,7 +424,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
* verified that we got the whole header.
*/
prec = (XLogRecord *) (state->readBuf +
- state->ReadRecPtr % XLOG_BLCKSZ);
+ state->ReadRecPtr % state->pageSize);
total_len = prec->xl_tot_len;
/*
@@ -432,7 +435,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
* ensure that we enter the XLREAD_CONTINUATION state below;
* otherwise we might fail to apply ValidXLogRecordHeader at all.
*/
- if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
+ if (targetRecOff <= state->pageSize - SizeOfXLogRecord)
{
if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
state->PrevRecPtr, prec))
@@ -477,10 +480,10 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
* available
*/
targetPagePtr =
- state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
- targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+ state->ReadRecPtr - (state->ReadRecPtr % state->pageSize);
+ targetRecOff = state->ReadRecPtr % state->pageSize;
- request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ);
+ request_len = Min(targetRecOff + total_len, state->pageSize);
record_len = request_len - targetRecOff;
/* ReadRecPtr contains page header */
@@ -549,7 +552,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
/* Calculate pointer to beginning of next page */
state->recordContRecPtr = state->ReadRecPtr + record_len;
- Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0);
+ Assert(state->recordContRecPtr % state->pageSize == 0);
state->readRecordState = XLREAD_CONTINUATION;
}
@@ -576,7 +579,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
/* this request contains page header */
Assert (targetPagePtr != 0);
if (XLogNeedData(state, targetPagePtr,
- Min(state->recordRemainLen, XLOG_BLCKSZ),
+ Min(state->recordRemainLen, state->pageSize),
false))
return XLREAD_NEED_DATA;
@@ -628,7 +631,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
Assert(state->readLen >= pageHeaderSize);
contdata = (char *) state->readBuf + pageHeaderSize;
- record_len = XLOG_BLCKSZ - pageHeaderSize;
+ record_len = state->pageSize - pageHeaderSize;
if (pageHeader->xlp_rem_len < record_len)
record_len = pageHeader->xlp_rem_len;
@@ -655,7 +658,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
}
/* Calculate pointer to beginning of next page, and continue */
- state->recordContRecPtr += XLOG_BLCKSZ;
+ state->recordContRecPtr += state->pageSize;
}
/* targetPagePtr is pointing the last-read page here */
@@ -789,10 +792,10 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
XLogPageHeaderSize((XLogPageHeader) state->readBuf);
addLen = pageHeaderSize;
- if (reqLen + pageHeaderSize <= XLOG_BLCKSZ)
+ if (reqLen + pageHeaderSize <= state->pageSize)
addLen = pageHeaderSize;
else
- addLen = XLOG_BLCKSZ - reqLen;
+ addLen = state->pageSize - reqLen;
Assert(addLen >= 0);
}
@@ -805,7 +808,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
/* Data is not in our buffer, request the caller for it. */
XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize);
targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize);
- Assert((pageptr % XLOG_BLCKSZ) == 0);
+ Assert((pageptr % state->pageSize) == 0);
/*
* Every time we request to load new data of a page to the caller, even if
@@ -829,7 +832,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
* will not come back here, but will request the actual target page.
*/
state->readPagePtr = pageptr - targetPageOff;
- state->readLen = XLOG_BLCKSZ;
+ state->readLen = state->pageSize;
return true;
}
@@ -962,7 +965,7 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
int32 offset;
XLogPageHeader hdr = (XLogPageHeader) phdr;
- Assert((recptr % XLOG_BLCKSZ) == 0);
+ Assert((recptr % state->pageSize) == 0);
XLByteToSeg(recptr, segno, state->segcxt.ws_segsize);
offset = XLogSegmentOffset(recptr, state->segcxt.ws_segsize);
@@ -1016,10 +1019,10 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
"WAL file is from different database system: incorrect segment size in page header");
return false;
}
- else if (longhdr->xlp_xlog_blcksz != XLOG_BLCKSZ)
+ else if (longhdr->xlp_xlog_blcksz != state->pageSize)
{
report_invalid_record(state,
- "WAL file is from different database system: incorrect XLOG_BLCKSZ in page header");
+ "WAL file is from different database system: incorrect block size in page header");
return false;
}
}
@@ -1140,7 +1143,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
* XLogNeedData() is prepared to handle that and will read at
* least short page-header worth of data
*/
- targetRecOff = tmpRecPtr % XLOG_BLCKSZ;
+ targetRecOff = tmpRecPtr % state->pageSize;
/* scroll back to page boundary */
targetPagePtr = tmpRecPtr - targetRecOff;
@@ -1174,8 +1177,8 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
*
* Note that record headers are MAXALIGN'ed
*/
- if (MAXALIGN(header->xlp_rem_len) >= (XLOG_BLCKSZ - pageHeaderSize))
- tmpRecPtr = targetPagePtr + XLOG_BLCKSZ;
+ if (MAXALIGN(header->xlp_rem_len) >= (state->pageSize - pageHeaderSize))
+ tmpRecPtr = targetPagePtr + state->pageSize;
else
{
/*
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index 9cbd8afa49..f2dd495310 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -199,11 +199,14 @@ StartupDecodingContext(List *output_plugin_options,
ctx->slot = slot;
- ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, cleanup_cb);
+ ctx->reader =
+ XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, cleanup_cb);
+
if (!ctx->reader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
errmsg("out of memory")));
+ ctx->reader->readBuf = palloc(XLOG_BLCKSZ);
ctx->page_read = page_read;
ctx->reorder = ReorderBufferAllocate();
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index 062093727d..8e95600ef7 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -579,8 +579,8 @@ StartReplication(StartReplicationCmd *cmd)
errmsg("IDENTIFY_SYSTEM has not been run before START_REPLICATION")));
/* create xlogreader for physical replication */
- xlogreader =
- XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+ xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+ NULL, wal_segment_close);
if (!xlogreader)
ereport(ERROR,
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index 712c85281c..8a6209b061 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -61,7 +61,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
XLogReaderState *xlogreader;
char *errormsg;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+ xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
@@ -121,7 +121,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex,
char *errormsg;
XLogRecPtr endptr;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+ xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
@@ -181,7 +181,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
forkptr += SizeOfXLogShortPHD;
}
- xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+ xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index f514de6b13..b635537659 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -1041,7 +1041,7 @@ main(int argc, char **argv)
/* we have everything we need, start reading */
xlogreader_state =
- XLogReaderAllocate(WalSegSz, waldir, WALDumpCloseSegment);
+ XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, waldir, WALDumpCloseSegment);
if (!xlogreader_state)
fatal_error("out of memory");
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index b9dad7ee51..1853516333 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -151,6 +151,7 @@ struct XLogReaderState
* ----------------------------------------
*/
/* variables to communicate with page reader */
+ int pageSize; /* size of a page */
XLogRecPtr readPagePtr; /* page pointer to read */
int32 readLen; /* bytes requested to reader, or actual bytes
* read by reader, which must be larger than
@@ -243,7 +244,7 @@ struct XLogReaderState
/* Get a new XLogReader */
extern XLogReaderState *XLogReaderAllocate(int wal_segment_size,
- const char *waldir,
+ int page_size, const char *waldir,
WALSegmentCleanupCB cleanup_cb);
/* Free an XLogReader */
--
2.27.0
----Next_Part(Thu_Mar__4_11_28_53_2021_014)----
^ permalink raw reply [nested|flat] 30+ messages in thread
* [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize.
@ 2019-09-05 12:29 Kyotaro Horiguchi <[email protected]>
0 siblings, 0 replies; 30+ messages in thread
From: Kyotaro Horiguchi @ 2019-09-05 12:29 UTC (permalink / raw)
XLog reader facility and its callers currently relies on the same
constant XLOG_BLCKSZ. This patch moves the responsibility to determine
the block size to the callers of the facility.
---
src/backend/access/transam/twophase.c | 3 +-
src/backend/access/transam/xlog.c | 9 ++--
src/backend/access/transam/xlogreader.c | 57 ++++++++++++-----------
src/backend/replication/logical/logical.c | 5 +-
src/backend/replication/walsender.c | 4 +-
src/bin/pg_rewind/parsexlog.c | 6 +--
src/bin/pg_waldump/pg_waldump.c | 2 +-
src/include/access/xlogreader.h | 3 +-
8 files changed, 50 insertions(+), 39 deletions(-)
diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index 7d5a96a5f5..9f9e7de253 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -1325,7 +1325,8 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
XLogReaderState *xlogreader;
char *errormsg;
- xlogreader = XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+ xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+ NULL, wal_segment_close);
if (!xlogreader)
ereport(ERROR,
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index b517ca85ad..0822ac70cb 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -1222,7 +1222,10 @@ XLogInsertRecord(XLogRecData *rdata,
appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len);
if (!debug_reader)
- debug_reader = XLogReaderAllocate(wal_segment_size, NULL, NULL);
+ {
+ debug_reader =
+ XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, NULL);
+ }
if (!debug_reader)
{
@@ -6531,8 +6534,8 @@ StartupXLOG(void)
OwnLatch(&XLogCtl->recoveryWakeupLatch);
/* Set up XLOG reader facility */
- xlogreader =
- XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+ xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+ NULL, wal_segment_close);
if (!xlogreader)
ereport(ERROR,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index ba3e4868f7..e5baf13086 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -72,7 +72,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...)
* Returns NULL if the xlogreader couldn't be allocated.
*/
XLogReaderState *
-XLogReaderAllocate(int wal_segment_size, const char *waldir,
+XLogReaderAllocate(int wal_segment_size, int page_size, const char *waldir,
WALSegmentCleanupCB cleanup_cb)
{
XLogReaderState *state;
@@ -87,6 +87,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
state->cleanup_cb = cleanup_cb;
state->max_block_id = -1;
+ state->pageSize = page_size;
/*
* Permanently allocate readBuf. We do it this way, rather than just
@@ -95,7 +96,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
* isn't guaranteed to have any particular alignment, whereas
* palloc_extended() will provide MAXALIGN'd storage.
*/
- state->readBuf = (char *) palloc_extended(XLOG_BLCKSZ,
+ state->readBuf = (char *) palloc_extended(state->pageSize,
MCXT_ALLOC_NO_OOM);
if (!state->readBuf)
{
@@ -163,7 +164,7 @@ XLogReaderFree(XLogReaderState *state)
* readRecordBufSize is set to the new buffer size.
*
* To avoid useless small increases, round its size to a multiple of
- * XLOG_BLCKSZ, and make sure it's at least 5*Max(BLCKSZ, XLOG_BLCKSZ) to start
+ * page size, and make sure it's at least 5*Max(BLCKSZ, <page size>) to start
* with. (That is enough for all "normal" records, but very large commit or
* abort records might need more space.)
*/
@@ -172,8 +173,8 @@ allocate_recordbuf(XLogReaderState *state, uint32 reclength)
{
uint32 newSize = reclength;
- newSize += XLOG_BLCKSZ - (newSize % XLOG_BLCKSZ);
- newSize = Max(newSize, 5 * Max(BLCKSZ, XLOG_BLCKSZ));
+ newSize += state->pageSize - (newSize % state->pageSize);
+ newSize = Max(newSize, 5 * Max(BLCKSZ, state->pageSize));
#ifndef FRONTEND
@@ -238,6 +239,8 @@ void
XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
{
Assert(!XLogRecPtrIsInvalid(RecPtr));
+ Assert(state->readBuf != NULL &&
+ state->readBuf == (void *) MAXALIGN(state->readBuf));
ResetDecoder(state);
@@ -366,15 +369,15 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
XLogPageHeader pageHeader;
targetPagePtr =
- state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
- targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+ state->ReadRecPtr - (state->ReadRecPtr % state->pageSize);
+ targetRecOff = state->ReadRecPtr % state->pageSize;
/*
* Check if we have enough data. For the first record in the page,
* the requesting length doesn't contain page header.
*/
if (XLogNeedData(state, targetPagePtr,
- Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+ Min(targetRecOff + SizeOfXLogRecord, state->pageSize),
targetRecOff != 0))
return XLREAD_NEED_DATA;
@@ -421,7 +424,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
* verified that we got the whole header.
*/
prec = (XLogRecord *) (state->readBuf +
- state->ReadRecPtr % XLOG_BLCKSZ);
+ state->ReadRecPtr % state->pageSize);
total_len = prec->xl_tot_len;
/*
@@ -432,7 +435,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
* ensure that we enter the XLREAD_CONTINUATION state below;
* otherwise we might fail to apply ValidXLogRecordHeader at all.
*/
- if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
+ if (targetRecOff <= state->pageSize - SizeOfXLogRecord)
{
if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
state->PrevRecPtr, prec))
@@ -477,10 +480,10 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
* available
*/
targetPagePtr =
- state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
- targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+ state->ReadRecPtr - (state->ReadRecPtr % state->pageSize);
+ targetRecOff = state->ReadRecPtr % state->pageSize;
- request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ);
+ request_len = Min(targetRecOff + total_len, state->pageSize);
record_len = request_len - targetRecOff;
/* ReadRecPtr contains page header */
@@ -549,7 +552,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
/* Calculate pointer to beginning of next page */
state->recordContRecPtr = state->ReadRecPtr + record_len;
- Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0);
+ Assert(state->recordContRecPtr % state->pageSize == 0);
state->readRecordState = XLREAD_CONTINUATION;
}
@@ -576,7 +579,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
/* this request contains page header */
Assert (targetPagePtr != 0);
if (XLogNeedData(state, targetPagePtr,
- Min(state->recordRemainLen, XLOG_BLCKSZ),
+ Min(state->recordRemainLen, state->pageSize),
false))
return XLREAD_NEED_DATA;
@@ -628,7 +631,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
Assert(state->readLen >= pageHeaderSize);
contdata = (char *) state->readBuf + pageHeaderSize;
- record_len = XLOG_BLCKSZ - pageHeaderSize;
+ record_len = state->pageSize - pageHeaderSize;
if (pageHeader->xlp_rem_len < record_len)
record_len = pageHeader->xlp_rem_len;
@@ -655,7 +658,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
}
/* Calculate pointer to beginning of next page, and continue */
- state->recordContRecPtr += XLOG_BLCKSZ;
+ state->recordContRecPtr += state->pageSize;
}
/* targetPagePtr is pointing the last-read page here */
@@ -789,10 +792,10 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
XLogPageHeaderSize((XLogPageHeader) state->readBuf);
addLen = pageHeaderSize;
- if (reqLen + pageHeaderSize <= XLOG_BLCKSZ)
+ if (reqLen + pageHeaderSize <= state->pageSize)
addLen = pageHeaderSize;
else
- addLen = XLOG_BLCKSZ - reqLen;
+ addLen = state->pageSize - reqLen;
Assert(addLen >= 0);
}
@@ -805,7 +808,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
/* Data is not in our buffer, request the caller for it. */
XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize);
targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize);
- Assert((pageptr % XLOG_BLCKSZ) == 0);
+ Assert((pageptr % state->pageSize) == 0);
/*
* Every time we request to load new data of a page to the caller, even if
@@ -829,7 +832,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
* will not come back here, but will request the actual target page.
*/
state->readPagePtr = pageptr - targetPageOff;
- state->readLen = XLOG_BLCKSZ;
+ state->readLen = state->pageSize;
return true;
}
@@ -962,7 +965,7 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
int32 offset;
XLogPageHeader hdr = (XLogPageHeader) phdr;
- Assert((recptr % XLOG_BLCKSZ) == 0);
+ Assert((recptr % state->pageSize) == 0);
XLByteToSeg(recptr, segno, state->segcxt.ws_segsize);
offset = XLogSegmentOffset(recptr, state->segcxt.ws_segsize);
@@ -1016,10 +1019,10 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
"WAL file is from different database system: incorrect segment size in page header");
return false;
}
- else if (longhdr->xlp_xlog_blcksz != XLOG_BLCKSZ)
+ else if (longhdr->xlp_xlog_blcksz != state->pageSize)
{
report_invalid_record(state,
- "WAL file is from different database system: incorrect XLOG_BLCKSZ in page header");
+ "WAL file is from different database system: incorrect block size in page header");
return false;
}
}
@@ -1140,7 +1143,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
* XLogNeedData() is prepared to handle that and will read at
* least short page-header worth of data
*/
- targetRecOff = tmpRecPtr % XLOG_BLCKSZ;
+ targetRecOff = tmpRecPtr % state->pageSize;
/* scroll back to page boundary */
targetPagePtr = tmpRecPtr - targetRecOff;
@@ -1174,8 +1177,8 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
*
* Note that record headers are MAXALIGN'ed
*/
- if (MAXALIGN(header->xlp_rem_len) >= (XLOG_BLCKSZ - pageHeaderSize))
- tmpRecPtr = targetPagePtr + XLOG_BLCKSZ;
+ if (MAXALIGN(header->xlp_rem_len) >= (state->pageSize - pageHeaderSize))
+ tmpRecPtr = targetPagePtr + state->pageSize;
else
{
/*
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index 9cbd8afa49..f2dd495310 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -199,11 +199,14 @@ StartupDecodingContext(List *output_plugin_options,
ctx->slot = slot;
- ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, cleanup_cb);
+ ctx->reader =
+ XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, cleanup_cb);
+
if (!ctx->reader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
errmsg("out of memory")));
+ ctx->reader->readBuf = palloc(XLOG_BLCKSZ);
ctx->page_read = page_read;
ctx->reorder = ReorderBufferAllocate();
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index 062093727d..8e95600ef7 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -579,8 +579,8 @@ StartReplication(StartReplicationCmd *cmd)
errmsg("IDENTIFY_SYSTEM has not been run before START_REPLICATION")));
/* create xlogreader for physical replication */
- xlogreader =
- XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+ xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+ NULL, wal_segment_close);
if (!xlogreader)
ereport(ERROR,
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index 712c85281c..8a6209b061 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -61,7 +61,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
XLogReaderState *xlogreader;
char *errormsg;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+ xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
@@ -121,7 +121,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex,
char *errormsg;
XLogRecPtr endptr;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+ xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
@@ -181,7 +181,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
forkptr += SizeOfXLogShortPHD;
}
- xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+ xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index f514de6b13..b635537659 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -1041,7 +1041,7 @@ main(int argc, char **argv)
/* we have everything we need, start reading */
xlogreader_state =
- XLogReaderAllocate(WalSegSz, waldir, WALDumpCloseSegment);
+ XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, waldir, WALDumpCloseSegment);
if (!xlogreader_state)
fatal_error("out of memory");
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index b9dad7ee51..1853516333 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -151,6 +151,7 @@ struct XLogReaderState
* ----------------------------------------
*/
/* variables to communicate with page reader */
+ int pageSize; /* size of a page */
XLogRecPtr readPagePtr; /* page pointer to read */
int32 readLen; /* bytes requested to reader, or actual bytes
* read by reader, which must be larger than
@@ -243,7 +244,7 @@ struct XLogReaderState
/* Get a new XLogReader */
extern XLogReaderState *XLogReaderAllocate(int wal_segment_size,
- const char *waldir,
+ int page_size, const char *waldir,
WALSegmentCleanupCB cleanup_cb);
/* Free an XLogReader */
--
2.27.0
----Next_Part(Thu_Mar__4_11_28_53_2021_014)----
^ permalink raw reply [nested|flat] 30+ messages in thread
* [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize.
@ 2019-09-05 12:29 Kyotaro Horiguchi <[email protected]>
0 siblings, 0 replies; 30+ messages in thread
From: Kyotaro Horiguchi @ 2019-09-05 12:29 UTC (permalink / raw)
XLog reader facility and its callers currently relies on the same
constant XLOG_BLCKSZ. This patch moves the responsibility to determine
the block size to the callers of the facility.
---
src/backend/access/transam/twophase.c | 3 +-
src/backend/access/transam/xlog.c | 9 ++--
src/backend/access/transam/xlogreader.c | 57 ++++++++++++-----------
src/backend/replication/logical/logical.c | 5 +-
src/backend/replication/walsender.c | 4 +-
src/bin/pg_rewind/parsexlog.c | 6 +--
src/bin/pg_waldump/pg_waldump.c | 2 +-
src/include/access/xlogreader.h | 3 +-
8 files changed, 50 insertions(+), 39 deletions(-)
diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index 7d5a96a5f5..9f9e7de253 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -1325,7 +1325,8 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
XLogReaderState *xlogreader;
char *errormsg;
- xlogreader = XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+ xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+ NULL, wal_segment_close);
if (!xlogreader)
ereport(ERROR,
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index b517ca85ad..0822ac70cb 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -1222,7 +1222,10 @@ XLogInsertRecord(XLogRecData *rdata,
appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len);
if (!debug_reader)
- debug_reader = XLogReaderAllocate(wal_segment_size, NULL, NULL);
+ {
+ debug_reader =
+ XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, NULL);
+ }
if (!debug_reader)
{
@@ -6531,8 +6534,8 @@ StartupXLOG(void)
OwnLatch(&XLogCtl->recoveryWakeupLatch);
/* Set up XLOG reader facility */
- xlogreader =
- XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+ xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+ NULL, wal_segment_close);
if (!xlogreader)
ereport(ERROR,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index ba3e4868f7..e5baf13086 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -72,7 +72,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...)
* Returns NULL if the xlogreader couldn't be allocated.
*/
XLogReaderState *
-XLogReaderAllocate(int wal_segment_size, const char *waldir,
+XLogReaderAllocate(int wal_segment_size, int page_size, const char *waldir,
WALSegmentCleanupCB cleanup_cb)
{
XLogReaderState *state;
@@ -87,6 +87,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
state->cleanup_cb = cleanup_cb;
state->max_block_id = -1;
+ state->pageSize = page_size;
/*
* Permanently allocate readBuf. We do it this way, rather than just
@@ -95,7 +96,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
* isn't guaranteed to have any particular alignment, whereas
* palloc_extended() will provide MAXALIGN'd storage.
*/
- state->readBuf = (char *) palloc_extended(XLOG_BLCKSZ,
+ state->readBuf = (char *) palloc_extended(state->pageSize,
MCXT_ALLOC_NO_OOM);
if (!state->readBuf)
{
@@ -163,7 +164,7 @@ XLogReaderFree(XLogReaderState *state)
* readRecordBufSize is set to the new buffer size.
*
* To avoid useless small increases, round its size to a multiple of
- * XLOG_BLCKSZ, and make sure it's at least 5*Max(BLCKSZ, XLOG_BLCKSZ) to start
+ * page size, and make sure it's at least 5*Max(BLCKSZ, <page size>) to start
* with. (That is enough for all "normal" records, but very large commit or
* abort records might need more space.)
*/
@@ -172,8 +173,8 @@ allocate_recordbuf(XLogReaderState *state, uint32 reclength)
{
uint32 newSize = reclength;
- newSize += XLOG_BLCKSZ - (newSize % XLOG_BLCKSZ);
- newSize = Max(newSize, 5 * Max(BLCKSZ, XLOG_BLCKSZ));
+ newSize += state->pageSize - (newSize % state->pageSize);
+ newSize = Max(newSize, 5 * Max(BLCKSZ, state->pageSize));
#ifndef FRONTEND
@@ -238,6 +239,8 @@ void
XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
{
Assert(!XLogRecPtrIsInvalid(RecPtr));
+ Assert(state->readBuf != NULL &&
+ state->readBuf == (void *) MAXALIGN(state->readBuf));
ResetDecoder(state);
@@ -366,15 +369,15 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
XLogPageHeader pageHeader;
targetPagePtr =
- state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
- targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+ state->ReadRecPtr - (state->ReadRecPtr % state->pageSize);
+ targetRecOff = state->ReadRecPtr % state->pageSize;
/*
* Check if we have enough data. For the first record in the page,
* the requesting length doesn't contain page header.
*/
if (XLogNeedData(state, targetPagePtr,
- Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+ Min(targetRecOff + SizeOfXLogRecord, state->pageSize),
targetRecOff != 0))
return XLREAD_NEED_DATA;
@@ -421,7 +424,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
* verified that we got the whole header.
*/
prec = (XLogRecord *) (state->readBuf +
- state->ReadRecPtr % XLOG_BLCKSZ);
+ state->ReadRecPtr % state->pageSize);
total_len = prec->xl_tot_len;
/*
@@ -432,7 +435,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
* ensure that we enter the XLREAD_CONTINUATION state below;
* otherwise we might fail to apply ValidXLogRecordHeader at all.
*/
- if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
+ if (targetRecOff <= state->pageSize - SizeOfXLogRecord)
{
if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
state->PrevRecPtr, prec))
@@ -477,10 +480,10 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
* available
*/
targetPagePtr =
- state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
- targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+ state->ReadRecPtr - (state->ReadRecPtr % state->pageSize);
+ targetRecOff = state->ReadRecPtr % state->pageSize;
- request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ);
+ request_len = Min(targetRecOff + total_len, state->pageSize);
record_len = request_len - targetRecOff;
/* ReadRecPtr contains page header */
@@ -549,7 +552,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
/* Calculate pointer to beginning of next page */
state->recordContRecPtr = state->ReadRecPtr + record_len;
- Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0);
+ Assert(state->recordContRecPtr % state->pageSize == 0);
state->readRecordState = XLREAD_CONTINUATION;
}
@@ -576,7 +579,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
/* this request contains page header */
Assert (targetPagePtr != 0);
if (XLogNeedData(state, targetPagePtr,
- Min(state->recordRemainLen, XLOG_BLCKSZ),
+ Min(state->recordRemainLen, state->pageSize),
false))
return XLREAD_NEED_DATA;
@@ -628,7 +631,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
Assert(state->readLen >= pageHeaderSize);
contdata = (char *) state->readBuf + pageHeaderSize;
- record_len = XLOG_BLCKSZ - pageHeaderSize;
+ record_len = state->pageSize - pageHeaderSize;
if (pageHeader->xlp_rem_len < record_len)
record_len = pageHeader->xlp_rem_len;
@@ -655,7 +658,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
}
/* Calculate pointer to beginning of next page, and continue */
- state->recordContRecPtr += XLOG_BLCKSZ;
+ state->recordContRecPtr += state->pageSize;
}
/* targetPagePtr is pointing the last-read page here */
@@ -789,10 +792,10 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
XLogPageHeaderSize((XLogPageHeader) state->readBuf);
addLen = pageHeaderSize;
- if (reqLen + pageHeaderSize <= XLOG_BLCKSZ)
+ if (reqLen + pageHeaderSize <= state->pageSize)
addLen = pageHeaderSize;
else
- addLen = XLOG_BLCKSZ - reqLen;
+ addLen = state->pageSize - reqLen;
Assert(addLen >= 0);
}
@@ -805,7 +808,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
/* Data is not in our buffer, request the caller for it. */
XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize);
targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize);
- Assert((pageptr % XLOG_BLCKSZ) == 0);
+ Assert((pageptr % state->pageSize) == 0);
/*
* Every time we request to load new data of a page to the caller, even if
@@ -829,7 +832,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
* will not come back here, but will request the actual target page.
*/
state->readPagePtr = pageptr - targetPageOff;
- state->readLen = XLOG_BLCKSZ;
+ state->readLen = state->pageSize;
return true;
}
@@ -962,7 +965,7 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
int32 offset;
XLogPageHeader hdr = (XLogPageHeader) phdr;
- Assert((recptr % XLOG_BLCKSZ) == 0);
+ Assert((recptr % state->pageSize) == 0);
XLByteToSeg(recptr, segno, state->segcxt.ws_segsize);
offset = XLogSegmentOffset(recptr, state->segcxt.ws_segsize);
@@ -1016,10 +1019,10 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
"WAL file is from different database system: incorrect segment size in page header");
return false;
}
- else if (longhdr->xlp_xlog_blcksz != XLOG_BLCKSZ)
+ else if (longhdr->xlp_xlog_blcksz != state->pageSize)
{
report_invalid_record(state,
- "WAL file is from different database system: incorrect XLOG_BLCKSZ in page header");
+ "WAL file is from different database system: incorrect block size in page header");
return false;
}
}
@@ -1140,7 +1143,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
* XLogNeedData() is prepared to handle that and will read at
* least short page-header worth of data
*/
- targetRecOff = tmpRecPtr % XLOG_BLCKSZ;
+ targetRecOff = tmpRecPtr % state->pageSize;
/* scroll back to page boundary */
targetPagePtr = tmpRecPtr - targetRecOff;
@@ -1174,8 +1177,8 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
*
* Note that record headers are MAXALIGN'ed
*/
- if (MAXALIGN(header->xlp_rem_len) >= (XLOG_BLCKSZ - pageHeaderSize))
- tmpRecPtr = targetPagePtr + XLOG_BLCKSZ;
+ if (MAXALIGN(header->xlp_rem_len) >= (state->pageSize - pageHeaderSize))
+ tmpRecPtr = targetPagePtr + state->pageSize;
else
{
/*
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index 9cbd8afa49..f2dd495310 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -199,11 +199,14 @@ StartupDecodingContext(List *output_plugin_options,
ctx->slot = slot;
- ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, cleanup_cb);
+ ctx->reader =
+ XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, cleanup_cb);
+
if (!ctx->reader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
errmsg("out of memory")));
+ ctx->reader->readBuf = palloc(XLOG_BLCKSZ);
ctx->page_read = page_read;
ctx->reorder = ReorderBufferAllocate();
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index 062093727d..8e95600ef7 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -579,8 +579,8 @@ StartReplication(StartReplicationCmd *cmd)
errmsg("IDENTIFY_SYSTEM has not been run before START_REPLICATION")));
/* create xlogreader for physical replication */
- xlogreader =
- XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+ xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+ NULL, wal_segment_close);
if (!xlogreader)
ereport(ERROR,
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index 712c85281c..8a6209b061 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -61,7 +61,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
XLogReaderState *xlogreader;
char *errormsg;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+ xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
@@ -121,7 +121,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex,
char *errormsg;
XLogRecPtr endptr;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+ xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
@@ -181,7 +181,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
forkptr += SizeOfXLogShortPHD;
}
- xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+ xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index f514de6b13..b635537659 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -1041,7 +1041,7 @@ main(int argc, char **argv)
/* we have everything we need, start reading */
xlogreader_state =
- XLogReaderAllocate(WalSegSz, waldir, WALDumpCloseSegment);
+ XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, waldir, WALDumpCloseSegment);
if (!xlogreader_state)
fatal_error("out of memory");
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index b9dad7ee51..1853516333 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -151,6 +151,7 @@ struct XLogReaderState
* ----------------------------------------
*/
/* variables to communicate with page reader */
+ int pageSize; /* size of a page */
XLogRecPtr readPagePtr; /* page pointer to read */
int32 readLen; /* bytes requested to reader, or actual bytes
* read by reader, which must be larger than
@@ -243,7 +244,7 @@ struct XLogReaderState
/* Get a new XLogReader */
extern XLogReaderState *XLogReaderAllocate(int wal_segment_size,
- const char *waldir,
+ int page_size, const char *waldir,
WALSegmentCleanupCB cleanup_cb);
/* Free an XLogReader */
--
2.27.0
----Next_Part(Thu_Mar__4_11_28_53_2021_014)----
^ permalink raw reply [nested|flat] 30+ messages in thread
* [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize.
@ 2019-09-05 12:29 Kyotaro Horiguchi <[email protected]>
0 siblings, 0 replies; 30+ messages in thread
From: Kyotaro Horiguchi @ 2019-09-05 12:29 UTC (permalink / raw)
XLog reader facility and its callers currently relies on the same
constant XLOG_BLCKSZ. This patch moves the responsibility to determine
the block size to the callers of the facility.
---
src/backend/access/transam/twophase.c | 3 +-
src/backend/access/transam/xlog.c | 9 ++--
src/backend/access/transam/xlogreader.c | 57 ++++++++++++-----------
src/backend/replication/logical/logical.c | 5 +-
src/backend/replication/walsender.c | 4 +-
src/bin/pg_rewind/parsexlog.c | 6 +--
src/bin/pg_waldump/pg_waldump.c | 2 +-
src/include/access/xlogreader.h | 3 +-
8 files changed, 50 insertions(+), 39 deletions(-)
diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index 7d5a96a5f5..9f9e7de253 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -1325,7 +1325,8 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
XLogReaderState *xlogreader;
char *errormsg;
- xlogreader = XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+ xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+ NULL, wal_segment_close);
if (!xlogreader)
ereport(ERROR,
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index b517ca85ad..0822ac70cb 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -1222,7 +1222,10 @@ XLogInsertRecord(XLogRecData *rdata,
appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len);
if (!debug_reader)
- debug_reader = XLogReaderAllocate(wal_segment_size, NULL, NULL);
+ {
+ debug_reader =
+ XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, NULL);
+ }
if (!debug_reader)
{
@@ -6531,8 +6534,8 @@ StartupXLOG(void)
OwnLatch(&XLogCtl->recoveryWakeupLatch);
/* Set up XLOG reader facility */
- xlogreader =
- XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+ xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+ NULL, wal_segment_close);
if (!xlogreader)
ereport(ERROR,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index ba3e4868f7..e5baf13086 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -72,7 +72,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...)
* Returns NULL if the xlogreader couldn't be allocated.
*/
XLogReaderState *
-XLogReaderAllocate(int wal_segment_size, const char *waldir,
+XLogReaderAllocate(int wal_segment_size, int page_size, const char *waldir,
WALSegmentCleanupCB cleanup_cb)
{
XLogReaderState *state;
@@ -87,6 +87,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
state->cleanup_cb = cleanup_cb;
state->max_block_id = -1;
+ state->pageSize = page_size;
/*
* Permanently allocate readBuf. We do it this way, rather than just
@@ -95,7 +96,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
* isn't guaranteed to have any particular alignment, whereas
* palloc_extended() will provide MAXALIGN'd storage.
*/
- state->readBuf = (char *) palloc_extended(XLOG_BLCKSZ,
+ state->readBuf = (char *) palloc_extended(state->pageSize,
MCXT_ALLOC_NO_OOM);
if (!state->readBuf)
{
@@ -163,7 +164,7 @@ XLogReaderFree(XLogReaderState *state)
* readRecordBufSize is set to the new buffer size.
*
* To avoid useless small increases, round its size to a multiple of
- * XLOG_BLCKSZ, and make sure it's at least 5*Max(BLCKSZ, XLOG_BLCKSZ) to start
+ * page size, and make sure it's at least 5*Max(BLCKSZ, <page size>) to start
* with. (That is enough for all "normal" records, but very large commit or
* abort records might need more space.)
*/
@@ -172,8 +173,8 @@ allocate_recordbuf(XLogReaderState *state, uint32 reclength)
{
uint32 newSize = reclength;
- newSize += XLOG_BLCKSZ - (newSize % XLOG_BLCKSZ);
- newSize = Max(newSize, 5 * Max(BLCKSZ, XLOG_BLCKSZ));
+ newSize += state->pageSize - (newSize % state->pageSize);
+ newSize = Max(newSize, 5 * Max(BLCKSZ, state->pageSize));
#ifndef FRONTEND
@@ -238,6 +239,8 @@ void
XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
{
Assert(!XLogRecPtrIsInvalid(RecPtr));
+ Assert(state->readBuf != NULL &&
+ state->readBuf == (void *) MAXALIGN(state->readBuf));
ResetDecoder(state);
@@ -366,15 +369,15 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
XLogPageHeader pageHeader;
targetPagePtr =
- state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
- targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+ state->ReadRecPtr - (state->ReadRecPtr % state->pageSize);
+ targetRecOff = state->ReadRecPtr % state->pageSize;
/*
* Check if we have enough data. For the first record in the page,
* the requesting length doesn't contain page header.
*/
if (XLogNeedData(state, targetPagePtr,
- Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+ Min(targetRecOff + SizeOfXLogRecord, state->pageSize),
targetRecOff != 0))
return XLREAD_NEED_DATA;
@@ -421,7 +424,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
* verified that we got the whole header.
*/
prec = (XLogRecord *) (state->readBuf +
- state->ReadRecPtr % XLOG_BLCKSZ);
+ state->ReadRecPtr % state->pageSize);
total_len = prec->xl_tot_len;
/*
@@ -432,7 +435,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
* ensure that we enter the XLREAD_CONTINUATION state below;
* otherwise we might fail to apply ValidXLogRecordHeader at all.
*/
- if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
+ if (targetRecOff <= state->pageSize - SizeOfXLogRecord)
{
if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
state->PrevRecPtr, prec))
@@ -477,10 +480,10 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
* available
*/
targetPagePtr =
- state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
- targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+ state->ReadRecPtr - (state->ReadRecPtr % state->pageSize);
+ targetRecOff = state->ReadRecPtr % state->pageSize;
- request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ);
+ request_len = Min(targetRecOff + total_len, state->pageSize);
record_len = request_len - targetRecOff;
/* ReadRecPtr contains page header */
@@ -549,7 +552,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
/* Calculate pointer to beginning of next page */
state->recordContRecPtr = state->ReadRecPtr + record_len;
- Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0);
+ Assert(state->recordContRecPtr % state->pageSize == 0);
state->readRecordState = XLREAD_CONTINUATION;
}
@@ -576,7 +579,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
/* this request contains page header */
Assert (targetPagePtr != 0);
if (XLogNeedData(state, targetPagePtr,
- Min(state->recordRemainLen, XLOG_BLCKSZ),
+ Min(state->recordRemainLen, state->pageSize),
false))
return XLREAD_NEED_DATA;
@@ -628,7 +631,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
Assert(state->readLen >= pageHeaderSize);
contdata = (char *) state->readBuf + pageHeaderSize;
- record_len = XLOG_BLCKSZ - pageHeaderSize;
+ record_len = state->pageSize - pageHeaderSize;
if (pageHeader->xlp_rem_len < record_len)
record_len = pageHeader->xlp_rem_len;
@@ -655,7 +658,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
}
/* Calculate pointer to beginning of next page, and continue */
- state->recordContRecPtr += XLOG_BLCKSZ;
+ state->recordContRecPtr += state->pageSize;
}
/* targetPagePtr is pointing the last-read page here */
@@ -789,10 +792,10 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
XLogPageHeaderSize((XLogPageHeader) state->readBuf);
addLen = pageHeaderSize;
- if (reqLen + pageHeaderSize <= XLOG_BLCKSZ)
+ if (reqLen + pageHeaderSize <= state->pageSize)
addLen = pageHeaderSize;
else
- addLen = XLOG_BLCKSZ - reqLen;
+ addLen = state->pageSize - reqLen;
Assert(addLen >= 0);
}
@@ -805,7 +808,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
/* Data is not in our buffer, request the caller for it. */
XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize);
targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize);
- Assert((pageptr % XLOG_BLCKSZ) == 0);
+ Assert((pageptr % state->pageSize) == 0);
/*
* Every time we request to load new data of a page to the caller, even if
@@ -829,7 +832,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
* will not come back here, but will request the actual target page.
*/
state->readPagePtr = pageptr - targetPageOff;
- state->readLen = XLOG_BLCKSZ;
+ state->readLen = state->pageSize;
return true;
}
@@ -962,7 +965,7 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
int32 offset;
XLogPageHeader hdr = (XLogPageHeader) phdr;
- Assert((recptr % XLOG_BLCKSZ) == 0);
+ Assert((recptr % state->pageSize) == 0);
XLByteToSeg(recptr, segno, state->segcxt.ws_segsize);
offset = XLogSegmentOffset(recptr, state->segcxt.ws_segsize);
@@ -1016,10 +1019,10 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
"WAL file is from different database system: incorrect segment size in page header");
return false;
}
- else if (longhdr->xlp_xlog_blcksz != XLOG_BLCKSZ)
+ else if (longhdr->xlp_xlog_blcksz != state->pageSize)
{
report_invalid_record(state,
- "WAL file is from different database system: incorrect XLOG_BLCKSZ in page header");
+ "WAL file is from different database system: incorrect block size in page header");
return false;
}
}
@@ -1140,7 +1143,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
* XLogNeedData() is prepared to handle that and will read at
* least short page-header worth of data
*/
- targetRecOff = tmpRecPtr % XLOG_BLCKSZ;
+ targetRecOff = tmpRecPtr % state->pageSize;
/* scroll back to page boundary */
targetPagePtr = tmpRecPtr - targetRecOff;
@@ -1174,8 +1177,8 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
*
* Note that record headers are MAXALIGN'ed
*/
- if (MAXALIGN(header->xlp_rem_len) >= (XLOG_BLCKSZ - pageHeaderSize))
- tmpRecPtr = targetPagePtr + XLOG_BLCKSZ;
+ if (MAXALIGN(header->xlp_rem_len) >= (state->pageSize - pageHeaderSize))
+ tmpRecPtr = targetPagePtr + state->pageSize;
else
{
/*
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index 9cbd8afa49..f2dd495310 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -199,11 +199,14 @@ StartupDecodingContext(List *output_plugin_options,
ctx->slot = slot;
- ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, cleanup_cb);
+ ctx->reader =
+ XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, cleanup_cb);
+
if (!ctx->reader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
errmsg("out of memory")));
+ ctx->reader->readBuf = palloc(XLOG_BLCKSZ);
ctx->page_read = page_read;
ctx->reorder = ReorderBufferAllocate();
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index 062093727d..8e95600ef7 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -579,8 +579,8 @@ StartReplication(StartReplicationCmd *cmd)
errmsg("IDENTIFY_SYSTEM has not been run before START_REPLICATION")));
/* create xlogreader for physical replication */
- xlogreader =
- XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+ xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+ NULL, wal_segment_close);
if (!xlogreader)
ereport(ERROR,
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index 712c85281c..8a6209b061 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -61,7 +61,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
XLogReaderState *xlogreader;
char *errormsg;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+ xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
@@ -121,7 +121,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex,
char *errormsg;
XLogRecPtr endptr;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+ xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
@@ -181,7 +181,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
forkptr += SizeOfXLogShortPHD;
}
- xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+ xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index f514de6b13..b635537659 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -1041,7 +1041,7 @@ main(int argc, char **argv)
/* we have everything we need, start reading */
xlogreader_state =
- XLogReaderAllocate(WalSegSz, waldir, WALDumpCloseSegment);
+ XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, waldir, WALDumpCloseSegment);
if (!xlogreader_state)
fatal_error("out of memory");
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index b9dad7ee51..1853516333 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -151,6 +151,7 @@ struct XLogReaderState
* ----------------------------------------
*/
/* variables to communicate with page reader */
+ int pageSize; /* size of a page */
XLogRecPtr readPagePtr; /* page pointer to read */
int32 readLen; /* bytes requested to reader, or actual bytes
* read by reader, which must be larger than
@@ -243,7 +244,7 @@ struct XLogReaderState
/* Get a new XLogReader */
extern XLogReaderState *XLogReaderAllocate(int wal_segment_size,
- const char *waldir,
+ int page_size, const char *waldir,
WALSegmentCleanupCB cleanup_cb);
/* Free an XLogReader */
--
2.27.0
----Next_Part(Thu_Mar__4_11_28_53_2021_014)----
^ permalink raw reply [nested|flat] 30+ messages in thread
* [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize.
@ 2019-09-05 12:29 Kyotaro Horiguchi <[email protected]>
0 siblings, 0 replies; 30+ messages in thread
From: Kyotaro Horiguchi @ 2019-09-05 12:29 UTC (permalink / raw)
XLog reader facility and its callers currently relies on the same
constant XLOG_BLCKSZ. This patch moves the responsibility to determine
the block size to the callers of the facility.
---
src/backend/access/transam/twophase.c | 3 +-
src/backend/access/transam/xlog.c | 9 ++--
src/backend/access/transam/xlogreader.c | 57 ++++++++++++-----------
src/backend/replication/logical/logical.c | 5 +-
src/backend/replication/walsender.c | 4 +-
src/bin/pg_rewind/parsexlog.c | 6 +--
src/bin/pg_waldump/pg_waldump.c | 2 +-
src/include/access/xlogreader.h | 3 +-
8 files changed, 50 insertions(+), 39 deletions(-)
diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index 7d5a96a5f5..9f9e7de253 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -1325,7 +1325,8 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
XLogReaderState *xlogreader;
char *errormsg;
- xlogreader = XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+ xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+ NULL, wal_segment_close);
if (!xlogreader)
ereport(ERROR,
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index b517ca85ad..0822ac70cb 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -1222,7 +1222,10 @@ XLogInsertRecord(XLogRecData *rdata,
appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len);
if (!debug_reader)
- debug_reader = XLogReaderAllocate(wal_segment_size, NULL, NULL);
+ {
+ debug_reader =
+ XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, NULL);
+ }
if (!debug_reader)
{
@@ -6531,8 +6534,8 @@ StartupXLOG(void)
OwnLatch(&XLogCtl->recoveryWakeupLatch);
/* Set up XLOG reader facility */
- xlogreader =
- XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+ xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+ NULL, wal_segment_close);
if (!xlogreader)
ereport(ERROR,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index ba3e4868f7..e5baf13086 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -72,7 +72,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...)
* Returns NULL if the xlogreader couldn't be allocated.
*/
XLogReaderState *
-XLogReaderAllocate(int wal_segment_size, const char *waldir,
+XLogReaderAllocate(int wal_segment_size, int page_size, const char *waldir,
WALSegmentCleanupCB cleanup_cb)
{
XLogReaderState *state;
@@ -87,6 +87,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
state->cleanup_cb = cleanup_cb;
state->max_block_id = -1;
+ state->pageSize = page_size;
/*
* Permanently allocate readBuf. We do it this way, rather than just
@@ -95,7 +96,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
* isn't guaranteed to have any particular alignment, whereas
* palloc_extended() will provide MAXALIGN'd storage.
*/
- state->readBuf = (char *) palloc_extended(XLOG_BLCKSZ,
+ state->readBuf = (char *) palloc_extended(state->pageSize,
MCXT_ALLOC_NO_OOM);
if (!state->readBuf)
{
@@ -163,7 +164,7 @@ XLogReaderFree(XLogReaderState *state)
* readRecordBufSize is set to the new buffer size.
*
* To avoid useless small increases, round its size to a multiple of
- * XLOG_BLCKSZ, and make sure it's at least 5*Max(BLCKSZ, XLOG_BLCKSZ) to start
+ * page size, and make sure it's at least 5*Max(BLCKSZ, <page size>) to start
* with. (That is enough for all "normal" records, but very large commit or
* abort records might need more space.)
*/
@@ -172,8 +173,8 @@ allocate_recordbuf(XLogReaderState *state, uint32 reclength)
{
uint32 newSize = reclength;
- newSize += XLOG_BLCKSZ - (newSize % XLOG_BLCKSZ);
- newSize = Max(newSize, 5 * Max(BLCKSZ, XLOG_BLCKSZ));
+ newSize += state->pageSize - (newSize % state->pageSize);
+ newSize = Max(newSize, 5 * Max(BLCKSZ, state->pageSize));
#ifndef FRONTEND
@@ -238,6 +239,8 @@ void
XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
{
Assert(!XLogRecPtrIsInvalid(RecPtr));
+ Assert(state->readBuf != NULL &&
+ state->readBuf == (void *) MAXALIGN(state->readBuf));
ResetDecoder(state);
@@ -366,15 +369,15 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
XLogPageHeader pageHeader;
targetPagePtr =
- state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
- targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+ state->ReadRecPtr - (state->ReadRecPtr % state->pageSize);
+ targetRecOff = state->ReadRecPtr % state->pageSize;
/*
* Check if we have enough data. For the first record in the page,
* the requesting length doesn't contain page header.
*/
if (XLogNeedData(state, targetPagePtr,
- Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+ Min(targetRecOff + SizeOfXLogRecord, state->pageSize),
targetRecOff != 0))
return XLREAD_NEED_DATA;
@@ -421,7 +424,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
* verified that we got the whole header.
*/
prec = (XLogRecord *) (state->readBuf +
- state->ReadRecPtr % XLOG_BLCKSZ);
+ state->ReadRecPtr % state->pageSize);
total_len = prec->xl_tot_len;
/*
@@ -432,7 +435,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
* ensure that we enter the XLREAD_CONTINUATION state below;
* otherwise we might fail to apply ValidXLogRecordHeader at all.
*/
- if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
+ if (targetRecOff <= state->pageSize - SizeOfXLogRecord)
{
if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
state->PrevRecPtr, prec))
@@ -477,10 +480,10 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
* available
*/
targetPagePtr =
- state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
- targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+ state->ReadRecPtr - (state->ReadRecPtr % state->pageSize);
+ targetRecOff = state->ReadRecPtr % state->pageSize;
- request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ);
+ request_len = Min(targetRecOff + total_len, state->pageSize);
record_len = request_len - targetRecOff;
/* ReadRecPtr contains page header */
@@ -549,7 +552,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
/* Calculate pointer to beginning of next page */
state->recordContRecPtr = state->ReadRecPtr + record_len;
- Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0);
+ Assert(state->recordContRecPtr % state->pageSize == 0);
state->readRecordState = XLREAD_CONTINUATION;
}
@@ -576,7 +579,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
/* this request contains page header */
Assert (targetPagePtr != 0);
if (XLogNeedData(state, targetPagePtr,
- Min(state->recordRemainLen, XLOG_BLCKSZ),
+ Min(state->recordRemainLen, state->pageSize),
false))
return XLREAD_NEED_DATA;
@@ -628,7 +631,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
Assert(state->readLen >= pageHeaderSize);
contdata = (char *) state->readBuf + pageHeaderSize;
- record_len = XLOG_BLCKSZ - pageHeaderSize;
+ record_len = state->pageSize - pageHeaderSize;
if (pageHeader->xlp_rem_len < record_len)
record_len = pageHeader->xlp_rem_len;
@@ -655,7 +658,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
}
/* Calculate pointer to beginning of next page, and continue */
- state->recordContRecPtr += XLOG_BLCKSZ;
+ state->recordContRecPtr += state->pageSize;
}
/* targetPagePtr is pointing the last-read page here */
@@ -789,10 +792,10 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
XLogPageHeaderSize((XLogPageHeader) state->readBuf);
addLen = pageHeaderSize;
- if (reqLen + pageHeaderSize <= XLOG_BLCKSZ)
+ if (reqLen + pageHeaderSize <= state->pageSize)
addLen = pageHeaderSize;
else
- addLen = XLOG_BLCKSZ - reqLen;
+ addLen = state->pageSize - reqLen;
Assert(addLen >= 0);
}
@@ -805,7 +808,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
/* Data is not in our buffer, request the caller for it. */
XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize);
targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize);
- Assert((pageptr % XLOG_BLCKSZ) == 0);
+ Assert((pageptr % state->pageSize) == 0);
/*
* Every time we request to load new data of a page to the caller, even if
@@ -829,7 +832,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
* will not come back here, but will request the actual target page.
*/
state->readPagePtr = pageptr - targetPageOff;
- state->readLen = XLOG_BLCKSZ;
+ state->readLen = state->pageSize;
return true;
}
@@ -962,7 +965,7 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
int32 offset;
XLogPageHeader hdr = (XLogPageHeader) phdr;
- Assert((recptr % XLOG_BLCKSZ) == 0);
+ Assert((recptr % state->pageSize) == 0);
XLByteToSeg(recptr, segno, state->segcxt.ws_segsize);
offset = XLogSegmentOffset(recptr, state->segcxt.ws_segsize);
@@ -1016,10 +1019,10 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
"WAL file is from different database system: incorrect segment size in page header");
return false;
}
- else if (longhdr->xlp_xlog_blcksz != XLOG_BLCKSZ)
+ else if (longhdr->xlp_xlog_blcksz != state->pageSize)
{
report_invalid_record(state,
- "WAL file is from different database system: incorrect XLOG_BLCKSZ in page header");
+ "WAL file is from different database system: incorrect block size in page header");
return false;
}
}
@@ -1140,7 +1143,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
* XLogNeedData() is prepared to handle that and will read at
* least short page-header worth of data
*/
- targetRecOff = tmpRecPtr % XLOG_BLCKSZ;
+ targetRecOff = tmpRecPtr % state->pageSize;
/* scroll back to page boundary */
targetPagePtr = tmpRecPtr - targetRecOff;
@@ -1174,8 +1177,8 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
*
* Note that record headers are MAXALIGN'ed
*/
- if (MAXALIGN(header->xlp_rem_len) >= (XLOG_BLCKSZ - pageHeaderSize))
- tmpRecPtr = targetPagePtr + XLOG_BLCKSZ;
+ if (MAXALIGN(header->xlp_rem_len) >= (state->pageSize - pageHeaderSize))
+ tmpRecPtr = targetPagePtr + state->pageSize;
else
{
/*
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index 9cbd8afa49..f2dd495310 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -199,11 +199,14 @@ StartupDecodingContext(List *output_plugin_options,
ctx->slot = slot;
- ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, cleanup_cb);
+ ctx->reader =
+ XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, cleanup_cb);
+
if (!ctx->reader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
errmsg("out of memory")));
+ ctx->reader->readBuf = palloc(XLOG_BLCKSZ);
ctx->page_read = page_read;
ctx->reorder = ReorderBufferAllocate();
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index 062093727d..8e95600ef7 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -579,8 +579,8 @@ StartReplication(StartReplicationCmd *cmd)
errmsg("IDENTIFY_SYSTEM has not been run before START_REPLICATION")));
/* create xlogreader for physical replication */
- xlogreader =
- XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+ xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+ NULL, wal_segment_close);
if (!xlogreader)
ereport(ERROR,
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index 712c85281c..8a6209b061 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -61,7 +61,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
XLogReaderState *xlogreader;
char *errormsg;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+ xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
@@ -121,7 +121,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex,
char *errormsg;
XLogRecPtr endptr;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+ xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
@@ -181,7 +181,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
forkptr += SizeOfXLogShortPHD;
}
- xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+ xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index f514de6b13..b635537659 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -1041,7 +1041,7 @@ main(int argc, char **argv)
/* we have everything we need, start reading */
xlogreader_state =
- XLogReaderAllocate(WalSegSz, waldir, WALDumpCloseSegment);
+ XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, waldir, WALDumpCloseSegment);
if (!xlogreader_state)
fatal_error("out of memory");
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index b9dad7ee51..1853516333 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -151,6 +151,7 @@ struct XLogReaderState
* ----------------------------------------
*/
/* variables to communicate with page reader */
+ int pageSize; /* size of a page */
XLogRecPtr readPagePtr; /* page pointer to read */
int32 readLen; /* bytes requested to reader, or actual bytes
* read by reader, which must be larger than
@@ -243,7 +244,7 @@ struct XLogReaderState
/* Get a new XLogReader */
extern XLogReaderState *XLogReaderAllocate(int wal_segment_size,
- const char *waldir,
+ int page_size, const char *waldir,
WALSegmentCleanupCB cleanup_cb);
/* Free an XLogReader */
--
2.27.0
----Next_Part(Thu_Mar__4_11_28_53_2021_014)----
^ permalink raw reply [nested|flat] 30+ messages in thread
* [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize.
@ 2019-09-05 12:29 Kyotaro Horiguchi <[email protected]>
0 siblings, 0 replies; 30+ messages in thread
From: Kyotaro Horiguchi @ 2019-09-05 12:29 UTC (permalink / raw)
XLog reader facility and its callers currently relies on the same
constant XLOG_BLCKSZ. This patch moves the responsibility to determine
the block size to the callers of the facility.
---
src/backend/access/transam/twophase.c | 3 +-
src/backend/access/transam/xlog.c | 9 ++--
src/backend/access/transam/xlogreader.c | 57 ++++++++++++-----------
src/backend/replication/logical/logical.c | 5 +-
src/backend/replication/walsender.c | 4 +-
src/bin/pg_rewind/parsexlog.c | 6 +--
src/bin/pg_waldump/pg_waldump.c | 2 +-
src/include/access/xlogreader.h | 3 +-
8 files changed, 50 insertions(+), 39 deletions(-)
diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index 7d5a96a5f5..9f9e7de253 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -1325,7 +1325,8 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
XLogReaderState *xlogreader;
char *errormsg;
- xlogreader = XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+ xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+ NULL, wal_segment_close);
if (!xlogreader)
ereport(ERROR,
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index b517ca85ad..0822ac70cb 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -1222,7 +1222,10 @@ XLogInsertRecord(XLogRecData *rdata,
appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len);
if (!debug_reader)
- debug_reader = XLogReaderAllocate(wal_segment_size, NULL, NULL);
+ {
+ debug_reader =
+ XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, NULL);
+ }
if (!debug_reader)
{
@@ -6531,8 +6534,8 @@ StartupXLOG(void)
OwnLatch(&XLogCtl->recoveryWakeupLatch);
/* Set up XLOG reader facility */
- xlogreader =
- XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+ xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+ NULL, wal_segment_close);
if (!xlogreader)
ereport(ERROR,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index ba3e4868f7..e5baf13086 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -72,7 +72,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...)
* Returns NULL if the xlogreader couldn't be allocated.
*/
XLogReaderState *
-XLogReaderAllocate(int wal_segment_size, const char *waldir,
+XLogReaderAllocate(int wal_segment_size, int page_size, const char *waldir,
WALSegmentCleanupCB cleanup_cb)
{
XLogReaderState *state;
@@ -87,6 +87,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
state->cleanup_cb = cleanup_cb;
state->max_block_id = -1;
+ state->pageSize = page_size;
/*
* Permanently allocate readBuf. We do it this way, rather than just
@@ -95,7 +96,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
* isn't guaranteed to have any particular alignment, whereas
* palloc_extended() will provide MAXALIGN'd storage.
*/
- state->readBuf = (char *) palloc_extended(XLOG_BLCKSZ,
+ state->readBuf = (char *) palloc_extended(state->pageSize,
MCXT_ALLOC_NO_OOM);
if (!state->readBuf)
{
@@ -163,7 +164,7 @@ XLogReaderFree(XLogReaderState *state)
* readRecordBufSize is set to the new buffer size.
*
* To avoid useless small increases, round its size to a multiple of
- * XLOG_BLCKSZ, and make sure it's at least 5*Max(BLCKSZ, XLOG_BLCKSZ) to start
+ * page size, and make sure it's at least 5*Max(BLCKSZ, <page size>) to start
* with. (That is enough for all "normal" records, but very large commit or
* abort records might need more space.)
*/
@@ -172,8 +173,8 @@ allocate_recordbuf(XLogReaderState *state, uint32 reclength)
{
uint32 newSize = reclength;
- newSize += XLOG_BLCKSZ - (newSize % XLOG_BLCKSZ);
- newSize = Max(newSize, 5 * Max(BLCKSZ, XLOG_BLCKSZ));
+ newSize += state->pageSize - (newSize % state->pageSize);
+ newSize = Max(newSize, 5 * Max(BLCKSZ, state->pageSize));
#ifndef FRONTEND
@@ -238,6 +239,8 @@ void
XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
{
Assert(!XLogRecPtrIsInvalid(RecPtr));
+ Assert(state->readBuf != NULL &&
+ state->readBuf == (void *) MAXALIGN(state->readBuf));
ResetDecoder(state);
@@ -366,15 +369,15 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
XLogPageHeader pageHeader;
targetPagePtr =
- state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
- targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+ state->ReadRecPtr - (state->ReadRecPtr % state->pageSize);
+ targetRecOff = state->ReadRecPtr % state->pageSize;
/*
* Check if we have enough data. For the first record in the page,
* the requesting length doesn't contain page header.
*/
if (XLogNeedData(state, targetPagePtr,
- Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+ Min(targetRecOff + SizeOfXLogRecord, state->pageSize),
targetRecOff != 0))
return XLREAD_NEED_DATA;
@@ -421,7 +424,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
* verified that we got the whole header.
*/
prec = (XLogRecord *) (state->readBuf +
- state->ReadRecPtr % XLOG_BLCKSZ);
+ state->ReadRecPtr % state->pageSize);
total_len = prec->xl_tot_len;
/*
@@ -432,7 +435,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
* ensure that we enter the XLREAD_CONTINUATION state below;
* otherwise we might fail to apply ValidXLogRecordHeader at all.
*/
- if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
+ if (targetRecOff <= state->pageSize - SizeOfXLogRecord)
{
if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
state->PrevRecPtr, prec))
@@ -477,10 +480,10 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
* available
*/
targetPagePtr =
- state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
- targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+ state->ReadRecPtr - (state->ReadRecPtr % state->pageSize);
+ targetRecOff = state->ReadRecPtr % state->pageSize;
- request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ);
+ request_len = Min(targetRecOff + total_len, state->pageSize);
record_len = request_len - targetRecOff;
/* ReadRecPtr contains page header */
@@ -549,7 +552,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
/* Calculate pointer to beginning of next page */
state->recordContRecPtr = state->ReadRecPtr + record_len;
- Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0);
+ Assert(state->recordContRecPtr % state->pageSize == 0);
state->readRecordState = XLREAD_CONTINUATION;
}
@@ -576,7 +579,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
/* this request contains page header */
Assert (targetPagePtr != 0);
if (XLogNeedData(state, targetPagePtr,
- Min(state->recordRemainLen, XLOG_BLCKSZ),
+ Min(state->recordRemainLen, state->pageSize),
false))
return XLREAD_NEED_DATA;
@@ -628,7 +631,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
Assert(state->readLen >= pageHeaderSize);
contdata = (char *) state->readBuf + pageHeaderSize;
- record_len = XLOG_BLCKSZ - pageHeaderSize;
+ record_len = state->pageSize - pageHeaderSize;
if (pageHeader->xlp_rem_len < record_len)
record_len = pageHeader->xlp_rem_len;
@@ -655,7 +658,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
}
/* Calculate pointer to beginning of next page, and continue */
- state->recordContRecPtr += XLOG_BLCKSZ;
+ state->recordContRecPtr += state->pageSize;
}
/* targetPagePtr is pointing the last-read page here */
@@ -789,10 +792,10 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
XLogPageHeaderSize((XLogPageHeader) state->readBuf);
addLen = pageHeaderSize;
- if (reqLen + pageHeaderSize <= XLOG_BLCKSZ)
+ if (reqLen + pageHeaderSize <= state->pageSize)
addLen = pageHeaderSize;
else
- addLen = XLOG_BLCKSZ - reqLen;
+ addLen = state->pageSize - reqLen;
Assert(addLen >= 0);
}
@@ -805,7 +808,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
/* Data is not in our buffer, request the caller for it. */
XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize);
targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize);
- Assert((pageptr % XLOG_BLCKSZ) == 0);
+ Assert((pageptr % state->pageSize) == 0);
/*
* Every time we request to load new data of a page to the caller, even if
@@ -829,7 +832,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
* will not come back here, but will request the actual target page.
*/
state->readPagePtr = pageptr - targetPageOff;
- state->readLen = XLOG_BLCKSZ;
+ state->readLen = state->pageSize;
return true;
}
@@ -962,7 +965,7 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
int32 offset;
XLogPageHeader hdr = (XLogPageHeader) phdr;
- Assert((recptr % XLOG_BLCKSZ) == 0);
+ Assert((recptr % state->pageSize) == 0);
XLByteToSeg(recptr, segno, state->segcxt.ws_segsize);
offset = XLogSegmentOffset(recptr, state->segcxt.ws_segsize);
@@ -1016,10 +1019,10 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
"WAL file is from different database system: incorrect segment size in page header");
return false;
}
- else if (longhdr->xlp_xlog_blcksz != XLOG_BLCKSZ)
+ else if (longhdr->xlp_xlog_blcksz != state->pageSize)
{
report_invalid_record(state,
- "WAL file is from different database system: incorrect XLOG_BLCKSZ in page header");
+ "WAL file is from different database system: incorrect block size in page header");
return false;
}
}
@@ -1140,7 +1143,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
* XLogNeedData() is prepared to handle that and will read at
* least short page-header worth of data
*/
- targetRecOff = tmpRecPtr % XLOG_BLCKSZ;
+ targetRecOff = tmpRecPtr % state->pageSize;
/* scroll back to page boundary */
targetPagePtr = tmpRecPtr - targetRecOff;
@@ -1174,8 +1177,8 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
*
* Note that record headers are MAXALIGN'ed
*/
- if (MAXALIGN(header->xlp_rem_len) >= (XLOG_BLCKSZ - pageHeaderSize))
- tmpRecPtr = targetPagePtr + XLOG_BLCKSZ;
+ if (MAXALIGN(header->xlp_rem_len) >= (state->pageSize - pageHeaderSize))
+ tmpRecPtr = targetPagePtr + state->pageSize;
else
{
/*
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index 9cbd8afa49..f2dd495310 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -199,11 +199,14 @@ StartupDecodingContext(List *output_plugin_options,
ctx->slot = slot;
- ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, cleanup_cb);
+ ctx->reader =
+ XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, cleanup_cb);
+
if (!ctx->reader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
errmsg("out of memory")));
+ ctx->reader->readBuf = palloc(XLOG_BLCKSZ);
ctx->page_read = page_read;
ctx->reorder = ReorderBufferAllocate();
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index 062093727d..8e95600ef7 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -579,8 +579,8 @@ StartReplication(StartReplicationCmd *cmd)
errmsg("IDENTIFY_SYSTEM has not been run before START_REPLICATION")));
/* create xlogreader for physical replication */
- xlogreader =
- XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+ xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+ NULL, wal_segment_close);
if (!xlogreader)
ereport(ERROR,
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index 712c85281c..8a6209b061 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -61,7 +61,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
XLogReaderState *xlogreader;
char *errormsg;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+ xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
@@ -121,7 +121,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex,
char *errormsg;
XLogRecPtr endptr;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+ xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
@@ -181,7 +181,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
forkptr += SizeOfXLogShortPHD;
}
- xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+ xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index f514de6b13..b635537659 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -1041,7 +1041,7 @@ main(int argc, char **argv)
/* we have everything we need, start reading */
xlogreader_state =
- XLogReaderAllocate(WalSegSz, waldir, WALDumpCloseSegment);
+ XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, waldir, WALDumpCloseSegment);
if (!xlogreader_state)
fatal_error("out of memory");
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index b9dad7ee51..1853516333 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -151,6 +151,7 @@ struct XLogReaderState
* ----------------------------------------
*/
/* variables to communicate with page reader */
+ int pageSize; /* size of a page */
XLogRecPtr readPagePtr; /* page pointer to read */
int32 readLen; /* bytes requested to reader, or actual bytes
* read by reader, which must be larger than
@@ -243,7 +244,7 @@ struct XLogReaderState
/* Get a new XLogReader */
extern XLogReaderState *XLogReaderAllocate(int wal_segment_size,
- const char *waldir,
+ int page_size, const char *waldir,
WALSegmentCleanupCB cleanup_cb);
/* Free an XLogReader */
--
2.27.0
----Next_Part(Thu_Mar__4_11_28_53_2021_014)----
^ permalink raw reply [nested|flat] 30+ messages in thread
* [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize.
@ 2019-09-05 12:29 Kyotaro Horiguchi <[email protected]>
0 siblings, 0 replies; 30+ messages in thread
From: Kyotaro Horiguchi @ 2019-09-05 12:29 UTC (permalink / raw)
XLog reader facility and its callers currently relies on the same
constant XLOG_BLCKSZ. This patch moves the responsibility to determine
the block size to the callers of the facility.
---
src/backend/access/transam/twophase.c | 3 +-
src/backend/access/transam/xlog.c | 9 ++--
src/backend/access/transam/xlogreader.c | 57 ++++++++++++-----------
src/backend/replication/logical/logical.c | 5 +-
src/backend/replication/walsender.c | 4 +-
src/bin/pg_rewind/parsexlog.c | 6 +--
src/bin/pg_waldump/pg_waldump.c | 2 +-
src/include/access/xlogreader.h | 3 +-
8 files changed, 50 insertions(+), 39 deletions(-)
diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index 7d5a96a5f5..9f9e7de253 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -1325,7 +1325,8 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
XLogReaderState *xlogreader;
char *errormsg;
- xlogreader = XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+ xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+ NULL, wal_segment_close);
if (!xlogreader)
ereport(ERROR,
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index b517ca85ad..0822ac70cb 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -1222,7 +1222,10 @@ XLogInsertRecord(XLogRecData *rdata,
appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len);
if (!debug_reader)
- debug_reader = XLogReaderAllocate(wal_segment_size, NULL, NULL);
+ {
+ debug_reader =
+ XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, NULL);
+ }
if (!debug_reader)
{
@@ -6531,8 +6534,8 @@ StartupXLOG(void)
OwnLatch(&XLogCtl->recoveryWakeupLatch);
/* Set up XLOG reader facility */
- xlogreader =
- XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+ xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+ NULL, wal_segment_close);
if (!xlogreader)
ereport(ERROR,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index ba3e4868f7..e5baf13086 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -72,7 +72,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...)
* Returns NULL if the xlogreader couldn't be allocated.
*/
XLogReaderState *
-XLogReaderAllocate(int wal_segment_size, const char *waldir,
+XLogReaderAllocate(int wal_segment_size, int page_size, const char *waldir,
WALSegmentCleanupCB cleanup_cb)
{
XLogReaderState *state;
@@ -87,6 +87,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
state->cleanup_cb = cleanup_cb;
state->max_block_id = -1;
+ state->pageSize = page_size;
/*
* Permanently allocate readBuf. We do it this way, rather than just
@@ -95,7 +96,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
* isn't guaranteed to have any particular alignment, whereas
* palloc_extended() will provide MAXALIGN'd storage.
*/
- state->readBuf = (char *) palloc_extended(XLOG_BLCKSZ,
+ state->readBuf = (char *) palloc_extended(state->pageSize,
MCXT_ALLOC_NO_OOM);
if (!state->readBuf)
{
@@ -163,7 +164,7 @@ XLogReaderFree(XLogReaderState *state)
* readRecordBufSize is set to the new buffer size.
*
* To avoid useless small increases, round its size to a multiple of
- * XLOG_BLCKSZ, and make sure it's at least 5*Max(BLCKSZ, XLOG_BLCKSZ) to start
+ * page size, and make sure it's at least 5*Max(BLCKSZ, <page size>) to start
* with. (That is enough for all "normal" records, but very large commit or
* abort records might need more space.)
*/
@@ -172,8 +173,8 @@ allocate_recordbuf(XLogReaderState *state, uint32 reclength)
{
uint32 newSize = reclength;
- newSize += XLOG_BLCKSZ - (newSize % XLOG_BLCKSZ);
- newSize = Max(newSize, 5 * Max(BLCKSZ, XLOG_BLCKSZ));
+ newSize += state->pageSize - (newSize % state->pageSize);
+ newSize = Max(newSize, 5 * Max(BLCKSZ, state->pageSize));
#ifndef FRONTEND
@@ -238,6 +239,8 @@ void
XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
{
Assert(!XLogRecPtrIsInvalid(RecPtr));
+ Assert(state->readBuf != NULL &&
+ state->readBuf == (void *) MAXALIGN(state->readBuf));
ResetDecoder(state);
@@ -366,15 +369,15 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
XLogPageHeader pageHeader;
targetPagePtr =
- state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
- targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+ state->ReadRecPtr - (state->ReadRecPtr % state->pageSize);
+ targetRecOff = state->ReadRecPtr % state->pageSize;
/*
* Check if we have enough data. For the first record in the page,
* the requesting length doesn't contain page header.
*/
if (XLogNeedData(state, targetPagePtr,
- Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+ Min(targetRecOff + SizeOfXLogRecord, state->pageSize),
targetRecOff != 0))
return XLREAD_NEED_DATA;
@@ -421,7 +424,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
* verified that we got the whole header.
*/
prec = (XLogRecord *) (state->readBuf +
- state->ReadRecPtr % XLOG_BLCKSZ);
+ state->ReadRecPtr % state->pageSize);
total_len = prec->xl_tot_len;
/*
@@ -432,7 +435,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
* ensure that we enter the XLREAD_CONTINUATION state below;
* otherwise we might fail to apply ValidXLogRecordHeader at all.
*/
- if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
+ if (targetRecOff <= state->pageSize - SizeOfXLogRecord)
{
if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
state->PrevRecPtr, prec))
@@ -477,10 +480,10 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
* available
*/
targetPagePtr =
- state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
- targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+ state->ReadRecPtr - (state->ReadRecPtr % state->pageSize);
+ targetRecOff = state->ReadRecPtr % state->pageSize;
- request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ);
+ request_len = Min(targetRecOff + total_len, state->pageSize);
record_len = request_len - targetRecOff;
/* ReadRecPtr contains page header */
@@ -549,7 +552,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
/* Calculate pointer to beginning of next page */
state->recordContRecPtr = state->ReadRecPtr + record_len;
- Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0);
+ Assert(state->recordContRecPtr % state->pageSize == 0);
state->readRecordState = XLREAD_CONTINUATION;
}
@@ -576,7 +579,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
/* this request contains page header */
Assert (targetPagePtr != 0);
if (XLogNeedData(state, targetPagePtr,
- Min(state->recordRemainLen, XLOG_BLCKSZ),
+ Min(state->recordRemainLen, state->pageSize),
false))
return XLREAD_NEED_DATA;
@@ -628,7 +631,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
Assert(state->readLen >= pageHeaderSize);
contdata = (char *) state->readBuf + pageHeaderSize;
- record_len = XLOG_BLCKSZ - pageHeaderSize;
+ record_len = state->pageSize - pageHeaderSize;
if (pageHeader->xlp_rem_len < record_len)
record_len = pageHeader->xlp_rem_len;
@@ -655,7 +658,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
}
/* Calculate pointer to beginning of next page, and continue */
- state->recordContRecPtr += XLOG_BLCKSZ;
+ state->recordContRecPtr += state->pageSize;
}
/* targetPagePtr is pointing the last-read page here */
@@ -789,10 +792,10 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
XLogPageHeaderSize((XLogPageHeader) state->readBuf);
addLen = pageHeaderSize;
- if (reqLen + pageHeaderSize <= XLOG_BLCKSZ)
+ if (reqLen + pageHeaderSize <= state->pageSize)
addLen = pageHeaderSize;
else
- addLen = XLOG_BLCKSZ - reqLen;
+ addLen = state->pageSize - reqLen;
Assert(addLen >= 0);
}
@@ -805,7 +808,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
/* Data is not in our buffer, request the caller for it. */
XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize);
targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize);
- Assert((pageptr % XLOG_BLCKSZ) == 0);
+ Assert((pageptr % state->pageSize) == 0);
/*
* Every time we request to load new data of a page to the caller, even if
@@ -829,7 +832,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
* will not come back here, but will request the actual target page.
*/
state->readPagePtr = pageptr - targetPageOff;
- state->readLen = XLOG_BLCKSZ;
+ state->readLen = state->pageSize;
return true;
}
@@ -962,7 +965,7 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
int32 offset;
XLogPageHeader hdr = (XLogPageHeader) phdr;
- Assert((recptr % XLOG_BLCKSZ) == 0);
+ Assert((recptr % state->pageSize) == 0);
XLByteToSeg(recptr, segno, state->segcxt.ws_segsize);
offset = XLogSegmentOffset(recptr, state->segcxt.ws_segsize);
@@ -1016,10 +1019,10 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
"WAL file is from different database system: incorrect segment size in page header");
return false;
}
- else if (longhdr->xlp_xlog_blcksz != XLOG_BLCKSZ)
+ else if (longhdr->xlp_xlog_blcksz != state->pageSize)
{
report_invalid_record(state,
- "WAL file is from different database system: incorrect XLOG_BLCKSZ in page header");
+ "WAL file is from different database system: incorrect block size in page header");
return false;
}
}
@@ -1140,7 +1143,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
* XLogNeedData() is prepared to handle that and will read at
* least short page-header worth of data
*/
- targetRecOff = tmpRecPtr % XLOG_BLCKSZ;
+ targetRecOff = tmpRecPtr % state->pageSize;
/* scroll back to page boundary */
targetPagePtr = tmpRecPtr - targetRecOff;
@@ -1174,8 +1177,8 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
*
* Note that record headers are MAXALIGN'ed
*/
- if (MAXALIGN(header->xlp_rem_len) >= (XLOG_BLCKSZ - pageHeaderSize))
- tmpRecPtr = targetPagePtr + XLOG_BLCKSZ;
+ if (MAXALIGN(header->xlp_rem_len) >= (state->pageSize - pageHeaderSize))
+ tmpRecPtr = targetPagePtr + state->pageSize;
else
{
/*
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index 9cbd8afa49..f2dd495310 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -199,11 +199,14 @@ StartupDecodingContext(List *output_plugin_options,
ctx->slot = slot;
- ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, cleanup_cb);
+ ctx->reader =
+ XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, cleanup_cb);
+
if (!ctx->reader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
errmsg("out of memory")));
+ ctx->reader->readBuf = palloc(XLOG_BLCKSZ);
ctx->page_read = page_read;
ctx->reorder = ReorderBufferAllocate();
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index 062093727d..8e95600ef7 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -579,8 +579,8 @@ StartReplication(StartReplicationCmd *cmd)
errmsg("IDENTIFY_SYSTEM has not been run before START_REPLICATION")));
/* create xlogreader for physical replication */
- xlogreader =
- XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+ xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+ NULL, wal_segment_close);
if (!xlogreader)
ereport(ERROR,
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index 712c85281c..8a6209b061 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -61,7 +61,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
XLogReaderState *xlogreader;
char *errormsg;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+ xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
@@ -121,7 +121,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex,
char *errormsg;
XLogRecPtr endptr;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+ xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
@@ -181,7 +181,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
forkptr += SizeOfXLogShortPHD;
}
- xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+ xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index f514de6b13..b635537659 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -1041,7 +1041,7 @@ main(int argc, char **argv)
/* we have everything we need, start reading */
xlogreader_state =
- XLogReaderAllocate(WalSegSz, waldir, WALDumpCloseSegment);
+ XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, waldir, WALDumpCloseSegment);
if (!xlogreader_state)
fatal_error("out of memory");
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index b9dad7ee51..1853516333 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -151,6 +151,7 @@ struct XLogReaderState
* ----------------------------------------
*/
/* variables to communicate with page reader */
+ int pageSize; /* size of a page */
XLogRecPtr readPagePtr; /* page pointer to read */
int32 readLen; /* bytes requested to reader, or actual bytes
* read by reader, which must be larger than
@@ -243,7 +244,7 @@ struct XLogReaderState
/* Get a new XLogReader */
extern XLogReaderState *XLogReaderAllocate(int wal_segment_size,
- const char *waldir,
+ int page_size, const char *waldir,
WALSegmentCleanupCB cleanup_cb);
/* Free an XLogReader */
--
2.27.0
----Next_Part(Thu_Mar__4_11_28_53_2021_014)----
^ permalink raw reply [nested|flat] 30+ messages in thread
* Re: Can't compile PG 17 (master) from git under Msys2 autoconf
@ 2024-04-03 17:52 Alvaro Herrera <[email protected]>
2024-04-03 20:34 ` RE: Can't compile PG 17 (master) from git under Msys2 autoconf Regina Obe <[email protected]>
2024-04-05 02:36 ` RE: Can't compile PG 17 (master) from git under Msys2 autoconf Regina Obe <[email protected]>
0 siblings, 2 replies; 30+ messages in thread
From: Alvaro Herrera @ 2024-04-03 17:52 UTC (permalink / raw)
To: Regina Obe <[email protected]>; +Cc: 'Postgres hackers' <[email protected]>
Hi Regina,
On 2024-Mar-27, Regina Obe wrote:
> The error is
>
> rm -f '../../src/include/storage/lwlocknames.h'
> cp -pR ../../backend/storage/lmgr/lwlocknames.h
> '../../src/include/storage/lwlocknames.h'
> cp: cannot stat '../../backend/storage/lmgr/lwlocknames.h': No such file or
> directory
> make[1]: *** [Makefile:143: ../../src/include/storage/lwlocknames.h] Error 1
> make[1]: Leaving directory '/projects/postgresql/postgresql-git/src/backend'
> make: *** [../../src/Makefile.global:382: submake-generated-headers] Error 2
Hmm, I changed these rules again in commit da952b415f44, maybe your
problem is with that one? I wonder if changing the references to
../include/storage/lwlocklist.h to
$(topdir)/src/include/storage/lwlocklist.h (and similar things in
src/backend/storage/lmgr/Makefile) would fix it.
Do you run configure in the source directory or a separate build one?
--
Álvaro Herrera Breisgau, Deutschland — https://www.EnterpriseDB.com/
"If it is not right, do not do it.
If it is not true, do not say it." (Marcus Aurelius, Meditations)
^ permalink raw reply [nested|flat] 30+ messages in thread
* RE: Can't compile PG 17 (master) from git under Msys2 autoconf
2024-04-03 17:52 Re: Can't compile PG 17 (master) from git under Msys2 autoconf Alvaro Herrera <[email protected]>
@ 2024-04-03 20:34 ` Regina Obe <[email protected]>
1 sibling, 0 replies; 30+ messages in thread
From: Regina Obe @ 2024-04-03 20:34 UTC (permalink / raw)
To: 'Alvaro Herrera' <[email protected]>; +Cc: 'Postgres hackers' <[email protected]>
> Hi Regina,
>
> On 2024-Mar-27, Regina Obe wrote:
>
> > The error is
> >
> > rm -f '../../src/include/storage/lwlocknames.h'
> > cp -pR ../../backend/storage/lmgr/lwlocknames.h
> > '../../src/include/storage/lwlocknames.h'
> > cp: cannot stat '../../backend/storage/lmgr/lwlocknames.h': No such
> > file or directory
> > make[1]: *** [Makefile:143: ../../src/include/storage/lwlocknames.h]
> > Error 1
> > make[1]: Leaving directory '/projects/postgresql/postgresql-
> git/src/backend'
> > make: *** [../../src/Makefile.global:382: submake-generated-headers]
> > Error 2
>
> Hmm, I changed these rules again in commit da952b415f44, maybe your
> problem is with that one? I wonder if changing the references to
> ../include/storage/lwlocklist.h to $(topdir)/src/include/storage/lwlocklist.h
> (and similar things in
> src/backend/storage/lmgr/Makefile) would fix it.
>
> Do you run configure in the source directory or a separate build one?
>
> --
> Álvaro Herrera Breisgau, Deutschland —
> https://www.EnterpriseDB.com/
> "If it is not right, do not do it.
> If it is not true, do not say it." (Marcus Aurelius, Meditations)
I run in the source directory, but tried doing in a separate build directory and ran into the same issue.
Let me look at that commit and get back to you if that makes a difference.
Thanks,
Regina
^ permalink raw reply [nested|flat] 30+ messages in thread
* RE: Can't compile PG 17 (master) from git under Msys2 autoconf
2024-04-03 17:52 Re: Can't compile PG 17 (master) from git under Msys2 autoconf Alvaro Herrera <[email protected]>
@ 2024-04-05 02:36 ` Regina Obe <[email protected]>
2024-04-05 17:41 ` Re: Can't compile PG 17 (master) from git under Msys2 autoconf 'Alvaro Herrera' <[email protected]>
1 sibling, 1 reply; 30+ messages in thread
From: Regina Obe @ 2024-04-05 02:36 UTC (permalink / raw)
To: 'Alvaro Herrera' <[email protected]>; +Cc: 'Postgres hackers' <[email protected]>
> > Hi Regina,
> >
> > On 2024-Mar-27, Regina Obe wrote:
> >
> > > The error is
> > >
> > > rm -f '../../src/include/storage/lwlocknames.h'
> > > cp -pR ../../backend/storage/lmgr/lwlocknames.h
> > > '../../src/include/storage/lwlocknames.h'
> > > cp: cannot stat '../../backend/storage/lmgr/lwlocknames.h': No such
> > > file or directory
> > > make[1]: *** [Makefile:143: ../../src/include/storage/lwlocknames.h]
> > > Error 1
> > > make[1]: Leaving directory '/projects/postgresql/postgresql-
> > git/src/backend'
> > > make: *** [../../src/Makefile.global:382: submake-generated-headers]
> > > Error 2
> >
> > Hmm, I changed these rules again in commit da952b415f44, maybe your
> > problem is with that one? I wonder if changing the references to
> > ../include/storage/lwlocklist.h to
> > $(topdir)/src/include/storage/lwlocklist.h
> > (and similar things in
> > src/backend/storage/lmgr/Makefile) would fix it.
> >
> > Do you run configure in the source directory or a separate build one?
> >
> > --
> > Álvaro Herrera Breisgau, Deutschland —
> > https://www.EnterpriseDB.com/
> > "If it is not right, do not do it.
> > If it is not true, do not say it." (Marcus Aurelius, Meditations)
>
> I tried the change
> > ../include/storage/lwlocklist.h to
> > $(top_builddir)/src/include/storage/lwlocklist.h
>
> I assume you meant that instead of $(topdir)
>
> But nah that made no difference. Your change was already in my patched
> version so isn't causing any issues.
I think I got something not too far off from what's there now that works under my msys2 setup again. This is partly using your idea of using $(top_builddir) to qualify the path but in the LN_S section that is causing me grief.
This seems to work okay building in tree and out of tree.
By changing these lines in src/backend/Makefile
$(top_builddir)/src/include/storage/lwlocknames.h: storage/lmgr/lwlocknames.h
rm -f '$@'
- $(LN_S) ../../backend/$< '$@'
+ $(LN_S) $(top_builddir)/src/backend/$< '$@'
$(top_builddir)/src/include/utils/wait_event_types.h: utils/activity/wait_event_types.h
rm -f '$@'
- $(LN_S) ../../backend/$< '$@'
+ $(LN_S) $(top_builddir)/src/backend/$< '$@'
I've also attached as a patch.
Thanks,
Regina
Attachments:
[application/octet-stream] v1-0001-FIX-autoconf-build-under-Msys2-Mingw64.patch (1.1K, ../../[email protected]/2-v1-0001-FIX-autoconf-build-under-Msys2-Mingw64.patch)
download | inline diff:
From 4be3327532d3f8bcc91a9d33fbef86251aced9a5 Mon Sep 17 00:00:00 2001
From: Regina Obe <[email protected]>
Date: Thu, 4 Apr 2024 22:26:53 -0400
Subject: [PATCH] FIX autoconf build under Msys2/Mingw64
Discussion: https://www.postgresql.org/message-id/flat/000d01da8606%245f63c230%241e2b4690%24%40pcorp.us
---
src/backend/Makefile | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/src/backend/Makefile b/src/backend/Makefile
index 6700aec039..f4b8da6482 100644
--- a/src/backend/Makefile
+++ b/src/backend/Makefile
@@ -140,11 +140,11 @@ generated-headers: $(top_builddir)/src/include/storage/lwlocknames.h $(top_build
$(top_builddir)/src/include/storage/lwlocknames.h: storage/lmgr/lwlocknames.h
rm -f '$@'
- $(LN_S) ../../backend/$< '$@'
+ $(LN_S) $(top_builddir)/src/backend/$< '$@'
$(top_builddir)/src/include/utils/wait_event_types.h: utils/activity/wait_event_types.h
rm -f '$@'
- $(LN_S) ../../backend/$< '$@'
+ $(LN_S) $(top_builddir)/src/backend/$< '$@'
utils/probes.o: utils/probes.d $(SUBDIROBJS)
$(DTRACE) $(DTRACEFLAGS) -C -G -s $(call expand_subsys,$^) -o $@
--
2.42.0.windows.2
^ permalink raw reply [nested|flat] 30+ messages in thread
* Re: Can't compile PG 17 (master) from git under Msys2 autoconf
2024-04-03 17:52 Re: Can't compile PG 17 (master) from git under Msys2 autoconf Alvaro Herrera <[email protected]>
2024-04-05 02:36 ` RE: Can't compile PG 17 (master) from git under Msys2 autoconf Regina Obe <[email protected]>
@ 2024-04-05 17:41 ` 'Alvaro Herrera' <[email protected]>
2024-04-05 17:59 ` RE: Can't compile PG 17 (master) from git under Msys2 autoconf Regina Obe <[email protected]>
0 siblings, 1 reply; 30+ messages in thread
From: 'Alvaro Herrera' @ 2024-04-05 17:41 UTC (permalink / raw)
To: Regina Obe <[email protected]>; +Cc: 'Postgres hackers' <[email protected]>
On 2024-Apr-04, Regina Obe wrote:
> I think I got something not too far off from what's there now that works under my msys2 setup again. This is partly using your idea of using $(top_builddir) to qualify the path but in the LN_S section that is causing me grief.
> This seems to work okay building in tree and out of tree.
> By changing these lines in src/backend/Makefile
>
> $(top_builddir)/src/include/storage/lwlocknames.h: storage/lmgr/lwlocknames.h
> rm -f '$@'
> - $(LN_S) ../../backend/$< '$@'
> + $(LN_S) $(top_builddir)/src/backend/$< '$@'
>
> $(top_builddir)/src/include/utils/wait_event_types.h: utils/activity/wait_event_types.h
> rm -f '$@'
> - $(LN_S) ../../backend/$< '$@'
> + $(LN_S) $(top_builddir)/src/backend/$< '$@'
>
> I've also attached as a patch.
Hmm, that's quite strange ... it certainly doesn't work for me. Maybe
the LN_S utility is resolving the symlink at creation time, rather than
letting it be a reference to be resolved later. Apparently, the only Msys2 buildfarm animal is now using Meson,
so we don't have any representative animal for your situation.
What does LN_S do for you anyway? Looking at
https://stackoverflow.com/questions/61594025/symlink-in-msys2-copy-or-hard-link
it sounds like this would work if the MSYS environment variable was set
to winsymlinks (or maybe not. I don't know if a "windows shortcut" would
be usable in this case.)
--
Álvaro Herrera PostgreSQL Developer — https://www.EnterpriseDB.com/
"Nadie está tan esclavizado como el que se cree libre no siéndolo" (Goethe)
^ permalink raw reply [nested|flat] 30+ messages in thread
* RE: Can't compile PG 17 (master) from git under Msys2 autoconf
2024-04-03 17:52 Re: Can't compile PG 17 (master) from git under Msys2 autoconf Alvaro Herrera <[email protected]>
2024-04-05 02:36 ` RE: Can't compile PG 17 (master) from git under Msys2 autoconf Regina Obe <[email protected]>
2024-04-05 17:41 ` Re: Can't compile PG 17 (master) from git under Msys2 autoconf 'Alvaro Herrera' <[email protected]>
@ 2024-04-05 17:59 ` Regina Obe <[email protected]>
0 siblings, 0 replies; 30+ messages in thread
From: Regina Obe @ 2024-04-05 17:59 UTC (permalink / raw)
To: 'Alvaro Herrera' <[email protected]>; +Cc: 'Postgres hackers' <[email protected]>
> > I think I got something not too far off from what's there now that works
> under my msys2 setup again. This is partly using your idea of using
> $(top_builddir) to qualify the path but in the LN_S section that is causing me
> grief.
> > This seems to work okay building in tree and out of tree.
> > By changing these lines in src/backend/Makefile
> >
> > $(top_builddir)/src/include/storage/lwlocknames.h:
> storage/lmgr/lwlocknames.h
> > rm -f '$@'
> > - $(LN_S) ../../backend/$< '$@'
> > + $(LN_S) $(top_builddir)/src/backend/$< '$@'
> >
> > $(top_builddir)/src/include/utils/wait_event_types.h:
> utils/activity/wait_event_types.h
> > rm -f '$@'
> > - $(LN_S) ../../backend/$< '$@'
> > + $(LN_S) $(top_builddir)/src/backend/$< '$@'
> >
> > I've also attached as a patch.
>
> Hmm, that's quite strange ... it certainly doesn't work for me. Maybe the
> LN_S utility is resolving the symlink at creation time, rather than letting it be a
> reference to be resolved later. Apparently, the only Msys2 buildfarm animal is
> now using Meson, so we don't have any representative animal for your
> situation.
>
> What does LN_S do for you anyway? Looking at
> https://stackoverflow.com/questions/61594025/symlink-in-msys2-copy-or-
> hard-link
> it sounds like this would work if the MSYS environment variable was set to
> winsymlinks (or maybe not. I don't know if a "windows shortcut" would be
> usable in this case.)
>
> --
> Álvaro Herrera PostgreSQL Developer —
> https://www.EnterpriseDB.com/
> "Nadie está tan esclavizado como el que se cree libre no siéndolo" (Goethe)
I think it ends up doing a copy thus the copy error in my log failures which don't exist anywhere in the Makefil
cp -pR ../../backend/storage/lmgr/lwlocknames.h
Sorry for not checking on a linux system. I was thinking I should have done that first.
Thanks,
Regina
^ permalink raw reply [nested|flat] 30+ messages in thread
end of thread, other threads:[~2024-04-05 17:59 UTC | newest]
Thread overview: 30+ messages (download: mbox mbox.gz follow: Atom feed)
-- links below jump to the message on this page --
2019-09-05 12:29 [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize. Kyotaro Horiguchi <[email protected]>
2019-09-05 12:29 [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize. Kyotaro Horiguchi <[email protected]>
2019-09-05 12:29 [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize. Kyotaro Horiguchi <[email protected]>
2019-09-05 12:29 [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize. Kyotaro Horiguchi <[email protected]>
2019-09-05 12:29 [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize. Kyotaro Horiguchi <[email protected]>
2019-09-05 12:29 [PATCH v16 4/4] Allow xlogreader to use different xlog blocksize. Kyotaro Horiguchi <[email protected]>
2019-09-05 12:29 [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize. Kyotaro Horiguchi <[email protected]>
2019-09-05 12:29 [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize. Kyotaro Horiguchi <[email protected]>
2019-09-05 12:29 [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize. Kyotaro Horiguchi <[email protected]>
2019-09-05 12:29 [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize. Kyotaro Horiguchi <[email protected]>
2019-09-05 12:29 [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize. Kyotaro Horiguchi <[email protected]>
2019-09-05 12:29 [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize. Kyotaro Horiguchi <[email protected]>
2019-09-05 12:29 [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize. Kyotaro Horiguchi <[email protected]>
2019-09-05 12:29 [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize. Kyotaro Horiguchi <[email protected]>
2019-09-05 12:29 [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize. Kyotaro Horiguchi <[email protected]>
2019-09-05 12:29 [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize. Kyotaro Horiguchi <[email protected]>
2019-09-05 12:29 [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize. Kyotaro Horiguchi <[email protected]>
2019-09-05 12:29 [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize. Kyotaro Horiguchi <[email protected]>
2019-09-05 12:29 [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize. Kyotaro Horiguchi <[email protected]>
2019-09-05 12:29 [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize. Kyotaro Horiguchi <[email protected]>
2019-09-05 12:29 [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize. Kyotaro Horiguchi <[email protected]>
2019-09-05 12:29 [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize. Kyotaro Horiguchi <[email protected]>
2019-09-05 12:29 [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize. Kyotaro Horiguchi <[email protected]>
2019-09-05 12:29 [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize. Kyotaro Horiguchi <[email protected]>
2019-09-05 12:29 [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize. Kyotaro Horiguchi <[email protected]>
2024-04-03 17:52 Re: Can't compile PG 17 (master) from git under Msys2 autoconf Alvaro Herrera <[email protected]>
2024-04-03 20:34 ` RE: Can't compile PG 17 (master) from git under Msys2 autoconf Regina Obe <[email protected]>
2024-04-05 02:36 ` RE: Can't compile PG 17 (master) from git under Msys2 autoconf Regina Obe <[email protected]>
2024-04-05 17:41 ` Re: Can't compile PG 17 (master) from git under Msys2 autoconf 'Alvaro Herrera' <[email protected]>
2024-04-05 17:59 ` RE: Can't compile PG 17 (master) from git under Msys2 autoconf Regina Obe <[email protected]>
This inbox is served by agora; see mirroring instructions
for how to clone and mirror all data and code used for this inbox