public inbox for [email protected]help / color / mirror / Atom feed
[PATCH v17 4/4] Allow xlogreader to use different xlog blocksize. 26+ messages / 2 participants [nested] [flat]
* [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize. @ 2019-09-05 12:29 Kyotaro Horiguchi <[email protected]> 0 siblings, 0 replies; 26+ messages in thread From: Kyotaro Horiguchi @ 2019-09-05 12:29 UTC (permalink / raw) XLog reader facility and its callers currently relies on the same constant XLOG_BLCKSZ. This patch moves the responsibility to determine the block size to the callers of the facility. --- src/backend/access/transam/twophase.c | 3 +- src/backend/access/transam/xlog.c | 9 ++-- src/backend/access/transam/xlogreader.c | 57 ++++++++++++----------- src/backend/replication/logical/logical.c | 5 +- src/backend/replication/walsender.c | 4 +- src/bin/pg_rewind/parsexlog.c | 6 +-- src/bin/pg_waldump/pg_waldump.c | 2 +- src/include/access/xlogreader.h | 3 +- 8 files changed, 50 insertions(+), 39 deletions(-) diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c index 7d5a96a5f5..9f9e7de253 100644 --- a/src/backend/access/transam/twophase.c +++ b/src/backend/access/transam/twophase.c @@ -1325,7 +1325,8 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len) XLogReaderState *xlogreader; char *errormsg; - xlogreader = XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close); + xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, + NULL, wal_segment_close); if (!xlogreader) ereport(ERROR, diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index b517ca85ad..0822ac70cb 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -1222,7 +1222,10 @@ XLogInsertRecord(XLogRecData *rdata, appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len); if (!debug_reader) - debug_reader = XLogReaderAllocate(wal_segment_size, NULL, NULL); + { + debug_reader = + XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, NULL); + } if (!debug_reader) { @@ -6531,8 +6534,8 @@ StartupXLOG(void) OwnLatch(&XLogCtl->recoveryWakeupLatch); /* Set up XLOG reader facility */ - xlogreader = - XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close); + xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, + NULL, wal_segment_close); if (!xlogreader) ereport(ERROR, diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c index ba3e4868f7..e5baf13086 100644 --- a/src/backend/access/transam/xlogreader.c +++ b/src/backend/access/transam/xlogreader.c @@ -72,7 +72,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...) * Returns NULL if the xlogreader couldn't be allocated. */ XLogReaderState * -XLogReaderAllocate(int wal_segment_size, const char *waldir, +XLogReaderAllocate(int wal_segment_size, int page_size, const char *waldir, WALSegmentCleanupCB cleanup_cb) { XLogReaderState *state; @@ -87,6 +87,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir, state->cleanup_cb = cleanup_cb; state->max_block_id = -1; + state->pageSize = page_size; /* * Permanently allocate readBuf. We do it this way, rather than just @@ -95,7 +96,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir, * isn't guaranteed to have any particular alignment, whereas * palloc_extended() will provide MAXALIGN'd storage. */ - state->readBuf = (char *) palloc_extended(XLOG_BLCKSZ, + state->readBuf = (char *) palloc_extended(state->pageSize, MCXT_ALLOC_NO_OOM); if (!state->readBuf) { @@ -163,7 +164,7 @@ XLogReaderFree(XLogReaderState *state) * readRecordBufSize is set to the new buffer size. * * To avoid useless small increases, round its size to a multiple of - * XLOG_BLCKSZ, and make sure it's at least 5*Max(BLCKSZ, XLOG_BLCKSZ) to start + * page size, and make sure it's at least 5*Max(BLCKSZ, <page size>) to start * with. (That is enough for all "normal" records, but very large commit or * abort records might need more space.) */ @@ -172,8 +173,8 @@ allocate_recordbuf(XLogReaderState *state, uint32 reclength) { uint32 newSize = reclength; - newSize += XLOG_BLCKSZ - (newSize % XLOG_BLCKSZ); - newSize = Max(newSize, 5 * Max(BLCKSZ, XLOG_BLCKSZ)); + newSize += state->pageSize - (newSize % state->pageSize); + newSize = Max(newSize, 5 * Max(BLCKSZ, state->pageSize)); #ifndef FRONTEND @@ -238,6 +239,8 @@ void XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr) { Assert(!XLogRecPtrIsInvalid(RecPtr)); + Assert(state->readBuf != NULL && + state->readBuf == (void *) MAXALIGN(state->readBuf)); ResetDecoder(state); @@ -366,15 +369,15 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) XLogPageHeader pageHeader; targetPagePtr = - state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ); - targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ; + state->ReadRecPtr - (state->ReadRecPtr % state->pageSize); + targetRecOff = state->ReadRecPtr % state->pageSize; /* * Check if we have enough data. For the first record in the page, * the requesting length doesn't contain page header. */ if (XLogNeedData(state, targetPagePtr, - Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ), + Min(targetRecOff + SizeOfXLogRecord, state->pageSize), targetRecOff != 0)) return XLREAD_NEED_DATA; @@ -421,7 +424,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) * verified that we got the whole header. */ prec = (XLogRecord *) (state->readBuf + - state->ReadRecPtr % XLOG_BLCKSZ); + state->ReadRecPtr % state->pageSize); total_len = prec->xl_tot_len; /* @@ -432,7 +435,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) * ensure that we enter the XLREAD_CONTINUATION state below; * otherwise we might fail to apply ValidXLogRecordHeader at all. */ - if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord) + if (targetRecOff <= state->pageSize - SizeOfXLogRecord) { if (!ValidXLogRecordHeader(state, state->ReadRecPtr, state->PrevRecPtr, prec)) @@ -477,10 +480,10 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) * available */ targetPagePtr = - state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ); - targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ; + state->ReadRecPtr - (state->ReadRecPtr % state->pageSize); + targetRecOff = state->ReadRecPtr % state->pageSize; - request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ); + request_len = Min(targetRecOff + total_len, state->pageSize); record_len = request_len - targetRecOff; /* ReadRecPtr contains page header */ @@ -549,7 +552,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) /* Calculate pointer to beginning of next page */ state->recordContRecPtr = state->ReadRecPtr + record_len; - Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0); + Assert(state->recordContRecPtr % state->pageSize == 0); state->readRecordState = XLREAD_CONTINUATION; } @@ -576,7 +579,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) /* this request contains page header */ Assert (targetPagePtr != 0); if (XLogNeedData(state, targetPagePtr, - Min(state->recordRemainLen, XLOG_BLCKSZ), + Min(state->recordRemainLen, state->pageSize), false)) return XLREAD_NEED_DATA; @@ -628,7 +631,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) Assert(state->readLen >= pageHeaderSize); contdata = (char *) state->readBuf + pageHeaderSize; - record_len = XLOG_BLCKSZ - pageHeaderSize; + record_len = state->pageSize - pageHeaderSize; if (pageHeader->xlp_rem_len < record_len) record_len = pageHeader->xlp_rem_len; @@ -655,7 +658,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) } /* Calculate pointer to beginning of next page, and continue */ - state->recordContRecPtr += XLOG_BLCKSZ; + state->recordContRecPtr += state->pageSize; } /* targetPagePtr is pointing the last-read page here */ @@ -789,10 +792,10 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen, XLogPageHeaderSize((XLogPageHeader) state->readBuf); addLen = pageHeaderSize; - if (reqLen + pageHeaderSize <= XLOG_BLCKSZ) + if (reqLen + pageHeaderSize <= state->pageSize) addLen = pageHeaderSize; else - addLen = XLOG_BLCKSZ - reqLen; + addLen = state->pageSize - reqLen; Assert(addLen >= 0); } @@ -805,7 +808,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen, /* Data is not in our buffer, request the caller for it. */ XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize); targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize); - Assert((pageptr % XLOG_BLCKSZ) == 0); + Assert((pageptr % state->pageSize) == 0); /* * Every time we request to load new data of a page to the caller, even if @@ -829,7 +832,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen, * will not come back here, but will request the actual target page. */ state->readPagePtr = pageptr - targetPageOff; - state->readLen = XLOG_BLCKSZ; + state->readLen = state->pageSize; return true; } @@ -962,7 +965,7 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr, int32 offset; XLogPageHeader hdr = (XLogPageHeader) phdr; - Assert((recptr % XLOG_BLCKSZ) == 0); + Assert((recptr % state->pageSize) == 0); XLByteToSeg(recptr, segno, state->segcxt.ws_segsize); offset = XLogSegmentOffset(recptr, state->segcxt.ws_segsize); @@ -1016,10 +1019,10 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr, "WAL file is from different database system: incorrect segment size in page header"); return false; } - else if (longhdr->xlp_xlog_blcksz != XLOG_BLCKSZ) + else if (longhdr->xlp_xlog_blcksz != state->pageSize) { report_invalid_record(state, - "WAL file is from different database system: incorrect XLOG_BLCKSZ in page header"); + "WAL file is from different database system: incorrect block size in page header"); return false; } } @@ -1140,7 +1143,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr, * XLogNeedData() is prepared to handle that and will read at * least short page-header worth of data */ - targetRecOff = tmpRecPtr % XLOG_BLCKSZ; + targetRecOff = tmpRecPtr % state->pageSize; /* scroll back to page boundary */ targetPagePtr = tmpRecPtr - targetRecOff; @@ -1174,8 +1177,8 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr, * * Note that record headers are MAXALIGN'ed */ - if (MAXALIGN(header->xlp_rem_len) >= (XLOG_BLCKSZ - pageHeaderSize)) - tmpRecPtr = targetPagePtr + XLOG_BLCKSZ; + if (MAXALIGN(header->xlp_rem_len) >= (state->pageSize - pageHeaderSize)) + tmpRecPtr = targetPagePtr + state->pageSize; else { /* diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c index 9cbd8afa49..f2dd495310 100644 --- a/src/backend/replication/logical/logical.c +++ b/src/backend/replication/logical/logical.c @@ -199,11 +199,14 @@ StartupDecodingContext(List *output_plugin_options, ctx->slot = slot; - ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, cleanup_cb); + ctx->reader = + XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, cleanup_cb); + if (!ctx->reader) ereport(ERROR, (errcode(ERRCODE_OUT_OF_MEMORY), errmsg("out of memory"))); + ctx->reader->readBuf = palloc(XLOG_BLCKSZ); ctx->page_read = page_read; ctx->reorder = ReorderBufferAllocate(); diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c index 062093727d..8e95600ef7 100644 --- a/src/backend/replication/walsender.c +++ b/src/backend/replication/walsender.c @@ -579,8 +579,8 @@ StartReplication(StartReplicationCmd *cmd) errmsg("IDENTIFY_SYSTEM has not been run before START_REPLICATION"))); /* create xlogreader for physical replication */ - xlogreader = - XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close); + xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, + NULL, wal_segment_close); if (!xlogreader) ereport(ERROR, diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c index 712c85281c..8a6209b061 100644 --- a/src/bin/pg_rewind/parsexlog.c +++ b/src/bin/pg_rewind/parsexlog.c @@ -61,7 +61,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex, XLogReaderState *xlogreader; char *errormsg; - xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL); + xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL); if (xlogreader == NULL) pg_fatal("out of memory"); @@ -121,7 +121,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex, char *errormsg; XLogRecPtr endptr; - xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL); + xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL); if (xlogreader == NULL) pg_fatal("out of memory"); @@ -181,7 +181,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex, forkptr += SizeOfXLogShortPHD; } - xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL); + xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL); if (xlogreader == NULL) pg_fatal("out of memory"); diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c index f514de6b13..b635537659 100644 --- a/src/bin/pg_waldump/pg_waldump.c +++ b/src/bin/pg_waldump/pg_waldump.c @@ -1041,7 +1041,7 @@ main(int argc, char **argv) /* we have everything we need, start reading */ xlogreader_state = - XLogReaderAllocate(WalSegSz, waldir, WALDumpCloseSegment); + XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, waldir, WALDumpCloseSegment); if (!xlogreader_state) fatal_error("out of memory"); diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h index b9dad7ee51..1853516333 100644 --- a/src/include/access/xlogreader.h +++ b/src/include/access/xlogreader.h @@ -151,6 +151,7 @@ struct XLogReaderState * ---------------------------------------- */ /* variables to communicate with page reader */ + int pageSize; /* size of a page */ XLogRecPtr readPagePtr; /* page pointer to read */ int32 readLen; /* bytes requested to reader, or actual bytes * read by reader, which must be larger than @@ -243,7 +244,7 @@ struct XLogReaderState /* Get a new XLogReader */ extern XLogReaderState *XLogReaderAllocate(int wal_segment_size, - const char *waldir, + int page_size, const char *waldir, WALSegmentCleanupCB cleanup_cb); /* Free an XLogReader */ -- 2.27.0 ----Next_Part(Thu_Mar__4_11_28_53_2021_014)---- ^ permalink raw reply [nested|flat] 26+ messages in thread
* [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize. @ 2019-09-05 12:29 Kyotaro Horiguchi <[email protected]> 0 siblings, 0 replies; 26+ messages in thread From: Kyotaro Horiguchi @ 2019-09-05 12:29 UTC (permalink / raw) XLog reader facility and its callers currently relies on the same constant XLOG_BLCKSZ. This patch moves the responsibility to determine the block size to the callers of the facility. --- src/backend/access/transam/twophase.c | 3 +- src/backend/access/transam/xlog.c | 9 ++-- src/backend/access/transam/xlogreader.c | 57 ++++++++++++----------- src/backend/replication/logical/logical.c | 5 +- src/backend/replication/walsender.c | 4 +- src/bin/pg_rewind/parsexlog.c | 6 +-- src/bin/pg_waldump/pg_waldump.c | 2 +- src/include/access/xlogreader.h | 3 +- 8 files changed, 50 insertions(+), 39 deletions(-) diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c index 7d5a96a5f5..9f9e7de253 100644 --- a/src/backend/access/transam/twophase.c +++ b/src/backend/access/transam/twophase.c @@ -1325,7 +1325,8 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len) XLogReaderState *xlogreader; char *errormsg; - xlogreader = XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close); + xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, + NULL, wal_segment_close); if (!xlogreader) ereport(ERROR, diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index b517ca85ad..0822ac70cb 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -1222,7 +1222,10 @@ XLogInsertRecord(XLogRecData *rdata, appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len); if (!debug_reader) - debug_reader = XLogReaderAllocate(wal_segment_size, NULL, NULL); + { + debug_reader = + XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, NULL); + } if (!debug_reader) { @@ -6531,8 +6534,8 @@ StartupXLOG(void) OwnLatch(&XLogCtl->recoveryWakeupLatch); /* Set up XLOG reader facility */ - xlogreader = - XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close); + xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, + NULL, wal_segment_close); if (!xlogreader) ereport(ERROR, diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c index ba3e4868f7..e5baf13086 100644 --- a/src/backend/access/transam/xlogreader.c +++ b/src/backend/access/transam/xlogreader.c @@ -72,7 +72,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...) * Returns NULL if the xlogreader couldn't be allocated. */ XLogReaderState * -XLogReaderAllocate(int wal_segment_size, const char *waldir, +XLogReaderAllocate(int wal_segment_size, int page_size, const char *waldir, WALSegmentCleanupCB cleanup_cb) { XLogReaderState *state; @@ -87,6 +87,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir, state->cleanup_cb = cleanup_cb; state->max_block_id = -1; + state->pageSize = page_size; /* * Permanently allocate readBuf. We do it this way, rather than just @@ -95,7 +96,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir, * isn't guaranteed to have any particular alignment, whereas * palloc_extended() will provide MAXALIGN'd storage. */ - state->readBuf = (char *) palloc_extended(XLOG_BLCKSZ, + state->readBuf = (char *) palloc_extended(state->pageSize, MCXT_ALLOC_NO_OOM); if (!state->readBuf) { @@ -163,7 +164,7 @@ XLogReaderFree(XLogReaderState *state) * readRecordBufSize is set to the new buffer size. * * To avoid useless small increases, round its size to a multiple of - * XLOG_BLCKSZ, and make sure it's at least 5*Max(BLCKSZ, XLOG_BLCKSZ) to start + * page size, and make sure it's at least 5*Max(BLCKSZ, <page size>) to start * with. (That is enough for all "normal" records, but very large commit or * abort records might need more space.) */ @@ -172,8 +173,8 @@ allocate_recordbuf(XLogReaderState *state, uint32 reclength) { uint32 newSize = reclength; - newSize += XLOG_BLCKSZ - (newSize % XLOG_BLCKSZ); - newSize = Max(newSize, 5 * Max(BLCKSZ, XLOG_BLCKSZ)); + newSize += state->pageSize - (newSize % state->pageSize); + newSize = Max(newSize, 5 * Max(BLCKSZ, state->pageSize)); #ifndef FRONTEND @@ -238,6 +239,8 @@ void XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr) { Assert(!XLogRecPtrIsInvalid(RecPtr)); + Assert(state->readBuf != NULL && + state->readBuf == (void *) MAXALIGN(state->readBuf)); ResetDecoder(state); @@ -366,15 +369,15 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) XLogPageHeader pageHeader; targetPagePtr = - state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ); - targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ; + state->ReadRecPtr - (state->ReadRecPtr % state->pageSize); + targetRecOff = state->ReadRecPtr % state->pageSize; /* * Check if we have enough data. For the first record in the page, * the requesting length doesn't contain page header. */ if (XLogNeedData(state, targetPagePtr, - Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ), + Min(targetRecOff + SizeOfXLogRecord, state->pageSize), targetRecOff != 0)) return XLREAD_NEED_DATA; @@ -421,7 +424,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) * verified that we got the whole header. */ prec = (XLogRecord *) (state->readBuf + - state->ReadRecPtr % XLOG_BLCKSZ); + state->ReadRecPtr % state->pageSize); total_len = prec->xl_tot_len; /* @@ -432,7 +435,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) * ensure that we enter the XLREAD_CONTINUATION state below; * otherwise we might fail to apply ValidXLogRecordHeader at all. */ - if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord) + if (targetRecOff <= state->pageSize - SizeOfXLogRecord) { if (!ValidXLogRecordHeader(state, state->ReadRecPtr, state->PrevRecPtr, prec)) @@ -477,10 +480,10 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) * available */ targetPagePtr = - state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ); - targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ; + state->ReadRecPtr - (state->ReadRecPtr % state->pageSize); + targetRecOff = state->ReadRecPtr % state->pageSize; - request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ); + request_len = Min(targetRecOff + total_len, state->pageSize); record_len = request_len - targetRecOff; /* ReadRecPtr contains page header */ @@ -549,7 +552,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) /* Calculate pointer to beginning of next page */ state->recordContRecPtr = state->ReadRecPtr + record_len; - Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0); + Assert(state->recordContRecPtr % state->pageSize == 0); state->readRecordState = XLREAD_CONTINUATION; } @@ -576,7 +579,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) /* this request contains page header */ Assert (targetPagePtr != 0); if (XLogNeedData(state, targetPagePtr, - Min(state->recordRemainLen, XLOG_BLCKSZ), + Min(state->recordRemainLen, state->pageSize), false)) return XLREAD_NEED_DATA; @@ -628,7 +631,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) Assert(state->readLen >= pageHeaderSize); contdata = (char *) state->readBuf + pageHeaderSize; - record_len = XLOG_BLCKSZ - pageHeaderSize; + record_len = state->pageSize - pageHeaderSize; if (pageHeader->xlp_rem_len < record_len) record_len = pageHeader->xlp_rem_len; @@ -655,7 +658,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) } /* Calculate pointer to beginning of next page, and continue */ - state->recordContRecPtr += XLOG_BLCKSZ; + state->recordContRecPtr += state->pageSize; } /* targetPagePtr is pointing the last-read page here */ @@ -789,10 +792,10 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen, XLogPageHeaderSize((XLogPageHeader) state->readBuf); addLen = pageHeaderSize; - if (reqLen + pageHeaderSize <= XLOG_BLCKSZ) + if (reqLen + pageHeaderSize <= state->pageSize) addLen = pageHeaderSize; else - addLen = XLOG_BLCKSZ - reqLen; + addLen = state->pageSize - reqLen; Assert(addLen >= 0); } @@ -805,7 +808,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen, /* Data is not in our buffer, request the caller for it. */ XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize); targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize); - Assert((pageptr % XLOG_BLCKSZ) == 0); + Assert((pageptr % state->pageSize) == 0); /* * Every time we request to load new data of a page to the caller, even if @@ -829,7 +832,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen, * will not come back here, but will request the actual target page. */ state->readPagePtr = pageptr - targetPageOff; - state->readLen = XLOG_BLCKSZ; + state->readLen = state->pageSize; return true; } @@ -962,7 +965,7 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr, int32 offset; XLogPageHeader hdr = (XLogPageHeader) phdr; - Assert((recptr % XLOG_BLCKSZ) == 0); + Assert((recptr % state->pageSize) == 0); XLByteToSeg(recptr, segno, state->segcxt.ws_segsize); offset = XLogSegmentOffset(recptr, state->segcxt.ws_segsize); @@ -1016,10 +1019,10 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr, "WAL file is from different database system: incorrect segment size in page header"); return false; } - else if (longhdr->xlp_xlog_blcksz != XLOG_BLCKSZ) + else if (longhdr->xlp_xlog_blcksz != state->pageSize) { report_invalid_record(state, - "WAL file is from different database system: incorrect XLOG_BLCKSZ in page header"); + "WAL file is from different database system: incorrect block size in page header"); return false; } } @@ -1140,7 +1143,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr, * XLogNeedData() is prepared to handle that and will read at * least short page-header worth of data */ - targetRecOff = tmpRecPtr % XLOG_BLCKSZ; + targetRecOff = tmpRecPtr % state->pageSize; /* scroll back to page boundary */ targetPagePtr = tmpRecPtr - targetRecOff; @@ -1174,8 +1177,8 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr, * * Note that record headers are MAXALIGN'ed */ - if (MAXALIGN(header->xlp_rem_len) >= (XLOG_BLCKSZ - pageHeaderSize)) - tmpRecPtr = targetPagePtr + XLOG_BLCKSZ; + if (MAXALIGN(header->xlp_rem_len) >= (state->pageSize - pageHeaderSize)) + tmpRecPtr = targetPagePtr + state->pageSize; else { /* diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c index 9cbd8afa49..f2dd495310 100644 --- a/src/backend/replication/logical/logical.c +++ b/src/backend/replication/logical/logical.c @@ -199,11 +199,14 @@ StartupDecodingContext(List *output_plugin_options, ctx->slot = slot; - ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, cleanup_cb); + ctx->reader = + XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, cleanup_cb); + if (!ctx->reader) ereport(ERROR, (errcode(ERRCODE_OUT_OF_MEMORY), errmsg("out of memory"))); + ctx->reader->readBuf = palloc(XLOG_BLCKSZ); ctx->page_read = page_read; ctx->reorder = ReorderBufferAllocate(); diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c index 062093727d..8e95600ef7 100644 --- a/src/backend/replication/walsender.c +++ b/src/backend/replication/walsender.c @@ -579,8 +579,8 @@ StartReplication(StartReplicationCmd *cmd) errmsg("IDENTIFY_SYSTEM has not been run before START_REPLICATION"))); /* create xlogreader for physical replication */ - xlogreader = - XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close); + xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, + NULL, wal_segment_close); if (!xlogreader) ereport(ERROR, diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c index 712c85281c..8a6209b061 100644 --- a/src/bin/pg_rewind/parsexlog.c +++ b/src/bin/pg_rewind/parsexlog.c @@ -61,7 +61,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex, XLogReaderState *xlogreader; char *errormsg; - xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL); + xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL); if (xlogreader == NULL) pg_fatal("out of memory"); @@ -121,7 +121,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex, char *errormsg; XLogRecPtr endptr; - xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL); + xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL); if (xlogreader == NULL) pg_fatal("out of memory"); @@ -181,7 +181,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex, forkptr += SizeOfXLogShortPHD; } - xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL); + xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL); if (xlogreader == NULL) pg_fatal("out of memory"); diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c index f514de6b13..b635537659 100644 --- a/src/bin/pg_waldump/pg_waldump.c +++ b/src/bin/pg_waldump/pg_waldump.c @@ -1041,7 +1041,7 @@ main(int argc, char **argv) /* we have everything we need, start reading */ xlogreader_state = - XLogReaderAllocate(WalSegSz, waldir, WALDumpCloseSegment); + XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, waldir, WALDumpCloseSegment); if (!xlogreader_state) fatal_error("out of memory"); diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h index b9dad7ee51..1853516333 100644 --- a/src/include/access/xlogreader.h +++ b/src/include/access/xlogreader.h @@ -151,6 +151,7 @@ struct XLogReaderState * ---------------------------------------- */ /* variables to communicate with page reader */ + int pageSize; /* size of a page */ XLogRecPtr readPagePtr; /* page pointer to read */ int32 readLen; /* bytes requested to reader, or actual bytes * read by reader, which must be larger than @@ -243,7 +244,7 @@ struct XLogReaderState /* Get a new XLogReader */ extern XLogReaderState *XLogReaderAllocate(int wal_segment_size, - const char *waldir, + int page_size, const char *waldir, WALSegmentCleanupCB cleanup_cb); /* Free an XLogReader */ -- 2.27.0 ----Next_Part(Thu_Mar__4_11_28_53_2021_014)---- ^ permalink raw reply [nested|flat] 26+ messages in thread
* [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize. @ 2019-09-05 12:29 Kyotaro Horiguchi <[email protected]> 0 siblings, 0 replies; 26+ messages in thread From: Kyotaro Horiguchi @ 2019-09-05 12:29 UTC (permalink / raw) XLog reader facility and its callers currently relies on the same constant XLOG_BLCKSZ. This patch moves the responsibility to determine the block size to the callers of the facility. --- src/backend/access/transam/twophase.c | 3 +- src/backend/access/transam/xlog.c | 9 ++-- src/backend/access/transam/xlogreader.c | 57 ++++++++++++----------- src/backend/replication/logical/logical.c | 5 +- src/backend/replication/walsender.c | 4 +- src/bin/pg_rewind/parsexlog.c | 6 +-- src/bin/pg_waldump/pg_waldump.c | 2 +- src/include/access/xlogreader.h | 3 +- 8 files changed, 50 insertions(+), 39 deletions(-) diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c index 7d5a96a5f5..9f9e7de253 100644 --- a/src/backend/access/transam/twophase.c +++ b/src/backend/access/transam/twophase.c @@ -1325,7 +1325,8 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len) XLogReaderState *xlogreader; char *errormsg; - xlogreader = XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close); + xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, + NULL, wal_segment_close); if (!xlogreader) ereport(ERROR, diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index b517ca85ad..0822ac70cb 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -1222,7 +1222,10 @@ XLogInsertRecord(XLogRecData *rdata, appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len); if (!debug_reader) - debug_reader = XLogReaderAllocate(wal_segment_size, NULL, NULL); + { + debug_reader = + XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, NULL); + } if (!debug_reader) { @@ -6531,8 +6534,8 @@ StartupXLOG(void) OwnLatch(&XLogCtl->recoveryWakeupLatch); /* Set up XLOG reader facility */ - xlogreader = - XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close); + xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, + NULL, wal_segment_close); if (!xlogreader) ereport(ERROR, diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c index ba3e4868f7..e5baf13086 100644 --- a/src/backend/access/transam/xlogreader.c +++ b/src/backend/access/transam/xlogreader.c @@ -72,7 +72,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...) * Returns NULL if the xlogreader couldn't be allocated. */ XLogReaderState * -XLogReaderAllocate(int wal_segment_size, const char *waldir, +XLogReaderAllocate(int wal_segment_size, int page_size, const char *waldir, WALSegmentCleanupCB cleanup_cb) { XLogReaderState *state; @@ -87,6 +87,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir, state->cleanup_cb = cleanup_cb; state->max_block_id = -1; + state->pageSize = page_size; /* * Permanently allocate readBuf. We do it this way, rather than just @@ -95,7 +96,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir, * isn't guaranteed to have any particular alignment, whereas * palloc_extended() will provide MAXALIGN'd storage. */ - state->readBuf = (char *) palloc_extended(XLOG_BLCKSZ, + state->readBuf = (char *) palloc_extended(state->pageSize, MCXT_ALLOC_NO_OOM); if (!state->readBuf) { @@ -163,7 +164,7 @@ XLogReaderFree(XLogReaderState *state) * readRecordBufSize is set to the new buffer size. * * To avoid useless small increases, round its size to a multiple of - * XLOG_BLCKSZ, and make sure it's at least 5*Max(BLCKSZ, XLOG_BLCKSZ) to start + * page size, and make sure it's at least 5*Max(BLCKSZ, <page size>) to start * with. (That is enough for all "normal" records, but very large commit or * abort records might need more space.) */ @@ -172,8 +173,8 @@ allocate_recordbuf(XLogReaderState *state, uint32 reclength) { uint32 newSize = reclength; - newSize += XLOG_BLCKSZ - (newSize % XLOG_BLCKSZ); - newSize = Max(newSize, 5 * Max(BLCKSZ, XLOG_BLCKSZ)); + newSize += state->pageSize - (newSize % state->pageSize); + newSize = Max(newSize, 5 * Max(BLCKSZ, state->pageSize)); #ifndef FRONTEND @@ -238,6 +239,8 @@ void XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr) { Assert(!XLogRecPtrIsInvalid(RecPtr)); + Assert(state->readBuf != NULL && + state->readBuf == (void *) MAXALIGN(state->readBuf)); ResetDecoder(state); @@ -366,15 +369,15 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) XLogPageHeader pageHeader; targetPagePtr = - state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ); - targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ; + state->ReadRecPtr - (state->ReadRecPtr % state->pageSize); + targetRecOff = state->ReadRecPtr % state->pageSize; /* * Check if we have enough data. For the first record in the page, * the requesting length doesn't contain page header. */ if (XLogNeedData(state, targetPagePtr, - Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ), + Min(targetRecOff + SizeOfXLogRecord, state->pageSize), targetRecOff != 0)) return XLREAD_NEED_DATA; @@ -421,7 +424,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) * verified that we got the whole header. */ prec = (XLogRecord *) (state->readBuf + - state->ReadRecPtr % XLOG_BLCKSZ); + state->ReadRecPtr % state->pageSize); total_len = prec->xl_tot_len; /* @@ -432,7 +435,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) * ensure that we enter the XLREAD_CONTINUATION state below; * otherwise we might fail to apply ValidXLogRecordHeader at all. */ - if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord) + if (targetRecOff <= state->pageSize - SizeOfXLogRecord) { if (!ValidXLogRecordHeader(state, state->ReadRecPtr, state->PrevRecPtr, prec)) @@ -477,10 +480,10 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) * available */ targetPagePtr = - state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ); - targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ; + state->ReadRecPtr - (state->ReadRecPtr % state->pageSize); + targetRecOff = state->ReadRecPtr % state->pageSize; - request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ); + request_len = Min(targetRecOff + total_len, state->pageSize); record_len = request_len - targetRecOff; /* ReadRecPtr contains page header */ @@ -549,7 +552,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) /* Calculate pointer to beginning of next page */ state->recordContRecPtr = state->ReadRecPtr + record_len; - Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0); + Assert(state->recordContRecPtr % state->pageSize == 0); state->readRecordState = XLREAD_CONTINUATION; } @@ -576,7 +579,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) /* this request contains page header */ Assert (targetPagePtr != 0); if (XLogNeedData(state, targetPagePtr, - Min(state->recordRemainLen, XLOG_BLCKSZ), + Min(state->recordRemainLen, state->pageSize), false)) return XLREAD_NEED_DATA; @@ -628,7 +631,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) Assert(state->readLen >= pageHeaderSize); contdata = (char *) state->readBuf + pageHeaderSize; - record_len = XLOG_BLCKSZ - pageHeaderSize; + record_len = state->pageSize - pageHeaderSize; if (pageHeader->xlp_rem_len < record_len) record_len = pageHeader->xlp_rem_len; @@ -655,7 +658,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) } /* Calculate pointer to beginning of next page, and continue */ - state->recordContRecPtr += XLOG_BLCKSZ; + state->recordContRecPtr += state->pageSize; } /* targetPagePtr is pointing the last-read page here */ @@ -789,10 +792,10 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen, XLogPageHeaderSize((XLogPageHeader) state->readBuf); addLen = pageHeaderSize; - if (reqLen + pageHeaderSize <= XLOG_BLCKSZ) + if (reqLen + pageHeaderSize <= state->pageSize) addLen = pageHeaderSize; else - addLen = XLOG_BLCKSZ - reqLen; + addLen = state->pageSize - reqLen; Assert(addLen >= 0); } @@ -805,7 +808,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen, /* Data is not in our buffer, request the caller for it. */ XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize); targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize); - Assert((pageptr % XLOG_BLCKSZ) == 0); + Assert((pageptr % state->pageSize) == 0); /* * Every time we request to load new data of a page to the caller, even if @@ -829,7 +832,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen, * will not come back here, but will request the actual target page. */ state->readPagePtr = pageptr - targetPageOff; - state->readLen = XLOG_BLCKSZ; + state->readLen = state->pageSize; return true; } @@ -962,7 +965,7 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr, int32 offset; XLogPageHeader hdr = (XLogPageHeader) phdr; - Assert((recptr % XLOG_BLCKSZ) == 0); + Assert((recptr % state->pageSize) == 0); XLByteToSeg(recptr, segno, state->segcxt.ws_segsize); offset = XLogSegmentOffset(recptr, state->segcxt.ws_segsize); @@ -1016,10 +1019,10 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr, "WAL file is from different database system: incorrect segment size in page header"); return false; } - else if (longhdr->xlp_xlog_blcksz != XLOG_BLCKSZ) + else if (longhdr->xlp_xlog_blcksz != state->pageSize) { report_invalid_record(state, - "WAL file is from different database system: incorrect XLOG_BLCKSZ in page header"); + "WAL file is from different database system: incorrect block size in page header"); return false; } } @@ -1140,7 +1143,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr, * XLogNeedData() is prepared to handle that and will read at * least short page-header worth of data */ - targetRecOff = tmpRecPtr % XLOG_BLCKSZ; + targetRecOff = tmpRecPtr % state->pageSize; /* scroll back to page boundary */ targetPagePtr = tmpRecPtr - targetRecOff; @@ -1174,8 +1177,8 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr, * * Note that record headers are MAXALIGN'ed */ - if (MAXALIGN(header->xlp_rem_len) >= (XLOG_BLCKSZ - pageHeaderSize)) - tmpRecPtr = targetPagePtr + XLOG_BLCKSZ; + if (MAXALIGN(header->xlp_rem_len) >= (state->pageSize - pageHeaderSize)) + tmpRecPtr = targetPagePtr + state->pageSize; else { /* diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c index 9cbd8afa49..f2dd495310 100644 --- a/src/backend/replication/logical/logical.c +++ b/src/backend/replication/logical/logical.c @@ -199,11 +199,14 @@ StartupDecodingContext(List *output_plugin_options, ctx->slot = slot; - ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, cleanup_cb); + ctx->reader = + XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, cleanup_cb); + if (!ctx->reader) ereport(ERROR, (errcode(ERRCODE_OUT_OF_MEMORY), errmsg("out of memory"))); + ctx->reader->readBuf = palloc(XLOG_BLCKSZ); ctx->page_read = page_read; ctx->reorder = ReorderBufferAllocate(); diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c index 062093727d..8e95600ef7 100644 --- a/src/backend/replication/walsender.c +++ b/src/backend/replication/walsender.c @@ -579,8 +579,8 @@ StartReplication(StartReplicationCmd *cmd) errmsg("IDENTIFY_SYSTEM has not been run before START_REPLICATION"))); /* create xlogreader for physical replication */ - xlogreader = - XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close); + xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, + NULL, wal_segment_close); if (!xlogreader) ereport(ERROR, diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c index 712c85281c..8a6209b061 100644 --- a/src/bin/pg_rewind/parsexlog.c +++ b/src/bin/pg_rewind/parsexlog.c @@ -61,7 +61,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex, XLogReaderState *xlogreader; char *errormsg; - xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL); + xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL); if (xlogreader == NULL) pg_fatal("out of memory"); @@ -121,7 +121,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex, char *errormsg; XLogRecPtr endptr; - xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL); + xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL); if (xlogreader == NULL) pg_fatal("out of memory"); @@ -181,7 +181,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex, forkptr += SizeOfXLogShortPHD; } - xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL); + xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL); if (xlogreader == NULL) pg_fatal("out of memory"); diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c index f514de6b13..b635537659 100644 --- a/src/bin/pg_waldump/pg_waldump.c +++ b/src/bin/pg_waldump/pg_waldump.c @@ -1041,7 +1041,7 @@ main(int argc, char **argv) /* we have everything we need, start reading */ xlogreader_state = - XLogReaderAllocate(WalSegSz, waldir, WALDumpCloseSegment); + XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, waldir, WALDumpCloseSegment); if (!xlogreader_state) fatal_error("out of memory"); diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h index b9dad7ee51..1853516333 100644 --- a/src/include/access/xlogreader.h +++ b/src/include/access/xlogreader.h @@ -151,6 +151,7 @@ struct XLogReaderState * ---------------------------------------- */ /* variables to communicate with page reader */ + int pageSize; /* size of a page */ XLogRecPtr readPagePtr; /* page pointer to read */ int32 readLen; /* bytes requested to reader, or actual bytes * read by reader, which must be larger than @@ -243,7 +244,7 @@ struct XLogReaderState /* Get a new XLogReader */ extern XLogReaderState *XLogReaderAllocate(int wal_segment_size, - const char *waldir, + int page_size, const char *waldir, WALSegmentCleanupCB cleanup_cb); /* Free an XLogReader */ -- 2.27.0 ----Next_Part(Thu_Mar__4_11_28_53_2021_014)---- ^ permalink raw reply [nested|flat] 26+ messages in thread
* [PATCH v16 4/4] Allow xlogreader to use different xlog blocksize. @ 2019-09-05 12:29 Kyotaro Horiguchi <[email protected]> 0 siblings, 0 replies; 26+ messages in thread From: Kyotaro Horiguchi @ 2019-09-05 12:29 UTC (permalink / raw) XLog reader facility and its callers currently relies on the same constant XLOG_BLCKSZ. This patch moves the responsibility to determine the block size to the callers of the facility. --- src/backend/access/transam/twophase.c | 3 +- src/backend/access/transam/xlog.c | 9 ++-- src/backend/access/transam/xlogreader.c | 57 ++++++++++++----------- src/backend/replication/logical/logical.c | 5 +- src/backend/replication/walsender.c | 4 +- src/bin/pg_rewind/parsexlog.c | 6 +-- src/bin/pg_waldump/pg_waldump.c | 2 +- src/include/access/xlogreader.h | 3 +- 8 files changed, 50 insertions(+), 39 deletions(-) diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c index 9a56b3487e..b64c61f742 100644 --- a/src/backend/access/transam/twophase.c +++ b/src/backend/access/transam/twophase.c @@ -1324,7 +1324,8 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len) XLogReaderState *xlogreader; char *errormsg; - xlogreader = XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close); + xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, + NULL, wal_segment_close); if (!xlogreader) ereport(ERROR, diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index eccb88b611..ec78de5dd9 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -1213,7 +1213,10 @@ XLogInsertRecord(XLogRecData *rdata, appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len); if (!debug_reader) - debug_reader = XLogReaderAllocate(wal_segment_size, NULL, NULL); + { + debug_reader = + XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, NULL); + } if (!debug_reader) { @@ -6469,8 +6472,8 @@ StartupXLOG(void) OwnLatch(&XLogCtl->recoveryWakeupLatch); /* Set up XLOG reader facility */ - xlogreader = - XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close); + xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, + NULL, wal_segment_close); if (!xlogreader) ereport(ERROR, diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c index d829e57baa..582b1dd11b 100644 --- a/src/backend/access/transam/xlogreader.c +++ b/src/backend/access/transam/xlogreader.c @@ -72,7 +72,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...) * Returns NULL if the xlogreader couldn't be allocated. */ XLogReaderState * -XLogReaderAllocate(int wal_segment_size, const char *waldir, +XLogReaderAllocate(int wal_segment_size, int page_size, const char *waldir, WALSegmentCleanupCB cleanup_cb) { XLogReaderState *state; @@ -87,6 +87,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir, state->cleanup_cb = cleanup_cb; state->max_block_id = -1; + state->pageSize = page_size; /* * Permanently allocate readBuf. We do it this way, rather than just @@ -95,7 +96,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir, * isn't guaranteed to have any particular alignment, whereas * palloc_extended() will provide MAXALIGN'd storage. */ - state->readBuf = (char *) palloc_extended(XLOG_BLCKSZ, + state->readBuf = (char *) palloc_extended(state->pageSize, MCXT_ALLOC_NO_OOM); if (!state->readBuf) { @@ -163,7 +164,7 @@ XLogReaderFree(XLogReaderState *state) * readRecordBufSize is set to the new buffer size. * * To avoid useless small increases, round its size to a multiple of - * XLOG_BLCKSZ, and make sure it's at least 5*Max(BLCKSZ, XLOG_BLCKSZ) to start + * page size, and make sure it's at least 5*Max(BLCKSZ, <page size>) to start * with. (That is enough for all "normal" records, but very large commit or * abort records might need more space.) */ @@ -172,8 +173,8 @@ allocate_recordbuf(XLogReaderState *state, uint32 reclength) { uint32 newSize = reclength; - newSize += XLOG_BLCKSZ - (newSize % XLOG_BLCKSZ); - newSize = Max(newSize, 5 * Max(BLCKSZ, XLOG_BLCKSZ)); + newSize += state->pageSize - (newSize % state->pageSize); + newSize = Max(newSize, 5 * Max(BLCKSZ, state->pageSize)); #ifndef FRONTEND @@ -238,6 +239,8 @@ void XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr) { Assert(!XLogRecPtrIsInvalid(RecPtr)); + Assert(state->readBuf != NULL && + state->readBuf == (void *) MAXALIGN(state->readBuf)); ResetDecoder(state); @@ -366,15 +369,15 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) XLogPageHeader pageHeader; targetPagePtr = - state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ); - targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ; + state->ReadRecPtr - (state->ReadRecPtr % state->pageSize); + targetRecOff = state->ReadRecPtr % state->pageSize; /* * Check if we have enough data. For the first record in the page, * the requesting length doesn't contain page header. */ if (XLogNeedData(state, targetPagePtr, - Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ), + Min(targetRecOff + SizeOfXLogRecord, state->pageSize), targetRecOff != 0)) return XLREAD_NEED_DATA; @@ -422,7 +425,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) * verified that we got the whole header. */ prec = (XLogRecord *) (state->readBuf + - state->ReadRecPtr % XLOG_BLCKSZ); + state->ReadRecPtr % state->pageSize); total_len = prec->xl_tot_len; /* @@ -433,7 +436,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) * ensure that we enter the XLREAD_CONTINUATION state below; * otherwise we might fail to apply ValidXLogRecordHeader at all. */ - if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord) + if (targetRecOff <= state->pageSize - SizeOfXLogRecord) { if (!ValidXLogRecordHeader(state, state->ReadRecPtr, state->PrevRecPtr, prec)) @@ -479,10 +482,10 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) * available */ targetPagePtr = - state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ); - targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ; + state->ReadRecPtr - (state->ReadRecPtr % state->pageSize); + targetRecOff = state->ReadRecPtr % state->pageSize; - request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ); + request_len = Min(targetRecOff + total_len, state->pageSize); record_len = request_len - targetRecOff; /* ReadRecPtr contains page header */ @@ -552,7 +555,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) /* Calculate pointer to beginning of next page */ state->recordContRecPtr = state->ReadRecPtr + record_len; - Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0); + Assert(state->recordContRecPtr % state->pageSize == 0); state->readRecordState = XLREAD_CONTINUATION; } @@ -579,7 +582,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) /* this request contains page header */ Assert (targetPagePtr != 0); if (XLogNeedData(state, targetPagePtr, - Min(state->recordRemainLen, XLOG_BLCKSZ), + Min(state->recordRemainLen, state->pageSize), false)) return XLREAD_NEED_DATA; @@ -631,7 +634,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) Assert(state->readLen >= pageHeaderSize); contdata = (char *) state->readBuf + pageHeaderSize; - record_len = XLOG_BLCKSZ - pageHeaderSize; + record_len = state->pageSize - pageHeaderSize; if (pageHeader->xlp_rem_len < record_len) record_len = pageHeader->xlp_rem_len; @@ -658,7 +661,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) } /* Calculate pointer to beginning of next page, and continue */ - state->recordContRecPtr += XLOG_BLCKSZ; + state->recordContRecPtr += state->pageSize; } /* targetPagePtr is pointing the last-read page here */ @@ -792,10 +795,10 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen, XLogPageHeaderSize((XLogPageHeader) state->readBuf); addLen = pageHeaderSize; - if (reqLen + pageHeaderSize <= XLOG_BLCKSZ) + if (reqLen + pageHeaderSize <= state->pageSize) addLen = pageHeaderSize; else - addLen = XLOG_BLCKSZ - reqLen; + addLen = state->pageSize - reqLen; Assert(addLen >= 0); } @@ -808,7 +811,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen, /* Data is not in our buffer, request the caller for it. */ XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize); targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize); - Assert((pageptr % XLOG_BLCKSZ) == 0); + Assert((pageptr % state->pageSize) == 0); /* * Every time we request to load new data of a page to the caller, even if @@ -832,7 +835,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen, * will not come back here, but will request the actual target page. */ state->readPagePtr = pageptr - targetPageOff; - state->readLen = XLOG_BLCKSZ; + state->readLen = state->pageSize; return true; } @@ -968,7 +971,7 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr, int32 offset; XLogPageHeader hdr = (XLogPageHeader) phdr; - Assert((recptr % XLOG_BLCKSZ) == 0); + Assert((recptr % state->pageSize) == 0); XLByteToSeg(recptr, segno, state->segcxt.ws_segsize); offset = XLogSegmentOffset(recptr, state->segcxt.ws_segsize); @@ -1022,10 +1025,10 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr, "WAL file is from different database system: incorrect segment size in page header"); return false; } - else if (longhdr->xlp_xlog_blcksz != XLOG_BLCKSZ) + else if (longhdr->xlp_xlog_blcksz != state->pageSize) { report_invalid_record(state, - "WAL file is from different database system: incorrect XLOG_BLCKSZ in page header"); + "WAL file is from different database system: incorrect block size in page header"); return false; } } @@ -1146,7 +1149,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr, * XLogNeedData() is prepared to handle that and will read at * least short page-header worth of data */ - targetRecOff = tmpRecPtr % XLOG_BLCKSZ; + targetRecOff = tmpRecPtr % state->pageSize; /* scroll back to page boundary */ targetPagePtr = tmpRecPtr - targetRecOff; @@ -1180,8 +1183,8 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr, * * Note that record headers are MAXALIGN'ed */ - if (MAXALIGN(header->xlp_rem_len) >= (XLOG_BLCKSZ - pageHeaderSize)) - tmpRecPtr = targetPagePtr + XLOG_BLCKSZ; + if (MAXALIGN(header->xlp_rem_len) >= (state->pageSize - pageHeaderSize)) + tmpRecPtr = targetPagePtr + state->pageSize; else { /* diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c index 97def6f781..ee322ba796 100644 --- a/src/backend/replication/logical/logical.c +++ b/src/backend/replication/logical/logical.c @@ -189,11 +189,14 @@ StartupDecodingContext(List *output_plugin_options, ctx->slot = slot; - ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, cleanup_cb); + ctx->reader = + XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, cleanup_cb); + if (!ctx->reader) ereport(ERROR, (errcode(ERRCODE_OUT_OF_MEMORY), errmsg("out of memory"))); + ctx->reader->readBuf = palloc(XLOG_BLCKSZ); ctx->page_read = page_read; ctx->reorder = ReorderBufferAllocate(); diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c index 33d952f9fd..a0914ddb25 100644 --- a/src/backend/replication/walsender.c +++ b/src/backend/replication/walsender.c @@ -578,8 +578,8 @@ StartReplication(StartReplicationCmd *cmd) errmsg("IDENTIFY_SYSTEM has not been run before START_REPLICATION"))); /* create xlogreader for physical replication */ - xlogreader = - XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close); + xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, + NULL, wal_segment_close); if (!xlogreader) ereport(ERROR, diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c index 7344a7c692..c5881594b7 100644 --- a/src/bin/pg_rewind/parsexlog.c +++ b/src/bin/pg_rewind/parsexlog.c @@ -58,7 +58,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex, XLogReaderState *xlogreader; char *errormsg; - xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL); + xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL); if (xlogreader == NULL) pg_fatal("out of memory"); @@ -112,7 +112,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex, char *errormsg; XLogRecPtr endptr; - xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL); + xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL); if (xlogreader == NULL) pg_fatal("out of memory"); @@ -172,7 +172,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex, forkptr += SizeOfXLogShortPHD; } - xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL); + xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL); if (xlogreader == NULL) pg_fatal("out of memory"); diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c index 9329a6a6d6..025479431d 100644 --- a/src/bin/pg_waldump/pg_waldump.c +++ b/src/bin/pg_waldump/pg_waldump.c @@ -1043,7 +1043,7 @@ main(int argc, char **argv) /* we have everything we need, start reading */ xlogreader_state = - XLogReaderAllocate(WalSegSz, waldir, WALDumpCloseSegment); + XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, waldir, WALDumpCloseSegment); if (!xlogreader_state) fatal_error("out of memory"); diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h index 732d463a49..e9d7d70db7 100644 --- a/src/include/access/xlogreader.h +++ b/src/include/access/xlogreader.h @@ -151,6 +151,7 @@ struct XLogReaderState * ---------------------------------------- */ /* variables to communicate with page reader */ + int pageSize; /* size of a page */ XLogRecPtr readPagePtr; /* page pointer to read */ int32 readLen; /* bytes requested to reader, or actual bytes * read by reader, which must be larger than @@ -243,7 +244,7 @@ struct XLogReaderState /* Get a new XLogReader */ extern XLogReaderState *XLogReaderAllocate(int wal_segment_size, - const char *waldir, + int page_size, const char *waldir, WALSegmentCleanupCB cleanup_cb); /* Free an XLogReader */ -- 2.18.4 ----Next_Part(Tue_Sep__8_16_35_16_2020_701)---- ^ permalink raw reply [nested|flat] 26+ messages in thread
* [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize. @ 2019-09-05 12:29 Kyotaro Horiguchi <[email protected]> 0 siblings, 0 replies; 26+ messages in thread From: Kyotaro Horiguchi @ 2019-09-05 12:29 UTC (permalink / raw) XLog reader facility and its callers currently relies on the same constant XLOG_BLCKSZ. This patch moves the responsibility to determine the block size to the callers of the facility. --- src/backend/access/transam/twophase.c | 3 +- src/backend/access/transam/xlog.c | 9 ++-- src/backend/access/transam/xlogreader.c | 57 ++++++++++++----------- src/backend/replication/logical/logical.c | 5 +- src/backend/replication/walsender.c | 4 +- src/bin/pg_rewind/parsexlog.c | 6 +-- src/bin/pg_waldump/pg_waldump.c | 2 +- src/include/access/xlogreader.h | 3 +- 8 files changed, 50 insertions(+), 39 deletions(-) diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c index 7d5a96a5f5..9f9e7de253 100644 --- a/src/backend/access/transam/twophase.c +++ b/src/backend/access/transam/twophase.c @@ -1325,7 +1325,8 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len) XLogReaderState *xlogreader; char *errormsg; - xlogreader = XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close); + xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, + NULL, wal_segment_close); if (!xlogreader) ereport(ERROR, diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index b517ca85ad..0822ac70cb 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -1222,7 +1222,10 @@ XLogInsertRecord(XLogRecData *rdata, appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len); if (!debug_reader) - debug_reader = XLogReaderAllocate(wal_segment_size, NULL, NULL); + { + debug_reader = + XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, NULL); + } if (!debug_reader) { @@ -6531,8 +6534,8 @@ StartupXLOG(void) OwnLatch(&XLogCtl->recoveryWakeupLatch); /* Set up XLOG reader facility */ - xlogreader = - XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close); + xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, + NULL, wal_segment_close); if (!xlogreader) ereport(ERROR, diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c index ba3e4868f7..e5baf13086 100644 --- a/src/backend/access/transam/xlogreader.c +++ b/src/backend/access/transam/xlogreader.c @@ -72,7 +72,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...) * Returns NULL if the xlogreader couldn't be allocated. */ XLogReaderState * -XLogReaderAllocate(int wal_segment_size, const char *waldir, +XLogReaderAllocate(int wal_segment_size, int page_size, const char *waldir, WALSegmentCleanupCB cleanup_cb) { XLogReaderState *state; @@ -87,6 +87,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir, state->cleanup_cb = cleanup_cb; state->max_block_id = -1; + state->pageSize = page_size; /* * Permanently allocate readBuf. We do it this way, rather than just @@ -95,7 +96,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir, * isn't guaranteed to have any particular alignment, whereas * palloc_extended() will provide MAXALIGN'd storage. */ - state->readBuf = (char *) palloc_extended(XLOG_BLCKSZ, + state->readBuf = (char *) palloc_extended(state->pageSize, MCXT_ALLOC_NO_OOM); if (!state->readBuf) { @@ -163,7 +164,7 @@ XLogReaderFree(XLogReaderState *state) * readRecordBufSize is set to the new buffer size. * * To avoid useless small increases, round its size to a multiple of - * XLOG_BLCKSZ, and make sure it's at least 5*Max(BLCKSZ, XLOG_BLCKSZ) to start + * page size, and make sure it's at least 5*Max(BLCKSZ, <page size>) to start * with. (That is enough for all "normal" records, but very large commit or * abort records might need more space.) */ @@ -172,8 +173,8 @@ allocate_recordbuf(XLogReaderState *state, uint32 reclength) { uint32 newSize = reclength; - newSize += XLOG_BLCKSZ - (newSize % XLOG_BLCKSZ); - newSize = Max(newSize, 5 * Max(BLCKSZ, XLOG_BLCKSZ)); + newSize += state->pageSize - (newSize % state->pageSize); + newSize = Max(newSize, 5 * Max(BLCKSZ, state->pageSize)); #ifndef FRONTEND @@ -238,6 +239,8 @@ void XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr) { Assert(!XLogRecPtrIsInvalid(RecPtr)); + Assert(state->readBuf != NULL && + state->readBuf == (void *) MAXALIGN(state->readBuf)); ResetDecoder(state); @@ -366,15 +369,15 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) XLogPageHeader pageHeader; targetPagePtr = - state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ); - targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ; + state->ReadRecPtr - (state->ReadRecPtr % state->pageSize); + targetRecOff = state->ReadRecPtr % state->pageSize; /* * Check if we have enough data. For the first record in the page, * the requesting length doesn't contain page header. */ if (XLogNeedData(state, targetPagePtr, - Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ), + Min(targetRecOff + SizeOfXLogRecord, state->pageSize), targetRecOff != 0)) return XLREAD_NEED_DATA; @@ -421,7 +424,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) * verified that we got the whole header. */ prec = (XLogRecord *) (state->readBuf + - state->ReadRecPtr % XLOG_BLCKSZ); + state->ReadRecPtr % state->pageSize); total_len = prec->xl_tot_len; /* @@ -432,7 +435,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) * ensure that we enter the XLREAD_CONTINUATION state below; * otherwise we might fail to apply ValidXLogRecordHeader at all. */ - if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord) + if (targetRecOff <= state->pageSize - SizeOfXLogRecord) { if (!ValidXLogRecordHeader(state, state->ReadRecPtr, state->PrevRecPtr, prec)) @@ -477,10 +480,10 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) * available */ targetPagePtr = - state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ); - targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ; + state->ReadRecPtr - (state->ReadRecPtr % state->pageSize); + targetRecOff = state->ReadRecPtr % state->pageSize; - request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ); + request_len = Min(targetRecOff + total_len, state->pageSize); record_len = request_len - targetRecOff; /* ReadRecPtr contains page header */ @@ -549,7 +552,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) /* Calculate pointer to beginning of next page */ state->recordContRecPtr = state->ReadRecPtr + record_len; - Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0); + Assert(state->recordContRecPtr % state->pageSize == 0); state->readRecordState = XLREAD_CONTINUATION; } @@ -576,7 +579,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) /* this request contains page header */ Assert (targetPagePtr != 0); if (XLogNeedData(state, targetPagePtr, - Min(state->recordRemainLen, XLOG_BLCKSZ), + Min(state->recordRemainLen, state->pageSize), false)) return XLREAD_NEED_DATA; @@ -628,7 +631,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) Assert(state->readLen >= pageHeaderSize); contdata = (char *) state->readBuf + pageHeaderSize; - record_len = XLOG_BLCKSZ - pageHeaderSize; + record_len = state->pageSize - pageHeaderSize; if (pageHeader->xlp_rem_len < record_len) record_len = pageHeader->xlp_rem_len; @@ -655,7 +658,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) } /* Calculate pointer to beginning of next page, and continue */ - state->recordContRecPtr += XLOG_BLCKSZ; + state->recordContRecPtr += state->pageSize; } /* targetPagePtr is pointing the last-read page here */ @@ -789,10 +792,10 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen, XLogPageHeaderSize((XLogPageHeader) state->readBuf); addLen = pageHeaderSize; - if (reqLen + pageHeaderSize <= XLOG_BLCKSZ) + if (reqLen + pageHeaderSize <= state->pageSize) addLen = pageHeaderSize; else - addLen = XLOG_BLCKSZ - reqLen; + addLen = state->pageSize - reqLen; Assert(addLen >= 0); } @@ -805,7 +808,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen, /* Data is not in our buffer, request the caller for it. */ XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize); targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize); - Assert((pageptr % XLOG_BLCKSZ) == 0); + Assert((pageptr % state->pageSize) == 0); /* * Every time we request to load new data of a page to the caller, even if @@ -829,7 +832,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen, * will not come back here, but will request the actual target page. */ state->readPagePtr = pageptr - targetPageOff; - state->readLen = XLOG_BLCKSZ; + state->readLen = state->pageSize; return true; } @@ -962,7 +965,7 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr, int32 offset; XLogPageHeader hdr = (XLogPageHeader) phdr; - Assert((recptr % XLOG_BLCKSZ) == 0); + Assert((recptr % state->pageSize) == 0); XLByteToSeg(recptr, segno, state->segcxt.ws_segsize); offset = XLogSegmentOffset(recptr, state->segcxt.ws_segsize); @@ -1016,10 +1019,10 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr, "WAL file is from different database system: incorrect segment size in page header"); return false; } - else if (longhdr->xlp_xlog_blcksz != XLOG_BLCKSZ) + else if (longhdr->xlp_xlog_blcksz != state->pageSize) { report_invalid_record(state, - "WAL file is from different database system: incorrect XLOG_BLCKSZ in page header"); + "WAL file is from different database system: incorrect block size in page header"); return false; } } @@ -1140,7 +1143,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr, * XLogNeedData() is prepared to handle that and will read at * least short page-header worth of data */ - targetRecOff = tmpRecPtr % XLOG_BLCKSZ; + targetRecOff = tmpRecPtr % state->pageSize; /* scroll back to page boundary */ targetPagePtr = tmpRecPtr - targetRecOff; @@ -1174,8 +1177,8 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr, * * Note that record headers are MAXALIGN'ed */ - if (MAXALIGN(header->xlp_rem_len) >= (XLOG_BLCKSZ - pageHeaderSize)) - tmpRecPtr = targetPagePtr + XLOG_BLCKSZ; + if (MAXALIGN(header->xlp_rem_len) >= (state->pageSize - pageHeaderSize)) + tmpRecPtr = targetPagePtr + state->pageSize; else { /* diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c index 9cbd8afa49..f2dd495310 100644 --- a/src/backend/replication/logical/logical.c +++ b/src/backend/replication/logical/logical.c @@ -199,11 +199,14 @@ StartupDecodingContext(List *output_plugin_options, ctx->slot = slot; - ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, cleanup_cb); + ctx->reader = + XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, cleanup_cb); + if (!ctx->reader) ereport(ERROR, (errcode(ERRCODE_OUT_OF_MEMORY), errmsg("out of memory"))); + ctx->reader->readBuf = palloc(XLOG_BLCKSZ); ctx->page_read = page_read; ctx->reorder = ReorderBufferAllocate(); diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c index 062093727d..8e95600ef7 100644 --- a/src/backend/replication/walsender.c +++ b/src/backend/replication/walsender.c @@ -579,8 +579,8 @@ StartReplication(StartReplicationCmd *cmd) errmsg("IDENTIFY_SYSTEM has not been run before START_REPLICATION"))); /* create xlogreader for physical replication */ - xlogreader = - XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close); + xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, + NULL, wal_segment_close); if (!xlogreader) ereport(ERROR, diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c index 712c85281c..8a6209b061 100644 --- a/src/bin/pg_rewind/parsexlog.c +++ b/src/bin/pg_rewind/parsexlog.c @@ -61,7 +61,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex, XLogReaderState *xlogreader; char *errormsg; - xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL); + xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL); if (xlogreader == NULL) pg_fatal("out of memory"); @@ -121,7 +121,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex, char *errormsg; XLogRecPtr endptr; - xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL); + xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL); if (xlogreader == NULL) pg_fatal("out of memory"); @@ -181,7 +181,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex, forkptr += SizeOfXLogShortPHD; } - xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL); + xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL); if (xlogreader == NULL) pg_fatal("out of memory"); diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c index f514de6b13..b635537659 100644 --- a/src/bin/pg_waldump/pg_waldump.c +++ b/src/bin/pg_waldump/pg_waldump.c @@ -1041,7 +1041,7 @@ main(int argc, char **argv) /* we have everything we need, start reading */ xlogreader_state = - XLogReaderAllocate(WalSegSz, waldir, WALDumpCloseSegment); + XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, waldir, WALDumpCloseSegment); if (!xlogreader_state) fatal_error("out of memory"); diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h index b9dad7ee51..1853516333 100644 --- a/src/include/access/xlogreader.h +++ b/src/include/access/xlogreader.h @@ -151,6 +151,7 @@ struct XLogReaderState * ---------------------------------------- */ /* variables to communicate with page reader */ + int pageSize; /* size of a page */ XLogRecPtr readPagePtr; /* page pointer to read */ int32 readLen; /* bytes requested to reader, or actual bytes * read by reader, which must be larger than @@ -243,7 +244,7 @@ struct XLogReaderState /* Get a new XLogReader */ extern XLogReaderState *XLogReaderAllocate(int wal_segment_size, - const char *waldir, + int page_size, const char *waldir, WALSegmentCleanupCB cleanup_cb); /* Free an XLogReader */ -- 2.27.0 ----Next_Part(Thu_Mar__4_11_28_53_2021_014)---- ^ permalink raw reply [nested|flat] 26+ messages in thread
* [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize. @ 2019-09-05 12:29 Kyotaro Horiguchi <[email protected]> 0 siblings, 0 replies; 26+ messages in thread From: Kyotaro Horiguchi @ 2019-09-05 12:29 UTC (permalink / raw) XLog reader facility and its callers currently relies on the same constant XLOG_BLCKSZ. This patch moves the responsibility to determine the block size to the callers of the facility. --- src/backend/access/transam/twophase.c | 3 +- src/backend/access/transam/xlog.c | 9 ++-- src/backend/access/transam/xlogreader.c | 57 ++++++++++++----------- src/backend/replication/logical/logical.c | 5 +- src/backend/replication/walsender.c | 4 +- src/bin/pg_rewind/parsexlog.c | 6 +-- src/bin/pg_waldump/pg_waldump.c | 2 +- src/include/access/xlogreader.h | 3 +- 8 files changed, 50 insertions(+), 39 deletions(-) diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c index 7d5a96a5f5..9f9e7de253 100644 --- a/src/backend/access/transam/twophase.c +++ b/src/backend/access/transam/twophase.c @@ -1325,7 +1325,8 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len) XLogReaderState *xlogreader; char *errormsg; - xlogreader = XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close); + xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, + NULL, wal_segment_close); if (!xlogreader) ereport(ERROR, diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index b517ca85ad..0822ac70cb 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -1222,7 +1222,10 @@ XLogInsertRecord(XLogRecData *rdata, appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len); if (!debug_reader) - debug_reader = XLogReaderAllocate(wal_segment_size, NULL, NULL); + { + debug_reader = + XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, NULL); + } if (!debug_reader) { @@ -6531,8 +6534,8 @@ StartupXLOG(void) OwnLatch(&XLogCtl->recoveryWakeupLatch); /* Set up XLOG reader facility */ - xlogreader = - XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close); + xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, + NULL, wal_segment_close); if (!xlogreader) ereport(ERROR, diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c index ba3e4868f7..e5baf13086 100644 --- a/src/backend/access/transam/xlogreader.c +++ b/src/backend/access/transam/xlogreader.c @@ -72,7 +72,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...) * Returns NULL if the xlogreader couldn't be allocated. */ XLogReaderState * -XLogReaderAllocate(int wal_segment_size, const char *waldir, +XLogReaderAllocate(int wal_segment_size, int page_size, const char *waldir, WALSegmentCleanupCB cleanup_cb) { XLogReaderState *state; @@ -87,6 +87,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir, state->cleanup_cb = cleanup_cb; state->max_block_id = -1; + state->pageSize = page_size; /* * Permanently allocate readBuf. We do it this way, rather than just @@ -95,7 +96,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir, * isn't guaranteed to have any particular alignment, whereas * palloc_extended() will provide MAXALIGN'd storage. */ - state->readBuf = (char *) palloc_extended(XLOG_BLCKSZ, + state->readBuf = (char *) palloc_extended(state->pageSize, MCXT_ALLOC_NO_OOM); if (!state->readBuf) { @@ -163,7 +164,7 @@ XLogReaderFree(XLogReaderState *state) * readRecordBufSize is set to the new buffer size. * * To avoid useless small increases, round its size to a multiple of - * XLOG_BLCKSZ, and make sure it's at least 5*Max(BLCKSZ, XLOG_BLCKSZ) to start + * page size, and make sure it's at least 5*Max(BLCKSZ, <page size>) to start * with. (That is enough for all "normal" records, but very large commit or * abort records might need more space.) */ @@ -172,8 +173,8 @@ allocate_recordbuf(XLogReaderState *state, uint32 reclength) { uint32 newSize = reclength; - newSize += XLOG_BLCKSZ - (newSize % XLOG_BLCKSZ); - newSize = Max(newSize, 5 * Max(BLCKSZ, XLOG_BLCKSZ)); + newSize += state->pageSize - (newSize % state->pageSize); + newSize = Max(newSize, 5 * Max(BLCKSZ, state->pageSize)); #ifndef FRONTEND @@ -238,6 +239,8 @@ void XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr) { Assert(!XLogRecPtrIsInvalid(RecPtr)); + Assert(state->readBuf != NULL && + state->readBuf == (void *) MAXALIGN(state->readBuf)); ResetDecoder(state); @@ -366,15 +369,15 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) XLogPageHeader pageHeader; targetPagePtr = - state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ); - targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ; + state->ReadRecPtr - (state->ReadRecPtr % state->pageSize); + targetRecOff = state->ReadRecPtr % state->pageSize; /* * Check if we have enough data. For the first record in the page, * the requesting length doesn't contain page header. */ if (XLogNeedData(state, targetPagePtr, - Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ), + Min(targetRecOff + SizeOfXLogRecord, state->pageSize), targetRecOff != 0)) return XLREAD_NEED_DATA; @@ -421,7 +424,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) * verified that we got the whole header. */ prec = (XLogRecord *) (state->readBuf + - state->ReadRecPtr % XLOG_BLCKSZ); + state->ReadRecPtr % state->pageSize); total_len = prec->xl_tot_len; /* @@ -432,7 +435,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) * ensure that we enter the XLREAD_CONTINUATION state below; * otherwise we might fail to apply ValidXLogRecordHeader at all. */ - if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord) + if (targetRecOff <= state->pageSize - SizeOfXLogRecord) { if (!ValidXLogRecordHeader(state, state->ReadRecPtr, state->PrevRecPtr, prec)) @@ -477,10 +480,10 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) * available */ targetPagePtr = - state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ); - targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ; + state->ReadRecPtr - (state->ReadRecPtr % state->pageSize); + targetRecOff = state->ReadRecPtr % state->pageSize; - request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ); + request_len = Min(targetRecOff + total_len, state->pageSize); record_len = request_len - targetRecOff; /* ReadRecPtr contains page header */ @@ -549,7 +552,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) /* Calculate pointer to beginning of next page */ state->recordContRecPtr = state->ReadRecPtr + record_len; - Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0); + Assert(state->recordContRecPtr % state->pageSize == 0); state->readRecordState = XLREAD_CONTINUATION; } @@ -576,7 +579,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) /* this request contains page header */ Assert (targetPagePtr != 0); if (XLogNeedData(state, targetPagePtr, - Min(state->recordRemainLen, XLOG_BLCKSZ), + Min(state->recordRemainLen, state->pageSize), false)) return XLREAD_NEED_DATA; @@ -628,7 +631,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) Assert(state->readLen >= pageHeaderSize); contdata = (char *) state->readBuf + pageHeaderSize; - record_len = XLOG_BLCKSZ - pageHeaderSize; + record_len = state->pageSize - pageHeaderSize; if (pageHeader->xlp_rem_len < record_len) record_len = pageHeader->xlp_rem_len; @@ -655,7 +658,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) } /* Calculate pointer to beginning of next page, and continue */ - state->recordContRecPtr += XLOG_BLCKSZ; + state->recordContRecPtr += state->pageSize; } /* targetPagePtr is pointing the last-read page here */ @@ -789,10 +792,10 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen, XLogPageHeaderSize((XLogPageHeader) state->readBuf); addLen = pageHeaderSize; - if (reqLen + pageHeaderSize <= XLOG_BLCKSZ) + if (reqLen + pageHeaderSize <= state->pageSize) addLen = pageHeaderSize; else - addLen = XLOG_BLCKSZ - reqLen; + addLen = state->pageSize - reqLen; Assert(addLen >= 0); } @@ -805,7 +808,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen, /* Data is not in our buffer, request the caller for it. */ XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize); targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize); - Assert((pageptr % XLOG_BLCKSZ) == 0); + Assert((pageptr % state->pageSize) == 0); /* * Every time we request to load new data of a page to the caller, even if @@ -829,7 +832,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen, * will not come back here, but will request the actual target page. */ state->readPagePtr = pageptr - targetPageOff; - state->readLen = XLOG_BLCKSZ; + state->readLen = state->pageSize; return true; } @@ -962,7 +965,7 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr, int32 offset; XLogPageHeader hdr = (XLogPageHeader) phdr; - Assert((recptr % XLOG_BLCKSZ) == 0); + Assert((recptr % state->pageSize) == 0); XLByteToSeg(recptr, segno, state->segcxt.ws_segsize); offset = XLogSegmentOffset(recptr, state->segcxt.ws_segsize); @@ -1016,10 +1019,10 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr, "WAL file is from different database system: incorrect segment size in page header"); return false; } - else if (longhdr->xlp_xlog_blcksz != XLOG_BLCKSZ) + else if (longhdr->xlp_xlog_blcksz != state->pageSize) { report_invalid_record(state, - "WAL file is from different database system: incorrect XLOG_BLCKSZ in page header"); + "WAL file is from different database system: incorrect block size in page header"); return false; } } @@ -1140,7 +1143,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr, * XLogNeedData() is prepared to handle that and will read at * least short page-header worth of data */ - targetRecOff = tmpRecPtr % XLOG_BLCKSZ; + targetRecOff = tmpRecPtr % state->pageSize; /* scroll back to page boundary */ targetPagePtr = tmpRecPtr - targetRecOff; @@ -1174,8 +1177,8 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr, * * Note that record headers are MAXALIGN'ed */ - if (MAXALIGN(header->xlp_rem_len) >= (XLOG_BLCKSZ - pageHeaderSize)) - tmpRecPtr = targetPagePtr + XLOG_BLCKSZ; + if (MAXALIGN(header->xlp_rem_len) >= (state->pageSize - pageHeaderSize)) + tmpRecPtr = targetPagePtr + state->pageSize; else { /* diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c index 9cbd8afa49..f2dd495310 100644 --- a/src/backend/replication/logical/logical.c +++ b/src/backend/replication/logical/logical.c @@ -199,11 +199,14 @@ StartupDecodingContext(List *output_plugin_options, ctx->slot = slot; - ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, cleanup_cb); + ctx->reader = + XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, cleanup_cb); + if (!ctx->reader) ereport(ERROR, (errcode(ERRCODE_OUT_OF_MEMORY), errmsg("out of memory"))); + ctx->reader->readBuf = palloc(XLOG_BLCKSZ); ctx->page_read = page_read; ctx->reorder = ReorderBufferAllocate(); diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c index 062093727d..8e95600ef7 100644 --- a/src/backend/replication/walsender.c +++ b/src/backend/replication/walsender.c @@ -579,8 +579,8 @@ StartReplication(StartReplicationCmd *cmd) errmsg("IDENTIFY_SYSTEM has not been run before START_REPLICATION"))); /* create xlogreader for physical replication */ - xlogreader = - XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close); + xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, + NULL, wal_segment_close); if (!xlogreader) ereport(ERROR, diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c index 712c85281c..8a6209b061 100644 --- a/src/bin/pg_rewind/parsexlog.c +++ b/src/bin/pg_rewind/parsexlog.c @@ -61,7 +61,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex, XLogReaderState *xlogreader; char *errormsg; - xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL); + xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL); if (xlogreader == NULL) pg_fatal("out of memory"); @@ -121,7 +121,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex, char *errormsg; XLogRecPtr endptr; - xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL); + xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL); if (xlogreader == NULL) pg_fatal("out of memory"); @@ -181,7 +181,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex, forkptr += SizeOfXLogShortPHD; } - xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL); + xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL); if (xlogreader == NULL) pg_fatal("out of memory"); diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c index f514de6b13..b635537659 100644 --- a/src/bin/pg_waldump/pg_waldump.c +++ b/src/bin/pg_waldump/pg_waldump.c @@ -1041,7 +1041,7 @@ main(int argc, char **argv) /* we have everything we need, start reading */ xlogreader_state = - XLogReaderAllocate(WalSegSz, waldir, WALDumpCloseSegment); + XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, waldir, WALDumpCloseSegment); if (!xlogreader_state) fatal_error("out of memory"); diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h index b9dad7ee51..1853516333 100644 --- a/src/include/access/xlogreader.h +++ b/src/include/access/xlogreader.h @@ -151,6 +151,7 @@ struct XLogReaderState * ---------------------------------------- */ /* variables to communicate with page reader */ + int pageSize; /* size of a page */ XLogRecPtr readPagePtr; /* page pointer to read */ int32 readLen; /* bytes requested to reader, or actual bytes * read by reader, which must be larger than @@ -243,7 +244,7 @@ struct XLogReaderState /* Get a new XLogReader */ extern XLogReaderState *XLogReaderAllocate(int wal_segment_size, - const char *waldir, + int page_size, const char *waldir, WALSegmentCleanupCB cleanup_cb); /* Free an XLogReader */ -- 2.27.0 ----Next_Part(Thu_Mar__4_11_28_53_2021_014)---- ^ permalink raw reply [nested|flat] 26+ messages in thread
* [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize. @ 2019-09-05 12:29 Kyotaro Horiguchi <[email protected]> 0 siblings, 0 replies; 26+ messages in thread From: Kyotaro Horiguchi @ 2019-09-05 12:29 UTC (permalink / raw) XLog reader facility and its callers currently relies on the same constant XLOG_BLCKSZ. This patch moves the responsibility to determine the block size to the callers of the facility. --- src/backend/access/transam/twophase.c | 3 +- src/backend/access/transam/xlog.c | 9 ++-- src/backend/access/transam/xlogreader.c | 57 ++++++++++++----------- src/backend/replication/logical/logical.c | 5 +- src/backend/replication/walsender.c | 4 +- src/bin/pg_rewind/parsexlog.c | 6 +-- src/bin/pg_waldump/pg_waldump.c | 2 +- src/include/access/xlogreader.h | 3 +- 8 files changed, 50 insertions(+), 39 deletions(-) diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c index 7d5a96a5f5..9f9e7de253 100644 --- a/src/backend/access/transam/twophase.c +++ b/src/backend/access/transam/twophase.c @@ -1325,7 +1325,8 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len) XLogReaderState *xlogreader; char *errormsg; - xlogreader = XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close); + xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, + NULL, wal_segment_close); if (!xlogreader) ereport(ERROR, diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index b517ca85ad..0822ac70cb 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -1222,7 +1222,10 @@ XLogInsertRecord(XLogRecData *rdata, appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len); if (!debug_reader) - debug_reader = XLogReaderAllocate(wal_segment_size, NULL, NULL); + { + debug_reader = + XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, NULL); + } if (!debug_reader) { @@ -6531,8 +6534,8 @@ StartupXLOG(void) OwnLatch(&XLogCtl->recoveryWakeupLatch); /* Set up XLOG reader facility */ - xlogreader = - XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close); + xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, + NULL, wal_segment_close); if (!xlogreader) ereport(ERROR, diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c index ba3e4868f7..e5baf13086 100644 --- a/src/backend/access/transam/xlogreader.c +++ b/src/backend/access/transam/xlogreader.c @@ -72,7 +72,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...) * Returns NULL if the xlogreader couldn't be allocated. */ XLogReaderState * -XLogReaderAllocate(int wal_segment_size, const char *waldir, +XLogReaderAllocate(int wal_segment_size, int page_size, const char *waldir, WALSegmentCleanupCB cleanup_cb) { XLogReaderState *state; @@ -87,6 +87,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir, state->cleanup_cb = cleanup_cb; state->max_block_id = -1; + state->pageSize = page_size; /* * Permanently allocate readBuf. We do it this way, rather than just @@ -95,7 +96,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir, * isn't guaranteed to have any particular alignment, whereas * palloc_extended() will provide MAXALIGN'd storage. */ - state->readBuf = (char *) palloc_extended(XLOG_BLCKSZ, + state->readBuf = (char *) palloc_extended(state->pageSize, MCXT_ALLOC_NO_OOM); if (!state->readBuf) { @@ -163,7 +164,7 @@ XLogReaderFree(XLogReaderState *state) * readRecordBufSize is set to the new buffer size. * * To avoid useless small increases, round its size to a multiple of - * XLOG_BLCKSZ, and make sure it's at least 5*Max(BLCKSZ, XLOG_BLCKSZ) to start + * page size, and make sure it's at least 5*Max(BLCKSZ, <page size>) to start * with. (That is enough for all "normal" records, but very large commit or * abort records might need more space.) */ @@ -172,8 +173,8 @@ allocate_recordbuf(XLogReaderState *state, uint32 reclength) { uint32 newSize = reclength; - newSize += XLOG_BLCKSZ - (newSize % XLOG_BLCKSZ); - newSize = Max(newSize, 5 * Max(BLCKSZ, XLOG_BLCKSZ)); + newSize += state->pageSize - (newSize % state->pageSize); + newSize = Max(newSize, 5 * Max(BLCKSZ, state->pageSize)); #ifndef FRONTEND @@ -238,6 +239,8 @@ void XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr) { Assert(!XLogRecPtrIsInvalid(RecPtr)); + Assert(state->readBuf != NULL && + state->readBuf == (void *) MAXALIGN(state->readBuf)); ResetDecoder(state); @@ -366,15 +369,15 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) XLogPageHeader pageHeader; targetPagePtr = - state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ); - targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ; + state->ReadRecPtr - (state->ReadRecPtr % state->pageSize); + targetRecOff = state->ReadRecPtr % state->pageSize; /* * Check if we have enough data. For the first record in the page, * the requesting length doesn't contain page header. */ if (XLogNeedData(state, targetPagePtr, - Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ), + Min(targetRecOff + SizeOfXLogRecord, state->pageSize), targetRecOff != 0)) return XLREAD_NEED_DATA; @@ -421,7 +424,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) * verified that we got the whole header. */ prec = (XLogRecord *) (state->readBuf + - state->ReadRecPtr % XLOG_BLCKSZ); + state->ReadRecPtr % state->pageSize); total_len = prec->xl_tot_len; /* @@ -432,7 +435,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) * ensure that we enter the XLREAD_CONTINUATION state below; * otherwise we might fail to apply ValidXLogRecordHeader at all. */ - if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord) + if (targetRecOff <= state->pageSize - SizeOfXLogRecord) { if (!ValidXLogRecordHeader(state, state->ReadRecPtr, state->PrevRecPtr, prec)) @@ -477,10 +480,10 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) * available */ targetPagePtr = - state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ); - targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ; + state->ReadRecPtr - (state->ReadRecPtr % state->pageSize); + targetRecOff = state->ReadRecPtr % state->pageSize; - request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ); + request_len = Min(targetRecOff + total_len, state->pageSize); record_len = request_len - targetRecOff; /* ReadRecPtr contains page header */ @@ -549,7 +552,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) /* Calculate pointer to beginning of next page */ state->recordContRecPtr = state->ReadRecPtr + record_len; - Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0); + Assert(state->recordContRecPtr % state->pageSize == 0); state->readRecordState = XLREAD_CONTINUATION; } @@ -576,7 +579,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) /* this request contains page header */ Assert (targetPagePtr != 0); if (XLogNeedData(state, targetPagePtr, - Min(state->recordRemainLen, XLOG_BLCKSZ), + Min(state->recordRemainLen, state->pageSize), false)) return XLREAD_NEED_DATA; @@ -628,7 +631,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) Assert(state->readLen >= pageHeaderSize); contdata = (char *) state->readBuf + pageHeaderSize; - record_len = XLOG_BLCKSZ - pageHeaderSize; + record_len = state->pageSize - pageHeaderSize; if (pageHeader->xlp_rem_len < record_len) record_len = pageHeader->xlp_rem_len; @@ -655,7 +658,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) } /* Calculate pointer to beginning of next page, and continue */ - state->recordContRecPtr += XLOG_BLCKSZ; + state->recordContRecPtr += state->pageSize; } /* targetPagePtr is pointing the last-read page here */ @@ -789,10 +792,10 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen, XLogPageHeaderSize((XLogPageHeader) state->readBuf); addLen = pageHeaderSize; - if (reqLen + pageHeaderSize <= XLOG_BLCKSZ) + if (reqLen + pageHeaderSize <= state->pageSize) addLen = pageHeaderSize; else - addLen = XLOG_BLCKSZ - reqLen; + addLen = state->pageSize - reqLen; Assert(addLen >= 0); } @@ -805,7 +808,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen, /* Data is not in our buffer, request the caller for it. */ XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize); targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize); - Assert((pageptr % XLOG_BLCKSZ) == 0); + Assert((pageptr % state->pageSize) == 0); /* * Every time we request to load new data of a page to the caller, even if @@ -829,7 +832,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen, * will not come back here, but will request the actual target page. */ state->readPagePtr = pageptr - targetPageOff; - state->readLen = XLOG_BLCKSZ; + state->readLen = state->pageSize; return true; } @@ -962,7 +965,7 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr, int32 offset; XLogPageHeader hdr = (XLogPageHeader) phdr; - Assert((recptr % XLOG_BLCKSZ) == 0); + Assert((recptr % state->pageSize) == 0); XLByteToSeg(recptr, segno, state->segcxt.ws_segsize); offset = XLogSegmentOffset(recptr, state->segcxt.ws_segsize); @@ -1016,10 +1019,10 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr, "WAL file is from different database system: incorrect segment size in page header"); return false; } - else if (longhdr->xlp_xlog_blcksz != XLOG_BLCKSZ) + else if (longhdr->xlp_xlog_blcksz != state->pageSize) { report_invalid_record(state, - "WAL file is from different database system: incorrect XLOG_BLCKSZ in page header"); + "WAL file is from different database system: incorrect block size in page header"); return false; } } @@ -1140,7 +1143,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr, * XLogNeedData() is prepared to handle that and will read at * least short page-header worth of data */ - targetRecOff = tmpRecPtr % XLOG_BLCKSZ; + targetRecOff = tmpRecPtr % state->pageSize; /* scroll back to page boundary */ targetPagePtr = tmpRecPtr - targetRecOff; @@ -1174,8 +1177,8 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr, * * Note that record headers are MAXALIGN'ed */ - if (MAXALIGN(header->xlp_rem_len) >= (XLOG_BLCKSZ - pageHeaderSize)) - tmpRecPtr = targetPagePtr + XLOG_BLCKSZ; + if (MAXALIGN(header->xlp_rem_len) >= (state->pageSize - pageHeaderSize)) + tmpRecPtr = targetPagePtr + state->pageSize; else { /* diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c index 9cbd8afa49..f2dd495310 100644 --- a/src/backend/replication/logical/logical.c +++ b/src/backend/replication/logical/logical.c @@ -199,11 +199,14 @@ StartupDecodingContext(List *output_plugin_options, ctx->slot = slot; - ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, cleanup_cb); + ctx->reader = + XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, cleanup_cb); + if (!ctx->reader) ereport(ERROR, (errcode(ERRCODE_OUT_OF_MEMORY), errmsg("out of memory"))); + ctx->reader->readBuf = palloc(XLOG_BLCKSZ); ctx->page_read = page_read; ctx->reorder = ReorderBufferAllocate(); diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c index 062093727d..8e95600ef7 100644 --- a/src/backend/replication/walsender.c +++ b/src/backend/replication/walsender.c @@ -579,8 +579,8 @@ StartReplication(StartReplicationCmd *cmd) errmsg("IDENTIFY_SYSTEM has not been run before START_REPLICATION"))); /* create xlogreader for physical replication */ - xlogreader = - XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close); + xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, + NULL, wal_segment_close); if (!xlogreader) ereport(ERROR, diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c index 712c85281c..8a6209b061 100644 --- a/src/bin/pg_rewind/parsexlog.c +++ b/src/bin/pg_rewind/parsexlog.c @@ -61,7 +61,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex, XLogReaderState *xlogreader; char *errormsg; - xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL); + xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL); if (xlogreader == NULL) pg_fatal("out of memory"); @@ -121,7 +121,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex, char *errormsg; XLogRecPtr endptr; - xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL); + xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL); if (xlogreader == NULL) pg_fatal("out of memory"); @@ -181,7 +181,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex, forkptr += SizeOfXLogShortPHD; } - xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL); + xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL); if (xlogreader == NULL) pg_fatal("out of memory"); diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c index f514de6b13..b635537659 100644 --- a/src/bin/pg_waldump/pg_waldump.c +++ b/src/bin/pg_waldump/pg_waldump.c @@ -1041,7 +1041,7 @@ main(int argc, char **argv) /* we have everything we need, start reading */ xlogreader_state = - XLogReaderAllocate(WalSegSz, waldir, WALDumpCloseSegment); + XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, waldir, WALDumpCloseSegment); if (!xlogreader_state) fatal_error("out of memory"); diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h index b9dad7ee51..1853516333 100644 --- a/src/include/access/xlogreader.h +++ b/src/include/access/xlogreader.h @@ -151,6 +151,7 @@ struct XLogReaderState * ---------------------------------------- */ /* variables to communicate with page reader */ + int pageSize; /* size of a page */ XLogRecPtr readPagePtr; /* page pointer to read */ int32 readLen; /* bytes requested to reader, or actual bytes * read by reader, which must be larger than @@ -243,7 +244,7 @@ struct XLogReaderState /* Get a new XLogReader */ extern XLogReaderState *XLogReaderAllocate(int wal_segment_size, - const char *waldir, + int page_size, const char *waldir, WALSegmentCleanupCB cleanup_cb); /* Free an XLogReader */ -- 2.27.0 ----Next_Part(Thu_Mar__4_11_28_53_2021_014)---- ^ permalink raw reply [nested|flat] 26+ messages in thread
* [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize. @ 2019-09-05 12:29 Kyotaro Horiguchi <[email protected]> 0 siblings, 0 replies; 26+ messages in thread From: Kyotaro Horiguchi @ 2019-09-05 12:29 UTC (permalink / raw) XLog reader facility and its callers currently relies on the same constant XLOG_BLCKSZ. This patch moves the responsibility to determine the block size to the callers of the facility. --- src/backend/access/transam/twophase.c | 3 +- src/backend/access/transam/xlog.c | 9 ++-- src/backend/access/transam/xlogreader.c | 57 ++++++++++++----------- src/backend/replication/logical/logical.c | 5 +- src/backend/replication/walsender.c | 4 +- src/bin/pg_rewind/parsexlog.c | 6 +-- src/bin/pg_waldump/pg_waldump.c | 2 +- src/include/access/xlogreader.h | 3 +- 8 files changed, 50 insertions(+), 39 deletions(-) diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c index 7d5a96a5f5..9f9e7de253 100644 --- a/src/backend/access/transam/twophase.c +++ b/src/backend/access/transam/twophase.c @@ -1325,7 +1325,8 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len) XLogReaderState *xlogreader; char *errormsg; - xlogreader = XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close); + xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, + NULL, wal_segment_close); if (!xlogreader) ereport(ERROR, diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index b517ca85ad..0822ac70cb 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -1222,7 +1222,10 @@ XLogInsertRecord(XLogRecData *rdata, appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len); if (!debug_reader) - debug_reader = XLogReaderAllocate(wal_segment_size, NULL, NULL); + { + debug_reader = + XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, NULL); + } if (!debug_reader) { @@ -6531,8 +6534,8 @@ StartupXLOG(void) OwnLatch(&XLogCtl->recoveryWakeupLatch); /* Set up XLOG reader facility */ - xlogreader = - XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close); + xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, + NULL, wal_segment_close); if (!xlogreader) ereport(ERROR, diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c index ba3e4868f7..e5baf13086 100644 --- a/src/backend/access/transam/xlogreader.c +++ b/src/backend/access/transam/xlogreader.c @@ -72,7 +72,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...) * Returns NULL if the xlogreader couldn't be allocated. */ XLogReaderState * -XLogReaderAllocate(int wal_segment_size, const char *waldir, +XLogReaderAllocate(int wal_segment_size, int page_size, const char *waldir, WALSegmentCleanupCB cleanup_cb) { XLogReaderState *state; @@ -87,6 +87,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir, state->cleanup_cb = cleanup_cb; state->max_block_id = -1; + state->pageSize = page_size; /* * Permanently allocate readBuf. We do it this way, rather than just @@ -95,7 +96,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir, * isn't guaranteed to have any particular alignment, whereas * palloc_extended() will provide MAXALIGN'd storage. */ - state->readBuf = (char *) palloc_extended(XLOG_BLCKSZ, + state->readBuf = (char *) palloc_extended(state->pageSize, MCXT_ALLOC_NO_OOM); if (!state->readBuf) { @@ -163,7 +164,7 @@ XLogReaderFree(XLogReaderState *state) * readRecordBufSize is set to the new buffer size. * * To avoid useless small increases, round its size to a multiple of - * XLOG_BLCKSZ, and make sure it's at least 5*Max(BLCKSZ, XLOG_BLCKSZ) to start + * page size, and make sure it's at least 5*Max(BLCKSZ, <page size>) to start * with. (That is enough for all "normal" records, but very large commit or * abort records might need more space.) */ @@ -172,8 +173,8 @@ allocate_recordbuf(XLogReaderState *state, uint32 reclength) { uint32 newSize = reclength; - newSize += XLOG_BLCKSZ - (newSize % XLOG_BLCKSZ); - newSize = Max(newSize, 5 * Max(BLCKSZ, XLOG_BLCKSZ)); + newSize += state->pageSize - (newSize % state->pageSize); + newSize = Max(newSize, 5 * Max(BLCKSZ, state->pageSize)); #ifndef FRONTEND @@ -238,6 +239,8 @@ void XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr) { Assert(!XLogRecPtrIsInvalid(RecPtr)); + Assert(state->readBuf != NULL && + state->readBuf == (void *) MAXALIGN(state->readBuf)); ResetDecoder(state); @@ -366,15 +369,15 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) XLogPageHeader pageHeader; targetPagePtr = - state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ); - targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ; + state->ReadRecPtr - (state->ReadRecPtr % state->pageSize); + targetRecOff = state->ReadRecPtr % state->pageSize; /* * Check if we have enough data. For the first record in the page, * the requesting length doesn't contain page header. */ if (XLogNeedData(state, targetPagePtr, - Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ), + Min(targetRecOff + SizeOfXLogRecord, state->pageSize), targetRecOff != 0)) return XLREAD_NEED_DATA; @@ -421,7 +424,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) * verified that we got the whole header. */ prec = (XLogRecord *) (state->readBuf + - state->ReadRecPtr % XLOG_BLCKSZ); + state->ReadRecPtr % state->pageSize); total_len = prec->xl_tot_len; /* @@ -432,7 +435,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) * ensure that we enter the XLREAD_CONTINUATION state below; * otherwise we might fail to apply ValidXLogRecordHeader at all. */ - if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord) + if (targetRecOff <= state->pageSize - SizeOfXLogRecord) { if (!ValidXLogRecordHeader(state, state->ReadRecPtr, state->PrevRecPtr, prec)) @@ -477,10 +480,10 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) * available */ targetPagePtr = - state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ); - targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ; + state->ReadRecPtr - (state->ReadRecPtr % state->pageSize); + targetRecOff = state->ReadRecPtr % state->pageSize; - request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ); + request_len = Min(targetRecOff + total_len, state->pageSize); record_len = request_len - targetRecOff; /* ReadRecPtr contains page header */ @@ -549,7 +552,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) /* Calculate pointer to beginning of next page */ state->recordContRecPtr = state->ReadRecPtr + record_len; - Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0); + Assert(state->recordContRecPtr % state->pageSize == 0); state->readRecordState = XLREAD_CONTINUATION; } @@ -576,7 +579,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) /* this request contains page header */ Assert (targetPagePtr != 0); if (XLogNeedData(state, targetPagePtr, - Min(state->recordRemainLen, XLOG_BLCKSZ), + Min(state->recordRemainLen, state->pageSize), false)) return XLREAD_NEED_DATA; @@ -628,7 +631,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) Assert(state->readLen >= pageHeaderSize); contdata = (char *) state->readBuf + pageHeaderSize; - record_len = XLOG_BLCKSZ - pageHeaderSize; + record_len = state->pageSize - pageHeaderSize; if (pageHeader->xlp_rem_len < record_len) record_len = pageHeader->xlp_rem_len; @@ -655,7 +658,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) } /* Calculate pointer to beginning of next page, and continue */ - state->recordContRecPtr += XLOG_BLCKSZ; + state->recordContRecPtr += state->pageSize; } /* targetPagePtr is pointing the last-read page here */ @@ -789,10 +792,10 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen, XLogPageHeaderSize((XLogPageHeader) state->readBuf); addLen = pageHeaderSize; - if (reqLen + pageHeaderSize <= XLOG_BLCKSZ) + if (reqLen + pageHeaderSize <= state->pageSize) addLen = pageHeaderSize; else - addLen = XLOG_BLCKSZ - reqLen; + addLen = state->pageSize - reqLen; Assert(addLen >= 0); } @@ -805,7 +808,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen, /* Data is not in our buffer, request the caller for it. */ XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize); targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize); - Assert((pageptr % XLOG_BLCKSZ) == 0); + Assert((pageptr % state->pageSize) == 0); /* * Every time we request to load new data of a page to the caller, even if @@ -829,7 +832,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen, * will not come back here, but will request the actual target page. */ state->readPagePtr = pageptr - targetPageOff; - state->readLen = XLOG_BLCKSZ; + state->readLen = state->pageSize; return true; } @@ -962,7 +965,7 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr, int32 offset; XLogPageHeader hdr = (XLogPageHeader) phdr; - Assert((recptr % XLOG_BLCKSZ) == 0); + Assert((recptr % state->pageSize) == 0); XLByteToSeg(recptr, segno, state->segcxt.ws_segsize); offset = XLogSegmentOffset(recptr, state->segcxt.ws_segsize); @@ -1016,10 +1019,10 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr, "WAL file is from different database system: incorrect segment size in page header"); return false; } - else if (longhdr->xlp_xlog_blcksz != XLOG_BLCKSZ) + else if (longhdr->xlp_xlog_blcksz != state->pageSize) { report_invalid_record(state, - "WAL file is from different database system: incorrect XLOG_BLCKSZ in page header"); + "WAL file is from different database system: incorrect block size in page header"); return false; } } @@ -1140,7 +1143,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr, * XLogNeedData() is prepared to handle that and will read at * least short page-header worth of data */ - targetRecOff = tmpRecPtr % XLOG_BLCKSZ; + targetRecOff = tmpRecPtr % state->pageSize; /* scroll back to page boundary */ targetPagePtr = tmpRecPtr - targetRecOff; @@ -1174,8 +1177,8 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr, * * Note that record headers are MAXALIGN'ed */ - if (MAXALIGN(header->xlp_rem_len) >= (XLOG_BLCKSZ - pageHeaderSize)) - tmpRecPtr = targetPagePtr + XLOG_BLCKSZ; + if (MAXALIGN(header->xlp_rem_len) >= (state->pageSize - pageHeaderSize)) + tmpRecPtr = targetPagePtr + state->pageSize; else { /* diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c index 9cbd8afa49..f2dd495310 100644 --- a/src/backend/replication/logical/logical.c +++ b/src/backend/replication/logical/logical.c @@ -199,11 +199,14 @@ StartupDecodingContext(List *output_plugin_options, ctx->slot = slot; - ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, cleanup_cb); + ctx->reader = + XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, cleanup_cb); + if (!ctx->reader) ereport(ERROR, (errcode(ERRCODE_OUT_OF_MEMORY), errmsg("out of memory"))); + ctx->reader->readBuf = palloc(XLOG_BLCKSZ); ctx->page_read = page_read; ctx->reorder = ReorderBufferAllocate(); diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c index 062093727d..8e95600ef7 100644 --- a/src/backend/replication/walsender.c +++ b/src/backend/replication/walsender.c @@ -579,8 +579,8 @@ StartReplication(StartReplicationCmd *cmd) errmsg("IDENTIFY_SYSTEM has not been run before START_REPLICATION"))); /* create xlogreader for physical replication */ - xlogreader = - XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close); + xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, + NULL, wal_segment_close); if (!xlogreader) ereport(ERROR, diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c index 712c85281c..8a6209b061 100644 --- a/src/bin/pg_rewind/parsexlog.c +++ b/src/bin/pg_rewind/parsexlog.c @@ -61,7 +61,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex, XLogReaderState *xlogreader; char *errormsg; - xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL); + xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL); if (xlogreader == NULL) pg_fatal("out of memory"); @@ -121,7 +121,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex, char *errormsg; XLogRecPtr endptr; - xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL); + xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL); if (xlogreader == NULL) pg_fatal("out of memory"); @@ -181,7 +181,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex, forkptr += SizeOfXLogShortPHD; } - xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL); + xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL); if (xlogreader == NULL) pg_fatal("out of memory"); diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c index f514de6b13..b635537659 100644 --- a/src/bin/pg_waldump/pg_waldump.c +++ b/src/bin/pg_waldump/pg_waldump.c @@ -1041,7 +1041,7 @@ main(int argc, char **argv) /* we have everything we need, start reading */ xlogreader_state = - XLogReaderAllocate(WalSegSz, waldir, WALDumpCloseSegment); + XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, waldir, WALDumpCloseSegment); if (!xlogreader_state) fatal_error("out of memory"); diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h index b9dad7ee51..1853516333 100644 --- a/src/include/access/xlogreader.h +++ b/src/include/access/xlogreader.h @@ -151,6 +151,7 @@ struct XLogReaderState * ---------------------------------------- */ /* variables to communicate with page reader */ + int pageSize; /* size of a page */ XLogRecPtr readPagePtr; /* page pointer to read */ int32 readLen; /* bytes requested to reader, or actual bytes * read by reader, which must be larger than @@ -243,7 +244,7 @@ struct XLogReaderState /* Get a new XLogReader */ extern XLogReaderState *XLogReaderAllocate(int wal_segment_size, - const char *waldir, + int page_size, const char *waldir, WALSegmentCleanupCB cleanup_cb); /* Free an XLogReader */ -- 2.27.0 ----Next_Part(Thu_Mar__4_11_28_53_2021_014)---- ^ permalink raw reply [nested|flat] 26+ messages in thread
* [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize. @ 2019-09-05 12:29 Kyotaro Horiguchi <[email protected]> 0 siblings, 0 replies; 26+ messages in thread From: Kyotaro Horiguchi @ 2019-09-05 12:29 UTC (permalink / raw) XLog reader facility and its callers currently relies on the same constant XLOG_BLCKSZ. This patch moves the responsibility to determine the block size to the callers of the facility. --- src/backend/access/transam/twophase.c | 3 +- src/backend/access/transam/xlog.c | 9 ++-- src/backend/access/transam/xlogreader.c | 57 ++++++++++++----------- src/backend/replication/logical/logical.c | 5 +- src/backend/replication/walsender.c | 4 +- src/bin/pg_rewind/parsexlog.c | 6 +-- src/bin/pg_waldump/pg_waldump.c | 2 +- src/include/access/xlogreader.h | 3 +- 8 files changed, 50 insertions(+), 39 deletions(-) diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c index 7d5a96a5f5..9f9e7de253 100644 --- a/src/backend/access/transam/twophase.c +++ b/src/backend/access/transam/twophase.c @@ -1325,7 +1325,8 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len) XLogReaderState *xlogreader; char *errormsg; - xlogreader = XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close); + xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, + NULL, wal_segment_close); if (!xlogreader) ereport(ERROR, diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index b517ca85ad..0822ac70cb 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -1222,7 +1222,10 @@ XLogInsertRecord(XLogRecData *rdata, appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len); if (!debug_reader) - debug_reader = XLogReaderAllocate(wal_segment_size, NULL, NULL); + { + debug_reader = + XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, NULL); + } if (!debug_reader) { @@ -6531,8 +6534,8 @@ StartupXLOG(void) OwnLatch(&XLogCtl->recoveryWakeupLatch); /* Set up XLOG reader facility */ - xlogreader = - XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close); + xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, + NULL, wal_segment_close); if (!xlogreader) ereport(ERROR, diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c index ba3e4868f7..e5baf13086 100644 --- a/src/backend/access/transam/xlogreader.c +++ b/src/backend/access/transam/xlogreader.c @@ -72,7 +72,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...) * Returns NULL if the xlogreader couldn't be allocated. */ XLogReaderState * -XLogReaderAllocate(int wal_segment_size, const char *waldir, +XLogReaderAllocate(int wal_segment_size, int page_size, const char *waldir, WALSegmentCleanupCB cleanup_cb) { XLogReaderState *state; @@ -87,6 +87,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir, state->cleanup_cb = cleanup_cb; state->max_block_id = -1; + state->pageSize = page_size; /* * Permanently allocate readBuf. We do it this way, rather than just @@ -95,7 +96,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir, * isn't guaranteed to have any particular alignment, whereas * palloc_extended() will provide MAXALIGN'd storage. */ - state->readBuf = (char *) palloc_extended(XLOG_BLCKSZ, + state->readBuf = (char *) palloc_extended(state->pageSize, MCXT_ALLOC_NO_OOM); if (!state->readBuf) { @@ -163,7 +164,7 @@ XLogReaderFree(XLogReaderState *state) * readRecordBufSize is set to the new buffer size. * * To avoid useless small increases, round its size to a multiple of - * XLOG_BLCKSZ, and make sure it's at least 5*Max(BLCKSZ, XLOG_BLCKSZ) to start + * page size, and make sure it's at least 5*Max(BLCKSZ, <page size>) to start * with. (That is enough for all "normal" records, but very large commit or * abort records might need more space.) */ @@ -172,8 +173,8 @@ allocate_recordbuf(XLogReaderState *state, uint32 reclength) { uint32 newSize = reclength; - newSize += XLOG_BLCKSZ - (newSize % XLOG_BLCKSZ); - newSize = Max(newSize, 5 * Max(BLCKSZ, XLOG_BLCKSZ)); + newSize += state->pageSize - (newSize % state->pageSize); + newSize = Max(newSize, 5 * Max(BLCKSZ, state->pageSize)); #ifndef FRONTEND @@ -238,6 +239,8 @@ void XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr) { Assert(!XLogRecPtrIsInvalid(RecPtr)); + Assert(state->readBuf != NULL && + state->readBuf == (void *) MAXALIGN(state->readBuf)); ResetDecoder(state); @@ -366,15 +369,15 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) XLogPageHeader pageHeader; targetPagePtr = - state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ); - targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ; + state->ReadRecPtr - (state->ReadRecPtr % state->pageSize); + targetRecOff = state->ReadRecPtr % state->pageSize; /* * Check if we have enough data. For the first record in the page, * the requesting length doesn't contain page header. */ if (XLogNeedData(state, targetPagePtr, - Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ), + Min(targetRecOff + SizeOfXLogRecord, state->pageSize), targetRecOff != 0)) return XLREAD_NEED_DATA; @@ -421,7 +424,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) * verified that we got the whole header. */ prec = (XLogRecord *) (state->readBuf + - state->ReadRecPtr % XLOG_BLCKSZ); + state->ReadRecPtr % state->pageSize); total_len = prec->xl_tot_len; /* @@ -432,7 +435,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) * ensure that we enter the XLREAD_CONTINUATION state below; * otherwise we might fail to apply ValidXLogRecordHeader at all. */ - if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord) + if (targetRecOff <= state->pageSize - SizeOfXLogRecord) { if (!ValidXLogRecordHeader(state, state->ReadRecPtr, state->PrevRecPtr, prec)) @@ -477,10 +480,10 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) * available */ targetPagePtr = - state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ); - targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ; + state->ReadRecPtr - (state->ReadRecPtr % state->pageSize); + targetRecOff = state->ReadRecPtr % state->pageSize; - request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ); + request_len = Min(targetRecOff + total_len, state->pageSize); record_len = request_len - targetRecOff; /* ReadRecPtr contains page header */ @@ -549,7 +552,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) /* Calculate pointer to beginning of next page */ state->recordContRecPtr = state->ReadRecPtr + record_len; - Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0); + Assert(state->recordContRecPtr % state->pageSize == 0); state->readRecordState = XLREAD_CONTINUATION; } @@ -576,7 +579,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) /* this request contains page header */ Assert (targetPagePtr != 0); if (XLogNeedData(state, targetPagePtr, - Min(state->recordRemainLen, XLOG_BLCKSZ), + Min(state->recordRemainLen, state->pageSize), false)) return XLREAD_NEED_DATA; @@ -628,7 +631,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) Assert(state->readLen >= pageHeaderSize); contdata = (char *) state->readBuf + pageHeaderSize; - record_len = XLOG_BLCKSZ - pageHeaderSize; + record_len = state->pageSize - pageHeaderSize; if (pageHeader->xlp_rem_len < record_len) record_len = pageHeader->xlp_rem_len; @@ -655,7 +658,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) } /* Calculate pointer to beginning of next page, and continue */ - state->recordContRecPtr += XLOG_BLCKSZ; + state->recordContRecPtr += state->pageSize; } /* targetPagePtr is pointing the last-read page here */ @@ -789,10 +792,10 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen, XLogPageHeaderSize((XLogPageHeader) state->readBuf); addLen = pageHeaderSize; - if (reqLen + pageHeaderSize <= XLOG_BLCKSZ) + if (reqLen + pageHeaderSize <= state->pageSize) addLen = pageHeaderSize; else - addLen = XLOG_BLCKSZ - reqLen; + addLen = state->pageSize - reqLen; Assert(addLen >= 0); } @@ -805,7 +808,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen, /* Data is not in our buffer, request the caller for it. */ XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize); targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize); - Assert((pageptr % XLOG_BLCKSZ) == 0); + Assert((pageptr % state->pageSize) == 0); /* * Every time we request to load new data of a page to the caller, even if @@ -829,7 +832,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen, * will not come back here, but will request the actual target page. */ state->readPagePtr = pageptr - targetPageOff; - state->readLen = XLOG_BLCKSZ; + state->readLen = state->pageSize; return true; } @@ -962,7 +965,7 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr, int32 offset; XLogPageHeader hdr = (XLogPageHeader) phdr; - Assert((recptr % XLOG_BLCKSZ) == 0); + Assert((recptr % state->pageSize) == 0); XLByteToSeg(recptr, segno, state->segcxt.ws_segsize); offset = XLogSegmentOffset(recptr, state->segcxt.ws_segsize); @@ -1016,10 +1019,10 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr, "WAL file is from different database system: incorrect segment size in page header"); return false; } - else if (longhdr->xlp_xlog_blcksz != XLOG_BLCKSZ) + else if (longhdr->xlp_xlog_blcksz != state->pageSize) { report_invalid_record(state, - "WAL file is from different database system: incorrect XLOG_BLCKSZ in page header"); + "WAL file is from different database system: incorrect block size in page header"); return false; } } @@ -1140,7 +1143,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr, * XLogNeedData() is prepared to handle that and will read at * least short page-header worth of data */ - targetRecOff = tmpRecPtr % XLOG_BLCKSZ; + targetRecOff = tmpRecPtr % state->pageSize; /* scroll back to page boundary */ targetPagePtr = tmpRecPtr - targetRecOff; @@ -1174,8 +1177,8 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr, * * Note that record headers are MAXALIGN'ed */ - if (MAXALIGN(header->xlp_rem_len) >= (XLOG_BLCKSZ - pageHeaderSize)) - tmpRecPtr = targetPagePtr + XLOG_BLCKSZ; + if (MAXALIGN(header->xlp_rem_len) >= (state->pageSize - pageHeaderSize)) + tmpRecPtr = targetPagePtr + state->pageSize; else { /* diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c index 9cbd8afa49..f2dd495310 100644 --- a/src/backend/replication/logical/logical.c +++ b/src/backend/replication/logical/logical.c @@ -199,11 +199,14 @@ StartupDecodingContext(List *output_plugin_options, ctx->slot = slot; - ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, cleanup_cb); + ctx->reader = + XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, cleanup_cb); + if (!ctx->reader) ereport(ERROR, (errcode(ERRCODE_OUT_OF_MEMORY), errmsg("out of memory"))); + ctx->reader->readBuf = palloc(XLOG_BLCKSZ); ctx->page_read = page_read; ctx->reorder = ReorderBufferAllocate(); diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c index 062093727d..8e95600ef7 100644 --- a/src/backend/replication/walsender.c +++ b/src/backend/replication/walsender.c @@ -579,8 +579,8 @@ StartReplication(StartReplicationCmd *cmd) errmsg("IDENTIFY_SYSTEM has not been run before START_REPLICATION"))); /* create xlogreader for physical replication */ - xlogreader = - XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close); + xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, + NULL, wal_segment_close); if (!xlogreader) ereport(ERROR, diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c index 712c85281c..8a6209b061 100644 --- a/src/bin/pg_rewind/parsexlog.c +++ b/src/bin/pg_rewind/parsexlog.c @@ -61,7 +61,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex, XLogReaderState *xlogreader; char *errormsg; - xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL); + xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL); if (xlogreader == NULL) pg_fatal("out of memory"); @@ -121,7 +121,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex, char *errormsg; XLogRecPtr endptr; - xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL); + xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL); if (xlogreader == NULL) pg_fatal("out of memory"); @@ -181,7 +181,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex, forkptr += SizeOfXLogShortPHD; } - xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL); + xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL); if (xlogreader == NULL) pg_fatal("out of memory"); diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c index f514de6b13..b635537659 100644 --- a/src/bin/pg_waldump/pg_waldump.c +++ b/src/bin/pg_waldump/pg_waldump.c @@ -1041,7 +1041,7 @@ main(int argc, char **argv) /* we have everything we need, start reading */ xlogreader_state = - XLogReaderAllocate(WalSegSz, waldir, WALDumpCloseSegment); + XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, waldir, WALDumpCloseSegment); if (!xlogreader_state) fatal_error("out of memory"); diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h index b9dad7ee51..1853516333 100644 --- a/src/include/access/xlogreader.h +++ b/src/include/access/xlogreader.h @@ -151,6 +151,7 @@ struct XLogReaderState * ---------------------------------------- */ /* variables to communicate with page reader */ + int pageSize; /* size of a page */ XLogRecPtr readPagePtr; /* page pointer to read */ int32 readLen; /* bytes requested to reader, or actual bytes * read by reader, which must be larger than @@ -243,7 +244,7 @@ struct XLogReaderState /* Get a new XLogReader */ extern XLogReaderState *XLogReaderAllocate(int wal_segment_size, - const char *waldir, + int page_size, const char *waldir, WALSegmentCleanupCB cleanup_cb); /* Free an XLogReader */ -- 2.27.0 ----Next_Part(Thu_Mar__4_11_28_53_2021_014)---- ^ permalink raw reply [nested|flat] 26+ messages in thread
* [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize. @ 2019-09-05 12:29 Kyotaro Horiguchi <[email protected]> 0 siblings, 0 replies; 26+ messages in thread From: Kyotaro Horiguchi @ 2019-09-05 12:29 UTC (permalink / raw) XLog reader facility and its callers currently relies on the same constant XLOG_BLCKSZ. This patch moves the responsibility to determine the block size to the callers of the facility. --- src/backend/access/transam/twophase.c | 3 +- src/backend/access/transam/xlog.c | 9 ++-- src/backend/access/transam/xlogreader.c | 57 ++++++++++++----------- src/backend/replication/logical/logical.c | 5 +- src/backend/replication/walsender.c | 4 +- src/bin/pg_rewind/parsexlog.c | 6 +-- src/bin/pg_waldump/pg_waldump.c | 2 +- src/include/access/xlogreader.h | 3 +- 8 files changed, 50 insertions(+), 39 deletions(-) diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c index 7d5a96a5f5..9f9e7de253 100644 --- a/src/backend/access/transam/twophase.c +++ b/src/backend/access/transam/twophase.c @@ -1325,7 +1325,8 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len) XLogReaderState *xlogreader; char *errormsg; - xlogreader = XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close); + xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, + NULL, wal_segment_close); if (!xlogreader) ereport(ERROR, diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index b517ca85ad..0822ac70cb 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -1222,7 +1222,10 @@ XLogInsertRecord(XLogRecData *rdata, appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len); if (!debug_reader) - debug_reader = XLogReaderAllocate(wal_segment_size, NULL, NULL); + { + debug_reader = + XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, NULL); + } if (!debug_reader) { @@ -6531,8 +6534,8 @@ StartupXLOG(void) OwnLatch(&XLogCtl->recoveryWakeupLatch); /* Set up XLOG reader facility */ - xlogreader = - XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close); + xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, + NULL, wal_segment_close); if (!xlogreader) ereport(ERROR, diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c index ba3e4868f7..e5baf13086 100644 --- a/src/backend/access/transam/xlogreader.c +++ b/src/backend/access/transam/xlogreader.c @@ -72,7 +72,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...) * Returns NULL if the xlogreader couldn't be allocated. */ XLogReaderState * -XLogReaderAllocate(int wal_segment_size, const char *waldir, +XLogReaderAllocate(int wal_segment_size, int page_size, const char *waldir, WALSegmentCleanupCB cleanup_cb) { XLogReaderState *state; @@ -87,6 +87,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir, state->cleanup_cb = cleanup_cb; state->max_block_id = -1; + state->pageSize = page_size; /* * Permanently allocate readBuf. We do it this way, rather than just @@ -95,7 +96,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir, * isn't guaranteed to have any particular alignment, whereas * palloc_extended() will provide MAXALIGN'd storage. */ - state->readBuf = (char *) palloc_extended(XLOG_BLCKSZ, + state->readBuf = (char *) palloc_extended(state->pageSize, MCXT_ALLOC_NO_OOM); if (!state->readBuf) { @@ -163,7 +164,7 @@ XLogReaderFree(XLogReaderState *state) * readRecordBufSize is set to the new buffer size. * * To avoid useless small increases, round its size to a multiple of - * XLOG_BLCKSZ, and make sure it's at least 5*Max(BLCKSZ, XLOG_BLCKSZ) to start + * page size, and make sure it's at least 5*Max(BLCKSZ, <page size>) to start * with. (That is enough for all "normal" records, but very large commit or * abort records might need more space.) */ @@ -172,8 +173,8 @@ allocate_recordbuf(XLogReaderState *state, uint32 reclength) { uint32 newSize = reclength; - newSize += XLOG_BLCKSZ - (newSize % XLOG_BLCKSZ); - newSize = Max(newSize, 5 * Max(BLCKSZ, XLOG_BLCKSZ)); + newSize += state->pageSize - (newSize % state->pageSize); + newSize = Max(newSize, 5 * Max(BLCKSZ, state->pageSize)); #ifndef FRONTEND @@ -238,6 +239,8 @@ void XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr) { Assert(!XLogRecPtrIsInvalid(RecPtr)); + Assert(state->readBuf != NULL && + state->readBuf == (void *) MAXALIGN(state->readBuf)); ResetDecoder(state); @@ -366,15 +369,15 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) XLogPageHeader pageHeader; targetPagePtr = - state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ); - targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ; + state->ReadRecPtr - (state->ReadRecPtr % state->pageSize); + targetRecOff = state->ReadRecPtr % state->pageSize; /* * Check if we have enough data. For the first record in the page, * the requesting length doesn't contain page header. */ if (XLogNeedData(state, targetPagePtr, - Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ), + Min(targetRecOff + SizeOfXLogRecord, state->pageSize), targetRecOff != 0)) return XLREAD_NEED_DATA; @@ -421,7 +424,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) * verified that we got the whole header. */ prec = (XLogRecord *) (state->readBuf + - state->ReadRecPtr % XLOG_BLCKSZ); + state->ReadRecPtr % state->pageSize); total_len = prec->xl_tot_len; /* @@ -432,7 +435,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) * ensure that we enter the XLREAD_CONTINUATION state below; * otherwise we might fail to apply ValidXLogRecordHeader at all. */ - if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord) + if (targetRecOff <= state->pageSize - SizeOfXLogRecord) { if (!ValidXLogRecordHeader(state, state->ReadRecPtr, state->PrevRecPtr, prec)) @@ -477,10 +480,10 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) * available */ targetPagePtr = - state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ); - targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ; + state->ReadRecPtr - (state->ReadRecPtr % state->pageSize); + targetRecOff = state->ReadRecPtr % state->pageSize; - request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ); + request_len = Min(targetRecOff + total_len, state->pageSize); record_len = request_len - targetRecOff; /* ReadRecPtr contains page header */ @@ -549,7 +552,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) /* Calculate pointer to beginning of next page */ state->recordContRecPtr = state->ReadRecPtr + record_len; - Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0); + Assert(state->recordContRecPtr % state->pageSize == 0); state->readRecordState = XLREAD_CONTINUATION; } @@ -576,7 +579,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) /* this request contains page header */ Assert (targetPagePtr != 0); if (XLogNeedData(state, targetPagePtr, - Min(state->recordRemainLen, XLOG_BLCKSZ), + Min(state->recordRemainLen, state->pageSize), false)) return XLREAD_NEED_DATA; @@ -628,7 +631,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) Assert(state->readLen >= pageHeaderSize); contdata = (char *) state->readBuf + pageHeaderSize; - record_len = XLOG_BLCKSZ - pageHeaderSize; + record_len = state->pageSize - pageHeaderSize; if (pageHeader->xlp_rem_len < record_len) record_len = pageHeader->xlp_rem_len; @@ -655,7 +658,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) } /* Calculate pointer to beginning of next page, and continue */ - state->recordContRecPtr += XLOG_BLCKSZ; + state->recordContRecPtr += state->pageSize; } /* targetPagePtr is pointing the last-read page here */ @@ -789,10 +792,10 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen, XLogPageHeaderSize((XLogPageHeader) state->readBuf); addLen = pageHeaderSize; - if (reqLen + pageHeaderSize <= XLOG_BLCKSZ) + if (reqLen + pageHeaderSize <= state->pageSize) addLen = pageHeaderSize; else - addLen = XLOG_BLCKSZ - reqLen; + addLen = state->pageSize - reqLen; Assert(addLen >= 0); } @@ -805,7 +808,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen, /* Data is not in our buffer, request the caller for it. */ XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize); targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize); - Assert((pageptr % XLOG_BLCKSZ) == 0); + Assert((pageptr % state->pageSize) == 0); /* * Every time we request to load new data of a page to the caller, even if @@ -829,7 +832,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen, * will not come back here, but will request the actual target page. */ state->readPagePtr = pageptr - targetPageOff; - state->readLen = XLOG_BLCKSZ; + state->readLen = state->pageSize; return true; } @@ -962,7 +965,7 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr, int32 offset; XLogPageHeader hdr = (XLogPageHeader) phdr; - Assert((recptr % XLOG_BLCKSZ) == 0); + Assert((recptr % state->pageSize) == 0); XLByteToSeg(recptr, segno, state->segcxt.ws_segsize); offset = XLogSegmentOffset(recptr, state->segcxt.ws_segsize); @@ -1016,10 +1019,10 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr, "WAL file is from different database system: incorrect segment size in page header"); return false; } - else if (longhdr->xlp_xlog_blcksz != XLOG_BLCKSZ) + else if (longhdr->xlp_xlog_blcksz != state->pageSize) { report_invalid_record(state, - "WAL file is from different database system: incorrect XLOG_BLCKSZ in page header"); + "WAL file is from different database system: incorrect block size in page header"); return false; } } @@ -1140,7 +1143,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr, * XLogNeedData() is prepared to handle that and will read at * least short page-header worth of data */ - targetRecOff = tmpRecPtr % XLOG_BLCKSZ; + targetRecOff = tmpRecPtr % state->pageSize; /* scroll back to page boundary */ targetPagePtr = tmpRecPtr - targetRecOff; @@ -1174,8 +1177,8 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr, * * Note that record headers are MAXALIGN'ed */ - if (MAXALIGN(header->xlp_rem_len) >= (XLOG_BLCKSZ - pageHeaderSize)) - tmpRecPtr = targetPagePtr + XLOG_BLCKSZ; + if (MAXALIGN(header->xlp_rem_len) >= (state->pageSize - pageHeaderSize)) + tmpRecPtr = targetPagePtr + state->pageSize; else { /* diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c index 9cbd8afa49..f2dd495310 100644 --- a/src/backend/replication/logical/logical.c +++ b/src/backend/replication/logical/logical.c @@ -199,11 +199,14 @@ StartupDecodingContext(List *output_plugin_options, ctx->slot = slot; - ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, cleanup_cb); + ctx->reader = + XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, cleanup_cb); + if (!ctx->reader) ereport(ERROR, (errcode(ERRCODE_OUT_OF_MEMORY), errmsg("out of memory"))); + ctx->reader->readBuf = palloc(XLOG_BLCKSZ); ctx->page_read = page_read; ctx->reorder = ReorderBufferAllocate(); diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c index 062093727d..8e95600ef7 100644 --- a/src/backend/replication/walsender.c +++ b/src/backend/replication/walsender.c @@ -579,8 +579,8 @@ StartReplication(StartReplicationCmd *cmd) errmsg("IDENTIFY_SYSTEM has not been run before START_REPLICATION"))); /* create xlogreader for physical replication */ - xlogreader = - XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close); + xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, + NULL, wal_segment_close); if (!xlogreader) ereport(ERROR, diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c index 712c85281c..8a6209b061 100644 --- a/src/bin/pg_rewind/parsexlog.c +++ b/src/bin/pg_rewind/parsexlog.c @@ -61,7 +61,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex, XLogReaderState *xlogreader; char *errormsg; - xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL); + xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL); if (xlogreader == NULL) pg_fatal("out of memory"); @@ -121,7 +121,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex, char *errormsg; XLogRecPtr endptr; - xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL); + xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL); if (xlogreader == NULL) pg_fatal("out of memory"); @@ -181,7 +181,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex, forkptr += SizeOfXLogShortPHD; } - xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL); + xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL); if (xlogreader == NULL) pg_fatal("out of memory"); diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c index f514de6b13..b635537659 100644 --- a/src/bin/pg_waldump/pg_waldump.c +++ b/src/bin/pg_waldump/pg_waldump.c @@ -1041,7 +1041,7 @@ main(int argc, char **argv) /* we have everything we need, start reading */ xlogreader_state = - XLogReaderAllocate(WalSegSz, waldir, WALDumpCloseSegment); + XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, waldir, WALDumpCloseSegment); if (!xlogreader_state) fatal_error("out of memory"); diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h index b9dad7ee51..1853516333 100644 --- a/src/include/access/xlogreader.h +++ b/src/include/access/xlogreader.h @@ -151,6 +151,7 @@ struct XLogReaderState * ---------------------------------------- */ /* variables to communicate with page reader */ + int pageSize; /* size of a page */ XLogRecPtr readPagePtr; /* page pointer to read */ int32 readLen; /* bytes requested to reader, or actual bytes * read by reader, which must be larger than @@ -243,7 +244,7 @@ struct XLogReaderState /* Get a new XLogReader */ extern XLogReaderState *XLogReaderAllocate(int wal_segment_size, - const char *waldir, + int page_size, const char *waldir, WALSegmentCleanupCB cleanup_cb); /* Free an XLogReader */ -- 2.27.0 ----Next_Part(Thu_Mar__4_11_28_53_2021_014)---- ^ permalink raw reply [nested|flat] 26+ messages in thread
* [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize. @ 2019-09-05 12:29 Kyotaro Horiguchi <[email protected]> 0 siblings, 0 replies; 26+ messages in thread From: Kyotaro Horiguchi @ 2019-09-05 12:29 UTC (permalink / raw) XLog reader facility and its callers currently relies on the same constant XLOG_BLCKSZ. This patch moves the responsibility to determine the block size to the callers of the facility. --- src/backend/access/transam/twophase.c | 3 +- src/backend/access/transam/xlog.c | 9 ++-- src/backend/access/transam/xlogreader.c | 57 ++++++++++++----------- src/backend/replication/logical/logical.c | 5 +- src/backend/replication/walsender.c | 4 +- src/bin/pg_rewind/parsexlog.c | 6 +-- src/bin/pg_waldump/pg_waldump.c | 2 +- src/include/access/xlogreader.h | 3 +- 8 files changed, 50 insertions(+), 39 deletions(-) diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c index 7d5a96a5f5..9f9e7de253 100644 --- a/src/backend/access/transam/twophase.c +++ b/src/backend/access/transam/twophase.c @@ -1325,7 +1325,8 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len) XLogReaderState *xlogreader; char *errormsg; - xlogreader = XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close); + xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, + NULL, wal_segment_close); if (!xlogreader) ereport(ERROR, diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index b517ca85ad..0822ac70cb 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -1222,7 +1222,10 @@ XLogInsertRecord(XLogRecData *rdata, appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len); if (!debug_reader) - debug_reader = XLogReaderAllocate(wal_segment_size, NULL, NULL); + { + debug_reader = + XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, NULL); + } if (!debug_reader) { @@ -6531,8 +6534,8 @@ StartupXLOG(void) OwnLatch(&XLogCtl->recoveryWakeupLatch); /* Set up XLOG reader facility */ - xlogreader = - XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close); + xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, + NULL, wal_segment_close); if (!xlogreader) ereport(ERROR, diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c index ba3e4868f7..e5baf13086 100644 --- a/src/backend/access/transam/xlogreader.c +++ b/src/backend/access/transam/xlogreader.c @@ -72,7 +72,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...) * Returns NULL if the xlogreader couldn't be allocated. */ XLogReaderState * -XLogReaderAllocate(int wal_segment_size, const char *waldir, +XLogReaderAllocate(int wal_segment_size, int page_size, const char *waldir, WALSegmentCleanupCB cleanup_cb) { XLogReaderState *state; @@ -87,6 +87,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir, state->cleanup_cb = cleanup_cb; state->max_block_id = -1; + state->pageSize = page_size; /* * Permanently allocate readBuf. We do it this way, rather than just @@ -95,7 +96,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir, * isn't guaranteed to have any particular alignment, whereas * palloc_extended() will provide MAXALIGN'd storage. */ - state->readBuf = (char *) palloc_extended(XLOG_BLCKSZ, + state->readBuf = (char *) palloc_extended(state->pageSize, MCXT_ALLOC_NO_OOM); if (!state->readBuf) { @@ -163,7 +164,7 @@ XLogReaderFree(XLogReaderState *state) * readRecordBufSize is set to the new buffer size. * * To avoid useless small increases, round its size to a multiple of - * XLOG_BLCKSZ, and make sure it's at least 5*Max(BLCKSZ, XLOG_BLCKSZ) to start + * page size, and make sure it's at least 5*Max(BLCKSZ, <page size>) to start * with. (That is enough for all "normal" records, but very large commit or * abort records might need more space.) */ @@ -172,8 +173,8 @@ allocate_recordbuf(XLogReaderState *state, uint32 reclength) { uint32 newSize = reclength; - newSize += XLOG_BLCKSZ - (newSize % XLOG_BLCKSZ); - newSize = Max(newSize, 5 * Max(BLCKSZ, XLOG_BLCKSZ)); + newSize += state->pageSize - (newSize % state->pageSize); + newSize = Max(newSize, 5 * Max(BLCKSZ, state->pageSize)); #ifndef FRONTEND @@ -238,6 +239,8 @@ void XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr) { Assert(!XLogRecPtrIsInvalid(RecPtr)); + Assert(state->readBuf != NULL && + state->readBuf == (void *) MAXALIGN(state->readBuf)); ResetDecoder(state); @@ -366,15 +369,15 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) XLogPageHeader pageHeader; targetPagePtr = - state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ); - targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ; + state->ReadRecPtr - (state->ReadRecPtr % state->pageSize); + targetRecOff = state->ReadRecPtr % state->pageSize; /* * Check if we have enough data. For the first record in the page, * the requesting length doesn't contain page header. */ if (XLogNeedData(state, targetPagePtr, - Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ), + Min(targetRecOff + SizeOfXLogRecord, state->pageSize), targetRecOff != 0)) return XLREAD_NEED_DATA; @@ -421,7 +424,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) * verified that we got the whole header. */ prec = (XLogRecord *) (state->readBuf + - state->ReadRecPtr % XLOG_BLCKSZ); + state->ReadRecPtr % state->pageSize); total_len = prec->xl_tot_len; /* @@ -432,7 +435,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) * ensure that we enter the XLREAD_CONTINUATION state below; * otherwise we might fail to apply ValidXLogRecordHeader at all. */ - if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord) + if (targetRecOff <= state->pageSize - SizeOfXLogRecord) { if (!ValidXLogRecordHeader(state, state->ReadRecPtr, state->PrevRecPtr, prec)) @@ -477,10 +480,10 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) * available */ targetPagePtr = - state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ); - targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ; + state->ReadRecPtr - (state->ReadRecPtr % state->pageSize); + targetRecOff = state->ReadRecPtr % state->pageSize; - request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ); + request_len = Min(targetRecOff + total_len, state->pageSize); record_len = request_len - targetRecOff; /* ReadRecPtr contains page header */ @@ -549,7 +552,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) /* Calculate pointer to beginning of next page */ state->recordContRecPtr = state->ReadRecPtr + record_len; - Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0); + Assert(state->recordContRecPtr % state->pageSize == 0); state->readRecordState = XLREAD_CONTINUATION; } @@ -576,7 +579,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) /* this request contains page header */ Assert (targetPagePtr != 0); if (XLogNeedData(state, targetPagePtr, - Min(state->recordRemainLen, XLOG_BLCKSZ), + Min(state->recordRemainLen, state->pageSize), false)) return XLREAD_NEED_DATA; @@ -628,7 +631,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) Assert(state->readLen >= pageHeaderSize); contdata = (char *) state->readBuf + pageHeaderSize; - record_len = XLOG_BLCKSZ - pageHeaderSize; + record_len = state->pageSize - pageHeaderSize; if (pageHeader->xlp_rem_len < record_len) record_len = pageHeader->xlp_rem_len; @@ -655,7 +658,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) } /* Calculate pointer to beginning of next page, and continue */ - state->recordContRecPtr += XLOG_BLCKSZ; + state->recordContRecPtr += state->pageSize; } /* targetPagePtr is pointing the last-read page here */ @@ -789,10 +792,10 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen, XLogPageHeaderSize((XLogPageHeader) state->readBuf); addLen = pageHeaderSize; - if (reqLen + pageHeaderSize <= XLOG_BLCKSZ) + if (reqLen + pageHeaderSize <= state->pageSize) addLen = pageHeaderSize; else - addLen = XLOG_BLCKSZ - reqLen; + addLen = state->pageSize - reqLen; Assert(addLen >= 0); } @@ -805,7 +808,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen, /* Data is not in our buffer, request the caller for it. */ XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize); targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize); - Assert((pageptr % XLOG_BLCKSZ) == 0); + Assert((pageptr % state->pageSize) == 0); /* * Every time we request to load new data of a page to the caller, even if @@ -829,7 +832,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen, * will not come back here, but will request the actual target page. */ state->readPagePtr = pageptr - targetPageOff; - state->readLen = XLOG_BLCKSZ; + state->readLen = state->pageSize; return true; } @@ -962,7 +965,7 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr, int32 offset; XLogPageHeader hdr = (XLogPageHeader) phdr; - Assert((recptr % XLOG_BLCKSZ) == 0); + Assert((recptr % state->pageSize) == 0); XLByteToSeg(recptr, segno, state->segcxt.ws_segsize); offset = XLogSegmentOffset(recptr, state->segcxt.ws_segsize); @@ -1016,10 +1019,10 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr, "WAL file is from different database system: incorrect segment size in page header"); return false; } - else if (longhdr->xlp_xlog_blcksz != XLOG_BLCKSZ) + else if (longhdr->xlp_xlog_blcksz != state->pageSize) { report_invalid_record(state, - "WAL file is from different database system: incorrect XLOG_BLCKSZ in page header"); + "WAL file is from different database system: incorrect block size in page header"); return false; } } @@ -1140,7 +1143,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr, * XLogNeedData() is prepared to handle that and will read at * least short page-header worth of data */ - targetRecOff = tmpRecPtr % XLOG_BLCKSZ; + targetRecOff = tmpRecPtr % state->pageSize; /* scroll back to page boundary */ targetPagePtr = tmpRecPtr - targetRecOff; @@ -1174,8 +1177,8 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr, * * Note that record headers are MAXALIGN'ed */ - if (MAXALIGN(header->xlp_rem_len) >= (XLOG_BLCKSZ - pageHeaderSize)) - tmpRecPtr = targetPagePtr + XLOG_BLCKSZ; + if (MAXALIGN(header->xlp_rem_len) >= (state->pageSize - pageHeaderSize)) + tmpRecPtr = targetPagePtr + state->pageSize; else { /* diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c index 9cbd8afa49..f2dd495310 100644 --- a/src/backend/replication/logical/logical.c +++ b/src/backend/replication/logical/logical.c @@ -199,11 +199,14 @@ StartupDecodingContext(List *output_plugin_options, ctx->slot = slot; - ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, cleanup_cb); + ctx->reader = + XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, cleanup_cb); + if (!ctx->reader) ereport(ERROR, (errcode(ERRCODE_OUT_OF_MEMORY), errmsg("out of memory"))); + ctx->reader->readBuf = palloc(XLOG_BLCKSZ); ctx->page_read = page_read; ctx->reorder = ReorderBufferAllocate(); diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c index 062093727d..8e95600ef7 100644 --- a/src/backend/replication/walsender.c +++ b/src/backend/replication/walsender.c @@ -579,8 +579,8 @@ StartReplication(StartReplicationCmd *cmd) errmsg("IDENTIFY_SYSTEM has not been run before START_REPLICATION"))); /* create xlogreader for physical replication */ - xlogreader = - XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close); + xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, + NULL, wal_segment_close); if (!xlogreader) ereport(ERROR, diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c index 712c85281c..8a6209b061 100644 --- a/src/bin/pg_rewind/parsexlog.c +++ b/src/bin/pg_rewind/parsexlog.c @@ -61,7 +61,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex, XLogReaderState *xlogreader; char *errormsg; - xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL); + xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL); if (xlogreader == NULL) pg_fatal("out of memory"); @@ -121,7 +121,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex, char *errormsg; XLogRecPtr endptr; - xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL); + xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL); if (xlogreader == NULL) pg_fatal("out of memory"); @@ -181,7 +181,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex, forkptr += SizeOfXLogShortPHD; } - xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL); + xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL); if (xlogreader == NULL) pg_fatal("out of memory"); diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c index f514de6b13..b635537659 100644 --- a/src/bin/pg_waldump/pg_waldump.c +++ b/src/bin/pg_waldump/pg_waldump.c @@ -1041,7 +1041,7 @@ main(int argc, char **argv) /* we have everything we need, start reading */ xlogreader_state = - XLogReaderAllocate(WalSegSz, waldir, WALDumpCloseSegment); + XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, waldir, WALDumpCloseSegment); if (!xlogreader_state) fatal_error("out of memory"); diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h index b9dad7ee51..1853516333 100644 --- a/src/include/access/xlogreader.h +++ b/src/include/access/xlogreader.h @@ -151,6 +151,7 @@ struct XLogReaderState * ---------------------------------------- */ /* variables to communicate with page reader */ + int pageSize; /* size of a page */ XLogRecPtr readPagePtr; /* page pointer to read */ int32 readLen; /* bytes requested to reader, or actual bytes * read by reader, which must be larger than @@ -243,7 +244,7 @@ struct XLogReaderState /* Get a new XLogReader */ extern XLogReaderState *XLogReaderAllocate(int wal_segment_size, - const char *waldir, + int page_size, const char *waldir, WALSegmentCleanupCB cleanup_cb); /* Free an XLogReader */ -- 2.27.0 ----Next_Part(Thu_Mar__4_11_28_53_2021_014)---- ^ permalink raw reply [nested|flat] 26+ messages in thread
* [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize. @ 2019-09-05 12:29 Kyotaro Horiguchi <[email protected]> 0 siblings, 0 replies; 26+ messages in thread From: Kyotaro Horiguchi @ 2019-09-05 12:29 UTC (permalink / raw) XLog reader facility and its callers currently relies on the same constant XLOG_BLCKSZ. This patch moves the responsibility to determine the block size to the callers of the facility. --- src/backend/access/transam/twophase.c | 3 +- src/backend/access/transam/xlog.c | 9 ++-- src/backend/access/transam/xlogreader.c | 57 ++++++++++++----------- src/backend/replication/logical/logical.c | 5 +- src/backend/replication/walsender.c | 4 +- src/bin/pg_rewind/parsexlog.c | 6 +-- src/bin/pg_waldump/pg_waldump.c | 2 +- src/include/access/xlogreader.h | 3 +- 8 files changed, 50 insertions(+), 39 deletions(-) diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c index 7d5a96a5f5..9f9e7de253 100644 --- a/src/backend/access/transam/twophase.c +++ b/src/backend/access/transam/twophase.c @@ -1325,7 +1325,8 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len) XLogReaderState *xlogreader; char *errormsg; - xlogreader = XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close); + xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, + NULL, wal_segment_close); if (!xlogreader) ereport(ERROR, diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index b517ca85ad..0822ac70cb 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -1222,7 +1222,10 @@ XLogInsertRecord(XLogRecData *rdata, appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len); if (!debug_reader) - debug_reader = XLogReaderAllocate(wal_segment_size, NULL, NULL); + { + debug_reader = + XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, NULL); + } if (!debug_reader) { @@ -6531,8 +6534,8 @@ StartupXLOG(void) OwnLatch(&XLogCtl->recoveryWakeupLatch); /* Set up XLOG reader facility */ - xlogreader = - XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close); + xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, + NULL, wal_segment_close); if (!xlogreader) ereport(ERROR, diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c index ba3e4868f7..e5baf13086 100644 --- a/src/backend/access/transam/xlogreader.c +++ b/src/backend/access/transam/xlogreader.c @@ -72,7 +72,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...) * Returns NULL if the xlogreader couldn't be allocated. */ XLogReaderState * -XLogReaderAllocate(int wal_segment_size, const char *waldir, +XLogReaderAllocate(int wal_segment_size, int page_size, const char *waldir, WALSegmentCleanupCB cleanup_cb) { XLogReaderState *state; @@ -87,6 +87,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir, state->cleanup_cb = cleanup_cb; state->max_block_id = -1; + state->pageSize = page_size; /* * Permanently allocate readBuf. We do it this way, rather than just @@ -95,7 +96,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir, * isn't guaranteed to have any particular alignment, whereas * palloc_extended() will provide MAXALIGN'd storage. */ - state->readBuf = (char *) palloc_extended(XLOG_BLCKSZ, + state->readBuf = (char *) palloc_extended(state->pageSize, MCXT_ALLOC_NO_OOM); if (!state->readBuf) { @@ -163,7 +164,7 @@ XLogReaderFree(XLogReaderState *state) * readRecordBufSize is set to the new buffer size. * * To avoid useless small increases, round its size to a multiple of - * XLOG_BLCKSZ, and make sure it's at least 5*Max(BLCKSZ, XLOG_BLCKSZ) to start + * page size, and make sure it's at least 5*Max(BLCKSZ, <page size>) to start * with. (That is enough for all "normal" records, but very large commit or * abort records might need more space.) */ @@ -172,8 +173,8 @@ allocate_recordbuf(XLogReaderState *state, uint32 reclength) { uint32 newSize = reclength; - newSize += XLOG_BLCKSZ - (newSize % XLOG_BLCKSZ); - newSize = Max(newSize, 5 * Max(BLCKSZ, XLOG_BLCKSZ)); + newSize += state->pageSize - (newSize % state->pageSize); + newSize = Max(newSize, 5 * Max(BLCKSZ, state->pageSize)); #ifndef FRONTEND @@ -238,6 +239,8 @@ void XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr) { Assert(!XLogRecPtrIsInvalid(RecPtr)); + Assert(state->readBuf != NULL && + state->readBuf == (void *) MAXALIGN(state->readBuf)); ResetDecoder(state); @@ -366,15 +369,15 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) XLogPageHeader pageHeader; targetPagePtr = - state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ); - targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ; + state->ReadRecPtr - (state->ReadRecPtr % state->pageSize); + targetRecOff = state->ReadRecPtr % state->pageSize; /* * Check if we have enough data. For the first record in the page, * the requesting length doesn't contain page header. */ if (XLogNeedData(state, targetPagePtr, - Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ), + Min(targetRecOff + SizeOfXLogRecord, state->pageSize), targetRecOff != 0)) return XLREAD_NEED_DATA; @@ -421,7 +424,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) * verified that we got the whole header. */ prec = (XLogRecord *) (state->readBuf + - state->ReadRecPtr % XLOG_BLCKSZ); + state->ReadRecPtr % state->pageSize); total_len = prec->xl_tot_len; /* @@ -432,7 +435,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) * ensure that we enter the XLREAD_CONTINUATION state below; * otherwise we might fail to apply ValidXLogRecordHeader at all. */ - if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord) + if (targetRecOff <= state->pageSize - SizeOfXLogRecord) { if (!ValidXLogRecordHeader(state, state->ReadRecPtr, state->PrevRecPtr, prec)) @@ -477,10 +480,10 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) * available */ targetPagePtr = - state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ); - targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ; + state->ReadRecPtr - (state->ReadRecPtr % state->pageSize); + targetRecOff = state->ReadRecPtr % state->pageSize; - request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ); + request_len = Min(targetRecOff + total_len, state->pageSize); record_len = request_len - targetRecOff; /* ReadRecPtr contains page header */ @@ -549,7 +552,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) /* Calculate pointer to beginning of next page */ state->recordContRecPtr = state->ReadRecPtr + record_len; - Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0); + Assert(state->recordContRecPtr % state->pageSize == 0); state->readRecordState = XLREAD_CONTINUATION; } @@ -576,7 +579,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) /* this request contains page header */ Assert (targetPagePtr != 0); if (XLogNeedData(state, targetPagePtr, - Min(state->recordRemainLen, XLOG_BLCKSZ), + Min(state->recordRemainLen, state->pageSize), false)) return XLREAD_NEED_DATA; @@ -628,7 +631,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) Assert(state->readLen >= pageHeaderSize); contdata = (char *) state->readBuf + pageHeaderSize; - record_len = XLOG_BLCKSZ - pageHeaderSize; + record_len = state->pageSize - pageHeaderSize; if (pageHeader->xlp_rem_len < record_len) record_len = pageHeader->xlp_rem_len; @@ -655,7 +658,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) } /* Calculate pointer to beginning of next page, and continue */ - state->recordContRecPtr += XLOG_BLCKSZ; + state->recordContRecPtr += state->pageSize; } /* targetPagePtr is pointing the last-read page here */ @@ -789,10 +792,10 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen, XLogPageHeaderSize((XLogPageHeader) state->readBuf); addLen = pageHeaderSize; - if (reqLen + pageHeaderSize <= XLOG_BLCKSZ) + if (reqLen + pageHeaderSize <= state->pageSize) addLen = pageHeaderSize; else - addLen = XLOG_BLCKSZ - reqLen; + addLen = state->pageSize - reqLen; Assert(addLen >= 0); } @@ -805,7 +808,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen, /* Data is not in our buffer, request the caller for it. */ XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize); targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize); - Assert((pageptr % XLOG_BLCKSZ) == 0); + Assert((pageptr % state->pageSize) == 0); /* * Every time we request to load new data of a page to the caller, even if @@ -829,7 +832,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen, * will not come back here, but will request the actual target page. */ state->readPagePtr = pageptr - targetPageOff; - state->readLen = XLOG_BLCKSZ; + state->readLen = state->pageSize; return true; } @@ -962,7 +965,7 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr, int32 offset; XLogPageHeader hdr = (XLogPageHeader) phdr; - Assert((recptr % XLOG_BLCKSZ) == 0); + Assert((recptr % state->pageSize) == 0); XLByteToSeg(recptr, segno, state->segcxt.ws_segsize); offset = XLogSegmentOffset(recptr, state->segcxt.ws_segsize); @@ -1016,10 +1019,10 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr, "WAL file is from different database system: incorrect segment size in page header"); return false; } - else if (longhdr->xlp_xlog_blcksz != XLOG_BLCKSZ) + else if (longhdr->xlp_xlog_blcksz != state->pageSize) { report_invalid_record(state, - "WAL file is from different database system: incorrect XLOG_BLCKSZ in page header"); + "WAL file is from different database system: incorrect block size in page header"); return false; } } @@ -1140,7 +1143,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr, * XLogNeedData() is prepared to handle that and will read at * least short page-header worth of data */ - targetRecOff = tmpRecPtr % XLOG_BLCKSZ; + targetRecOff = tmpRecPtr % state->pageSize; /* scroll back to page boundary */ targetPagePtr = tmpRecPtr - targetRecOff; @@ -1174,8 +1177,8 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr, * * Note that record headers are MAXALIGN'ed */ - if (MAXALIGN(header->xlp_rem_len) >= (XLOG_BLCKSZ - pageHeaderSize)) - tmpRecPtr = targetPagePtr + XLOG_BLCKSZ; + if (MAXALIGN(header->xlp_rem_len) >= (state->pageSize - pageHeaderSize)) + tmpRecPtr = targetPagePtr + state->pageSize; else { /* diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c index 9cbd8afa49..f2dd495310 100644 --- a/src/backend/replication/logical/logical.c +++ b/src/backend/replication/logical/logical.c @@ -199,11 +199,14 @@ StartupDecodingContext(List *output_plugin_options, ctx->slot = slot; - ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, cleanup_cb); + ctx->reader = + XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, cleanup_cb); + if (!ctx->reader) ereport(ERROR, (errcode(ERRCODE_OUT_OF_MEMORY), errmsg("out of memory"))); + ctx->reader->readBuf = palloc(XLOG_BLCKSZ); ctx->page_read = page_read; ctx->reorder = ReorderBufferAllocate(); diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c index 062093727d..8e95600ef7 100644 --- a/src/backend/replication/walsender.c +++ b/src/backend/replication/walsender.c @@ -579,8 +579,8 @@ StartReplication(StartReplicationCmd *cmd) errmsg("IDENTIFY_SYSTEM has not been run before START_REPLICATION"))); /* create xlogreader for physical replication */ - xlogreader = - XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close); + xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, + NULL, wal_segment_close); if (!xlogreader) ereport(ERROR, diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c index 712c85281c..8a6209b061 100644 --- a/src/bin/pg_rewind/parsexlog.c +++ b/src/bin/pg_rewind/parsexlog.c @@ -61,7 +61,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex, XLogReaderState *xlogreader; char *errormsg; - xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL); + xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL); if (xlogreader == NULL) pg_fatal("out of memory"); @@ -121,7 +121,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex, char *errormsg; XLogRecPtr endptr; - xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL); + xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL); if (xlogreader == NULL) pg_fatal("out of memory"); @@ -181,7 +181,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex, forkptr += SizeOfXLogShortPHD; } - xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL); + xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL); if (xlogreader == NULL) pg_fatal("out of memory"); diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c index f514de6b13..b635537659 100644 --- a/src/bin/pg_waldump/pg_waldump.c +++ b/src/bin/pg_waldump/pg_waldump.c @@ -1041,7 +1041,7 @@ main(int argc, char **argv) /* we have everything we need, start reading */ xlogreader_state = - XLogReaderAllocate(WalSegSz, waldir, WALDumpCloseSegment); + XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, waldir, WALDumpCloseSegment); if (!xlogreader_state) fatal_error("out of memory"); diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h index b9dad7ee51..1853516333 100644 --- a/src/include/access/xlogreader.h +++ b/src/include/access/xlogreader.h @@ -151,6 +151,7 @@ struct XLogReaderState * ---------------------------------------- */ /* variables to communicate with page reader */ + int pageSize; /* size of a page */ XLogRecPtr readPagePtr; /* page pointer to read */ int32 readLen; /* bytes requested to reader, or actual bytes * read by reader, which must be larger than @@ -243,7 +244,7 @@ struct XLogReaderState /* Get a new XLogReader */ extern XLogReaderState *XLogReaderAllocate(int wal_segment_size, - const char *waldir, + int page_size, const char *waldir, WALSegmentCleanupCB cleanup_cb); /* Free an XLogReader */ -- 2.27.0 ----Next_Part(Thu_Mar__4_11_28_53_2021_014)---- ^ permalink raw reply [nested|flat] 26+ messages in thread
* [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize. @ 2019-09-05 12:29 Kyotaro Horiguchi <[email protected]> 0 siblings, 0 replies; 26+ messages in thread From: Kyotaro Horiguchi @ 2019-09-05 12:29 UTC (permalink / raw) XLog reader facility and its callers currently relies on the same constant XLOG_BLCKSZ. This patch moves the responsibility to determine the block size to the callers of the facility. --- src/backend/access/transam/twophase.c | 3 +- src/backend/access/transam/xlog.c | 9 ++-- src/backend/access/transam/xlogreader.c | 57 ++++++++++++----------- src/backend/replication/logical/logical.c | 5 +- src/backend/replication/walsender.c | 4 +- src/bin/pg_rewind/parsexlog.c | 6 +-- src/bin/pg_waldump/pg_waldump.c | 2 +- src/include/access/xlogreader.h | 3 +- 8 files changed, 50 insertions(+), 39 deletions(-) diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c index 7d5a96a5f5..9f9e7de253 100644 --- a/src/backend/access/transam/twophase.c +++ b/src/backend/access/transam/twophase.c @@ -1325,7 +1325,8 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len) XLogReaderState *xlogreader; char *errormsg; - xlogreader = XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close); + xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, + NULL, wal_segment_close); if (!xlogreader) ereport(ERROR, diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index b517ca85ad..0822ac70cb 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -1222,7 +1222,10 @@ XLogInsertRecord(XLogRecData *rdata, appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len); if (!debug_reader) - debug_reader = XLogReaderAllocate(wal_segment_size, NULL, NULL); + { + debug_reader = + XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, NULL); + } if (!debug_reader) { @@ -6531,8 +6534,8 @@ StartupXLOG(void) OwnLatch(&XLogCtl->recoveryWakeupLatch); /* Set up XLOG reader facility */ - xlogreader = - XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close); + xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, + NULL, wal_segment_close); if (!xlogreader) ereport(ERROR, diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c index ba3e4868f7..e5baf13086 100644 --- a/src/backend/access/transam/xlogreader.c +++ b/src/backend/access/transam/xlogreader.c @@ -72,7 +72,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...) * Returns NULL if the xlogreader couldn't be allocated. */ XLogReaderState * -XLogReaderAllocate(int wal_segment_size, const char *waldir, +XLogReaderAllocate(int wal_segment_size, int page_size, const char *waldir, WALSegmentCleanupCB cleanup_cb) { XLogReaderState *state; @@ -87,6 +87,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir, state->cleanup_cb = cleanup_cb; state->max_block_id = -1; + state->pageSize = page_size; /* * Permanently allocate readBuf. We do it this way, rather than just @@ -95,7 +96,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir, * isn't guaranteed to have any particular alignment, whereas * palloc_extended() will provide MAXALIGN'd storage. */ - state->readBuf = (char *) palloc_extended(XLOG_BLCKSZ, + state->readBuf = (char *) palloc_extended(state->pageSize, MCXT_ALLOC_NO_OOM); if (!state->readBuf) { @@ -163,7 +164,7 @@ XLogReaderFree(XLogReaderState *state) * readRecordBufSize is set to the new buffer size. * * To avoid useless small increases, round its size to a multiple of - * XLOG_BLCKSZ, and make sure it's at least 5*Max(BLCKSZ, XLOG_BLCKSZ) to start + * page size, and make sure it's at least 5*Max(BLCKSZ, <page size>) to start * with. (That is enough for all "normal" records, but very large commit or * abort records might need more space.) */ @@ -172,8 +173,8 @@ allocate_recordbuf(XLogReaderState *state, uint32 reclength) { uint32 newSize = reclength; - newSize += XLOG_BLCKSZ - (newSize % XLOG_BLCKSZ); - newSize = Max(newSize, 5 * Max(BLCKSZ, XLOG_BLCKSZ)); + newSize += state->pageSize - (newSize % state->pageSize); + newSize = Max(newSize, 5 * Max(BLCKSZ, state->pageSize)); #ifndef FRONTEND @@ -238,6 +239,8 @@ void XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr) { Assert(!XLogRecPtrIsInvalid(RecPtr)); + Assert(state->readBuf != NULL && + state->readBuf == (void *) MAXALIGN(state->readBuf)); ResetDecoder(state); @@ -366,15 +369,15 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) XLogPageHeader pageHeader; targetPagePtr = - state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ); - targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ; + state->ReadRecPtr - (state->ReadRecPtr % state->pageSize); + targetRecOff = state->ReadRecPtr % state->pageSize; /* * Check if we have enough data. For the first record in the page, * the requesting length doesn't contain page header. */ if (XLogNeedData(state, targetPagePtr, - Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ), + Min(targetRecOff + SizeOfXLogRecord, state->pageSize), targetRecOff != 0)) return XLREAD_NEED_DATA; @@ -421,7 +424,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) * verified that we got the whole header. */ prec = (XLogRecord *) (state->readBuf + - state->ReadRecPtr % XLOG_BLCKSZ); + state->ReadRecPtr % state->pageSize); total_len = prec->xl_tot_len; /* @@ -432,7 +435,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) * ensure that we enter the XLREAD_CONTINUATION state below; * otherwise we might fail to apply ValidXLogRecordHeader at all. */ - if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord) + if (targetRecOff <= state->pageSize - SizeOfXLogRecord) { if (!ValidXLogRecordHeader(state, state->ReadRecPtr, state->PrevRecPtr, prec)) @@ -477,10 +480,10 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) * available */ targetPagePtr = - state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ); - targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ; + state->ReadRecPtr - (state->ReadRecPtr % state->pageSize); + targetRecOff = state->ReadRecPtr % state->pageSize; - request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ); + request_len = Min(targetRecOff + total_len, state->pageSize); record_len = request_len - targetRecOff; /* ReadRecPtr contains page header */ @@ -549,7 +552,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) /* Calculate pointer to beginning of next page */ state->recordContRecPtr = state->ReadRecPtr + record_len; - Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0); + Assert(state->recordContRecPtr % state->pageSize == 0); state->readRecordState = XLREAD_CONTINUATION; } @@ -576,7 +579,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) /* this request contains page header */ Assert (targetPagePtr != 0); if (XLogNeedData(state, targetPagePtr, - Min(state->recordRemainLen, XLOG_BLCKSZ), + Min(state->recordRemainLen, state->pageSize), false)) return XLREAD_NEED_DATA; @@ -628,7 +631,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) Assert(state->readLen >= pageHeaderSize); contdata = (char *) state->readBuf + pageHeaderSize; - record_len = XLOG_BLCKSZ - pageHeaderSize; + record_len = state->pageSize - pageHeaderSize; if (pageHeader->xlp_rem_len < record_len) record_len = pageHeader->xlp_rem_len; @@ -655,7 +658,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) } /* Calculate pointer to beginning of next page, and continue */ - state->recordContRecPtr += XLOG_BLCKSZ; + state->recordContRecPtr += state->pageSize; } /* targetPagePtr is pointing the last-read page here */ @@ -789,10 +792,10 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen, XLogPageHeaderSize((XLogPageHeader) state->readBuf); addLen = pageHeaderSize; - if (reqLen + pageHeaderSize <= XLOG_BLCKSZ) + if (reqLen + pageHeaderSize <= state->pageSize) addLen = pageHeaderSize; else - addLen = XLOG_BLCKSZ - reqLen; + addLen = state->pageSize - reqLen; Assert(addLen >= 0); } @@ -805,7 +808,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen, /* Data is not in our buffer, request the caller for it. */ XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize); targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize); - Assert((pageptr % XLOG_BLCKSZ) == 0); + Assert((pageptr % state->pageSize) == 0); /* * Every time we request to load new data of a page to the caller, even if @@ -829,7 +832,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen, * will not come back here, but will request the actual target page. */ state->readPagePtr = pageptr - targetPageOff; - state->readLen = XLOG_BLCKSZ; + state->readLen = state->pageSize; return true; } @@ -962,7 +965,7 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr, int32 offset; XLogPageHeader hdr = (XLogPageHeader) phdr; - Assert((recptr % XLOG_BLCKSZ) == 0); + Assert((recptr % state->pageSize) == 0); XLByteToSeg(recptr, segno, state->segcxt.ws_segsize); offset = XLogSegmentOffset(recptr, state->segcxt.ws_segsize); @@ -1016,10 +1019,10 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr, "WAL file is from different database system: incorrect segment size in page header"); return false; } - else if (longhdr->xlp_xlog_blcksz != XLOG_BLCKSZ) + else if (longhdr->xlp_xlog_blcksz != state->pageSize) { report_invalid_record(state, - "WAL file is from different database system: incorrect XLOG_BLCKSZ in page header"); + "WAL file is from different database system: incorrect block size in page header"); return false; } } @@ -1140,7 +1143,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr, * XLogNeedData() is prepared to handle that and will read at * least short page-header worth of data */ - targetRecOff = tmpRecPtr % XLOG_BLCKSZ; + targetRecOff = tmpRecPtr % state->pageSize; /* scroll back to page boundary */ targetPagePtr = tmpRecPtr - targetRecOff; @@ -1174,8 +1177,8 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr, * * Note that record headers are MAXALIGN'ed */ - if (MAXALIGN(header->xlp_rem_len) >= (XLOG_BLCKSZ - pageHeaderSize)) - tmpRecPtr = targetPagePtr + XLOG_BLCKSZ; + if (MAXALIGN(header->xlp_rem_len) >= (state->pageSize - pageHeaderSize)) + tmpRecPtr = targetPagePtr + state->pageSize; else { /* diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c index 9cbd8afa49..f2dd495310 100644 --- a/src/backend/replication/logical/logical.c +++ b/src/backend/replication/logical/logical.c @@ -199,11 +199,14 @@ StartupDecodingContext(List *output_plugin_options, ctx->slot = slot; - ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, cleanup_cb); + ctx->reader = + XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, cleanup_cb); + if (!ctx->reader) ereport(ERROR, (errcode(ERRCODE_OUT_OF_MEMORY), errmsg("out of memory"))); + ctx->reader->readBuf = palloc(XLOG_BLCKSZ); ctx->page_read = page_read; ctx->reorder = ReorderBufferAllocate(); diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c index 062093727d..8e95600ef7 100644 --- a/src/backend/replication/walsender.c +++ b/src/backend/replication/walsender.c @@ -579,8 +579,8 @@ StartReplication(StartReplicationCmd *cmd) errmsg("IDENTIFY_SYSTEM has not been run before START_REPLICATION"))); /* create xlogreader for physical replication */ - xlogreader = - XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close); + xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, + NULL, wal_segment_close); if (!xlogreader) ereport(ERROR, diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c index 712c85281c..8a6209b061 100644 --- a/src/bin/pg_rewind/parsexlog.c +++ b/src/bin/pg_rewind/parsexlog.c @@ -61,7 +61,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex, XLogReaderState *xlogreader; char *errormsg; - xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL); + xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL); if (xlogreader == NULL) pg_fatal("out of memory"); @@ -121,7 +121,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex, char *errormsg; XLogRecPtr endptr; - xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL); + xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL); if (xlogreader == NULL) pg_fatal("out of memory"); @@ -181,7 +181,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex, forkptr += SizeOfXLogShortPHD; } - xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL); + xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL); if (xlogreader == NULL) pg_fatal("out of memory"); diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c index f514de6b13..b635537659 100644 --- a/src/bin/pg_waldump/pg_waldump.c +++ b/src/bin/pg_waldump/pg_waldump.c @@ -1041,7 +1041,7 @@ main(int argc, char **argv) /* we have everything we need, start reading */ xlogreader_state = - XLogReaderAllocate(WalSegSz, waldir, WALDumpCloseSegment); + XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, waldir, WALDumpCloseSegment); if (!xlogreader_state) fatal_error("out of memory"); diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h index b9dad7ee51..1853516333 100644 --- a/src/include/access/xlogreader.h +++ b/src/include/access/xlogreader.h @@ -151,6 +151,7 @@ struct XLogReaderState * ---------------------------------------- */ /* variables to communicate with page reader */ + int pageSize; /* size of a page */ XLogRecPtr readPagePtr; /* page pointer to read */ int32 readLen; /* bytes requested to reader, or actual bytes * read by reader, which must be larger than @@ -243,7 +244,7 @@ struct XLogReaderState /* Get a new XLogReader */ extern XLogReaderState *XLogReaderAllocate(int wal_segment_size, - const char *waldir, + int page_size, const char *waldir, WALSegmentCleanupCB cleanup_cb); /* Free an XLogReader */ -- 2.27.0 ----Next_Part(Thu_Mar__4_11_28_53_2021_014)---- ^ permalink raw reply [nested|flat] 26+ messages in thread
* [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize. @ 2019-09-05 12:29 Kyotaro Horiguchi <[email protected]> 0 siblings, 0 replies; 26+ messages in thread From: Kyotaro Horiguchi @ 2019-09-05 12:29 UTC (permalink / raw) XLog reader facility and its callers currently relies on the same constant XLOG_BLCKSZ. This patch moves the responsibility to determine the block size to the callers of the facility. --- src/backend/access/transam/twophase.c | 3 +- src/backend/access/transam/xlog.c | 9 ++-- src/backend/access/transam/xlogreader.c | 57 ++++++++++++----------- src/backend/replication/logical/logical.c | 5 +- src/backend/replication/walsender.c | 4 +- src/bin/pg_rewind/parsexlog.c | 6 +-- src/bin/pg_waldump/pg_waldump.c | 2 +- src/include/access/xlogreader.h | 3 +- 8 files changed, 50 insertions(+), 39 deletions(-) diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c index 7d5a96a5f5..9f9e7de253 100644 --- a/src/backend/access/transam/twophase.c +++ b/src/backend/access/transam/twophase.c @@ -1325,7 +1325,8 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len) XLogReaderState *xlogreader; char *errormsg; - xlogreader = XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close); + xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, + NULL, wal_segment_close); if (!xlogreader) ereport(ERROR, diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index b517ca85ad..0822ac70cb 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -1222,7 +1222,10 @@ XLogInsertRecord(XLogRecData *rdata, appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len); if (!debug_reader) - debug_reader = XLogReaderAllocate(wal_segment_size, NULL, NULL); + { + debug_reader = + XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, NULL); + } if (!debug_reader) { @@ -6531,8 +6534,8 @@ StartupXLOG(void) OwnLatch(&XLogCtl->recoveryWakeupLatch); /* Set up XLOG reader facility */ - xlogreader = - XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close); + xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, + NULL, wal_segment_close); if (!xlogreader) ereport(ERROR, diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c index ba3e4868f7..e5baf13086 100644 --- a/src/backend/access/transam/xlogreader.c +++ b/src/backend/access/transam/xlogreader.c @@ -72,7 +72,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...) * Returns NULL if the xlogreader couldn't be allocated. */ XLogReaderState * -XLogReaderAllocate(int wal_segment_size, const char *waldir, +XLogReaderAllocate(int wal_segment_size, int page_size, const char *waldir, WALSegmentCleanupCB cleanup_cb) { XLogReaderState *state; @@ -87,6 +87,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir, state->cleanup_cb = cleanup_cb; state->max_block_id = -1; + state->pageSize = page_size; /* * Permanently allocate readBuf. We do it this way, rather than just @@ -95,7 +96,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir, * isn't guaranteed to have any particular alignment, whereas * palloc_extended() will provide MAXALIGN'd storage. */ - state->readBuf = (char *) palloc_extended(XLOG_BLCKSZ, + state->readBuf = (char *) palloc_extended(state->pageSize, MCXT_ALLOC_NO_OOM); if (!state->readBuf) { @@ -163,7 +164,7 @@ XLogReaderFree(XLogReaderState *state) * readRecordBufSize is set to the new buffer size. * * To avoid useless small increases, round its size to a multiple of - * XLOG_BLCKSZ, and make sure it's at least 5*Max(BLCKSZ, XLOG_BLCKSZ) to start + * page size, and make sure it's at least 5*Max(BLCKSZ, <page size>) to start * with. (That is enough for all "normal" records, but very large commit or * abort records might need more space.) */ @@ -172,8 +173,8 @@ allocate_recordbuf(XLogReaderState *state, uint32 reclength) { uint32 newSize = reclength; - newSize += XLOG_BLCKSZ - (newSize % XLOG_BLCKSZ); - newSize = Max(newSize, 5 * Max(BLCKSZ, XLOG_BLCKSZ)); + newSize += state->pageSize - (newSize % state->pageSize); + newSize = Max(newSize, 5 * Max(BLCKSZ, state->pageSize)); #ifndef FRONTEND @@ -238,6 +239,8 @@ void XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr) { Assert(!XLogRecPtrIsInvalid(RecPtr)); + Assert(state->readBuf != NULL && + state->readBuf == (void *) MAXALIGN(state->readBuf)); ResetDecoder(state); @@ -366,15 +369,15 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) XLogPageHeader pageHeader; targetPagePtr = - state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ); - targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ; + state->ReadRecPtr - (state->ReadRecPtr % state->pageSize); + targetRecOff = state->ReadRecPtr % state->pageSize; /* * Check if we have enough data. For the first record in the page, * the requesting length doesn't contain page header. */ if (XLogNeedData(state, targetPagePtr, - Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ), + Min(targetRecOff + SizeOfXLogRecord, state->pageSize), targetRecOff != 0)) return XLREAD_NEED_DATA; @@ -421,7 +424,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) * verified that we got the whole header. */ prec = (XLogRecord *) (state->readBuf + - state->ReadRecPtr % XLOG_BLCKSZ); + state->ReadRecPtr % state->pageSize); total_len = prec->xl_tot_len; /* @@ -432,7 +435,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) * ensure that we enter the XLREAD_CONTINUATION state below; * otherwise we might fail to apply ValidXLogRecordHeader at all. */ - if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord) + if (targetRecOff <= state->pageSize - SizeOfXLogRecord) { if (!ValidXLogRecordHeader(state, state->ReadRecPtr, state->PrevRecPtr, prec)) @@ -477,10 +480,10 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) * available */ targetPagePtr = - state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ); - targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ; + state->ReadRecPtr - (state->ReadRecPtr % state->pageSize); + targetRecOff = state->ReadRecPtr % state->pageSize; - request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ); + request_len = Min(targetRecOff + total_len, state->pageSize); record_len = request_len - targetRecOff; /* ReadRecPtr contains page header */ @@ -549,7 +552,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) /* Calculate pointer to beginning of next page */ state->recordContRecPtr = state->ReadRecPtr + record_len; - Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0); + Assert(state->recordContRecPtr % state->pageSize == 0); state->readRecordState = XLREAD_CONTINUATION; } @@ -576,7 +579,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) /* this request contains page header */ Assert (targetPagePtr != 0); if (XLogNeedData(state, targetPagePtr, - Min(state->recordRemainLen, XLOG_BLCKSZ), + Min(state->recordRemainLen, state->pageSize), false)) return XLREAD_NEED_DATA; @@ -628,7 +631,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) Assert(state->readLen >= pageHeaderSize); contdata = (char *) state->readBuf + pageHeaderSize; - record_len = XLOG_BLCKSZ - pageHeaderSize; + record_len = state->pageSize - pageHeaderSize; if (pageHeader->xlp_rem_len < record_len) record_len = pageHeader->xlp_rem_len; @@ -655,7 +658,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) } /* Calculate pointer to beginning of next page, and continue */ - state->recordContRecPtr += XLOG_BLCKSZ; + state->recordContRecPtr += state->pageSize; } /* targetPagePtr is pointing the last-read page here */ @@ -789,10 +792,10 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen, XLogPageHeaderSize((XLogPageHeader) state->readBuf); addLen = pageHeaderSize; - if (reqLen + pageHeaderSize <= XLOG_BLCKSZ) + if (reqLen + pageHeaderSize <= state->pageSize) addLen = pageHeaderSize; else - addLen = XLOG_BLCKSZ - reqLen; + addLen = state->pageSize - reqLen; Assert(addLen >= 0); } @@ -805,7 +808,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen, /* Data is not in our buffer, request the caller for it. */ XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize); targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize); - Assert((pageptr % XLOG_BLCKSZ) == 0); + Assert((pageptr % state->pageSize) == 0); /* * Every time we request to load new data of a page to the caller, even if @@ -829,7 +832,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen, * will not come back here, but will request the actual target page. */ state->readPagePtr = pageptr - targetPageOff; - state->readLen = XLOG_BLCKSZ; + state->readLen = state->pageSize; return true; } @@ -962,7 +965,7 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr, int32 offset; XLogPageHeader hdr = (XLogPageHeader) phdr; - Assert((recptr % XLOG_BLCKSZ) == 0); + Assert((recptr % state->pageSize) == 0); XLByteToSeg(recptr, segno, state->segcxt.ws_segsize); offset = XLogSegmentOffset(recptr, state->segcxt.ws_segsize); @@ -1016,10 +1019,10 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr, "WAL file is from different database system: incorrect segment size in page header"); return false; } - else if (longhdr->xlp_xlog_blcksz != XLOG_BLCKSZ) + else if (longhdr->xlp_xlog_blcksz != state->pageSize) { report_invalid_record(state, - "WAL file is from different database system: incorrect XLOG_BLCKSZ in page header"); + "WAL file is from different database system: incorrect block size in page header"); return false; } } @@ -1140,7 +1143,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr, * XLogNeedData() is prepared to handle that and will read at * least short page-header worth of data */ - targetRecOff = tmpRecPtr % XLOG_BLCKSZ; + targetRecOff = tmpRecPtr % state->pageSize; /* scroll back to page boundary */ targetPagePtr = tmpRecPtr - targetRecOff; @@ -1174,8 +1177,8 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr, * * Note that record headers are MAXALIGN'ed */ - if (MAXALIGN(header->xlp_rem_len) >= (XLOG_BLCKSZ - pageHeaderSize)) - tmpRecPtr = targetPagePtr + XLOG_BLCKSZ; + if (MAXALIGN(header->xlp_rem_len) >= (state->pageSize - pageHeaderSize)) + tmpRecPtr = targetPagePtr + state->pageSize; else { /* diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c index 9cbd8afa49..f2dd495310 100644 --- a/src/backend/replication/logical/logical.c +++ b/src/backend/replication/logical/logical.c @@ -199,11 +199,14 @@ StartupDecodingContext(List *output_plugin_options, ctx->slot = slot; - ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, cleanup_cb); + ctx->reader = + XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, cleanup_cb); + if (!ctx->reader) ereport(ERROR, (errcode(ERRCODE_OUT_OF_MEMORY), errmsg("out of memory"))); + ctx->reader->readBuf = palloc(XLOG_BLCKSZ); ctx->page_read = page_read; ctx->reorder = ReorderBufferAllocate(); diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c index 062093727d..8e95600ef7 100644 --- a/src/backend/replication/walsender.c +++ b/src/backend/replication/walsender.c @@ -579,8 +579,8 @@ StartReplication(StartReplicationCmd *cmd) errmsg("IDENTIFY_SYSTEM has not been run before START_REPLICATION"))); /* create xlogreader for physical replication */ - xlogreader = - XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close); + xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, + NULL, wal_segment_close); if (!xlogreader) ereport(ERROR, diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c index 712c85281c..8a6209b061 100644 --- a/src/bin/pg_rewind/parsexlog.c +++ b/src/bin/pg_rewind/parsexlog.c @@ -61,7 +61,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex, XLogReaderState *xlogreader; char *errormsg; - xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL); + xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL); if (xlogreader == NULL) pg_fatal("out of memory"); @@ -121,7 +121,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex, char *errormsg; XLogRecPtr endptr; - xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL); + xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL); if (xlogreader == NULL) pg_fatal("out of memory"); @@ -181,7 +181,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex, forkptr += SizeOfXLogShortPHD; } - xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL); + xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL); if (xlogreader == NULL) pg_fatal("out of memory"); diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c index f514de6b13..b635537659 100644 --- a/src/bin/pg_waldump/pg_waldump.c +++ b/src/bin/pg_waldump/pg_waldump.c @@ -1041,7 +1041,7 @@ main(int argc, char **argv) /* we have everything we need, start reading */ xlogreader_state = - XLogReaderAllocate(WalSegSz, waldir, WALDumpCloseSegment); + XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, waldir, WALDumpCloseSegment); if (!xlogreader_state) fatal_error("out of memory"); diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h index b9dad7ee51..1853516333 100644 --- a/src/include/access/xlogreader.h +++ b/src/include/access/xlogreader.h @@ -151,6 +151,7 @@ struct XLogReaderState * ---------------------------------------- */ /* variables to communicate with page reader */ + int pageSize; /* size of a page */ XLogRecPtr readPagePtr; /* page pointer to read */ int32 readLen; /* bytes requested to reader, or actual bytes * read by reader, which must be larger than @@ -243,7 +244,7 @@ struct XLogReaderState /* Get a new XLogReader */ extern XLogReaderState *XLogReaderAllocate(int wal_segment_size, - const char *waldir, + int page_size, const char *waldir, WALSegmentCleanupCB cleanup_cb); /* Free an XLogReader */ -- 2.27.0 ----Next_Part(Thu_Mar__4_11_28_53_2021_014)---- ^ permalink raw reply [nested|flat] 26+ messages in thread
* [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize. @ 2019-09-05 12:29 Kyotaro Horiguchi <[email protected]> 0 siblings, 0 replies; 26+ messages in thread From: Kyotaro Horiguchi @ 2019-09-05 12:29 UTC (permalink / raw) XLog reader facility and its callers currently relies on the same constant XLOG_BLCKSZ. This patch moves the responsibility to determine the block size to the callers of the facility. --- src/backend/access/transam/twophase.c | 3 +- src/backend/access/transam/xlog.c | 9 ++-- src/backend/access/transam/xlogreader.c | 57 ++++++++++++----------- src/backend/replication/logical/logical.c | 5 +- src/backend/replication/walsender.c | 4 +- src/bin/pg_rewind/parsexlog.c | 6 +-- src/bin/pg_waldump/pg_waldump.c | 2 +- src/include/access/xlogreader.h | 3 +- 8 files changed, 50 insertions(+), 39 deletions(-) diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c index 7d5a96a5f5..9f9e7de253 100644 --- a/src/backend/access/transam/twophase.c +++ b/src/backend/access/transam/twophase.c @@ -1325,7 +1325,8 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len) XLogReaderState *xlogreader; char *errormsg; - xlogreader = XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close); + xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, + NULL, wal_segment_close); if (!xlogreader) ereport(ERROR, diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index b517ca85ad..0822ac70cb 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -1222,7 +1222,10 @@ XLogInsertRecord(XLogRecData *rdata, appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len); if (!debug_reader) - debug_reader = XLogReaderAllocate(wal_segment_size, NULL, NULL); + { + debug_reader = + XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, NULL); + } if (!debug_reader) { @@ -6531,8 +6534,8 @@ StartupXLOG(void) OwnLatch(&XLogCtl->recoveryWakeupLatch); /* Set up XLOG reader facility */ - xlogreader = - XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close); + xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, + NULL, wal_segment_close); if (!xlogreader) ereport(ERROR, diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c index ba3e4868f7..e5baf13086 100644 --- a/src/backend/access/transam/xlogreader.c +++ b/src/backend/access/transam/xlogreader.c @@ -72,7 +72,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...) * Returns NULL if the xlogreader couldn't be allocated. */ XLogReaderState * -XLogReaderAllocate(int wal_segment_size, const char *waldir, +XLogReaderAllocate(int wal_segment_size, int page_size, const char *waldir, WALSegmentCleanupCB cleanup_cb) { XLogReaderState *state; @@ -87,6 +87,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir, state->cleanup_cb = cleanup_cb; state->max_block_id = -1; + state->pageSize = page_size; /* * Permanently allocate readBuf. We do it this way, rather than just @@ -95,7 +96,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir, * isn't guaranteed to have any particular alignment, whereas * palloc_extended() will provide MAXALIGN'd storage. */ - state->readBuf = (char *) palloc_extended(XLOG_BLCKSZ, + state->readBuf = (char *) palloc_extended(state->pageSize, MCXT_ALLOC_NO_OOM); if (!state->readBuf) { @@ -163,7 +164,7 @@ XLogReaderFree(XLogReaderState *state) * readRecordBufSize is set to the new buffer size. * * To avoid useless small increases, round its size to a multiple of - * XLOG_BLCKSZ, and make sure it's at least 5*Max(BLCKSZ, XLOG_BLCKSZ) to start + * page size, and make sure it's at least 5*Max(BLCKSZ, <page size>) to start * with. (That is enough for all "normal" records, but very large commit or * abort records might need more space.) */ @@ -172,8 +173,8 @@ allocate_recordbuf(XLogReaderState *state, uint32 reclength) { uint32 newSize = reclength; - newSize += XLOG_BLCKSZ - (newSize % XLOG_BLCKSZ); - newSize = Max(newSize, 5 * Max(BLCKSZ, XLOG_BLCKSZ)); + newSize += state->pageSize - (newSize % state->pageSize); + newSize = Max(newSize, 5 * Max(BLCKSZ, state->pageSize)); #ifndef FRONTEND @@ -238,6 +239,8 @@ void XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr) { Assert(!XLogRecPtrIsInvalid(RecPtr)); + Assert(state->readBuf != NULL && + state->readBuf == (void *) MAXALIGN(state->readBuf)); ResetDecoder(state); @@ -366,15 +369,15 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) XLogPageHeader pageHeader; targetPagePtr = - state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ); - targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ; + state->ReadRecPtr - (state->ReadRecPtr % state->pageSize); + targetRecOff = state->ReadRecPtr % state->pageSize; /* * Check if we have enough data. For the first record in the page, * the requesting length doesn't contain page header. */ if (XLogNeedData(state, targetPagePtr, - Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ), + Min(targetRecOff + SizeOfXLogRecord, state->pageSize), targetRecOff != 0)) return XLREAD_NEED_DATA; @@ -421,7 +424,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) * verified that we got the whole header. */ prec = (XLogRecord *) (state->readBuf + - state->ReadRecPtr % XLOG_BLCKSZ); + state->ReadRecPtr % state->pageSize); total_len = prec->xl_tot_len; /* @@ -432,7 +435,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) * ensure that we enter the XLREAD_CONTINUATION state below; * otherwise we might fail to apply ValidXLogRecordHeader at all. */ - if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord) + if (targetRecOff <= state->pageSize - SizeOfXLogRecord) { if (!ValidXLogRecordHeader(state, state->ReadRecPtr, state->PrevRecPtr, prec)) @@ -477,10 +480,10 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) * available */ targetPagePtr = - state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ); - targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ; + state->ReadRecPtr - (state->ReadRecPtr % state->pageSize); + targetRecOff = state->ReadRecPtr % state->pageSize; - request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ); + request_len = Min(targetRecOff + total_len, state->pageSize); record_len = request_len - targetRecOff; /* ReadRecPtr contains page header */ @@ -549,7 +552,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) /* Calculate pointer to beginning of next page */ state->recordContRecPtr = state->ReadRecPtr + record_len; - Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0); + Assert(state->recordContRecPtr % state->pageSize == 0); state->readRecordState = XLREAD_CONTINUATION; } @@ -576,7 +579,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) /* this request contains page header */ Assert (targetPagePtr != 0); if (XLogNeedData(state, targetPagePtr, - Min(state->recordRemainLen, XLOG_BLCKSZ), + Min(state->recordRemainLen, state->pageSize), false)) return XLREAD_NEED_DATA; @@ -628,7 +631,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) Assert(state->readLen >= pageHeaderSize); contdata = (char *) state->readBuf + pageHeaderSize; - record_len = XLOG_BLCKSZ - pageHeaderSize; + record_len = state->pageSize - pageHeaderSize; if (pageHeader->xlp_rem_len < record_len) record_len = pageHeader->xlp_rem_len; @@ -655,7 +658,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) } /* Calculate pointer to beginning of next page, and continue */ - state->recordContRecPtr += XLOG_BLCKSZ; + state->recordContRecPtr += state->pageSize; } /* targetPagePtr is pointing the last-read page here */ @@ -789,10 +792,10 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen, XLogPageHeaderSize((XLogPageHeader) state->readBuf); addLen = pageHeaderSize; - if (reqLen + pageHeaderSize <= XLOG_BLCKSZ) + if (reqLen + pageHeaderSize <= state->pageSize) addLen = pageHeaderSize; else - addLen = XLOG_BLCKSZ - reqLen; + addLen = state->pageSize - reqLen; Assert(addLen >= 0); } @@ -805,7 +808,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen, /* Data is not in our buffer, request the caller for it. */ XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize); targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize); - Assert((pageptr % XLOG_BLCKSZ) == 0); + Assert((pageptr % state->pageSize) == 0); /* * Every time we request to load new data of a page to the caller, even if @@ -829,7 +832,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen, * will not come back here, but will request the actual target page. */ state->readPagePtr = pageptr - targetPageOff; - state->readLen = XLOG_BLCKSZ; + state->readLen = state->pageSize; return true; } @@ -962,7 +965,7 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr, int32 offset; XLogPageHeader hdr = (XLogPageHeader) phdr; - Assert((recptr % XLOG_BLCKSZ) == 0); + Assert((recptr % state->pageSize) == 0); XLByteToSeg(recptr, segno, state->segcxt.ws_segsize); offset = XLogSegmentOffset(recptr, state->segcxt.ws_segsize); @@ -1016,10 +1019,10 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr, "WAL file is from different database system: incorrect segment size in page header"); return false; } - else if (longhdr->xlp_xlog_blcksz != XLOG_BLCKSZ) + else if (longhdr->xlp_xlog_blcksz != state->pageSize) { report_invalid_record(state, - "WAL file is from different database system: incorrect XLOG_BLCKSZ in page header"); + "WAL file is from different database system: incorrect block size in page header"); return false; } } @@ -1140,7 +1143,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr, * XLogNeedData() is prepared to handle that and will read at * least short page-header worth of data */ - targetRecOff = tmpRecPtr % XLOG_BLCKSZ; + targetRecOff = tmpRecPtr % state->pageSize; /* scroll back to page boundary */ targetPagePtr = tmpRecPtr - targetRecOff; @@ -1174,8 +1177,8 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr, * * Note that record headers are MAXALIGN'ed */ - if (MAXALIGN(header->xlp_rem_len) >= (XLOG_BLCKSZ - pageHeaderSize)) - tmpRecPtr = targetPagePtr + XLOG_BLCKSZ; + if (MAXALIGN(header->xlp_rem_len) >= (state->pageSize - pageHeaderSize)) + tmpRecPtr = targetPagePtr + state->pageSize; else { /* diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c index 9cbd8afa49..f2dd495310 100644 --- a/src/backend/replication/logical/logical.c +++ b/src/backend/replication/logical/logical.c @@ -199,11 +199,14 @@ StartupDecodingContext(List *output_plugin_options, ctx->slot = slot; - ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, cleanup_cb); + ctx->reader = + XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, cleanup_cb); + if (!ctx->reader) ereport(ERROR, (errcode(ERRCODE_OUT_OF_MEMORY), errmsg("out of memory"))); + ctx->reader->readBuf = palloc(XLOG_BLCKSZ); ctx->page_read = page_read; ctx->reorder = ReorderBufferAllocate(); diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c index 062093727d..8e95600ef7 100644 --- a/src/backend/replication/walsender.c +++ b/src/backend/replication/walsender.c @@ -579,8 +579,8 @@ StartReplication(StartReplicationCmd *cmd) errmsg("IDENTIFY_SYSTEM has not been run before START_REPLICATION"))); /* create xlogreader for physical replication */ - xlogreader = - XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close); + xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, + NULL, wal_segment_close); if (!xlogreader) ereport(ERROR, diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c index 712c85281c..8a6209b061 100644 --- a/src/bin/pg_rewind/parsexlog.c +++ b/src/bin/pg_rewind/parsexlog.c @@ -61,7 +61,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex, XLogReaderState *xlogreader; char *errormsg; - xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL); + xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL); if (xlogreader == NULL) pg_fatal("out of memory"); @@ -121,7 +121,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex, char *errormsg; XLogRecPtr endptr; - xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL); + xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL); if (xlogreader == NULL) pg_fatal("out of memory"); @@ -181,7 +181,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex, forkptr += SizeOfXLogShortPHD; } - xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL); + xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL); if (xlogreader == NULL) pg_fatal("out of memory"); diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c index f514de6b13..b635537659 100644 --- a/src/bin/pg_waldump/pg_waldump.c +++ b/src/bin/pg_waldump/pg_waldump.c @@ -1041,7 +1041,7 @@ main(int argc, char **argv) /* we have everything we need, start reading */ xlogreader_state = - XLogReaderAllocate(WalSegSz, waldir, WALDumpCloseSegment); + XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, waldir, WALDumpCloseSegment); if (!xlogreader_state) fatal_error("out of memory"); diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h index b9dad7ee51..1853516333 100644 --- a/src/include/access/xlogreader.h +++ b/src/include/access/xlogreader.h @@ -151,6 +151,7 @@ struct XLogReaderState * ---------------------------------------- */ /* variables to communicate with page reader */ + int pageSize; /* size of a page */ XLogRecPtr readPagePtr; /* page pointer to read */ int32 readLen; /* bytes requested to reader, or actual bytes * read by reader, which must be larger than @@ -243,7 +244,7 @@ struct XLogReaderState /* Get a new XLogReader */ extern XLogReaderState *XLogReaderAllocate(int wal_segment_size, - const char *waldir, + int page_size, const char *waldir, WALSegmentCleanupCB cleanup_cb); /* Free an XLogReader */ -- 2.27.0 ----Next_Part(Thu_Mar__4_11_28_53_2021_014)---- ^ permalink raw reply [nested|flat] 26+ messages in thread
* [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize. @ 2019-09-05 12:29 Kyotaro Horiguchi <[email protected]> 0 siblings, 0 replies; 26+ messages in thread From: Kyotaro Horiguchi @ 2019-09-05 12:29 UTC (permalink / raw) XLog reader facility and its callers currently relies on the same constant XLOG_BLCKSZ. This patch moves the responsibility to determine the block size to the callers of the facility. --- src/backend/access/transam/twophase.c | 3 +- src/backend/access/transam/xlog.c | 9 ++-- src/backend/access/transam/xlogreader.c | 57 ++++++++++++----------- src/backend/replication/logical/logical.c | 5 +- src/backend/replication/walsender.c | 4 +- src/bin/pg_rewind/parsexlog.c | 6 +-- src/bin/pg_waldump/pg_waldump.c | 2 +- src/include/access/xlogreader.h | 3 +- 8 files changed, 50 insertions(+), 39 deletions(-) diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c index 7d5a96a5f5..9f9e7de253 100644 --- a/src/backend/access/transam/twophase.c +++ b/src/backend/access/transam/twophase.c @@ -1325,7 +1325,8 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len) XLogReaderState *xlogreader; char *errormsg; - xlogreader = XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close); + xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, + NULL, wal_segment_close); if (!xlogreader) ereport(ERROR, diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index b517ca85ad..0822ac70cb 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -1222,7 +1222,10 @@ XLogInsertRecord(XLogRecData *rdata, appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len); if (!debug_reader) - debug_reader = XLogReaderAllocate(wal_segment_size, NULL, NULL); + { + debug_reader = + XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, NULL); + } if (!debug_reader) { @@ -6531,8 +6534,8 @@ StartupXLOG(void) OwnLatch(&XLogCtl->recoveryWakeupLatch); /* Set up XLOG reader facility */ - xlogreader = - XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close); + xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, + NULL, wal_segment_close); if (!xlogreader) ereport(ERROR, diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c index ba3e4868f7..e5baf13086 100644 --- a/src/backend/access/transam/xlogreader.c +++ b/src/backend/access/transam/xlogreader.c @@ -72,7 +72,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...) * Returns NULL if the xlogreader couldn't be allocated. */ XLogReaderState * -XLogReaderAllocate(int wal_segment_size, const char *waldir, +XLogReaderAllocate(int wal_segment_size, int page_size, const char *waldir, WALSegmentCleanupCB cleanup_cb) { XLogReaderState *state; @@ -87,6 +87,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir, state->cleanup_cb = cleanup_cb; state->max_block_id = -1; + state->pageSize = page_size; /* * Permanently allocate readBuf. We do it this way, rather than just @@ -95,7 +96,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir, * isn't guaranteed to have any particular alignment, whereas * palloc_extended() will provide MAXALIGN'd storage. */ - state->readBuf = (char *) palloc_extended(XLOG_BLCKSZ, + state->readBuf = (char *) palloc_extended(state->pageSize, MCXT_ALLOC_NO_OOM); if (!state->readBuf) { @@ -163,7 +164,7 @@ XLogReaderFree(XLogReaderState *state) * readRecordBufSize is set to the new buffer size. * * To avoid useless small increases, round its size to a multiple of - * XLOG_BLCKSZ, and make sure it's at least 5*Max(BLCKSZ, XLOG_BLCKSZ) to start + * page size, and make sure it's at least 5*Max(BLCKSZ, <page size>) to start * with. (That is enough for all "normal" records, but very large commit or * abort records might need more space.) */ @@ -172,8 +173,8 @@ allocate_recordbuf(XLogReaderState *state, uint32 reclength) { uint32 newSize = reclength; - newSize += XLOG_BLCKSZ - (newSize % XLOG_BLCKSZ); - newSize = Max(newSize, 5 * Max(BLCKSZ, XLOG_BLCKSZ)); + newSize += state->pageSize - (newSize % state->pageSize); + newSize = Max(newSize, 5 * Max(BLCKSZ, state->pageSize)); #ifndef FRONTEND @@ -238,6 +239,8 @@ void XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr) { Assert(!XLogRecPtrIsInvalid(RecPtr)); + Assert(state->readBuf != NULL && + state->readBuf == (void *) MAXALIGN(state->readBuf)); ResetDecoder(state); @@ -366,15 +369,15 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) XLogPageHeader pageHeader; targetPagePtr = - state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ); - targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ; + state->ReadRecPtr - (state->ReadRecPtr % state->pageSize); + targetRecOff = state->ReadRecPtr % state->pageSize; /* * Check if we have enough data. For the first record in the page, * the requesting length doesn't contain page header. */ if (XLogNeedData(state, targetPagePtr, - Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ), + Min(targetRecOff + SizeOfXLogRecord, state->pageSize), targetRecOff != 0)) return XLREAD_NEED_DATA; @@ -421,7 +424,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) * verified that we got the whole header. */ prec = (XLogRecord *) (state->readBuf + - state->ReadRecPtr % XLOG_BLCKSZ); + state->ReadRecPtr % state->pageSize); total_len = prec->xl_tot_len; /* @@ -432,7 +435,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) * ensure that we enter the XLREAD_CONTINUATION state below; * otherwise we might fail to apply ValidXLogRecordHeader at all. */ - if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord) + if (targetRecOff <= state->pageSize - SizeOfXLogRecord) { if (!ValidXLogRecordHeader(state, state->ReadRecPtr, state->PrevRecPtr, prec)) @@ -477,10 +480,10 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) * available */ targetPagePtr = - state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ); - targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ; + state->ReadRecPtr - (state->ReadRecPtr % state->pageSize); + targetRecOff = state->ReadRecPtr % state->pageSize; - request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ); + request_len = Min(targetRecOff + total_len, state->pageSize); record_len = request_len - targetRecOff; /* ReadRecPtr contains page header */ @@ -549,7 +552,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) /* Calculate pointer to beginning of next page */ state->recordContRecPtr = state->ReadRecPtr + record_len; - Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0); + Assert(state->recordContRecPtr % state->pageSize == 0); state->readRecordState = XLREAD_CONTINUATION; } @@ -576,7 +579,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) /* this request contains page header */ Assert (targetPagePtr != 0); if (XLogNeedData(state, targetPagePtr, - Min(state->recordRemainLen, XLOG_BLCKSZ), + Min(state->recordRemainLen, state->pageSize), false)) return XLREAD_NEED_DATA; @@ -628,7 +631,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) Assert(state->readLen >= pageHeaderSize); contdata = (char *) state->readBuf + pageHeaderSize; - record_len = XLOG_BLCKSZ - pageHeaderSize; + record_len = state->pageSize - pageHeaderSize; if (pageHeader->xlp_rem_len < record_len) record_len = pageHeader->xlp_rem_len; @@ -655,7 +658,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) } /* Calculate pointer to beginning of next page, and continue */ - state->recordContRecPtr += XLOG_BLCKSZ; + state->recordContRecPtr += state->pageSize; } /* targetPagePtr is pointing the last-read page here */ @@ -789,10 +792,10 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen, XLogPageHeaderSize((XLogPageHeader) state->readBuf); addLen = pageHeaderSize; - if (reqLen + pageHeaderSize <= XLOG_BLCKSZ) + if (reqLen + pageHeaderSize <= state->pageSize) addLen = pageHeaderSize; else - addLen = XLOG_BLCKSZ - reqLen; + addLen = state->pageSize - reqLen; Assert(addLen >= 0); } @@ -805,7 +808,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen, /* Data is not in our buffer, request the caller for it. */ XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize); targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize); - Assert((pageptr % XLOG_BLCKSZ) == 0); + Assert((pageptr % state->pageSize) == 0); /* * Every time we request to load new data of a page to the caller, even if @@ -829,7 +832,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen, * will not come back here, but will request the actual target page. */ state->readPagePtr = pageptr - targetPageOff; - state->readLen = XLOG_BLCKSZ; + state->readLen = state->pageSize; return true; } @@ -962,7 +965,7 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr, int32 offset; XLogPageHeader hdr = (XLogPageHeader) phdr; - Assert((recptr % XLOG_BLCKSZ) == 0); + Assert((recptr % state->pageSize) == 0); XLByteToSeg(recptr, segno, state->segcxt.ws_segsize); offset = XLogSegmentOffset(recptr, state->segcxt.ws_segsize); @@ -1016,10 +1019,10 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr, "WAL file is from different database system: incorrect segment size in page header"); return false; } - else if (longhdr->xlp_xlog_blcksz != XLOG_BLCKSZ) + else if (longhdr->xlp_xlog_blcksz != state->pageSize) { report_invalid_record(state, - "WAL file is from different database system: incorrect XLOG_BLCKSZ in page header"); + "WAL file is from different database system: incorrect block size in page header"); return false; } } @@ -1140,7 +1143,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr, * XLogNeedData() is prepared to handle that and will read at * least short page-header worth of data */ - targetRecOff = tmpRecPtr % XLOG_BLCKSZ; + targetRecOff = tmpRecPtr % state->pageSize; /* scroll back to page boundary */ targetPagePtr = tmpRecPtr - targetRecOff; @@ -1174,8 +1177,8 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr, * * Note that record headers are MAXALIGN'ed */ - if (MAXALIGN(header->xlp_rem_len) >= (XLOG_BLCKSZ - pageHeaderSize)) - tmpRecPtr = targetPagePtr + XLOG_BLCKSZ; + if (MAXALIGN(header->xlp_rem_len) >= (state->pageSize - pageHeaderSize)) + tmpRecPtr = targetPagePtr + state->pageSize; else { /* diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c index 9cbd8afa49..f2dd495310 100644 --- a/src/backend/replication/logical/logical.c +++ b/src/backend/replication/logical/logical.c @@ -199,11 +199,14 @@ StartupDecodingContext(List *output_plugin_options, ctx->slot = slot; - ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, cleanup_cb); + ctx->reader = + XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, cleanup_cb); + if (!ctx->reader) ereport(ERROR, (errcode(ERRCODE_OUT_OF_MEMORY), errmsg("out of memory"))); + ctx->reader->readBuf = palloc(XLOG_BLCKSZ); ctx->page_read = page_read; ctx->reorder = ReorderBufferAllocate(); diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c index 062093727d..8e95600ef7 100644 --- a/src/backend/replication/walsender.c +++ b/src/backend/replication/walsender.c @@ -579,8 +579,8 @@ StartReplication(StartReplicationCmd *cmd) errmsg("IDENTIFY_SYSTEM has not been run before START_REPLICATION"))); /* create xlogreader for physical replication */ - xlogreader = - XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close); + xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, + NULL, wal_segment_close); if (!xlogreader) ereport(ERROR, diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c index 712c85281c..8a6209b061 100644 --- a/src/bin/pg_rewind/parsexlog.c +++ b/src/bin/pg_rewind/parsexlog.c @@ -61,7 +61,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex, XLogReaderState *xlogreader; char *errormsg; - xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL); + xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL); if (xlogreader == NULL) pg_fatal("out of memory"); @@ -121,7 +121,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex, char *errormsg; XLogRecPtr endptr; - xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL); + xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL); if (xlogreader == NULL) pg_fatal("out of memory"); @@ -181,7 +181,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex, forkptr += SizeOfXLogShortPHD; } - xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL); + xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL); if (xlogreader == NULL) pg_fatal("out of memory"); diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c index f514de6b13..b635537659 100644 --- a/src/bin/pg_waldump/pg_waldump.c +++ b/src/bin/pg_waldump/pg_waldump.c @@ -1041,7 +1041,7 @@ main(int argc, char **argv) /* we have everything we need, start reading */ xlogreader_state = - XLogReaderAllocate(WalSegSz, waldir, WALDumpCloseSegment); + XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, waldir, WALDumpCloseSegment); if (!xlogreader_state) fatal_error("out of memory"); diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h index b9dad7ee51..1853516333 100644 --- a/src/include/access/xlogreader.h +++ b/src/include/access/xlogreader.h @@ -151,6 +151,7 @@ struct XLogReaderState * ---------------------------------------- */ /* variables to communicate with page reader */ + int pageSize; /* size of a page */ XLogRecPtr readPagePtr; /* page pointer to read */ int32 readLen; /* bytes requested to reader, or actual bytes * read by reader, which must be larger than @@ -243,7 +244,7 @@ struct XLogReaderState /* Get a new XLogReader */ extern XLogReaderState *XLogReaderAllocate(int wal_segment_size, - const char *waldir, + int page_size, const char *waldir, WALSegmentCleanupCB cleanup_cb); /* Free an XLogReader */ -- 2.27.0 ----Next_Part(Thu_Mar__4_11_28_53_2021_014)---- ^ permalink raw reply [nested|flat] 26+ messages in thread
* [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize. @ 2019-09-05 12:29 Kyotaro Horiguchi <[email protected]> 0 siblings, 0 replies; 26+ messages in thread From: Kyotaro Horiguchi @ 2019-09-05 12:29 UTC (permalink / raw) XLog reader facility and its callers currently relies on the same constant XLOG_BLCKSZ. This patch moves the responsibility to determine the block size to the callers of the facility. --- src/backend/access/transam/twophase.c | 3 +- src/backend/access/transam/xlog.c | 9 ++-- src/backend/access/transam/xlogreader.c | 57 ++++++++++++----------- src/backend/replication/logical/logical.c | 5 +- src/backend/replication/walsender.c | 4 +- src/bin/pg_rewind/parsexlog.c | 6 +-- src/bin/pg_waldump/pg_waldump.c | 2 +- src/include/access/xlogreader.h | 3 +- 8 files changed, 50 insertions(+), 39 deletions(-) diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c index 7d5a96a5f5..9f9e7de253 100644 --- a/src/backend/access/transam/twophase.c +++ b/src/backend/access/transam/twophase.c @@ -1325,7 +1325,8 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len) XLogReaderState *xlogreader; char *errormsg; - xlogreader = XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close); + xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, + NULL, wal_segment_close); if (!xlogreader) ereport(ERROR, diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index b517ca85ad..0822ac70cb 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -1222,7 +1222,10 @@ XLogInsertRecord(XLogRecData *rdata, appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len); if (!debug_reader) - debug_reader = XLogReaderAllocate(wal_segment_size, NULL, NULL); + { + debug_reader = + XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, NULL); + } if (!debug_reader) { @@ -6531,8 +6534,8 @@ StartupXLOG(void) OwnLatch(&XLogCtl->recoveryWakeupLatch); /* Set up XLOG reader facility */ - xlogreader = - XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close); + xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, + NULL, wal_segment_close); if (!xlogreader) ereport(ERROR, diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c index ba3e4868f7..e5baf13086 100644 --- a/src/backend/access/transam/xlogreader.c +++ b/src/backend/access/transam/xlogreader.c @@ -72,7 +72,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...) * Returns NULL if the xlogreader couldn't be allocated. */ XLogReaderState * -XLogReaderAllocate(int wal_segment_size, const char *waldir, +XLogReaderAllocate(int wal_segment_size, int page_size, const char *waldir, WALSegmentCleanupCB cleanup_cb) { XLogReaderState *state; @@ -87,6 +87,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir, state->cleanup_cb = cleanup_cb; state->max_block_id = -1; + state->pageSize = page_size; /* * Permanently allocate readBuf. We do it this way, rather than just @@ -95,7 +96,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir, * isn't guaranteed to have any particular alignment, whereas * palloc_extended() will provide MAXALIGN'd storage. */ - state->readBuf = (char *) palloc_extended(XLOG_BLCKSZ, + state->readBuf = (char *) palloc_extended(state->pageSize, MCXT_ALLOC_NO_OOM); if (!state->readBuf) { @@ -163,7 +164,7 @@ XLogReaderFree(XLogReaderState *state) * readRecordBufSize is set to the new buffer size. * * To avoid useless small increases, round its size to a multiple of - * XLOG_BLCKSZ, and make sure it's at least 5*Max(BLCKSZ, XLOG_BLCKSZ) to start + * page size, and make sure it's at least 5*Max(BLCKSZ, <page size>) to start * with. (That is enough for all "normal" records, but very large commit or * abort records might need more space.) */ @@ -172,8 +173,8 @@ allocate_recordbuf(XLogReaderState *state, uint32 reclength) { uint32 newSize = reclength; - newSize += XLOG_BLCKSZ - (newSize % XLOG_BLCKSZ); - newSize = Max(newSize, 5 * Max(BLCKSZ, XLOG_BLCKSZ)); + newSize += state->pageSize - (newSize % state->pageSize); + newSize = Max(newSize, 5 * Max(BLCKSZ, state->pageSize)); #ifndef FRONTEND @@ -238,6 +239,8 @@ void XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr) { Assert(!XLogRecPtrIsInvalid(RecPtr)); + Assert(state->readBuf != NULL && + state->readBuf == (void *) MAXALIGN(state->readBuf)); ResetDecoder(state); @@ -366,15 +369,15 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) XLogPageHeader pageHeader; targetPagePtr = - state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ); - targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ; + state->ReadRecPtr - (state->ReadRecPtr % state->pageSize); + targetRecOff = state->ReadRecPtr % state->pageSize; /* * Check if we have enough data. For the first record in the page, * the requesting length doesn't contain page header. */ if (XLogNeedData(state, targetPagePtr, - Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ), + Min(targetRecOff + SizeOfXLogRecord, state->pageSize), targetRecOff != 0)) return XLREAD_NEED_DATA; @@ -421,7 +424,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) * verified that we got the whole header. */ prec = (XLogRecord *) (state->readBuf + - state->ReadRecPtr % XLOG_BLCKSZ); + state->ReadRecPtr % state->pageSize); total_len = prec->xl_tot_len; /* @@ -432,7 +435,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) * ensure that we enter the XLREAD_CONTINUATION state below; * otherwise we might fail to apply ValidXLogRecordHeader at all. */ - if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord) + if (targetRecOff <= state->pageSize - SizeOfXLogRecord) { if (!ValidXLogRecordHeader(state, state->ReadRecPtr, state->PrevRecPtr, prec)) @@ -477,10 +480,10 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) * available */ targetPagePtr = - state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ); - targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ; + state->ReadRecPtr - (state->ReadRecPtr % state->pageSize); + targetRecOff = state->ReadRecPtr % state->pageSize; - request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ); + request_len = Min(targetRecOff + total_len, state->pageSize); record_len = request_len - targetRecOff; /* ReadRecPtr contains page header */ @@ -549,7 +552,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) /* Calculate pointer to beginning of next page */ state->recordContRecPtr = state->ReadRecPtr + record_len; - Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0); + Assert(state->recordContRecPtr % state->pageSize == 0); state->readRecordState = XLREAD_CONTINUATION; } @@ -576,7 +579,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) /* this request contains page header */ Assert (targetPagePtr != 0); if (XLogNeedData(state, targetPagePtr, - Min(state->recordRemainLen, XLOG_BLCKSZ), + Min(state->recordRemainLen, state->pageSize), false)) return XLREAD_NEED_DATA; @@ -628,7 +631,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) Assert(state->readLen >= pageHeaderSize); contdata = (char *) state->readBuf + pageHeaderSize; - record_len = XLOG_BLCKSZ - pageHeaderSize; + record_len = state->pageSize - pageHeaderSize; if (pageHeader->xlp_rem_len < record_len) record_len = pageHeader->xlp_rem_len; @@ -655,7 +658,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) } /* Calculate pointer to beginning of next page, and continue */ - state->recordContRecPtr += XLOG_BLCKSZ; + state->recordContRecPtr += state->pageSize; } /* targetPagePtr is pointing the last-read page here */ @@ -789,10 +792,10 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen, XLogPageHeaderSize((XLogPageHeader) state->readBuf); addLen = pageHeaderSize; - if (reqLen + pageHeaderSize <= XLOG_BLCKSZ) + if (reqLen + pageHeaderSize <= state->pageSize) addLen = pageHeaderSize; else - addLen = XLOG_BLCKSZ - reqLen; + addLen = state->pageSize - reqLen; Assert(addLen >= 0); } @@ -805,7 +808,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen, /* Data is not in our buffer, request the caller for it. */ XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize); targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize); - Assert((pageptr % XLOG_BLCKSZ) == 0); + Assert((pageptr % state->pageSize) == 0); /* * Every time we request to load new data of a page to the caller, even if @@ -829,7 +832,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen, * will not come back here, but will request the actual target page. */ state->readPagePtr = pageptr - targetPageOff; - state->readLen = XLOG_BLCKSZ; + state->readLen = state->pageSize; return true; } @@ -962,7 +965,7 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr, int32 offset; XLogPageHeader hdr = (XLogPageHeader) phdr; - Assert((recptr % XLOG_BLCKSZ) == 0); + Assert((recptr % state->pageSize) == 0); XLByteToSeg(recptr, segno, state->segcxt.ws_segsize); offset = XLogSegmentOffset(recptr, state->segcxt.ws_segsize); @@ -1016,10 +1019,10 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr, "WAL file is from different database system: incorrect segment size in page header"); return false; } - else if (longhdr->xlp_xlog_blcksz != XLOG_BLCKSZ) + else if (longhdr->xlp_xlog_blcksz != state->pageSize) { report_invalid_record(state, - "WAL file is from different database system: incorrect XLOG_BLCKSZ in page header"); + "WAL file is from different database system: incorrect block size in page header"); return false; } } @@ -1140,7 +1143,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr, * XLogNeedData() is prepared to handle that and will read at * least short page-header worth of data */ - targetRecOff = tmpRecPtr % XLOG_BLCKSZ; + targetRecOff = tmpRecPtr % state->pageSize; /* scroll back to page boundary */ targetPagePtr = tmpRecPtr - targetRecOff; @@ -1174,8 +1177,8 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr, * * Note that record headers are MAXALIGN'ed */ - if (MAXALIGN(header->xlp_rem_len) >= (XLOG_BLCKSZ - pageHeaderSize)) - tmpRecPtr = targetPagePtr + XLOG_BLCKSZ; + if (MAXALIGN(header->xlp_rem_len) >= (state->pageSize - pageHeaderSize)) + tmpRecPtr = targetPagePtr + state->pageSize; else { /* diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c index 9cbd8afa49..f2dd495310 100644 --- a/src/backend/replication/logical/logical.c +++ b/src/backend/replication/logical/logical.c @@ -199,11 +199,14 @@ StartupDecodingContext(List *output_plugin_options, ctx->slot = slot; - ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, cleanup_cb); + ctx->reader = + XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, cleanup_cb); + if (!ctx->reader) ereport(ERROR, (errcode(ERRCODE_OUT_OF_MEMORY), errmsg("out of memory"))); + ctx->reader->readBuf = palloc(XLOG_BLCKSZ); ctx->page_read = page_read; ctx->reorder = ReorderBufferAllocate(); diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c index 062093727d..8e95600ef7 100644 --- a/src/backend/replication/walsender.c +++ b/src/backend/replication/walsender.c @@ -579,8 +579,8 @@ StartReplication(StartReplicationCmd *cmd) errmsg("IDENTIFY_SYSTEM has not been run before START_REPLICATION"))); /* create xlogreader for physical replication */ - xlogreader = - XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close); + xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, + NULL, wal_segment_close); if (!xlogreader) ereport(ERROR, diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c index 712c85281c..8a6209b061 100644 --- a/src/bin/pg_rewind/parsexlog.c +++ b/src/bin/pg_rewind/parsexlog.c @@ -61,7 +61,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex, XLogReaderState *xlogreader; char *errormsg; - xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL); + xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL); if (xlogreader == NULL) pg_fatal("out of memory"); @@ -121,7 +121,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex, char *errormsg; XLogRecPtr endptr; - xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL); + xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL); if (xlogreader == NULL) pg_fatal("out of memory"); @@ -181,7 +181,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex, forkptr += SizeOfXLogShortPHD; } - xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL); + xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL); if (xlogreader == NULL) pg_fatal("out of memory"); diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c index f514de6b13..b635537659 100644 --- a/src/bin/pg_waldump/pg_waldump.c +++ b/src/bin/pg_waldump/pg_waldump.c @@ -1041,7 +1041,7 @@ main(int argc, char **argv) /* we have everything we need, start reading */ xlogreader_state = - XLogReaderAllocate(WalSegSz, waldir, WALDumpCloseSegment); + XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, waldir, WALDumpCloseSegment); if (!xlogreader_state) fatal_error("out of memory"); diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h index b9dad7ee51..1853516333 100644 --- a/src/include/access/xlogreader.h +++ b/src/include/access/xlogreader.h @@ -151,6 +151,7 @@ struct XLogReaderState * ---------------------------------------- */ /* variables to communicate with page reader */ + int pageSize; /* size of a page */ XLogRecPtr readPagePtr; /* page pointer to read */ int32 readLen; /* bytes requested to reader, or actual bytes * read by reader, which must be larger than @@ -243,7 +244,7 @@ struct XLogReaderState /* Get a new XLogReader */ extern XLogReaderState *XLogReaderAllocate(int wal_segment_size, - const char *waldir, + int page_size, const char *waldir, WALSegmentCleanupCB cleanup_cb); /* Free an XLogReader */ -- 2.27.0 ----Next_Part(Thu_Mar__4_11_28_53_2021_014)---- ^ permalink raw reply [nested|flat] 26+ messages in thread
* [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize. @ 2019-09-05 12:29 Kyotaro Horiguchi <[email protected]> 0 siblings, 0 replies; 26+ messages in thread From: Kyotaro Horiguchi @ 2019-09-05 12:29 UTC (permalink / raw) XLog reader facility and its callers currently relies on the same constant XLOG_BLCKSZ. This patch moves the responsibility to determine the block size to the callers of the facility. --- src/backend/access/transam/twophase.c | 3 +- src/backend/access/transam/xlog.c | 9 ++-- src/backend/access/transam/xlogreader.c | 57 ++++++++++++----------- src/backend/replication/logical/logical.c | 5 +- src/backend/replication/walsender.c | 4 +- src/bin/pg_rewind/parsexlog.c | 6 +-- src/bin/pg_waldump/pg_waldump.c | 2 +- src/include/access/xlogreader.h | 3 +- 8 files changed, 50 insertions(+), 39 deletions(-) diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c index 7d5a96a5f5..9f9e7de253 100644 --- a/src/backend/access/transam/twophase.c +++ b/src/backend/access/transam/twophase.c @@ -1325,7 +1325,8 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len) XLogReaderState *xlogreader; char *errormsg; - xlogreader = XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close); + xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, + NULL, wal_segment_close); if (!xlogreader) ereport(ERROR, diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index b517ca85ad..0822ac70cb 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -1222,7 +1222,10 @@ XLogInsertRecord(XLogRecData *rdata, appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len); if (!debug_reader) - debug_reader = XLogReaderAllocate(wal_segment_size, NULL, NULL); + { + debug_reader = + XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, NULL); + } if (!debug_reader) { @@ -6531,8 +6534,8 @@ StartupXLOG(void) OwnLatch(&XLogCtl->recoveryWakeupLatch); /* Set up XLOG reader facility */ - xlogreader = - XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close); + xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, + NULL, wal_segment_close); if (!xlogreader) ereport(ERROR, diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c index ba3e4868f7..e5baf13086 100644 --- a/src/backend/access/transam/xlogreader.c +++ b/src/backend/access/transam/xlogreader.c @@ -72,7 +72,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...) * Returns NULL if the xlogreader couldn't be allocated. */ XLogReaderState * -XLogReaderAllocate(int wal_segment_size, const char *waldir, +XLogReaderAllocate(int wal_segment_size, int page_size, const char *waldir, WALSegmentCleanupCB cleanup_cb) { XLogReaderState *state; @@ -87,6 +87,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir, state->cleanup_cb = cleanup_cb; state->max_block_id = -1; + state->pageSize = page_size; /* * Permanently allocate readBuf. We do it this way, rather than just @@ -95,7 +96,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir, * isn't guaranteed to have any particular alignment, whereas * palloc_extended() will provide MAXALIGN'd storage. */ - state->readBuf = (char *) palloc_extended(XLOG_BLCKSZ, + state->readBuf = (char *) palloc_extended(state->pageSize, MCXT_ALLOC_NO_OOM); if (!state->readBuf) { @@ -163,7 +164,7 @@ XLogReaderFree(XLogReaderState *state) * readRecordBufSize is set to the new buffer size. * * To avoid useless small increases, round its size to a multiple of - * XLOG_BLCKSZ, and make sure it's at least 5*Max(BLCKSZ, XLOG_BLCKSZ) to start + * page size, and make sure it's at least 5*Max(BLCKSZ, <page size>) to start * with. (That is enough for all "normal" records, but very large commit or * abort records might need more space.) */ @@ -172,8 +173,8 @@ allocate_recordbuf(XLogReaderState *state, uint32 reclength) { uint32 newSize = reclength; - newSize += XLOG_BLCKSZ - (newSize % XLOG_BLCKSZ); - newSize = Max(newSize, 5 * Max(BLCKSZ, XLOG_BLCKSZ)); + newSize += state->pageSize - (newSize % state->pageSize); + newSize = Max(newSize, 5 * Max(BLCKSZ, state->pageSize)); #ifndef FRONTEND @@ -238,6 +239,8 @@ void XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr) { Assert(!XLogRecPtrIsInvalid(RecPtr)); + Assert(state->readBuf != NULL && + state->readBuf == (void *) MAXALIGN(state->readBuf)); ResetDecoder(state); @@ -366,15 +369,15 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) XLogPageHeader pageHeader; targetPagePtr = - state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ); - targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ; + state->ReadRecPtr - (state->ReadRecPtr % state->pageSize); + targetRecOff = state->ReadRecPtr % state->pageSize; /* * Check if we have enough data. For the first record in the page, * the requesting length doesn't contain page header. */ if (XLogNeedData(state, targetPagePtr, - Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ), + Min(targetRecOff + SizeOfXLogRecord, state->pageSize), targetRecOff != 0)) return XLREAD_NEED_DATA; @@ -421,7 +424,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) * verified that we got the whole header. */ prec = (XLogRecord *) (state->readBuf + - state->ReadRecPtr % XLOG_BLCKSZ); + state->ReadRecPtr % state->pageSize); total_len = prec->xl_tot_len; /* @@ -432,7 +435,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) * ensure that we enter the XLREAD_CONTINUATION state below; * otherwise we might fail to apply ValidXLogRecordHeader at all. */ - if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord) + if (targetRecOff <= state->pageSize - SizeOfXLogRecord) { if (!ValidXLogRecordHeader(state, state->ReadRecPtr, state->PrevRecPtr, prec)) @@ -477,10 +480,10 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) * available */ targetPagePtr = - state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ); - targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ; + state->ReadRecPtr - (state->ReadRecPtr % state->pageSize); + targetRecOff = state->ReadRecPtr % state->pageSize; - request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ); + request_len = Min(targetRecOff + total_len, state->pageSize); record_len = request_len - targetRecOff; /* ReadRecPtr contains page header */ @@ -549,7 +552,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) /* Calculate pointer to beginning of next page */ state->recordContRecPtr = state->ReadRecPtr + record_len; - Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0); + Assert(state->recordContRecPtr % state->pageSize == 0); state->readRecordState = XLREAD_CONTINUATION; } @@ -576,7 +579,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) /* this request contains page header */ Assert (targetPagePtr != 0); if (XLogNeedData(state, targetPagePtr, - Min(state->recordRemainLen, XLOG_BLCKSZ), + Min(state->recordRemainLen, state->pageSize), false)) return XLREAD_NEED_DATA; @@ -628,7 +631,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) Assert(state->readLen >= pageHeaderSize); contdata = (char *) state->readBuf + pageHeaderSize; - record_len = XLOG_BLCKSZ - pageHeaderSize; + record_len = state->pageSize - pageHeaderSize; if (pageHeader->xlp_rem_len < record_len) record_len = pageHeader->xlp_rem_len; @@ -655,7 +658,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) } /* Calculate pointer to beginning of next page, and continue */ - state->recordContRecPtr += XLOG_BLCKSZ; + state->recordContRecPtr += state->pageSize; } /* targetPagePtr is pointing the last-read page here */ @@ -789,10 +792,10 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen, XLogPageHeaderSize((XLogPageHeader) state->readBuf); addLen = pageHeaderSize; - if (reqLen + pageHeaderSize <= XLOG_BLCKSZ) + if (reqLen + pageHeaderSize <= state->pageSize) addLen = pageHeaderSize; else - addLen = XLOG_BLCKSZ - reqLen; + addLen = state->pageSize - reqLen; Assert(addLen >= 0); } @@ -805,7 +808,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen, /* Data is not in our buffer, request the caller for it. */ XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize); targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize); - Assert((pageptr % XLOG_BLCKSZ) == 0); + Assert((pageptr % state->pageSize) == 0); /* * Every time we request to load new data of a page to the caller, even if @@ -829,7 +832,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen, * will not come back here, but will request the actual target page. */ state->readPagePtr = pageptr - targetPageOff; - state->readLen = XLOG_BLCKSZ; + state->readLen = state->pageSize; return true; } @@ -962,7 +965,7 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr, int32 offset; XLogPageHeader hdr = (XLogPageHeader) phdr; - Assert((recptr % XLOG_BLCKSZ) == 0); + Assert((recptr % state->pageSize) == 0); XLByteToSeg(recptr, segno, state->segcxt.ws_segsize); offset = XLogSegmentOffset(recptr, state->segcxt.ws_segsize); @@ -1016,10 +1019,10 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr, "WAL file is from different database system: incorrect segment size in page header"); return false; } - else if (longhdr->xlp_xlog_blcksz != XLOG_BLCKSZ) + else if (longhdr->xlp_xlog_blcksz != state->pageSize) { report_invalid_record(state, - "WAL file is from different database system: incorrect XLOG_BLCKSZ in page header"); + "WAL file is from different database system: incorrect block size in page header"); return false; } } @@ -1140,7 +1143,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr, * XLogNeedData() is prepared to handle that and will read at * least short page-header worth of data */ - targetRecOff = tmpRecPtr % XLOG_BLCKSZ; + targetRecOff = tmpRecPtr % state->pageSize; /* scroll back to page boundary */ targetPagePtr = tmpRecPtr - targetRecOff; @@ -1174,8 +1177,8 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr, * * Note that record headers are MAXALIGN'ed */ - if (MAXALIGN(header->xlp_rem_len) >= (XLOG_BLCKSZ - pageHeaderSize)) - tmpRecPtr = targetPagePtr + XLOG_BLCKSZ; + if (MAXALIGN(header->xlp_rem_len) >= (state->pageSize - pageHeaderSize)) + tmpRecPtr = targetPagePtr + state->pageSize; else { /* diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c index 9cbd8afa49..f2dd495310 100644 --- a/src/backend/replication/logical/logical.c +++ b/src/backend/replication/logical/logical.c @@ -199,11 +199,14 @@ StartupDecodingContext(List *output_plugin_options, ctx->slot = slot; - ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, cleanup_cb); + ctx->reader = + XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, cleanup_cb); + if (!ctx->reader) ereport(ERROR, (errcode(ERRCODE_OUT_OF_MEMORY), errmsg("out of memory"))); + ctx->reader->readBuf = palloc(XLOG_BLCKSZ); ctx->page_read = page_read; ctx->reorder = ReorderBufferAllocate(); diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c index 062093727d..8e95600ef7 100644 --- a/src/backend/replication/walsender.c +++ b/src/backend/replication/walsender.c @@ -579,8 +579,8 @@ StartReplication(StartReplicationCmd *cmd) errmsg("IDENTIFY_SYSTEM has not been run before START_REPLICATION"))); /* create xlogreader for physical replication */ - xlogreader = - XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close); + xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, + NULL, wal_segment_close); if (!xlogreader) ereport(ERROR, diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c index 712c85281c..8a6209b061 100644 --- a/src/bin/pg_rewind/parsexlog.c +++ b/src/bin/pg_rewind/parsexlog.c @@ -61,7 +61,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex, XLogReaderState *xlogreader; char *errormsg; - xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL); + xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL); if (xlogreader == NULL) pg_fatal("out of memory"); @@ -121,7 +121,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex, char *errormsg; XLogRecPtr endptr; - xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL); + xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL); if (xlogreader == NULL) pg_fatal("out of memory"); @@ -181,7 +181,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex, forkptr += SizeOfXLogShortPHD; } - xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL); + xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL); if (xlogreader == NULL) pg_fatal("out of memory"); diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c index f514de6b13..b635537659 100644 --- a/src/bin/pg_waldump/pg_waldump.c +++ b/src/bin/pg_waldump/pg_waldump.c @@ -1041,7 +1041,7 @@ main(int argc, char **argv) /* we have everything we need, start reading */ xlogreader_state = - XLogReaderAllocate(WalSegSz, waldir, WALDumpCloseSegment); + XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, waldir, WALDumpCloseSegment); if (!xlogreader_state) fatal_error("out of memory"); diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h index b9dad7ee51..1853516333 100644 --- a/src/include/access/xlogreader.h +++ b/src/include/access/xlogreader.h @@ -151,6 +151,7 @@ struct XLogReaderState * ---------------------------------------- */ /* variables to communicate with page reader */ + int pageSize; /* size of a page */ XLogRecPtr readPagePtr; /* page pointer to read */ int32 readLen; /* bytes requested to reader, or actual bytes * read by reader, which must be larger than @@ -243,7 +244,7 @@ struct XLogReaderState /* Get a new XLogReader */ extern XLogReaderState *XLogReaderAllocate(int wal_segment_size, - const char *waldir, + int page_size, const char *waldir, WALSegmentCleanupCB cleanup_cb); /* Free an XLogReader */ -- 2.27.0 ----Next_Part(Thu_Mar__4_11_28_53_2021_014)---- ^ permalink raw reply [nested|flat] 26+ messages in thread
* [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize. @ 2019-09-05 12:29 Kyotaro Horiguchi <[email protected]> 0 siblings, 0 replies; 26+ messages in thread From: Kyotaro Horiguchi @ 2019-09-05 12:29 UTC (permalink / raw) XLog reader facility and its callers currently relies on the same constant XLOG_BLCKSZ. This patch moves the responsibility to determine the block size to the callers of the facility. --- src/backend/access/transam/twophase.c | 3 +- src/backend/access/transam/xlog.c | 9 ++-- src/backend/access/transam/xlogreader.c | 57 ++++++++++++----------- src/backend/replication/logical/logical.c | 5 +- src/backend/replication/walsender.c | 4 +- src/bin/pg_rewind/parsexlog.c | 6 +-- src/bin/pg_waldump/pg_waldump.c | 2 +- src/include/access/xlogreader.h | 3 +- 8 files changed, 50 insertions(+), 39 deletions(-) diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c index 7d5a96a5f5..9f9e7de253 100644 --- a/src/backend/access/transam/twophase.c +++ b/src/backend/access/transam/twophase.c @@ -1325,7 +1325,8 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len) XLogReaderState *xlogreader; char *errormsg; - xlogreader = XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close); + xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, + NULL, wal_segment_close); if (!xlogreader) ereport(ERROR, diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index b517ca85ad..0822ac70cb 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -1222,7 +1222,10 @@ XLogInsertRecord(XLogRecData *rdata, appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len); if (!debug_reader) - debug_reader = XLogReaderAllocate(wal_segment_size, NULL, NULL); + { + debug_reader = + XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, NULL); + } if (!debug_reader) { @@ -6531,8 +6534,8 @@ StartupXLOG(void) OwnLatch(&XLogCtl->recoveryWakeupLatch); /* Set up XLOG reader facility */ - xlogreader = - XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close); + xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, + NULL, wal_segment_close); if (!xlogreader) ereport(ERROR, diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c index ba3e4868f7..e5baf13086 100644 --- a/src/backend/access/transam/xlogreader.c +++ b/src/backend/access/transam/xlogreader.c @@ -72,7 +72,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...) * Returns NULL if the xlogreader couldn't be allocated. */ XLogReaderState * -XLogReaderAllocate(int wal_segment_size, const char *waldir, +XLogReaderAllocate(int wal_segment_size, int page_size, const char *waldir, WALSegmentCleanupCB cleanup_cb) { XLogReaderState *state; @@ -87,6 +87,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir, state->cleanup_cb = cleanup_cb; state->max_block_id = -1; + state->pageSize = page_size; /* * Permanently allocate readBuf. We do it this way, rather than just @@ -95,7 +96,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir, * isn't guaranteed to have any particular alignment, whereas * palloc_extended() will provide MAXALIGN'd storage. */ - state->readBuf = (char *) palloc_extended(XLOG_BLCKSZ, + state->readBuf = (char *) palloc_extended(state->pageSize, MCXT_ALLOC_NO_OOM); if (!state->readBuf) { @@ -163,7 +164,7 @@ XLogReaderFree(XLogReaderState *state) * readRecordBufSize is set to the new buffer size. * * To avoid useless small increases, round its size to a multiple of - * XLOG_BLCKSZ, and make sure it's at least 5*Max(BLCKSZ, XLOG_BLCKSZ) to start + * page size, and make sure it's at least 5*Max(BLCKSZ, <page size>) to start * with. (That is enough for all "normal" records, but very large commit or * abort records might need more space.) */ @@ -172,8 +173,8 @@ allocate_recordbuf(XLogReaderState *state, uint32 reclength) { uint32 newSize = reclength; - newSize += XLOG_BLCKSZ - (newSize % XLOG_BLCKSZ); - newSize = Max(newSize, 5 * Max(BLCKSZ, XLOG_BLCKSZ)); + newSize += state->pageSize - (newSize % state->pageSize); + newSize = Max(newSize, 5 * Max(BLCKSZ, state->pageSize)); #ifndef FRONTEND @@ -238,6 +239,8 @@ void XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr) { Assert(!XLogRecPtrIsInvalid(RecPtr)); + Assert(state->readBuf != NULL && + state->readBuf == (void *) MAXALIGN(state->readBuf)); ResetDecoder(state); @@ -366,15 +369,15 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) XLogPageHeader pageHeader; targetPagePtr = - state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ); - targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ; + state->ReadRecPtr - (state->ReadRecPtr % state->pageSize); + targetRecOff = state->ReadRecPtr % state->pageSize; /* * Check if we have enough data. For the first record in the page, * the requesting length doesn't contain page header. */ if (XLogNeedData(state, targetPagePtr, - Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ), + Min(targetRecOff + SizeOfXLogRecord, state->pageSize), targetRecOff != 0)) return XLREAD_NEED_DATA; @@ -421,7 +424,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) * verified that we got the whole header. */ prec = (XLogRecord *) (state->readBuf + - state->ReadRecPtr % XLOG_BLCKSZ); + state->ReadRecPtr % state->pageSize); total_len = prec->xl_tot_len; /* @@ -432,7 +435,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) * ensure that we enter the XLREAD_CONTINUATION state below; * otherwise we might fail to apply ValidXLogRecordHeader at all. */ - if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord) + if (targetRecOff <= state->pageSize - SizeOfXLogRecord) { if (!ValidXLogRecordHeader(state, state->ReadRecPtr, state->PrevRecPtr, prec)) @@ -477,10 +480,10 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) * available */ targetPagePtr = - state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ); - targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ; + state->ReadRecPtr - (state->ReadRecPtr % state->pageSize); + targetRecOff = state->ReadRecPtr % state->pageSize; - request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ); + request_len = Min(targetRecOff + total_len, state->pageSize); record_len = request_len - targetRecOff; /* ReadRecPtr contains page header */ @@ -549,7 +552,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) /* Calculate pointer to beginning of next page */ state->recordContRecPtr = state->ReadRecPtr + record_len; - Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0); + Assert(state->recordContRecPtr % state->pageSize == 0); state->readRecordState = XLREAD_CONTINUATION; } @@ -576,7 +579,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) /* this request contains page header */ Assert (targetPagePtr != 0); if (XLogNeedData(state, targetPagePtr, - Min(state->recordRemainLen, XLOG_BLCKSZ), + Min(state->recordRemainLen, state->pageSize), false)) return XLREAD_NEED_DATA; @@ -628,7 +631,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) Assert(state->readLen >= pageHeaderSize); contdata = (char *) state->readBuf + pageHeaderSize; - record_len = XLOG_BLCKSZ - pageHeaderSize; + record_len = state->pageSize - pageHeaderSize; if (pageHeader->xlp_rem_len < record_len) record_len = pageHeader->xlp_rem_len; @@ -655,7 +658,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) } /* Calculate pointer to beginning of next page, and continue */ - state->recordContRecPtr += XLOG_BLCKSZ; + state->recordContRecPtr += state->pageSize; } /* targetPagePtr is pointing the last-read page here */ @@ -789,10 +792,10 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen, XLogPageHeaderSize((XLogPageHeader) state->readBuf); addLen = pageHeaderSize; - if (reqLen + pageHeaderSize <= XLOG_BLCKSZ) + if (reqLen + pageHeaderSize <= state->pageSize) addLen = pageHeaderSize; else - addLen = XLOG_BLCKSZ - reqLen; + addLen = state->pageSize - reqLen; Assert(addLen >= 0); } @@ -805,7 +808,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen, /* Data is not in our buffer, request the caller for it. */ XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize); targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize); - Assert((pageptr % XLOG_BLCKSZ) == 0); + Assert((pageptr % state->pageSize) == 0); /* * Every time we request to load new data of a page to the caller, even if @@ -829,7 +832,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen, * will not come back here, but will request the actual target page. */ state->readPagePtr = pageptr - targetPageOff; - state->readLen = XLOG_BLCKSZ; + state->readLen = state->pageSize; return true; } @@ -962,7 +965,7 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr, int32 offset; XLogPageHeader hdr = (XLogPageHeader) phdr; - Assert((recptr % XLOG_BLCKSZ) == 0); + Assert((recptr % state->pageSize) == 0); XLByteToSeg(recptr, segno, state->segcxt.ws_segsize); offset = XLogSegmentOffset(recptr, state->segcxt.ws_segsize); @@ -1016,10 +1019,10 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr, "WAL file is from different database system: incorrect segment size in page header"); return false; } - else if (longhdr->xlp_xlog_blcksz != XLOG_BLCKSZ) + else if (longhdr->xlp_xlog_blcksz != state->pageSize) { report_invalid_record(state, - "WAL file is from different database system: incorrect XLOG_BLCKSZ in page header"); + "WAL file is from different database system: incorrect block size in page header"); return false; } } @@ -1140,7 +1143,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr, * XLogNeedData() is prepared to handle that and will read at * least short page-header worth of data */ - targetRecOff = tmpRecPtr % XLOG_BLCKSZ; + targetRecOff = tmpRecPtr % state->pageSize; /* scroll back to page boundary */ targetPagePtr = tmpRecPtr - targetRecOff; @@ -1174,8 +1177,8 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr, * * Note that record headers are MAXALIGN'ed */ - if (MAXALIGN(header->xlp_rem_len) >= (XLOG_BLCKSZ - pageHeaderSize)) - tmpRecPtr = targetPagePtr + XLOG_BLCKSZ; + if (MAXALIGN(header->xlp_rem_len) >= (state->pageSize - pageHeaderSize)) + tmpRecPtr = targetPagePtr + state->pageSize; else { /* diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c index 9cbd8afa49..f2dd495310 100644 --- a/src/backend/replication/logical/logical.c +++ b/src/backend/replication/logical/logical.c @@ -199,11 +199,14 @@ StartupDecodingContext(List *output_plugin_options, ctx->slot = slot; - ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, cleanup_cb); + ctx->reader = + XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, cleanup_cb); + if (!ctx->reader) ereport(ERROR, (errcode(ERRCODE_OUT_OF_MEMORY), errmsg("out of memory"))); + ctx->reader->readBuf = palloc(XLOG_BLCKSZ); ctx->page_read = page_read; ctx->reorder = ReorderBufferAllocate(); diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c index 062093727d..8e95600ef7 100644 --- a/src/backend/replication/walsender.c +++ b/src/backend/replication/walsender.c @@ -579,8 +579,8 @@ StartReplication(StartReplicationCmd *cmd) errmsg("IDENTIFY_SYSTEM has not been run before START_REPLICATION"))); /* create xlogreader for physical replication */ - xlogreader = - XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close); + xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, + NULL, wal_segment_close); if (!xlogreader) ereport(ERROR, diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c index 712c85281c..8a6209b061 100644 --- a/src/bin/pg_rewind/parsexlog.c +++ b/src/bin/pg_rewind/parsexlog.c @@ -61,7 +61,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex, XLogReaderState *xlogreader; char *errormsg; - xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL); + xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL); if (xlogreader == NULL) pg_fatal("out of memory"); @@ -121,7 +121,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex, char *errormsg; XLogRecPtr endptr; - xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL); + xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL); if (xlogreader == NULL) pg_fatal("out of memory"); @@ -181,7 +181,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex, forkptr += SizeOfXLogShortPHD; } - xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL); + xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL); if (xlogreader == NULL) pg_fatal("out of memory"); diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c index f514de6b13..b635537659 100644 --- a/src/bin/pg_waldump/pg_waldump.c +++ b/src/bin/pg_waldump/pg_waldump.c @@ -1041,7 +1041,7 @@ main(int argc, char **argv) /* we have everything we need, start reading */ xlogreader_state = - XLogReaderAllocate(WalSegSz, waldir, WALDumpCloseSegment); + XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, waldir, WALDumpCloseSegment); if (!xlogreader_state) fatal_error("out of memory"); diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h index b9dad7ee51..1853516333 100644 --- a/src/include/access/xlogreader.h +++ b/src/include/access/xlogreader.h @@ -151,6 +151,7 @@ struct XLogReaderState * ---------------------------------------- */ /* variables to communicate with page reader */ + int pageSize; /* size of a page */ XLogRecPtr readPagePtr; /* page pointer to read */ int32 readLen; /* bytes requested to reader, or actual bytes * read by reader, which must be larger than @@ -243,7 +244,7 @@ struct XLogReaderState /* Get a new XLogReader */ extern XLogReaderState *XLogReaderAllocate(int wal_segment_size, - const char *waldir, + int page_size, const char *waldir, WALSegmentCleanupCB cleanup_cb); /* Free an XLogReader */ -- 2.27.0 ----Next_Part(Thu_Mar__4_11_28_53_2021_014)---- ^ permalink raw reply [nested|flat] 26+ messages in thread
* [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize. @ 2019-09-05 12:29 Kyotaro Horiguchi <[email protected]> 0 siblings, 0 replies; 26+ messages in thread From: Kyotaro Horiguchi @ 2019-09-05 12:29 UTC (permalink / raw) XLog reader facility and its callers currently relies on the same constant XLOG_BLCKSZ. This patch moves the responsibility to determine the block size to the callers of the facility. --- src/backend/access/transam/twophase.c | 3 +- src/backend/access/transam/xlog.c | 9 ++-- src/backend/access/transam/xlogreader.c | 57 ++++++++++++----------- src/backend/replication/logical/logical.c | 5 +- src/backend/replication/walsender.c | 4 +- src/bin/pg_rewind/parsexlog.c | 6 +-- src/bin/pg_waldump/pg_waldump.c | 2 +- src/include/access/xlogreader.h | 3 +- 8 files changed, 50 insertions(+), 39 deletions(-) diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c index 7d5a96a5f5..9f9e7de253 100644 --- a/src/backend/access/transam/twophase.c +++ b/src/backend/access/transam/twophase.c @@ -1325,7 +1325,8 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len) XLogReaderState *xlogreader; char *errormsg; - xlogreader = XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close); + xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, + NULL, wal_segment_close); if (!xlogreader) ereport(ERROR, diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index b517ca85ad..0822ac70cb 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -1222,7 +1222,10 @@ XLogInsertRecord(XLogRecData *rdata, appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len); if (!debug_reader) - debug_reader = XLogReaderAllocate(wal_segment_size, NULL, NULL); + { + debug_reader = + XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, NULL); + } if (!debug_reader) { @@ -6531,8 +6534,8 @@ StartupXLOG(void) OwnLatch(&XLogCtl->recoveryWakeupLatch); /* Set up XLOG reader facility */ - xlogreader = - XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close); + xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, + NULL, wal_segment_close); if (!xlogreader) ereport(ERROR, diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c index ba3e4868f7..e5baf13086 100644 --- a/src/backend/access/transam/xlogreader.c +++ b/src/backend/access/transam/xlogreader.c @@ -72,7 +72,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...) * Returns NULL if the xlogreader couldn't be allocated. */ XLogReaderState * -XLogReaderAllocate(int wal_segment_size, const char *waldir, +XLogReaderAllocate(int wal_segment_size, int page_size, const char *waldir, WALSegmentCleanupCB cleanup_cb) { XLogReaderState *state; @@ -87,6 +87,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir, state->cleanup_cb = cleanup_cb; state->max_block_id = -1; + state->pageSize = page_size; /* * Permanently allocate readBuf. We do it this way, rather than just @@ -95,7 +96,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir, * isn't guaranteed to have any particular alignment, whereas * palloc_extended() will provide MAXALIGN'd storage. */ - state->readBuf = (char *) palloc_extended(XLOG_BLCKSZ, + state->readBuf = (char *) palloc_extended(state->pageSize, MCXT_ALLOC_NO_OOM); if (!state->readBuf) { @@ -163,7 +164,7 @@ XLogReaderFree(XLogReaderState *state) * readRecordBufSize is set to the new buffer size. * * To avoid useless small increases, round its size to a multiple of - * XLOG_BLCKSZ, and make sure it's at least 5*Max(BLCKSZ, XLOG_BLCKSZ) to start + * page size, and make sure it's at least 5*Max(BLCKSZ, <page size>) to start * with. (That is enough for all "normal" records, but very large commit or * abort records might need more space.) */ @@ -172,8 +173,8 @@ allocate_recordbuf(XLogReaderState *state, uint32 reclength) { uint32 newSize = reclength; - newSize += XLOG_BLCKSZ - (newSize % XLOG_BLCKSZ); - newSize = Max(newSize, 5 * Max(BLCKSZ, XLOG_BLCKSZ)); + newSize += state->pageSize - (newSize % state->pageSize); + newSize = Max(newSize, 5 * Max(BLCKSZ, state->pageSize)); #ifndef FRONTEND @@ -238,6 +239,8 @@ void XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr) { Assert(!XLogRecPtrIsInvalid(RecPtr)); + Assert(state->readBuf != NULL && + state->readBuf == (void *) MAXALIGN(state->readBuf)); ResetDecoder(state); @@ -366,15 +369,15 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) XLogPageHeader pageHeader; targetPagePtr = - state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ); - targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ; + state->ReadRecPtr - (state->ReadRecPtr % state->pageSize); + targetRecOff = state->ReadRecPtr % state->pageSize; /* * Check if we have enough data. For the first record in the page, * the requesting length doesn't contain page header. */ if (XLogNeedData(state, targetPagePtr, - Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ), + Min(targetRecOff + SizeOfXLogRecord, state->pageSize), targetRecOff != 0)) return XLREAD_NEED_DATA; @@ -421,7 +424,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) * verified that we got the whole header. */ prec = (XLogRecord *) (state->readBuf + - state->ReadRecPtr % XLOG_BLCKSZ); + state->ReadRecPtr % state->pageSize); total_len = prec->xl_tot_len; /* @@ -432,7 +435,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) * ensure that we enter the XLREAD_CONTINUATION state below; * otherwise we might fail to apply ValidXLogRecordHeader at all. */ - if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord) + if (targetRecOff <= state->pageSize - SizeOfXLogRecord) { if (!ValidXLogRecordHeader(state, state->ReadRecPtr, state->PrevRecPtr, prec)) @@ -477,10 +480,10 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) * available */ targetPagePtr = - state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ); - targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ; + state->ReadRecPtr - (state->ReadRecPtr % state->pageSize); + targetRecOff = state->ReadRecPtr % state->pageSize; - request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ); + request_len = Min(targetRecOff + total_len, state->pageSize); record_len = request_len - targetRecOff; /* ReadRecPtr contains page header */ @@ -549,7 +552,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) /* Calculate pointer to beginning of next page */ state->recordContRecPtr = state->ReadRecPtr + record_len; - Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0); + Assert(state->recordContRecPtr % state->pageSize == 0); state->readRecordState = XLREAD_CONTINUATION; } @@ -576,7 +579,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) /* this request contains page header */ Assert (targetPagePtr != 0); if (XLogNeedData(state, targetPagePtr, - Min(state->recordRemainLen, XLOG_BLCKSZ), + Min(state->recordRemainLen, state->pageSize), false)) return XLREAD_NEED_DATA; @@ -628,7 +631,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) Assert(state->readLen >= pageHeaderSize); contdata = (char *) state->readBuf + pageHeaderSize; - record_len = XLOG_BLCKSZ - pageHeaderSize; + record_len = state->pageSize - pageHeaderSize; if (pageHeader->xlp_rem_len < record_len) record_len = pageHeader->xlp_rem_len; @@ -655,7 +658,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) } /* Calculate pointer to beginning of next page, and continue */ - state->recordContRecPtr += XLOG_BLCKSZ; + state->recordContRecPtr += state->pageSize; } /* targetPagePtr is pointing the last-read page here */ @@ -789,10 +792,10 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen, XLogPageHeaderSize((XLogPageHeader) state->readBuf); addLen = pageHeaderSize; - if (reqLen + pageHeaderSize <= XLOG_BLCKSZ) + if (reqLen + pageHeaderSize <= state->pageSize) addLen = pageHeaderSize; else - addLen = XLOG_BLCKSZ - reqLen; + addLen = state->pageSize - reqLen; Assert(addLen >= 0); } @@ -805,7 +808,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen, /* Data is not in our buffer, request the caller for it. */ XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize); targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize); - Assert((pageptr % XLOG_BLCKSZ) == 0); + Assert((pageptr % state->pageSize) == 0); /* * Every time we request to load new data of a page to the caller, even if @@ -829,7 +832,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen, * will not come back here, but will request the actual target page. */ state->readPagePtr = pageptr - targetPageOff; - state->readLen = XLOG_BLCKSZ; + state->readLen = state->pageSize; return true; } @@ -962,7 +965,7 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr, int32 offset; XLogPageHeader hdr = (XLogPageHeader) phdr; - Assert((recptr % XLOG_BLCKSZ) == 0); + Assert((recptr % state->pageSize) == 0); XLByteToSeg(recptr, segno, state->segcxt.ws_segsize); offset = XLogSegmentOffset(recptr, state->segcxt.ws_segsize); @@ -1016,10 +1019,10 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr, "WAL file is from different database system: incorrect segment size in page header"); return false; } - else if (longhdr->xlp_xlog_blcksz != XLOG_BLCKSZ) + else if (longhdr->xlp_xlog_blcksz != state->pageSize) { report_invalid_record(state, - "WAL file is from different database system: incorrect XLOG_BLCKSZ in page header"); + "WAL file is from different database system: incorrect block size in page header"); return false; } } @@ -1140,7 +1143,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr, * XLogNeedData() is prepared to handle that and will read at * least short page-header worth of data */ - targetRecOff = tmpRecPtr % XLOG_BLCKSZ; + targetRecOff = tmpRecPtr % state->pageSize; /* scroll back to page boundary */ targetPagePtr = tmpRecPtr - targetRecOff; @@ -1174,8 +1177,8 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr, * * Note that record headers are MAXALIGN'ed */ - if (MAXALIGN(header->xlp_rem_len) >= (XLOG_BLCKSZ - pageHeaderSize)) - tmpRecPtr = targetPagePtr + XLOG_BLCKSZ; + if (MAXALIGN(header->xlp_rem_len) >= (state->pageSize - pageHeaderSize)) + tmpRecPtr = targetPagePtr + state->pageSize; else { /* diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c index 9cbd8afa49..f2dd495310 100644 --- a/src/backend/replication/logical/logical.c +++ b/src/backend/replication/logical/logical.c @@ -199,11 +199,14 @@ StartupDecodingContext(List *output_plugin_options, ctx->slot = slot; - ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, cleanup_cb); + ctx->reader = + XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, cleanup_cb); + if (!ctx->reader) ereport(ERROR, (errcode(ERRCODE_OUT_OF_MEMORY), errmsg("out of memory"))); + ctx->reader->readBuf = palloc(XLOG_BLCKSZ); ctx->page_read = page_read; ctx->reorder = ReorderBufferAllocate(); diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c index 062093727d..8e95600ef7 100644 --- a/src/backend/replication/walsender.c +++ b/src/backend/replication/walsender.c @@ -579,8 +579,8 @@ StartReplication(StartReplicationCmd *cmd) errmsg("IDENTIFY_SYSTEM has not been run before START_REPLICATION"))); /* create xlogreader for physical replication */ - xlogreader = - XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close); + xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, + NULL, wal_segment_close); if (!xlogreader) ereport(ERROR, diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c index 712c85281c..8a6209b061 100644 --- a/src/bin/pg_rewind/parsexlog.c +++ b/src/bin/pg_rewind/parsexlog.c @@ -61,7 +61,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex, XLogReaderState *xlogreader; char *errormsg; - xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL); + xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL); if (xlogreader == NULL) pg_fatal("out of memory"); @@ -121,7 +121,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex, char *errormsg; XLogRecPtr endptr; - xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL); + xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL); if (xlogreader == NULL) pg_fatal("out of memory"); @@ -181,7 +181,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex, forkptr += SizeOfXLogShortPHD; } - xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL); + xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL); if (xlogreader == NULL) pg_fatal("out of memory"); diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c index f514de6b13..b635537659 100644 --- a/src/bin/pg_waldump/pg_waldump.c +++ b/src/bin/pg_waldump/pg_waldump.c @@ -1041,7 +1041,7 @@ main(int argc, char **argv) /* we have everything we need, start reading */ xlogreader_state = - XLogReaderAllocate(WalSegSz, waldir, WALDumpCloseSegment); + XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, waldir, WALDumpCloseSegment); if (!xlogreader_state) fatal_error("out of memory"); diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h index b9dad7ee51..1853516333 100644 --- a/src/include/access/xlogreader.h +++ b/src/include/access/xlogreader.h @@ -151,6 +151,7 @@ struct XLogReaderState * ---------------------------------------- */ /* variables to communicate with page reader */ + int pageSize; /* size of a page */ XLogRecPtr readPagePtr; /* page pointer to read */ int32 readLen; /* bytes requested to reader, or actual bytes * read by reader, which must be larger than @@ -243,7 +244,7 @@ struct XLogReaderState /* Get a new XLogReader */ extern XLogReaderState *XLogReaderAllocate(int wal_segment_size, - const char *waldir, + int page_size, const char *waldir, WALSegmentCleanupCB cleanup_cb); /* Free an XLogReader */ -- 2.27.0 ----Next_Part(Thu_Mar__4_11_28_53_2021_014)---- ^ permalink raw reply [nested|flat] 26+ messages in thread
* [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize. @ 2019-09-05 12:29 Kyotaro Horiguchi <[email protected]> 0 siblings, 0 replies; 26+ messages in thread From: Kyotaro Horiguchi @ 2019-09-05 12:29 UTC (permalink / raw) XLog reader facility and its callers currently relies on the same constant XLOG_BLCKSZ. This patch moves the responsibility to determine the block size to the callers of the facility. --- src/backend/access/transam/twophase.c | 3 +- src/backend/access/transam/xlog.c | 9 ++-- src/backend/access/transam/xlogreader.c | 57 ++++++++++++----------- src/backend/replication/logical/logical.c | 5 +- src/backend/replication/walsender.c | 4 +- src/bin/pg_rewind/parsexlog.c | 6 +-- src/bin/pg_waldump/pg_waldump.c | 2 +- src/include/access/xlogreader.h | 3 +- 8 files changed, 50 insertions(+), 39 deletions(-) diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c index 7d5a96a5f5..9f9e7de253 100644 --- a/src/backend/access/transam/twophase.c +++ b/src/backend/access/transam/twophase.c @@ -1325,7 +1325,8 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len) XLogReaderState *xlogreader; char *errormsg; - xlogreader = XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close); + xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, + NULL, wal_segment_close); if (!xlogreader) ereport(ERROR, diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index b517ca85ad..0822ac70cb 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -1222,7 +1222,10 @@ XLogInsertRecord(XLogRecData *rdata, appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len); if (!debug_reader) - debug_reader = XLogReaderAllocate(wal_segment_size, NULL, NULL); + { + debug_reader = + XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, NULL); + } if (!debug_reader) { @@ -6531,8 +6534,8 @@ StartupXLOG(void) OwnLatch(&XLogCtl->recoveryWakeupLatch); /* Set up XLOG reader facility */ - xlogreader = - XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close); + xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, + NULL, wal_segment_close); if (!xlogreader) ereport(ERROR, diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c index ba3e4868f7..e5baf13086 100644 --- a/src/backend/access/transam/xlogreader.c +++ b/src/backend/access/transam/xlogreader.c @@ -72,7 +72,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...) * Returns NULL if the xlogreader couldn't be allocated. */ XLogReaderState * -XLogReaderAllocate(int wal_segment_size, const char *waldir, +XLogReaderAllocate(int wal_segment_size, int page_size, const char *waldir, WALSegmentCleanupCB cleanup_cb) { XLogReaderState *state; @@ -87,6 +87,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir, state->cleanup_cb = cleanup_cb; state->max_block_id = -1; + state->pageSize = page_size; /* * Permanently allocate readBuf. We do it this way, rather than just @@ -95,7 +96,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir, * isn't guaranteed to have any particular alignment, whereas * palloc_extended() will provide MAXALIGN'd storage. */ - state->readBuf = (char *) palloc_extended(XLOG_BLCKSZ, + state->readBuf = (char *) palloc_extended(state->pageSize, MCXT_ALLOC_NO_OOM); if (!state->readBuf) { @@ -163,7 +164,7 @@ XLogReaderFree(XLogReaderState *state) * readRecordBufSize is set to the new buffer size. * * To avoid useless small increases, round its size to a multiple of - * XLOG_BLCKSZ, and make sure it's at least 5*Max(BLCKSZ, XLOG_BLCKSZ) to start + * page size, and make sure it's at least 5*Max(BLCKSZ, <page size>) to start * with. (That is enough for all "normal" records, but very large commit or * abort records might need more space.) */ @@ -172,8 +173,8 @@ allocate_recordbuf(XLogReaderState *state, uint32 reclength) { uint32 newSize = reclength; - newSize += XLOG_BLCKSZ - (newSize % XLOG_BLCKSZ); - newSize = Max(newSize, 5 * Max(BLCKSZ, XLOG_BLCKSZ)); + newSize += state->pageSize - (newSize % state->pageSize); + newSize = Max(newSize, 5 * Max(BLCKSZ, state->pageSize)); #ifndef FRONTEND @@ -238,6 +239,8 @@ void XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr) { Assert(!XLogRecPtrIsInvalid(RecPtr)); + Assert(state->readBuf != NULL && + state->readBuf == (void *) MAXALIGN(state->readBuf)); ResetDecoder(state); @@ -366,15 +369,15 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) XLogPageHeader pageHeader; targetPagePtr = - state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ); - targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ; + state->ReadRecPtr - (state->ReadRecPtr % state->pageSize); + targetRecOff = state->ReadRecPtr % state->pageSize; /* * Check if we have enough data. For the first record in the page, * the requesting length doesn't contain page header. */ if (XLogNeedData(state, targetPagePtr, - Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ), + Min(targetRecOff + SizeOfXLogRecord, state->pageSize), targetRecOff != 0)) return XLREAD_NEED_DATA; @@ -421,7 +424,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) * verified that we got the whole header. */ prec = (XLogRecord *) (state->readBuf + - state->ReadRecPtr % XLOG_BLCKSZ); + state->ReadRecPtr % state->pageSize); total_len = prec->xl_tot_len; /* @@ -432,7 +435,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) * ensure that we enter the XLREAD_CONTINUATION state below; * otherwise we might fail to apply ValidXLogRecordHeader at all. */ - if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord) + if (targetRecOff <= state->pageSize - SizeOfXLogRecord) { if (!ValidXLogRecordHeader(state, state->ReadRecPtr, state->PrevRecPtr, prec)) @@ -477,10 +480,10 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) * available */ targetPagePtr = - state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ); - targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ; + state->ReadRecPtr - (state->ReadRecPtr % state->pageSize); + targetRecOff = state->ReadRecPtr % state->pageSize; - request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ); + request_len = Min(targetRecOff + total_len, state->pageSize); record_len = request_len - targetRecOff; /* ReadRecPtr contains page header */ @@ -549,7 +552,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) /* Calculate pointer to beginning of next page */ state->recordContRecPtr = state->ReadRecPtr + record_len; - Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0); + Assert(state->recordContRecPtr % state->pageSize == 0); state->readRecordState = XLREAD_CONTINUATION; } @@ -576,7 +579,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) /* this request contains page header */ Assert (targetPagePtr != 0); if (XLogNeedData(state, targetPagePtr, - Min(state->recordRemainLen, XLOG_BLCKSZ), + Min(state->recordRemainLen, state->pageSize), false)) return XLREAD_NEED_DATA; @@ -628,7 +631,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) Assert(state->readLen >= pageHeaderSize); contdata = (char *) state->readBuf + pageHeaderSize; - record_len = XLOG_BLCKSZ - pageHeaderSize; + record_len = state->pageSize - pageHeaderSize; if (pageHeader->xlp_rem_len < record_len) record_len = pageHeader->xlp_rem_len; @@ -655,7 +658,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) } /* Calculate pointer to beginning of next page, and continue */ - state->recordContRecPtr += XLOG_BLCKSZ; + state->recordContRecPtr += state->pageSize; } /* targetPagePtr is pointing the last-read page here */ @@ -789,10 +792,10 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen, XLogPageHeaderSize((XLogPageHeader) state->readBuf); addLen = pageHeaderSize; - if (reqLen + pageHeaderSize <= XLOG_BLCKSZ) + if (reqLen + pageHeaderSize <= state->pageSize) addLen = pageHeaderSize; else - addLen = XLOG_BLCKSZ - reqLen; + addLen = state->pageSize - reqLen; Assert(addLen >= 0); } @@ -805,7 +808,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen, /* Data is not in our buffer, request the caller for it. */ XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize); targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize); - Assert((pageptr % XLOG_BLCKSZ) == 0); + Assert((pageptr % state->pageSize) == 0); /* * Every time we request to load new data of a page to the caller, even if @@ -829,7 +832,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen, * will not come back here, but will request the actual target page. */ state->readPagePtr = pageptr - targetPageOff; - state->readLen = XLOG_BLCKSZ; + state->readLen = state->pageSize; return true; } @@ -962,7 +965,7 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr, int32 offset; XLogPageHeader hdr = (XLogPageHeader) phdr; - Assert((recptr % XLOG_BLCKSZ) == 0); + Assert((recptr % state->pageSize) == 0); XLByteToSeg(recptr, segno, state->segcxt.ws_segsize); offset = XLogSegmentOffset(recptr, state->segcxt.ws_segsize); @@ -1016,10 +1019,10 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr, "WAL file is from different database system: incorrect segment size in page header"); return false; } - else if (longhdr->xlp_xlog_blcksz != XLOG_BLCKSZ) + else if (longhdr->xlp_xlog_blcksz != state->pageSize) { report_invalid_record(state, - "WAL file is from different database system: incorrect XLOG_BLCKSZ in page header"); + "WAL file is from different database system: incorrect block size in page header"); return false; } } @@ -1140,7 +1143,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr, * XLogNeedData() is prepared to handle that and will read at * least short page-header worth of data */ - targetRecOff = tmpRecPtr % XLOG_BLCKSZ; + targetRecOff = tmpRecPtr % state->pageSize; /* scroll back to page boundary */ targetPagePtr = tmpRecPtr - targetRecOff; @@ -1174,8 +1177,8 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr, * * Note that record headers are MAXALIGN'ed */ - if (MAXALIGN(header->xlp_rem_len) >= (XLOG_BLCKSZ - pageHeaderSize)) - tmpRecPtr = targetPagePtr + XLOG_BLCKSZ; + if (MAXALIGN(header->xlp_rem_len) >= (state->pageSize - pageHeaderSize)) + tmpRecPtr = targetPagePtr + state->pageSize; else { /* diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c index 9cbd8afa49..f2dd495310 100644 --- a/src/backend/replication/logical/logical.c +++ b/src/backend/replication/logical/logical.c @@ -199,11 +199,14 @@ StartupDecodingContext(List *output_plugin_options, ctx->slot = slot; - ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, cleanup_cb); + ctx->reader = + XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, cleanup_cb); + if (!ctx->reader) ereport(ERROR, (errcode(ERRCODE_OUT_OF_MEMORY), errmsg("out of memory"))); + ctx->reader->readBuf = palloc(XLOG_BLCKSZ); ctx->page_read = page_read; ctx->reorder = ReorderBufferAllocate(); diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c index 062093727d..8e95600ef7 100644 --- a/src/backend/replication/walsender.c +++ b/src/backend/replication/walsender.c @@ -579,8 +579,8 @@ StartReplication(StartReplicationCmd *cmd) errmsg("IDENTIFY_SYSTEM has not been run before START_REPLICATION"))); /* create xlogreader for physical replication */ - xlogreader = - XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close); + xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, + NULL, wal_segment_close); if (!xlogreader) ereport(ERROR, diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c index 712c85281c..8a6209b061 100644 --- a/src/bin/pg_rewind/parsexlog.c +++ b/src/bin/pg_rewind/parsexlog.c @@ -61,7 +61,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex, XLogReaderState *xlogreader; char *errormsg; - xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL); + xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL); if (xlogreader == NULL) pg_fatal("out of memory"); @@ -121,7 +121,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex, char *errormsg; XLogRecPtr endptr; - xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL); + xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL); if (xlogreader == NULL) pg_fatal("out of memory"); @@ -181,7 +181,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex, forkptr += SizeOfXLogShortPHD; } - xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL); + xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL); if (xlogreader == NULL) pg_fatal("out of memory"); diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c index f514de6b13..b635537659 100644 --- a/src/bin/pg_waldump/pg_waldump.c +++ b/src/bin/pg_waldump/pg_waldump.c @@ -1041,7 +1041,7 @@ main(int argc, char **argv) /* we have everything we need, start reading */ xlogreader_state = - XLogReaderAllocate(WalSegSz, waldir, WALDumpCloseSegment); + XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, waldir, WALDumpCloseSegment); if (!xlogreader_state) fatal_error("out of memory"); diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h index b9dad7ee51..1853516333 100644 --- a/src/include/access/xlogreader.h +++ b/src/include/access/xlogreader.h @@ -151,6 +151,7 @@ struct XLogReaderState * ---------------------------------------- */ /* variables to communicate with page reader */ + int pageSize; /* size of a page */ XLogRecPtr readPagePtr; /* page pointer to read */ int32 readLen; /* bytes requested to reader, or actual bytes * read by reader, which must be larger than @@ -243,7 +244,7 @@ struct XLogReaderState /* Get a new XLogReader */ extern XLogReaderState *XLogReaderAllocate(int wal_segment_size, - const char *waldir, + int page_size, const char *waldir, WALSegmentCleanupCB cleanup_cb); /* Free an XLogReader */ -- 2.27.0 ----Next_Part(Thu_Mar__4_11_28_53_2021_014)---- ^ permalink raw reply [nested|flat] 26+ messages in thread
* [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize. @ 2019-09-05 12:29 Kyotaro Horiguchi <[email protected]> 0 siblings, 0 replies; 26+ messages in thread From: Kyotaro Horiguchi @ 2019-09-05 12:29 UTC (permalink / raw) XLog reader facility and its callers currently relies on the same constant XLOG_BLCKSZ. This patch moves the responsibility to determine the block size to the callers of the facility. --- src/backend/access/transam/twophase.c | 3 +- src/backend/access/transam/xlog.c | 9 ++-- src/backend/access/transam/xlogreader.c | 57 ++++++++++++----------- src/backend/replication/logical/logical.c | 5 +- src/backend/replication/walsender.c | 4 +- src/bin/pg_rewind/parsexlog.c | 6 +-- src/bin/pg_waldump/pg_waldump.c | 2 +- src/include/access/xlogreader.h | 3 +- 8 files changed, 50 insertions(+), 39 deletions(-) diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c index 7d5a96a5f5..9f9e7de253 100644 --- a/src/backend/access/transam/twophase.c +++ b/src/backend/access/transam/twophase.c @@ -1325,7 +1325,8 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len) XLogReaderState *xlogreader; char *errormsg; - xlogreader = XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close); + xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, + NULL, wal_segment_close); if (!xlogreader) ereport(ERROR, diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index b517ca85ad..0822ac70cb 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -1222,7 +1222,10 @@ XLogInsertRecord(XLogRecData *rdata, appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len); if (!debug_reader) - debug_reader = XLogReaderAllocate(wal_segment_size, NULL, NULL); + { + debug_reader = + XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, NULL); + } if (!debug_reader) { @@ -6531,8 +6534,8 @@ StartupXLOG(void) OwnLatch(&XLogCtl->recoveryWakeupLatch); /* Set up XLOG reader facility */ - xlogreader = - XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close); + xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, + NULL, wal_segment_close); if (!xlogreader) ereport(ERROR, diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c index ba3e4868f7..e5baf13086 100644 --- a/src/backend/access/transam/xlogreader.c +++ b/src/backend/access/transam/xlogreader.c @@ -72,7 +72,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...) * Returns NULL if the xlogreader couldn't be allocated. */ XLogReaderState * -XLogReaderAllocate(int wal_segment_size, const char *waldir, +XLogReaderAllocate(int wal_segment_size, int page_size, const char *waldir, WALSegmentCleanupCB cleanup_cb) { XLogReaderState *state; @@ -87,6 +87,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir, state->cleanup_cb = cleanup_cb; state->max_block_id = -1; + state->pageSize = page_size; /* * Permanently allocate readBuf. We do it this way, rather than just @@ -95,7 +96,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir, * isn't guaranteed to have any particular alignment, whereas * palloc_extended() will provide MAXALIGN'd storage. */ - state->readBuf = (char *) palloc_extended(XLOG_BLCKSZ, + state->readBuf = (char *) palloc_extended(state->pageSize, MCXT_ALLOC_NO_OOM); if (!state->readBuf) { @@ -163,7 +164,7 @@ XLogReaderFree(XLogReaderState *state) * readRecordBufSize is set to the new buffer size. * * To avoid useless small increases, round its size to a multiple of - * XLOG_BLCKSZ, and make sure it's at least 5*Max(BLCKSZ, XLOG_BLCKSZ) to start + * page size, and make sure it's at least 5*Max(BLCKSZ, <page size>) to start * with. (That is enough for all "normal" records, but very large commit or * abort records might need more space.) */ @@ -172,8 +173,8 @@ allocate_recordbuf(XLogReaderState *state, uint32 reclength) { uint32 newSize = reclength; - newSize += XLOG_BLCKSZ - (newSize % XLOG_BLCKSZ); - newSize = Max(newSize, 5 * Max(BLCKSZ, XLOG_BLCKSZ)); + newSize += state->pageSize - (newSize % state->pageSize); + newSize = Max(newSize, 5 * Max(BLCKSZ, state->pageSize)); #ifndef FRONTEND @@ -238,6 +239,8 @@ void XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr) { Assert(!XLogRecPtrIsInvalid(RecPtr)); + Assert(state->readBuf != NULL && + state->readBuf == (void *) MAXALIGN(state->readBuf)); ResetDecoder(state); @@ -366,15 +369,15 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) XLogPageHeader pageHeader; targetPagePtr = - state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ); - targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ; + state->ReadRecPtr - (state->ReadRecPtr % state->pageSize); + targetRecOff = state->ReadRecPtr % state->pageSize; /* * Check if we have enough data. For the first record in the page, * the requesting length doesn't contain page header. */ if (XLogNeedData(state, targetPagePtr, - Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ), + Min(targetRecOff + SizeOfXLogRecord, state->pageSize), targetRecOff != 0)) return XLREAD_NEED_DATA; @@ -421,7 +424,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) * verified that we got the whole header. */ prec = (XLogRecord *) (state->readBuf + - state->ReadRecPtr % XLOG_BLCKSZ); + state->ReadRecPtr % state->pageSize); total_len = prec->xl_tot_len; /* @@ -432,7 +435,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) * ensure that we enter the XLREAD_CONTINUATION state below; * otherwise we might fail to apply ValidXLogRecordHeader at all. */ - if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord) + if (targetRecOff <= state->pageSize - SizeOfXLogRecord) { if (!ValidXLogRecordHeader(state, state->ReadRecPtr, state->PrevRecPtr, prec)) @@ -477,10 +480,10 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) * available */ targetPagePtr = - state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ); - targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ; + state->ReadRecPtr - (state->ReadRecPtr % state->pageSize); + targetRecOff = state->ReadRecPtr % state->pageSize; - request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ); + request_len = Min(targetRecOff + total_len, state->pageSize); record_len = request_len - targetRecOff; /* ReadRecPtr contains page header */ @@ -549,7 +552,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) /* Calculate pointer to beginning of next page */ state->recordContRecPtr = state->ReadRecPtr + record_len; - Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0); + Assert(state->recordContRecPtr % state->pageSize == 0); state->readRecordState = XLREAD_CONTINUATION; } @@ -576,7 +579,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) /* this request contains page header */ Assert (targetPagePtr != 0); if (XLogNeedData(state, targetPagePtr, - Min(state->recordRemainLen, XLOG_BLCKSZ), + Min(state->recordRemainLen, state->pageSize), false)) return XLREAD_NEED_DATA; @@ -628,7 +631,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) Assert(state->readLen >= pageHeaderSize); contdata = (char *) state->readBuf + pageHeaderSize; - record_len = XLOG_BLCKSZ - pageHeaderSize; + record_len = state->pageSize - pageHeaderSize; if (pageHeader->xlp_rem_len < record_len) record_len = pageHeader->xlp_rem_len; @@ -655,7 +658,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) } /* Calculate pointer to beginning of next page, and continue */ - state->recordContRecPtr += XLOG_BLCKSZ; + state->recordContRecPtr += state->pageSize; } /* targetPagePtr is pointing the last-read page here */ @@ -789,10 +792,10 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen, XLogPageHeaderSize((XLogPageHeader) state->readBuf); addLen = pageHeaderSize; - if (reqLen + pageHeaderSize <= XLOG_BLCKSZ) + if (reqLen + pageHeaderSize <= state->pageSize) addLen = pageHeaderSize; else - addLen = XLOG_BLCKSZ - reqLen; + addLen = state->pageSize - reqLen; Assert(addLen >= 0); } @@ -805,7 +808,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen, /* Data is not in our buffer, request the caller for it. */ XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize); targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize); - Assert((pageptr % XLOG_BLCKSZ) == 0); + Assert((pageptr % state->pageSize) == 0); /* * Every time we request to load new data of a page to the caller, even if @@ -829,7 +832,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen, * will not come back here, but will request the actual target page. */ state->readPagePtr = pageptr - targetPageOff; - state->readLen = XLOG_BLCKSZ; + state->readLen = state->pageSize; return true; } @@ -962,7 +965,7 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr, int32 offset; XLogPageHeader hdr = (XLogPageHeader) phdr; - Assert((recptr % XLOG_BLCKSZ) == 0); + Assert((recptr % state->pageSize) == 0); XLByteToSeg(recptr, segno, state->segcxt.ws_segsize); offset = XLogSegmentOffset(recptr, state->segcxt.ws_segsize); @@ -1016,10 +1019,10 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr, "WAL file is from different database system: incorrect segment size in page header"); return false; } - else if (longhdr->xlp_xlog_blcksz != XLOG_BLCKSZ) + else if (longhdr->xlp_xlog_blcksz != state->pageSize) { report_invalid_record(state, - "WAL file is from different database system: incorrect XLOG_BLCKSZ in page header"); + "WAL file is from different database system: incorrect block size in page header"); return false; } } @@ -1140,7 +1143,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr, * XLogNeedData() is prepared to handle that and will read at * least short page-header worth of data */ - targetRecOff = tmpRecPtr % XLOG_BLCKSZ; + targetRecOff = tmpRecPtr % state->pageSize; /* scroll back to page boundary */ targetPagePtr = tmpRecPtr - targetRecOff; @@ -1174,8 +1177,8 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr, * * Note that record headers are MAXALIGN'ed */ - if (MAXALIGN(header->xlp_rem_len) >= (XLOG_BLCKSZ - pageHeaderSize)) - tmpRecPtr = targetPagePtr + XLOG_BLCKSZ; + if (MAXALIGN(header->xlp_rem_len) >= (state->pageSize - pageHeaderSize)) + tmpRecPtr = targetPagePtr + state->pageSize; else { /* diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c index 9cbd8afa49..f2dd495310 100644 --- a/src/backend/replication/logical/logical.c +++ b/src/backend/replication/logical/logical.c @@ -199,11 +199,14 @@ StartupDecodingContext(List *output_plugin_options, ctx->slot = slot; - ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, cleanup_cb); + ctx->reader = + XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, cleanup_cb); + if (!ctx->reader) ereport(ERROR, (errcode(ERRCODE_OUT_OF_MEMORY), errmsg("out of memory"))); + ctx->reader->readBuf = palloc(XLOG_BLCKSZ); ctx->page_read = page_read; ctx->reorder = ReorderBufferAllocate(); diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c index 062093727d..8e95600ef7 100644 --- a/src/backend/replication/walsender.c +++ b/src/backend/replication/walsender.c @@ -579,8 +579,8 @@ StartReplication(StartReplicationCmd *cmd) errmsg("IDENTIFY_SYSTEM has not been run before START_REPLICATION"))); /* create xlogreader for physical replication */ - xlogreader = - XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close); + xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, + NULL, wal_segment_close); if (!xlogreader) ereport(ERROR, diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c index 712c85281c..8a6209b061 100644 --- a/src/bin/pg_rewind/parsexlog.c +++ b/src/bin/pg_rewind/parsexlog.c @@ -61,7 +61,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex, XLogReaderState *xlogreader; char *errormsg; - xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL); + xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL); if (xlogreader == NULL) pg_fatal("out of memory"); @@ -121,7 +121,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex, char *errormsg; XLogRecPtr endptr; - xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL); + xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL); if (xlogreader == NULL) pg_fatal("out of memory"); @@ -181,7 +181,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex, forkptr += SizeOfXLogShortPHD; } - xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL); + xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL); if (xlogreader == NULL) pg_fatal("out of memory"); diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c index f514de6b13..b635537659 100644 --- a/src/bin/pg_waldump/pg_waldump.c +++ b/src/bin/pg_waldump/pg_waldump.c @@ -1041,7 +1041,7 @@ main(int argc, char **argv) /* we have everything we need, start reading */ xlogreader_state = - XLogReaderAllocate(WalSegSz, waldir, WALDumpCloseSegment); + XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, waldir, WALDumpCloseSegment); if (!xlogreader_state) fatal_error("out of memory"); diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h index b9dad7ee51..1853516333 100644 --- a/src/include/access/xlogreader.h +++ b/src/include/access/xlogreader.h @@ -151,6 +151,7 @@ struct XLogReaderState * ---------------------------------------- */ /* variables to communicate with page reader */ + int pageSize; /* size of a page */ XLogRecPtr readPagePtr; /* page pointer to read */ int32 readLen; /* bytes requested to reader, or actual bytes * read by reader, which must be larger than @@ -243,7 +244,7 @@ struct XLogReaderState /* Get a new XLogReader */ extern XLogReaderState *XLogReaderAllocate(int wal_segment_size, - const char *waldir, + int page_size, const char *waldir, WALSegmentCleanupCB cleanup_cb); /* Free an XLogReader */ -- 2.27.0 ----Next_Part(Thu_Mar__4_11_28_53_2021_014)---- ^ permalink raw reply [nested|flat] 26+ messages in thread
* [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize. @ 2019-09-05 12:29 Kyotaro Horiguchi <[email protected]> 0 siblings, 0 replies; 26+ messages in thread From: Kyotaro Horiguchi @ 2019-09-05 12:29 UTC (permalink / raw) XLog reader facility and its callers currently relies on the same constant XLOG_BLCKSZ. This patch moves the responsibility to determine the block size to the callers of the facility. --- src/backend/access/transam/twophase.c | 3 +- src/backend/access/transam/xlog.c | 9 ++-- src/backend/access/transam/xlogreader.c | 57 ++++++++++++----------- src/backend/replication/logical/logical.c | 5 +- src/backend/replication/walsender.c | 4 +- src/bin/pg_rewind/parsexlog.c | 6 +-- src/bin/pg_waldump/pg_waldump.c | 2 +- src/include/access/xlogreader.h | 3 +- 8 files changed, 50 insertions(+), 39 deletions(-) diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c index 7d5a96a5f5..9f9e7de253 100644 --- a/src/backend/access/transam/twophase.c +++ b/src/backend/access/transam/twophase.c @@ -1325,7 +1325,8 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len) XLogReaderState *xlogreader; char *errormsg; - xlogreader = XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close); + xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, + NULL, wal_segment_close); if (!xlogreader) ereport(ERROR, diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index b517ca85ad..0822ac70cb 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -1222,7 +1222,10 @@ XLogInsertRecord(XLogRecData *rdata, appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len); if (!debug_reader) - debug_reader = XLogReaderAllocate(wal_segment_size, NULL, NULL); + { + debug_reader = + XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, NULL); + } if (!debug_reader) { @@ -6531,8 +6534,8 @@ StartupXLOG(void) OwnLatch(&XLogCtl->recoveryWakeupLatch); /* Set up XLOG reader facility */ - xlogreader = - XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close); + xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, + NULL, wal_segment_close); if (!xlogreader) ereport(ERROR, diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c index ba3e4868f7..e5baf13086 100644 --- a/src/backend/access/transam/xlogreader.c +++ b/src/backend/access/transam/xlogreader.c @@ -72,7 +72,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...) * Returns NULL if the xlogreader couldn't be allocated. */ XLogReaderState * -XLogReaderAllocate(int wal_segment_size, const char *waldir, +XLogReaderAllocate(int wal_segment_size, int page_size, const char *waldir, WALSegmentCleanupCB cleanup_cb) { XLogReaderState *state; @@ -87,6 +87,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir, state->cleanup_cb = cleanup_cb; state->max_block_id = -1; + state->pageSize = page_size; /* * Permanently allocate readBuf. We do it this way, rather than just @@ -95,7 +96,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir, * isn't guaranteed to have any particular alignment, whereas * palloc_extended() will provide MAXALIGN'd storage. */ - state->readBuf = (char *) palloc_extended(XLOG_BLCKSZ, + state->readBuf = (char *) palloc_extended(state->pageSize, MCXT_ALLOC_NO_OOM); if (!state->readBuf) { @@ -163,7 +164,7 @@ XLogReaderFree(XLogReaderState *state) * readRecordBufSize is set to the new buffer size. * * To avoid useless small increases, round its size to a multiple of - * XLOG_BLCKSZ, and make sure it's at least 5*Max(BLCKSZ, XLOG_BLCKSZ) to start + * page size, and make sure it's at least 5*Max(BLCKSZ, <page size>) to start * with. (That is enough for all "normal" records, but very large commit or * abort records might need more space.) */ @@ -172,8 +173,8 @@ allocate_recordbuf(XLogReaderState *state, uint32 reclength) { uint32 newSize = reclength; - newSize += XLOG_BLCKSZ - (newSize % XLOG_BLCKSZ); - newSize = Max(newSize, 5 * Max(BLCKSZ, XLOG_BLCKSZ)); + newSize += state->pageSize - (newSize % state->pageSize); + newSize = Max(newSize, 5 * Max(BLCKSZ, state->pageSize)); #ifndef FRONTEND @@ -238,6 +239,8 @@ void XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr) { Assert(!XLogRecPtrIsInvalid(RecPtr)); + Assert(state->readBuf != NULL && + state->readBuf == (void *) MAXALIGN(state->readBuf)); ResetDecoder(state); @@ -366,15 +369,15 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) XLogPageHeader pageHeader; targetPagePtr = - state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ); - targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ; + state->ReadRecPtr - (state->ReadRecPtr % state->pageSize); + targetRecOff = state->ReadRecPtr % state->pageSize; /* * Check if we have enough data. For the first record in the page, * the requesting length doesn't contain page header. */ if (XLogNeedData(state, targetPagePtr, - Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ), + Min(targetRecOff + SizeOfXLogRecord, state->pageSize), targetRecOff != 0)) return XLREAD_NEED_DATA; @@ -421,7 +424,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) * verified that we got the whole header. */ prec = (XLogRecord *) (state->readBuf + - state->ReadRecPtr % XLOG_BLCKSZ); + state->ReadRecPtr % state->pageSize); total_len = prec->xl_tot_len; /* @@ -432,7 +435,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) * ensure that we enter the XLREAD_CONTINUATION state below; * otherwise we might fail to apply ValidXLogRecordHeader at all. */ - if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord) + if (targetRecOff <= state->pageSize - SizeOfXLogRecord) { if (!ValidXLogRecordHeader(state, state->ReadRecPtr, state->PrevRecPtr, prec)) @@ -477,10 +480,10 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) * available */ targetPagePtr = - state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ); - targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ; + state->ReadRecPtr - (state->ReadRecPtr % state->pageSize); + targetRecOff = state->ReadRecPtr % state->pageSize; - request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ); + request_len = Min(targetRecOff + total_len, state->pageSize); record_len = request_len - targetRecOff; /* ReadRecPtr contains page header */ @@ -549,7 +552,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) /* Calculate pointer to beginning of next page */ state->recordContRecPtr = state->ReadRecPtr + record_len; - Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0); + Assert(state->recordContRecPtr % state->pageSize == 0); state->readRecordState = XLREAD_CONTINUATION; } @@ -576,7 +579,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) /* this request contains page header */ Assert (targetPagePtr != 0); if (XLogNeedData(state, targetPagePtr, - Min(state->recordRemainLen, XLOG_BLCKSZ), + Min(state->recordRemainLen, state->pageSize), false)) return XLREAD_NEED_DATA; @@ -628,7 +631,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) Assert(state->readLen >= pageHeaderSize); contdata = (char *) state->readBuf + pageHeaderSize; - record_len = XLOG_BLCKSZ - pageHeaderSize; + record_len = state->pageSize - pageHeaderSize; if (pageHeader->xlp_rem_len < record_len) record_len = pageHeader->xlp_rem_len; @@ -655,7 +658,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) } /* Calculate pointer to beginning of next page, and continue */ - state->recordContRecPtr += XLOG_BLCKSZ; + state->recordContRecPtr += state->pageSize; } /* targetPagePtr is pointing the last-read page here */ @@ -789,10 +792,10 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen, XLogPageHeaderSize((XLogPageHeader) state->readBuf); addLen = pageHeaderSize; - if (reqLen + pageHeaderSize <= XLOG_BLCKSZ) + if (reqLen + pageHeaderSize <= state->pageSize) addLen = pageHeaderSize; else - addLen = XLOG_BLCKSZ - reqLen; + addLen = state->pageSize - reqLen; Assert(addLen >= 0); } @@ -805,7 +808,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen, /* Data is not in our buffer, request the caller for it. */ XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize); targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize); - Assert((pageptr % XLOG_BLCKSZ) == 0); + Assert((pageptr % state->pageSize) == 0); /* * Every time we request to load new data of a page to the caller, even if @@ -829,7 +832,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen, * will not come back here, but will request the actual target page. */ state->readPagePtr = pageptr - targetPageOff; - state->readLen = XLOG_BLCKSZ; + state->readLen = state->pageSize; return true; } @@ -962,7 +965,7 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr, int32 offset; XLogPageHeader hdr = (XLogPageHeader) phdr; - Assert((recptr % XLOG_BLCKSZ) == 0); + Assert((recptr % state->pageSize) == 0); XLByteToSeg(recptr, segno, state->segcxt.ws_segsize); offset = XLogSegmentOffset(recptr, state->segcxt.ws_segsize); @@ -1016,10 +1019,10 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr, "WAL file is from different database system: incorrect segment size in page header"); return false; } - else if (longhdr->xlp_xlog_blcksz != XLOG_BLCKSZ) + else if (longhdr->xlp_xlog_blcksz != state->pageSize) { report_invalid_record(state, - "WAL file is from different database system: incorrect XLOG_BLCKSZ in page header"); + "WAL file is from different database system: incorrect block size in page header"); return false; } } @@ -1140,7 +1143,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr, * XLogNeedData() is prepared to handle that and will read at * least short page-header worth of data */ - targetRecOff = tmpRecPtr % XLOG_BLCKSZ; + targetRecOff = tmpRecPtr % state->pageSize; /* scroll back to page boundary */ targetPagePtr = tmpRecPtr - targetRecOff; @@ -1174,8 +1177,8 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr, * * Note that record headers are MAXALIGN'ed */ - if (MAXALIGN(header->xlp_rem_len) >= (XLOG_BLCKSZ - pageHeaderSize)) - tmpRecPtr = targetPagePtr + XLOG_BLCKSZ; + if (MAXALIGN(header->xlp_rem_len) >= (state->pageSize - pageHeaderSize)) + tmpRecPtr = targetPagePtr + state->pageSize; else { /* diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c index 9cbd8afa49..f2dd495310 100644 --- a/src/backend/replication/logical/logical.c +++ b/src/backend/replication/logical/logical.c @@ -199,11 +199,14 @@ StartupDecodingContext(List *output_plugin_options, ctx->slot = slot; - ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, cleanup_cb); + ctx->reader = + XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, cleanup_cb); + if (!ctx->reader) ereport(ERROR, (errcode(ERRCODE_OUT_OF_MEMORY), errmsg("out of memory"))); + ctx->reader->readBuf = palloc(XLOG_BLCKSZ); ctx->page_read = page_read; ctx->reorder = ReorderBufferAllocate(); diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c index 062093727d..8e95600ef7 100644 --- a/src/backend/replication/walsender.c +++ b/src/backend/replication/walsender.c @@ -579,8 +579,8 @@ StartReplication(StartReplicationCmd *cmd) errmsg("IDENTIFY_SYSTEM has not been run before START_REPLICATION"))); /* create xlogreader for physical replication */ - xlogreader = - XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close); + xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, + NULL, wal_segment_close); if (!xlogreader) ereport(ERROR, diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c index 712c85281c..8a6209b061 100644 --- a/src/bin/pg_rewind/parsexlog.c +++ b/src/bin/pg_rewind/parsexlog.c @@ -61,7 +61,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex, XLogReaderState *xlogreader; char *errormsg; - xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL); + xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL); if (xlogreader == NULL) pg_fatal("out of memory"); @@ -121,7 +121,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex, char *errormsg; XLogRecPtr endptr; - xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL); + xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL); if (xlogreader == NULL) pg_fatal("out of memory"); @@ -181,7 +181,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex, forkptr += SizeOfXLogShortPHD; } - xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL); + xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL); if (xlogreader == NULL) pg_fatal("out of memory"); diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c index f514de6b13..b635537659 100644 --- a/src/bin/pg_waldump/pg_waldump.c +++ b/src/bin/pg_waldump/pg_waldump.c @@ -1041,7 +1041,7 @@ main(int argc, char **argv) /* we have everything we need, start reading */ xlogreader_state = - XLogReaderAllocate(WalSegSz, waldir, WALDumpCloseSegment); + XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, waldir, WALDumpCloseSegment); if (!xlogreader_state) fatal_error("out of memory"); diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h index b9dad7ee51..1853516333 100644 --- a/src/include/access/xlogreader.h +++ b/src/include/access/xlogreader.h @@ -151,6 +151,7 @@ struct XLogReaderState * ---------------------------------------- */ /* variables to communicate with page reader */ + int pageSize; /* size of a page */ XLogRecPtr readPagePtr; /* page pointer to read */ int32 readLen; /* bytes requested to reader, or actual bytes * read by reader, which must be larger than @@ -243,7 +244,7 @@ struct XLogReaderState /* Get a new XLogReader */ extern XLogReaderState *XLogReaderAllocate(int wal_segment_size, - const char *waldir, + int page_size, const char *waldir, WALSegmentCleanupCB cleanup_cb); /* Free an XLogReader */ -- 2.27.0 ----Next_Part(Thu_Mar__4_11_28_53_2021_014)---- ^ permalink raw reply [nested|flat] 26+ messages in thread
* [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize. @ 2019-09-05 12:29 Kyotaro Horiguchi <[email protected]> 0 siblings, 0 replies; 26+ messages in thread From: Kyotaro Horiguchi @ 2019-09-05 12:29 UTC (permalink / raw) XLog reader facility and its callers currently relies on the same constant XLOG_BLCKSZ. This patch moves the responsibility to determine the block size to the callers of the facility. --- src/backend/access/transam/twophase.c | 3 +- src/backend/access/transam/xlog.c | 9 ++-- src/backend/access/transam/xlogreader.c | 57 ++++++++++++----------- src/backend/replication/logical/logical.c | 5 +- src/backend/replication/walsender.c | 4 +- src/bin/pg_rewind/parsexlog.c | 6 +-- src/bin/pg_waldump/pg_waldump.c | 2 +- src/include/access/xlogreader.h | 3 +- 8 files changed, 50 insertions(+), 39 deletions(-) diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c index 7d5a96a5f5..9f9e7de253 100644 --- a/src/backend/access/transam/twophase.c +++ b/src/backend/access/transam/twophase.c @@ -1325,7 +1325,8 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len) XLogReaderState *xlogreader; char *errormsg; - xlogreader = XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close); + xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, + NULL, wal_segment_close); if (!xlogreader) ereport(ERROR, diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index b517ca85ad..0822ac70cb 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -1222,7 +1222,10 @@ XLogInsertRecord(XLogRecData *rdata, appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len); if (!debug_reader) - debug_reader = XLogReaderAllocate(wal_segment_size, NULL, NULL); + { + debug_reader = + XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, NULL); + } if (!debug_reader) { @@ -6531,8 +6534,8 @@ StartupXLOG(void) OwnLatch(&XLogCtl->recoveryWakeupLatch); /* Set up XLOG reader facility */ - xlogreader = - XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close); + xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, + NULL, wal_segment_close); if (!xlogreader) ereport(ERROR, diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c index ba3e4868f7..e5baf13086 100644 --- a/src/backend/access/transam/xlogreader.c +++ b/src/backend/access/transam/xlogreader.c @@ -72,7 +72,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...) * Returns NULL if the xlogreader couldn't be allocated. */ XLogReaderState * -XLogReaderAllocate(int wal_segment_size, const char *waldir, +XLogReaderAllocate(int wal_segment_size, int page_size, const char *waldir, WALSegmentCleanupCB cleanup_cb) { XLogReaderState *state; @@ -87,6 +87,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir, state->cleanup_cb = cleanup_cb; state->max_block_id = -1; + state->pageSize = page_size; /* * Permanently allocate readBuf. We do it this way, rather than just @@ -95,7 +96,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir, * isn't guaranteed to have any particular alignment, whereas * palloc_extended() will provide MAXALIGN'd storage. */ - state->readBuf = (char *) palloc_extended(XLOG_BLCKSZ, + state->readBuf = (char *) palloc_extended(state->pageSize, MCXT_ALLOC_NO_OOM); if (!state->readBuf) { @@ -163,7 +164,7 @@ XLogReaderFree(XLogReaderState *state) * readRecordBufSize is set to the new buffer size. * * To avoid useless small increases, round its size to a multiple of - * XLOG_BLCKSZ, and make sure it's at least 5*Max(BLCKSZ, XLOG_BLCKSZ) to start + * page size, and make sure it's at least 5*Max(BLCKSZ, <page size>) to start * with. (That is enough for all "normal" records, but very large commit or * abort records might need more space.) */ @@ -172,8 +173,8 @@ allocate_recordbuf(XLogReaderState *state, uint32 reclength) { uint32 newSize = reclength; - newSize += XLOG_BLCKSZ - (newSize % XLOG_BLCKSZ); - newSize = Max(newSize, 5 * Max(BLCKSZ, XLOG_BLCKSZ)); + newSize += state->pageSize - (newSize % state->pageSize); + newSize = Max(newSize, 5 * Max(BLCKSZ, state->pageSize)); #ifndef FRONTEND @@ -238,6 +239,8 @@ void XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr) { Assert(!XLogRecPtrIsInvalid(RecPtr)); + Assert(state->readBuf != NULL && + state->readBuf == (void *) MAXALIGN(state->readBuf)); ResetDecoder(state); @@ -366,15 +369,15 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) XLogPageHeader pageHeader; targetPagePtr = - state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ); - targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ; + state->ReadRecPtr - (state->ReadRecPtr % state->pageSize); + targetRecOff = state->ReadRecPtr % state->pageSize; /* * Check if we have enough data. For the first record in the page, * the requesting length doesn't contain page header. */ if (XLogNeedData(state, targetPagePtr, - Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ), + Min(targetRecOff + SizeOfXLogRecord, state->pageSize), targetRecOff != 0)) return XLREAD_NEED_DATA; @@ -421,7 +424,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) * verified that we got the whole header. */ prec = (XLogRecord *) (state->readBuf + - state->ReadRecPtr % XLOG_BLCKSZ); + state->ReadRecPtr % state->pageSize); total_len = prec->xl_tot_len; /* @@ -432,7 +435,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) * ensure that we enter the XLREAD_CONTINUATION state below; * otherwise we might fail to apply ValidXLogRecordHeader at all. */ - if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord) + if (targetRecOff <= state->pageSize - SizeOfXLogRecord) { if (!ValidXLogRecordHeader(state, state->ReadRecPtr, state->PrevRecPtr, prec)) @@ -477,10 +480,10 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) * available */ targetPagePtr = - state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ); - targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ; + state->ReadRecPtr - (state->ReadRecPtr % state->pageSize); + targetRecOff = state->ReadRecPtr % state->pageSize; - request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ); + request_len = Min(targetRecOff + total_len, state->pageSize); record_len = request_len - targetRecOff; /* ReadRecPtr contains page header */ @@ -549,7 +552,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) /* Calculate pointer to beginning of next page */ state->recordContRecPtr = state->ReadRecPtr + record_len; - Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0); + Assert(state->recordContRecPtr % state->pageSize == 0); state->readRecordState = XLREAD_CONTINUATION; } @@ -576,7 +579,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) /* this request contains page header */ Assert (targetPagePtr != 0); if (XLogNeedData(state, targetPagePtr, - Min(state->recordRemainLen, XLOG_BLCKSZ), + Min(state->recordRemainLen, state->pageSize), false)) return XLREAD_NEED_DATA; @@ -628,7 +631,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) Assert(state->readLen >= pageHeaderSize); contdata = (char *) state->readBuf + pageHeaderSize; - record_len = XLOG_BLCKSZ - pageHeaderSize; + record_len = state->pageSize - pageHeaderSize; if (pageHeader->xlp_rem_len < record_len) record_len = pageHeader->xlp_rem_len; @@ -655,7 +658,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) } /* Calculate pointer to beginning of next page, and continue */ - state->recordContRecPtr += XLOG_BLCKSZ; + state->recordContRecPtr += state->pageSize; } /* targetPagePtr is pointing the last-read page here */ @@ -789,10 +792,10 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen, XLogPageHeaderSize((XLogPageHeader) state->readBuf); addLen = pageHeaderSize; - if (reqLen + pageHeaderSize <= XLOG_BLCKSZ) + if (reqLen + pageHeaderSize <= state->pageSize) addLen = pageHeaderSize; else - addLen = XLOG_BLCKSZ - reqLen; + addLen = state->pageSize - reqLen; Assert(addLen >= 0); } @@ -805,7 +808,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen, /* Data is not in our buffer, request the caller for it. */ XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize); targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize); - Assert((pageptr % XLOG_BLCKSZ) == 0); + Assert((pageptr % state->pageSize) == 0); /* * Every time we request to load new data of a page to the caller, even if @@ -829,7 +832,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen, * will not come back here, but will request the actual target page. */ state->readPagePtr = pageptr - targetPageOff; - state->readLen = XLOG_BLCKSZ; + state->readLen = state->pageSize; return true; } @@ -962,7 +965,7 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr, int32 offset; XLogPageHeader hdr = (XLogPageHeader) phdr; - Assert((recptr % XLOG_BLCKSZ) == 0); + Assert((recptr % state->pageSize) == 0); XLByteToSeg(recptr, segno, state->segcxt.ws_segsize); offset = XLogSegmentOffset(recptr, state->segcxt.ws_segsize); @@ -1016,10 +1019,10 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr, "WAL file is from different database system: incorrect segment size in page header"); return false; } - else if (longhdr->xlp_xlog_blcksz != XLOG_BLCKSZ) + else if (longhdr->xlp_xlog_blcksz != state->pageSize) { report_invalid_record(state, - "WAL file is from different database system: incorrect XLOG_BLCKSZ in page header"); + "WAL file is from different database system: incorrect block size in page header"); return false; } } @@ -1140,7 +1143,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr, * XLogNeedData() is prepared to handle that and will read at * least short page-header worth of data */ - targetRecOff = tmpRecPtr % XLOG_BLCKSZ; + targetRecOff = tmpRecPtr % state->pageSize; /* scroll back to page boundary */ targetPagePtr = tmpRecPtr - targetRecOff; @@ -1174,8 +1177,8 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr, * * Note that record headers are MAXALIGN'ed */ - if (MAXALIGN(header->xlp_rem_len) >= (XLOG_BLCKSZ - pageHeaderSize)) - tmpRecPtr = targetPagePtr + XLOG_BLCKSZ; + if (MAXALIGN(header->xlp_rem_len) >= (state->pageSize - pageHeaderSize)) + tmpRecPtr = targetPagePtr + state->pageSize; else { /* diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c index 9cbd8afa49..f2dd495310 100644 --- a/src/backend/replication/logical/logical.c +++ b/src/backend/replication/logical/logical.c @@ -199,11 +199,14 @@ StartupDecodingContext(List *output_plugin_options, ctx->slot = slot; - ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, cleanup_cb); + ctx->reader = + XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, cleanup_cb); + if (!ctx->reader) ereport(ERROR, (errcode(ERRCODE_OUT_OF_MEMORY), errmsg("out of memory"))); + ctx->reader->readBuf = palloc(XLOG_BLCKSZ); ctx->page_read = page_read; ctx->reorder = ReorderBufferAllocate(); diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c index 062093727d..8e95600ef7 100644 --- a/src/backend/replication/walsender.c +++ b/src/backend/replication/walsender.c @@ -579,8 +579,8 @@ StartReplication(StartReplicationCmd *cmd) errmsg("IDENTIFY_SYSTEM has not been run before START_REPLICATION"))); /* create xlogreader for physical replication */ - xlogreader = - XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close); + xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, + NULL, wal_segment_close); if (!xlogreader) ereport(ERROR, diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c index 712c85281c..8a6209b061 100644 --- a/src/bin/pg_rewind/parsexlog.c +++ b/src/bin/pg_rewind/parsexlog.c @@ -61,7 +61,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex, XLogReaderState *xlogreader; char *errormsg; - xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL); + xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL); if (xlogreader == NULL) pg_fatal("out of memory"); @@ -121,7 +121,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex, char *errormsg; XLogRecPtr endptr; - xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL); + xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL); if (xlogreader == NULL) pg_fatal("out of memory"); @@ -181,7 +181,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex, forkptr += SizeOfXLogShortPHD; } - xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL); + xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL); if (xlogreader == NULL) pg_fatal("out of memory"); diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c index f514de6b13..b635537659 100644 --- a/src/bin/pg_waldump/pg_waldump.c +++ b/src/bin/pg_waldump/pg_waldump.c @@ -1041,7 +1041,7 @@ main(int argc, char **argv) /* we have everything we need, start reading */ xlogreader_state = - XLogReaderAllocate(WalSegSz, waldir, WALDumpCloseSegment); + XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, waldir, WALDumpCloseSegment); if (!xlogreader_state) fatal_error("out of memory"); diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h index b9dad7ee51..1853516333 100644 --- a/src/include/access/xlogreader.h +++ b/src/include/access/xlogreader.h @@ -151,6 +151,7 @@ struct XLogReaderState * ---------------------------------------- */ /* variables to communicate with page reader */ + int pageSize; /* size of a page */ XLogRecPtr readPagePtr; /* page pointer to read */ int32 readLen; /* bytes requested to reader, or actual bytes * read by reader, which must be larger than @@ -243,7 +244,7 @@ struct XLogReaderState /* Get a new XLogReader */ extern XLogReaderState *XLogReaderAllocate(int wal_segment_size, - const char *waldir, + int page_size, const char *waldir, WALSegmentCleanupCB cleanup_cb); /* Free an XLogReader */ -- 2.27.0 ----Next_Part(Thu_Mar__4_11_28_53_2021_014)---- ^ permalink raw reply [nested|flat] 26+ messages in thread
* [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize. @ 2019-09-05 12:29 Kyotaro Horiguchi <[email protected]> 0 siblings, 0 replies; 26+ messages in thread From: Kyotaro Horiguchi @ 2019-09-05 12:29 UTC (permalink / raw) XLog reader facility and its callers currently relies on the same constant XLOG_BLCKSZ. This patch moves the responsibility to determine the block size to the callers of the facility. --- src/backend/access/transam/twophase.c | 3 +- src/backend/access/transam/xlog.c | 9 ++-- src/backend/access/transam/xlogreader.c | 57 ++++++++++++----------- src/backend/replication/logical/logical.c | 5 +- src/backend/replication/walsender.c | 4 +- src/bin/pg_rewind/parsexlog.c | 6 +-- src/bin/pg_waldump/pg_waldump.c | 2 +- src/include/access/xlogreader.h | 3 +- 8 files changed, 50 insertions(+), 39 deletions(-) diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c index 7d5a96a5f5..9f9e7de253 100644 --- a/src/backend/access/transam/twophase.c +++ b/src/backend/access/transam/twophase.c @@ -1325,7 +1325,8 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len) XLogReaderState *xlogreader; char *errormsg; - xlogreader = XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close); + xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, + NULL, wal_segment_close); if (!xlogreader) ereport(ERROR, diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index b517ca85ad..0822ac70cb 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -1222,7 +1222,10 @@ XLogInsertRecord(XLogRecData *rdata, appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len); if (!debug_reader) - debug_reader = XLogReaderAllocate(wal_segment_size, NULL, NULL); + { + debug_reader = + XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, NULL); + } if (!debug_reader) { @@ -6531,8 +6534,8 @@ StartupXLOG(void) OwnLatch(&XLogCtl->recoveryWakeupLatch); /* Set up XLOG reader facility */ - xlogreader = - XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close); + xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, + NULL, wal_segment_close); if (!xlogreader) ereport(ERROR, diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c index ba3e4868f7..e5baf13086 100644 --- a/src/backend/access/transam/xlogreader.c +++ b/src/backend/access/transam/xlogreader.c @@ -72,7 +72,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...) * Returns NULL if the xlogreader couldn't be allocated. */ XLogReaderState * -XLogReaderAllocate(int wal_segment_size, const char *waldir, +XLogReaderAllocate(int wal_segment_size, int page_size, const char *waldir, WALSegmentCleanupCB cleanup_cb) { XLogReaderState *state; @@ -87,6 +87,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir, state->cleanup_cb = cleanup_cb; state->max_block_id = -1; + state->pageSize = page_size; /* * Permanently allocate readBuf. We do it this way, rather than just @@ -95,7 +96,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir, * isn't guaranteed to have any particular alignment, whereas * palloc_extended() will provide MAXALIGN'd storage. */ - state->readBuf = (char *) palloc_extended(XLOG_BLCKSZ, + state->readBuf = (char *) palloc_extended(state->pageSize, MCXT_ALLOC_NO_OOM); if (!state->readBuf) { @@ -163,7 +164,7 @@ XLogReaderFree(XLogReaderState *state) * readRecordBufSize is set to the new buffer size. * * To avoid useless small increases, round its size to a multiple of - * XLOG_BLCKSZ, and make sure it's at least 5*Max(BLCKSZ, XLOG_BLCKSZ) to start + * page size, and make sure it's at least 5*Max(BLCKSZ, <page size>) to start * with. (That is enough for all "normal" records, but very large commit or * abort records might need more space.) */ @@ -172,8 +173,8 @@ allocate_recordbuf(XLogReaderState *state, uint32 reclength) { uint32 newSize = reclength; - newSize += XLOG_BLCKSZ - (newSize % XLOG_BLCKSZ); - newSize = Max(newSize, 5 * Max(BLCKSZ, XLOG_BLCKSZ)); + newSize += state->pageSize - (newSize % state->pageSize); + newSize = Max(newSize, 5 * Max(BLCKSZ, state->pageSize)); #ifndef FRONTEND @@ -238,6 +239,8 @@ void XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr) { Assert(!XLogRecPtrIsInvalid(RecPtr)); + Assert(state->readBuf != NULL && + state->readBuf == (void *) MAXALIGN(state->readBuf)); ResetDecoder(state); @@ -366,15 +369,15 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) XLogPageHeader pageHeader; targetPagePtr = - state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ); - targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ; + state->ReadRecPtr - (state->ReadRecPtr % state->pageSize); + targetRecOff = state->ReadRecPtr % state->pageSize; /* * Check if we have enough data. For the first record in the page, * the requesting length doesn't contain page header. */ if (XLogNeedData(state, targetPagePtr, - Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ), + Min(targetRecOff + SizeOfXLogRecord, state->pageSize), targetRecOff != 0)) return XLREAD_NEED_DATA; @@ -421,7 +424,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) * verified that we got the whole header. */ prec = (XLogRecord *) (state->readBuf + - state->ReadRecPtr % XLOG_BLCKSZ); + state->ReadRecPtr % state->pageSize); total_len = prec->xl_tot_len; /* @@ -432,7 +435,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) * ensure that we enter the XLREAD_CONTINUATION state below; * otherwise we might fail to apply ValidXLogRecordHeader at all. */ - if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord) + if (targetRecOff <= state->pageSize - SizeOfXLogRecord) { if (!ValidXLogRecordHeader(state, state->ReadRecPtr, state->PrevRecPtr, prec)) @@ -477,10 +480,10 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) * available */ targetPagePtr = - state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ); - targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ; + state->ReadRecPtr - (state->ReadRecPtr % state->pageSize); + targetRecOff = state->ReadRecPtr % state->pageSize; - request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ); + request_len = Min(targetRecOff + total_len, state->pageSize); record_len = request_len - targetRecOff; /* ReadRecPtr contains page header */ @@ -549,7 +552,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) /* Calculate pointer to beginning of next page */ state->recordContRecPtr = state->ReadRecPtr + record_len; - Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0); + Assert(state->recordContRecPtr % state->pageSize == 0); state->readRecordState = XLREAD_CONTINUATION; } @@ -576,7 +579,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) /* this request contains page header */ Assert (targetPagePtr != 0); if (XLogNeedData(state, targetPagePtr, - Min(state->recordRemainLen, XLOG_BLCKSZ), + Min(state->recordRemainLen, state->pageSize), false)) return XLREAD_NEED_DATA; @@ -628,7 +631,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) Assert(state->readLen >= pageHeaderSize); contdata = (char *) state->readBuf + pageHeaderSize; - record_len = XLOG_BLCKSZ - pageHeaderSize; + record_len = state->pageSize - pageHeaderSize; if (pageHeader->xlp_rem_len < record_len) record_len = pageHeader->xlp_rem_len; @@ -655,7 +658,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) } /* Calculate pointer to beginning of next page, and continue */ - state->recordContRecPtr += XLOG_BLCKSZ; + state->recordContRecPtr += state->pageSize; } /* targetPagePtr is pointing the last-read page here */ @@ -789,10 +792,10 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen, XLogPageHeaderSize((XLogPageHeader) state->readBuf); addLen = pageHeaderSize; - if (reqLen + pageHeaderSize <= XLOG_BLCKSZ) + if (reqLen + pageHeaderSize <= state->pageSize) addLen = pageHeaderSize; else - addLen = XLOG_BLCKSZ - reqLen; + addLen = state->pageSize - reqLen; Assert(addLen >= 0); } @@ -805,7 +808,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen, /* Data is not in our buffer, request the caller for it. */ XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize); targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize); - Assert((pageptr % XLOG_BLCKSZ) == 0); + Assert((pageptr % state->pageSize) == 0); /* * Every time we request to load new data of a page to the caller, even if @@ -829,7 +832,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen, * will not come back here, but will request the actual target page. */ state->readPagePtr = pageptr - targetPageOff; - state->readLen = XLOG_BLCKSZ; + state->readLen = state->pageSize; return true; } @@ -962,7 +965,7 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr, int32 offset; XLogPageHeader hdr = (XLogPageHeader) phdr; - Assert((recptr % XLOG_BLCKSZ) == 0); + Assert((recptr % state->pageSize) == 0); XLByteToSeg(recptr, segno, state->segcxt.ws_segsize); offset = XLogSegmentOffset(recptr, state->segcxt.ws_segsize); @@ -1016,10 +1019,10 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr, "WAL file is from different database system: incorrect segment size in page header"); return false; } - else if (longhdr->xlp_xlog_blcksz != XLOG_BLCKSZ) + else if (longhdr->xlp_xlog_blcksz != state->pageSize) { report_invalid_record(state, - "WAL file is from different database system: incorrect XLOG_BLCKSZ in page header"); + "WAL file is from different database system: incorrect block size in page header"); return false; } } @@ -1140,7 +1143,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr, * XLogNeedData() is prepared to handle that and will read at * least short page-header worth of data */ - targetRecOff = tmpRecPtr % XLOG_BLCKSZ; + targetRecOff = tmpRecPtr % state->pageSize; /* scroll back to page boundary */ targetPagePtr = tmpRecPtr - targetRecOff; @@ -1174,8 +1177,8 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr, * * Note that record headers are MAXALIGN'ed */ - if (MAXALIGN(header->xlp_rem_len) >= (XLOG_BLCKSZ - pageHeaderSize)) - tmpRecPtr = targetPagePtr + XLOG_BLCKSZ; + if (MAXALIGN(header->xlp_rem_len) >= (state->pageSize - pageHeaderSize)) + tmpRecPtr = targetPagePtr + state->pageSize; else { /* diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c index 9cbd8afa49..f2dd495310 100644 --- a/src/backend/replication/logical/logical.c +++ b/src/backend/replication/logical/logical.c @@ -199,11 +199,14 @@ StartupDecodingContext(List *output_plugin_options, ctx->slot = slot; - ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, cleanup_cb); + ctx->reader = + XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, cleanup_cb); + if (!ctx->reader) ereport(ERROR, (errcode(ERRCODE_OUT_OF_MEMORY), errmsg("out of memory"))); + ctx->reader->readBuf = palloc(XLOG_BLCKSZ); ctx->page_read = page_read; ctx->reorder = ReorderBufferAllocate(); diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c index 062093727d..8e95600ef7 100644 --- a/src/backend/replication/walsender.c +++ b/src/backend/replication/walsender.c @@ -579,8 +579,8 @@ StartReplication(StartReplicationCmd *cmd) errmsg("IDENTIFY_SYSTEM has not been run before START_REPLICATION"))); /* create xlogreader for physical replication */ - xlogreader = - XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close); + xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, + NULL, wal_segment_close); if (!xlogreader) ereport(ERROR, diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c index 712c85281c..8a6209b061 100644 --- a/src/bin/pg_rewind/parsexlog.c +++ b/src/bin/pg_rewind/parsexlog.c @@ -61,7 +61,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex, XLogReaderState *xlogreader; char *errormsg; - xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL); + xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL); if (xlogreader == NULL) pg_fatal("out of memory"); @@ -121,7 +121,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex, char *errormsg; XLogRecPtr endptr; - xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL); + xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL); if (xlogreader == NULL) pg_fatal("out of memory"); @@ -181,7 +181,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex, forkptr += SizeOfXLogShortPHD; } - xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL); + xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL); if (xlogreader == NULL) pg_fatal("out of memory"); diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c index f514de6b13..b635537659 100644 --- a/src/bin/pg_waldump/pg_waldump.c +++ b/src/bin/pg_waldump/pg_waldump.c @@ -1041,7 +1041,7 @@ main(int argc, char **argv) /* we have everything we need, start reading */ xlogreader_state = - XLogReaderAllocate(WalSegSz, waldir, WALDumpCloseSegment); + XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, waldir, WALDumpCloseSegment); if (!xlogreader_state) fatal_error("out of memory"); diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h index b9dad7ee51..1853516333 100644 --- a/src/include/access/xlogreader.h +++ b/src/include/access/xlogreader.h @@ -151,6 +151,7 @@ struct XLogReaderState * ---------------------------------------- */ /* variables to communicate with page reader */ + int pageSize; /* size of a page */ XLogRecPtr readPagePtr; /* page pointer to read */ int32 readLen; /* bytes requested to reader, or actual bytes * read by reader, which must be larger than @@ -243,7 +244,7 @@ struct XLogReaderState /* Get a new XLogReader */ extern XLogReaderState *XLogReaderAllocate(int wal_segment_size, - const char *waldir, + int page_size, const char *waldir, WALSegmentCleanupCB cleanup_cb); /* Free an XLogReader */ -- 2.27.0 ----Next_Part(Thu_Mar__4_11_28_53_2021_014)---- ^ permalink raw reply [nested|flat] 26+ messages in thread
* Re: pg_plan_advice @ 2026-01-08 20:58 Robert Haas <[email protected]> 0 siblings, 0 replies; 26+ messages in thread From: Robert Haas @ 2026-01-08 20:58 UTC (permalink / raw) To: Jakub Wartak <[email protected]>; +Cc: Jacob Champion <[email protected]>; Dian Fay <[email protected]>; Matheus Alcantara <[email protected]>; PostgreSQL Hackers <[email protected]> On Thu, Dec 18, 2025 at 8:36 AM Robert Haas <[email protected]> wrote: > What must be happening here is that either pgpa_join.c (maybe with > complicity from pgpa_walker.c) is not populating the > pgpa_plan_walker_context's join_strategies[JSTRAT_NESTED_LOOP_PLAIN] > member correctly, or else pgpa_output.c is not serializing it to text > correctly. I suspect the former is a more likely but I'm not sure > exactly what's happening. I think I see the problem: pgpa_process_unrolled_join() returns a set called "all_relids" but it only returns the union of the inner relid sets, not including the outer relid set. In your example, we want to get: NESTED_LOOP_PLAIN((part partsupp) (supplier part partsupp)) But the join order is: JOIN_ORDER(nation (supplier (part partsupp))) So every table is the outer table of some unrolled join, except for the innermost table, which is partsupp. So all the others get omitted from the output, and we get the output you saw: NESTED_LOOP_PLAIN(partsupp partsupp) Proposed fix attached. -- Robert Haas EDB: http://www.enterprisedb.com commit aa085a812c2db077b86caff366d31624b9a06ecf Author: Robert Haas <[email protected]> Date: Thu Jan 8 15:57:34 2026 -0500 fix pgpa_process_unrolled_join return value diff --git a/contrib/pg_plan_advice/expected/join_order.out b/contrib/pg_plan_advice/expected/join_order.out index eaaa38f000c..f3dd7810484 100644 --- a/contrib/pg_plan_advice/expected/join_order.out +++ b/contrib/pg_plan_advice/expected/join_order.out @@ -159,7 +159,7 @@ SELECT * FROM jo_fact f JOIN_ORDER(f (d1 d2)) /* matched */ Generated Plan Advice: JOIN_ORDER(f (d1 d2)) - MERGE_JOIN_PLAIN(d2) + MERGE_JOIN_PLAIN((d1 d2)) NESTED_LOOP_MATERIALIZE(d2) SEQ_SCAN(f d1 d2) NO_GATHER(f d1 d2) diff --git a/contrib/pg_plan_advice/pgpa_walker.c b/contrib/pg_plan_advice/pgpa_walker.c index 4dffc60114a..fdc003ec509 100644 --- a/contrib/pg_plan_advice/pgpa_walker.c +++ b/contrib/pg_plan_advice/pgpa_walker.c @@ -438,7 +438,10 @@ static Bitmapset * pgpa_process_unrolled_join(pgpa_plan_walker_context *walker, pgpa_unrolled_join *ujoin) { - Bitmapset *all_relids = NULL; + Bitmapset *all_relids = bms_copy(ujoin->outer.scan->relids); + + /* If this fails, we didn't unroll properly. */ + Assert(ujoin->outer.unrolled_join == NULL); for (int k = 0; k < ujoin->ninner; ++k) { Attachments: [text/plain] unrolled_join.diff.txt (1.3K, ../../CA+TgmoZzBkd1BG8qusicUjme0kZuT8konQM_rcr0gMXs-TpK7A@mail.gmail.com/2-unrolled_join.diff.txt) download | inline diff: commit aa085a812c2db077b86caff366d31624b9a06ecf Author: Robert Haas <[email protected]> Date: Thu Jan 8 15:57:34 2026 -0500 fix pgpa_process_unrolled_join return value diff --git a/contrib/pg_plan_advice/expected/join_order.out b/contrib/pg_plan_advice/expected/join_order.out index eaaa38f000c..f3dd7810484 100644 --- a/contrib/pg_plan_advice/expected/join_order.out +++ b/contrib/pg_plan_advice/expected/join_order.out @@ -159,7 +159,7 @@ SELECT * FROM jo_fact f JOIN_ORDER(f (d1 d2)) /* matched */ Generated Plan Advice: JOIN_ORDER(f (d1 d2)) - MERGE_JOIN_PLAIN(d2) + MERGE_JOIN_PLAIN((d1 d2)) NESTED_LOOP_MATERIALIZE(d2) SEQ_SCAN(f d1 d2) NO_GATHER(f d1 d2) diff --git a/contrib/pg_plan_advice/pgpa_walker.c b/contrib/pg_plan_advice/pgpa_walker.c index 4dffc60114a..fdc003ec509 100644 --- a/contrib/pg_plan_advice/pgpa_walker.c +++ b/contrib/pg_plan_advice/pgpa_walker.c @@ -438,7 +438,10 @@ static Bitmapset * pgpa_process_unrolled_join(pgpa_plan_walker_context *walker, pgpa_unrolled_join *ujoin) { - Bitmapset *all_relids = NULL; + Bitmapset *all_relids = bms_copy(ujoin->outer.scan->relids); + + /* If this fails, we didn't unroll properly. */ + Assert(ujoin->outer.unrolled_join == NULL); for (int k = 0; k < ujoin->ninner; ++k) { ^ permalink raw reply [nested|flat] 26+ messages in thread
end of thread, other threads:[~2026-01-08 20:58 UTC | newest] Thread overview: 26+ messages (download: mbox mbox.gz follow: Atom feed) -- links below jump to the message on this page -- 2019-09-05 12:29 [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize. Kyotaro Horiguchi <[email protected]> 2019-09-05 12:29 [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize. Kyotaro Horiguchi <[email protected]> 2019-09-05 12:29 [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize. Kyotaro Horiguchi <[email protected]> 2019-09-05 12:29 [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize. Kyotaro Horiguchi <[email protected]> 2019-09-05 12:29 [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize. Kyotaro Horiguchi <[email protected]> 2019-09-05 12:29 [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize. Kyotaro Horiguchi <[email protected]> 2019-09-05 12:29 [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize. Kyotaro Horiguchi <[email protected]> 2019-09-05 12:29 [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize. Kyotaro Horiguchi <[email protected]> 2019-09-05 12:29 [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize. Kyotaro Horiguchi <[email protected]> 2019-09-05 12:29 [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize. Kyotaro Horiguchi <[email protected]> 2019-09-05 12:29 [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize. Kyotaro Horiguchi <[email protected]> 2019-09-05 12:29 [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize. Kyotaro Horiguchi <[email protected]> 2019-09-05 12:29 [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize. Kyotaro Horiguchi <[email protected]> 2019-09-05 12:29 [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize. Kyotaro Horiguchi <[email protected]> 2019-09-05 12:29 [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize. Kyotaro Horiguchi <[email protected]> 2019-09-05 12:29 [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize. Kyotaro Horiguchi <[email protected]> 2019-09-05 12:29 [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize. Kyotaro Horiguchi <[email protected]> 2019-09-05 12:29 [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize. Kyotaro Horiguchi <[email protected]> 2019-09-05 12:29 [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize. Kyotaro Horiguchi <[email protected]> 2019-09-05 12:29 [PATCH v16 4/4] Allow xlogreader to use different xlog blocksize. Kyotaro Horiguchi <[email protected]> 2019-09-05 12:29 [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize. Kyotaro Horiguchi <[email protected]> 2019-09-05 12:29 [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize. Kyotaro Horiguchi <[email protected]> 2019-09-05 12:29 [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize. Kyotaro Horiguchi <[email protected]> 2019-09-05 12:29 [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize. Kyotaro Horiguchi <[email protected]> 2019-09-05 12:29 [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize. Kyotaro Horiguchi <[email protected]> 2026-01-08 20:58 Re: pg_plan_advice Robert Haas <[email protected]>
This inbox is served by agora; see mirroring instructions for how to clone and mirror all data and code used for this inbox