public inbox for [email protected]help / color / mirror / Atom feed
[PATCH v17 4/4] Allow xlogreader to use different xlog blocksize. 25+ messages / 1 participants [nested] [flat]
* [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize. @ 2019-09-05 12:29 Kyotaro Horiguchi <[email protected]> 0 siblings, 0 replies; 25+ messages in thread From: Kyotaro Horiguchi @ 2019-09-05 12:29 UTC (permalink / raw) XLog reader facility and its callers currently relies on the same constant XLOG_BLCKSZ. This patch moves the responsibility to determine the block size to the callers of the facility. --- src/backend/access/transam/twophase.c | 3 +- src/backend/access/transam/xlog.c | 9 ++-- src/backend/access/transam/xlogreader.c | 57 ++++++++++++----------- src/backend/replication/logical/logical.c | 5 +- src/backend/replication/walsender.c | 4 +- src/bin/pg_rewind/parsexlog.c | 6 +-- src/bin/pg_waldump/pg_waldump.c | 2 +- src/include/access/xlogreader.h | 3 +- 8 files changed, 50 insertions(+), 39 deletions(-) diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c index 7d5a96a5f5..9f9e7de253 100644 --- a/src/backend/access/transam/twophase.c +++ b/src/backend/access/transam/twophase.c @@ -1325,7 +1325,8 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len) XLogReaderState *xlogreader; char *errormsg; - xlogreader = XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close); + xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, + NULL, wal_segment_close); if (!xlogreader) ereport(ERROR, diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index b517ca85ad..0822ac70cb 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -1222,7 +1222,10 @@ XLogInsertRecord(XLogRecData *rdata, appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len); if (!debug_reader) - debug_reader = XLogReaderAllocate(wal_segment_size, NULL, NULL); + { + debug_reader = + XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, NULL); + } if (!debug_reader) { @@ -6531,8 +6534,8 @@ StartupXLOG(void) OwnLatch(&XLogCtl->recoveryWakeupLatch); /* Set up XLOG reader facility */ - xlogreader = - XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close); + xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, + NULL, wal_segment_close); if (!xlogreader) ereport(ERROR, diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c index ba3e4868f7..e5baf13086 100644 --- a/src/backend/access/transam/xlogreader.c +++ b/src/backend/access/transam/xlogreader.c @@ -72,7 +72,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...) * Returns NULL if the xlogreader couldn't be allocated. */ XLogReaderState * -XLogReaderAllocate(int wal_segment_size, const char *waldir, +XLogReaderAllocate(int wal_segment_size, int page_size, const char *waldir, WALSegmentCleanupCB cleanup_cb) { XLogReaderState *state; @@ -87,6 +87,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir, state->cleanup_cb = cleanup_cb; state->max_block_id = -1; + state->pageSize = page_size; /* * Permanently allocate readBuf. We do it this way, rather than just @@ -95,7 +96,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir, * isn't guaranteed to have any particular alignment, whereas * palloc_extended() will provide MAXALIGN'd storage. */ - state->readBuf = (char *) palloc_extended(XLOG_BLCKSZ, + state->readBuf = (char *) palloc_extended(state->pageSize, MCXT_ALLOC_NO_OOM); if (!state->readBuf) { @@ -163,7 +164,7 @@ XLogReaderFree(XLogReaderState *state) * readRecordBufSize is set to the new buffer size. * * To avoid useless small increases, round its size to a multiple of - * XLOG_BLCKSZ, and make sure it's at least 5*Max(BLCKSZ, XLOG_BLCKSZ) to start + * page size, and make sure it's at least 5*Max(BLCKSZ, <page size>) to start * with. (That is enough for all "normal" records, but very large commit or * abort records might need more space.) */ @@ -172,8 +173,8 @@ allocate_recordbuf(XLogReaderState *state, uint32 reclength) { uint32 newSize = reclength; - newSize += XLOG_BLCKSZ - (newSize % XLOG_BLCKSZ); - newSize = Max(newSize, 5 * Max(BLCKSZ, XLOG_BLCKSZ)); + newSize += state->pageSize - (newSize % state->pageSize); + newSize = Max(newSize, 5 * Max(BLCKSZ, state->pageSize)); #ifndef FRONTEND @@ -238,6 +239,8 @@ void XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr) { Assert(!XLogRecPtrIsInvalid(RecPtr)); + Assert(state->readBuf != NULL && + state->readBuf == (void *) MAXALIGN(state->readBuf)); ResetDecoder(state); @@ -366,15 +369,15 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) XLogPageHeader pageHeader; targetPagePtr = - state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ); - targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ; + state->ReadRecPtr - (state->ReadRecPtr % state->pageSize); + targetRecOff = state->ReadRecPtr % state->pageSize; /* * Check if we have enough data. For the first record in the page, * the requesting length doesn't contain page header. */ if (XLogNeedData(state, targetPagePtr, - Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ), + Min(targetRecOff + SizeOfXLogRecord, state->pageSize), targetRecOff != 0)) return XLREAD_NEED_DATA; @@ -421,7 +424,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) * verified that we got the whole header. */ prec = (XLogRecord *) (state->readBuf + - state->ReadRecPtr % XLOG_BLCKSZ); + state->ReadRecPtr % state->pageSize); total_len = prec->xl_tot_len; /* @@ -432,7 +435,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) * ensure that we enter the XLREAD_CONTINUATION state below; * otherwise we might fail to apply ValidXLogRecordHeader at all. */ - if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord) + if (targetRecOff <= state->pageSize - SizeOfXLogRecord) { if (!ValidXLogRecordHeader(state, state->ReadRecPtr, state->PrevRecPtr, prec)) @@ -477,10 +480,10 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) * available */ targetPagePtr = - state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ); - targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ; + state->ReadRecPtr - (state->ReadRecPtr % state->pageSize); + targetRecOff = state->ReadRecPtr % state->pageSize; - request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ); + request_len = Min(targetRecOff + total_len, state->pageSize); record_len = request_len - targetRecOff; /* ReadRecPtr contains page header */ @@ -549,7 +552,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) /* Calculate pointer to beginning of next page */ state->recordContRecPtr = state->ReadRecPtr + record_len; - Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0); + Assert(state->recordContRecPtr % state->pageSize == 0); state->readRecordState = XLREAD_CONTINUATION; } @@ -576,7 +579,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) /* this request contains page header */ Assert (targetPagePtr != 0); if (XLogNeedData(state, targetPagePtr, - Min(state->recordRemainLen, XLOG_BLCKSZ), + Min(state->recordRemainLen, state->pageSize), false)) return XLREAD_NEED_DATA; @@ -628,7 +631,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) Assert(state->readLen >= pageHeaderSize); contdata = (char *) state->readBuf + pageHeaderSize; - record_len = XLOG_BLCKSZ - pageHeaderSize; + record_len = state->pageSize - pageHeaderSize; if (pageHeader->xlp_rem_len < record_len) record_len = pageHeader->xlp_rem_len; @@ -655,7 +658,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) } /* Calculate pointer to beginning of next page, and continue */ - state->recordContRecPtr += XLOG_BLCKSZ; + state->recordContRecPtr += state->pageSize; } /* targetPagePtr is pointing the last-read page here */ @@ -789,10 +792,10 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen, XLogPageHeaderSize((XLogPageHeader) state->readBuf); addLen = pageHeaderSize; - if (reqLen + pageHeaderSize <= XLOG_BLCKSZ) + if (reqLen + pageHeaderSize <= state->pageSize) addLen = pageHeaderSize; else - addLen = XLOG_BLCKSZ - reqLen; + addLen = state->pageSize - reqLen; Assert(addLen >= 0); } @@ -805,7 +808,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen, /* Data is not in our buffer, request the caller for it. */ XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize); targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize); - Assert((pageptr % XLOG_BLCKSZ) == 0); + Assert((pageptr % state->pageSize) == 0); /* * Every time we request to load new data of a page to the caller, even if @@ -829,7 +832,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen, * will not come back here, but will request the actual target page. */ state->readPagePtr = pageptr - targetPageOff; - state->readLen = XLOG_BLCKSZ; + state->readLen = state->pageSize; return true; } @@ -962,7 +965,7 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr, int32 offset; XLogPageHeader hdr = (XLogPageHeader) phdr; - Assert((recptr % XLOG_BLCKSZ) == 0); + Assert((recptr % state->pageSize) == 0); XLByteToSeg(recptr, segno, state->segcxt.ws_segsize); offset = XLogSegmentOffset(recptr, state->segcxt.ws_segsize); @@ -1016,10 +1019,10 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr, "WAL file is from different database system: incorrect segment size in page header"); return false; } - else if (longhdr->xlp_xlog_blcksz != XLOG_BLCKSZ) + else if (longhdr->xlp_xlog_blcksz != state->pageSize) { report_invalid_record(state, - "WAL file is from different database system: incorrect XLOG_BLCKSZ in page header"); + "WAL file is from different database system: incorrect block size in page header"); return false; } } @@ -1140,7 +1143,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr, * XLogNeedData() is prepared to handle that and will read at * least short page-header worth of data */ - targetRecOff = tmpRecPtr % XLOG_BLCKSZ; + targetRecOff = tmpRecPtr % state->pageSize; /* scroll back to page boundary */ targetPagePtr = tmpRecPtr - targetRecOff; @@ -1174,8 +1177,8 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr, * * Note that record headers are MAXALIGN'ed */ - if (MAXALIGN(header->xlp_rem_len) >= (XLOG_BLCKSZ - pageHeaderSize)) - tmpRecPtr = targetPagePtr + XLOG_BLCKSZ; + if (MAXALIGN(header->xlp_rem_len) >= (state->pageSize - pageHeaderSize)) + tmpRecPtr = targetPagePtr + state->pageSize; else { /* diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c index 9cbd8afa49..f2dd495310 100644 --- a/src/backend/replication/logical/logical.c +++ b/src/backend/replication/logical/logical.c @@ -199,11 +199,14 @@ StartupDecodingContext(List *output_plugin_options, ctx->slot = slot; - ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, cleanup_cb); + ctx->reader = + XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, cleanup_cb); + if (!ctx->reader) ereport(ERROR, (errcode(ERRCODE_OUT_OF_MEMORY), errmsg("out of memory"))); + ctx->reader->readBuf = palloc(XLOG_BLCKSZ); ctx->page_read = page_read; ctx->reorder = ReorderBufferAllocate(); diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c index 062093727d..8e95600ef7 100644 --- a/src/backend/replication/walsender.c +++ b/src/backend/replication/walsender.c @@ -579,8 +579,8 @@ StartReplication(StartReplicationCmd *cmd) errmsg("IDENTIFY_SYSTEM has not been run before START_REPLICATION"))); /* create xlogreader for physical replication */ - xlogreader = - XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close); + xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, + NULL, wal_segment_close); if (!xlogreader) ereport(ERROR, diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c index 712c85281c..8a6209b061 100644 --- a/src/bin/pg_rewind/parsexlog.c +++ b/src/bin/pg_rewind/parsexlog.c @@ -61,7 +61,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex, XLogReaderState *xlogreader; char *errormsg; - xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL); + xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL); if (xlogreader == NULL) pg_fatal("out of memory"); @@ -121,7 +121,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex, char *errormsg; XLogRecPtr endptr; - xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL); + xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL); if (xlogreader == NULL) pg_fatal("out of memory"); @@ -181,7 +181,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex, forkptr += SizeOfXLogShortPHD; } - xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL); + xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL); if (xlogreader == NULL) pg_fatal("out of memory"); diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c index f514de6b13..b635537659 100644 --- a/src/bin/pg_waldump/pg_waldump.c +++ b/src/bin/pg_waldump/pg_waldump.c @@ -1041,7 +1041,7 @@ main(int argc, char **argv) /* we have everything we need, start reading */ xlogreader_state = - XLogReaderAllocate(WalSegSz, waldir, WALDumpCloseSegment); + XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, waldir, WALDumpCloseSegment); if (!xlogreader_state) fatal_error("out of memory"); diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h index b9dad7ee51..1853516333 100644 --- a/src/include/access/xlogreader.h +++ b/src/include/access/xlogreader.h @@ -151,6 +151,7 @@ struct XLogReaderState * ---------------------------------------- */ /* variables to communicate with page reader */ + int pageSize; /* size of a page */ XLogRecPtr readPagePtr; /* page pointer to read */ int32 readLen; /* bytes requested to reader, or actual bytes * read by reader, which must be larger than @@ -243,7 +244,7 @@ struct XLogReaderState /* Get a new XLogReader */ extern XLogReaderState *XLogReaderAllocate(int wal_segment_size, - const char *waldir, + int page_size, const char *waldir, WALSegmentCleanupCB cleanup_cb); /* Free an XLogReader */ -- 2.27.0 ----Next_Part(Thu_Mar__4_11_28_53_2021_014)---- ^ permalink raw reply [nested|flat] 25+ messages in thread
* [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize. @ 2019-09-05 12:29 Kyotaro Horiguchi <[email protected]> 0 siblings, 0 replies; 25+ messages in thread From: Kyotaro Horiguchi @ 2019-09-05 12:29 UTC (permalink / raw) XLog reader facility and its callers currently relies on the same constant XLOG_BLCKSZ. This patch moves the responsibility to determine the block size to the callers of the facility. --- src/backend/access/transam/twophase.c | 3 +- src/backend/access/transam/xlog.c | 9 ++-- src/backend/access/transam/xlogreader.c | 57 ++++++++++++----------- src/backend/replication/logical/logical.c | 5 +- src/backend/replication/walsender.c | 4 +- src/bin/pg_rewind/parsexlog.c | 6 +-- src/bin/pg_waldump/pg_waldump.c | 2 +- src/include/access/xlogreader.h | 3 +- 8 files changed, 50 insertions(+), 39 deletions(-) diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c index 7d5a96a5f5..9f9e7de253 100644 --- a/src/backend/access/transam/twophase.c +++ b/src/backend/access/transam/twophase.c @@ -1325,7 +1325,8 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len) XLogReaderState *xlogreader; char *errormsg; - xlogreader = XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close); + xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, + NULL, wal_segment_close); if (!xlogreader) ereport(ERROR, diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index b517ca85ad..0822ac70cb 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -1222,7 +1222,10 @@ XLogInsertRecord(XLogRecData *rdata, appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len); if (!debug_reader) - debug_reader = XLogReaderAllocate(wal_segment_size, NULL, NULL); + { + debug_reader = + XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, NULL); + } if (!debug_reader) { @@ -6531,8 +6534,8 @@ StartupXLOG(void) OwnLatch(&XLogCtl->recoveryWakeupLatch); /* Set up XLOG reader facility */ - xlogreader = - XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close); + xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, + NULL, wal_segment_close); if (!xlogreader) ereport(ERROR, diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c index ba3e4868f7..e5baf13086 100644 --- a/src/backend/access/transam/xlogreader.c +++ b/src/backend/access/transam/xlogreader.c @@ -72,7 +72,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...) * Returns NULL if the xlogreader couldn't be allocated. */ XLogReaderState * -XLogReaderAllocate(int wal_segment_size, const char *waldir, +XLogReaderAllocate(int wal_segment_size, int page_size, const char *waldir, WALSegmentCleanupCB cleanup_cb) { XLogReaderState *state; @@ -87,6 +87,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir, state->cleanup_cb = cleanup_cb; state->max_block_id = -1; + state->pageSize = page_size; /* * Permanently allocate readBuf. We do it this way, rather than just @@ -95,7 +96,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir, * isn't guaranteed to have any particular alignment, whereas * palloc_extended() will provide MAXALIGN'd storage. */ - state->readBuf = (char *) palloc_extended(XLOG_BLCKSZ, + state->readBuf = (char *) palloc_extended(state->pageSize, MCXT_ALLOC_NO_OOM); if (!state->readBuf) { @@ -163,7 +164,7 @@ XLogReaderFree(XLogReaderState *state) * readRecordBufSize is set to the new buffer size. * * To avoid useless small increases, round its size to a multiple of - * XLOG_BLCKSZ, and make sure it's at least 5*Max(BLCKSZ, XLOG_BLCKSZ) to start + * page size, and make sure it's at least 5*Max(BLCKSZ, <page size>) to start * with. (That is enough for all "normal" records, but very large commit or * abort records might need more space.) */ @@ -172,8 +173,8 @@ allocate_recordbuf(XLogReaderState *state, uint32 reclength) { uint32 newSize = reclength; - newSize += XLOG_BLCKSZ - (newSize % XLOG_BLCKSZ); - newSize = Max(newSize, 5 * Max(BLCKSZ, XLOG_BLCKSZ)); + newSize += state->pageSize - (newSize % state->pageSize); + newSize = Max(newSize, 5 * Max(BLCKSZ, state->pageSize)); #ifndef FRONTEND @@ -238,6 +239,8 @@ void XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr) { Assert(!XLogRecPtrIsInvalid(RecPtr)); + Assert(state->readBuf != NULL && + state->readBuf == (void *) MAXALIGN(state->readBuf)); ResetDecoder(state); @@ -366,15 +369,15 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) XLogPageHeader pageHeader; targetPagePtr = - state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ); - targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ; + state->ReadRecPtr - (state->ReadRecPtr % state->pageSize); + targetRecOff = state->ReadRecPtr % state->pageSize; /* * Check if we have enough data. For the first record in the page, * the requesting length doesn't contain page header. */ if (XLogNeedData(state, targetPagePtr, - Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ), + Min(targetRecOff + SizeOfXLogRecord, state->pageSize), targetRecOff != 0)) return XLREAD_NEED_DATA; @@ -421,7 +424,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) * verified that we got the whole header. */ prec = (XLogRecord *) (state->readBuf + - state->ReadRecPtr % XLOG_BLCKSZ); + state->ReadRecPtr % state->pageSize); total_len = prec->xl_tot_len; /* @@ -432,7 +435,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) * ensure that we enter the XLREAD_CONTINUATION state below; * otherwise we might fail to apply ValidXLogRecordHeader at all. */ - if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord) + if (targetRecOff <= state->pageSize - SizeOfXLogRecord) { if (!ValidXLogRecordHeader(state, state->ReadRecPtr, state->PrevRecPtr, prec)) @@ -477,10 +480,10 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) * available */ targetPagePtr = - state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ); - targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ; + state->ReadRecPtr - (state->ReadRecPtr % state->pageSize); + targetRecOff = state->ReadRecPtr % state->pageSize; - request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ); + request_len = Min(targetRecOff + total_len, state->pageSize); record_len = request_len - targetRecOff; /* ReadRecPtr contains page header */ @@ -549,7 +552,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) /* Calculate pointer to beginning of next page */ state->recordContRecPtr = state->ReadRecPtr + record_len; - Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0); + Assert(state->recordContRecPtr % state->pageSize == 0); state->readRecordState = XLREAD_CONTINUATION; } @@ -576,7 +579,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) /* this request contains page header */ Assert (targetPagePtr != 0); if (XLogNeedData(state, targetPagePtr, - Min(state->recordRemainLen, XLOG_BLCKSZ), + Min(state->recordRemainLen, state->pageSize), false)) return XLREAD_NEED_DATA; @@ -628,7 +631,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) Assert(state->readLen >= pageHeaderSize); contdata = (char *) state->readBuf + pageHeaderSize; - record_len = XLOG_BLCKSZ - pageHeaderSize; + record_len = state->pageSize - pageHeaderSize; if (pageHeader->xlp_rem_len < record_len) record_len = pageHeader->xlp_rem_len; @@ -655,7 +658,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) } /* Calculate pointer to beginning of next page, and continue */ - state->recordContRecPtr += XLOG_BLCKSZ; + state->recordContRecPtr += state->pageSize; } /* targetPagePtr is pointing the last-read page here */ @@ -789,10 +792,10 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen, XLogPageHeaderSize((XLogPageHeader) state->readBuf); addLen = pageHeaderSize; - if (reqLen + pageHeaderSize <= XLOG_BLCKSZ) + if (reqLen + pageHeaderSize <= state->pageSize) addLen = pageHeaderSize; else - addLen = XLOG_BLCKSZ - reqLen; + addLen = state->pageSize - reqLen; Assert(addLen >= 0); } @@ -805,7 +808,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen, /* Data is not in our buffer, request the caller for it. */ XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize); targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize); - Assert((pageptr % XLOG_BLCKSZ) == 0); + Assert((pageptr % state->pageSize) == 0); /* * Every time we request to load new data of a page to the caller, even if @@ -829,7 +832,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen, * will not come back here, but will request the actual target page. */ state->readPagePtr = pageptr - targetPageOff; - state->readLen = XLOG_BLCKSZ; + state->readLen = state->pageSize; return true; } @@ -962,7 +965,7 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr, int32 offset; XLogPageHeader hdr = (XLogPageHeader) phdr; - Assert((recptr % XLOG_BLCKSZ) == 0); + Assert((recptr % state->pageSize) == 0); XLByteToSeg(recptr, segno, state->segcxt.ws_segsize); offset = XLogSegmentOffset(recptr, state->segcxt.ws_segsize); @@ -1016,10 +1019,10 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr, "WAL file is from different database system: incorrect segment size in page header"); return false; } - else if (longhdr->xlp_xlog_blcksz != XLOG_BLCKSZ) + else if (longhdr->xlp_xlog_blcksz != state->pageSize) { report_invalid_record(state, - "WAL file is from different database system: incorrect XLOG_BLCKSZ in page header"); + "WAL file is from different database system: incorrect block size in page header"); return false; } } @@ -1140,7 +1143,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr, * XLogNeedData() is prepared to handle that and will read at * least short page-header worth of data */ - targetRecOff = tmpRecPtr % XLOG_BLCKSZ; + targetRecOff = tmpRecPtr % state->pageSize; /* scroll back to page boundary */ targetPagePtr = tmpRecPtr - targetRecOff; @@ -1174,8 +1177,8 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr, * * Note that record headers are MAXALIGN'ed */ - if (MAXALIGN(header->xlp_rem_len) >= (XLOG_BLCKSZ - pageHeaderSize)) - tmpRecPtr = targetPagePtr + XLOG_BLCKSZ; + if (MAXALIGN(header->xlp_rem_len) >= (state->pageSize - pageHeaderSize)) + tmpRecPtr = targetPagePtr + state->pageSize; else { /* diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c index 9cbd8afa49..f2dd495310 100644 --- a/src/backend/replication/logical/logical.c +++ b/src/backend/replication/logical/logical.c @@ -199,11 +199,14 @@ StartupDecodingContext(List *output_plugin_options, ctx->slot = slot; - ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, cleanup_cb); + ctx->reader = + XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, cleanup_cb); + if (!ctx->reader) ereport(ERROR, (errcode(ERRCODE_OUT_OF_MEMORY), errmsg("out of memory"))); + ctx->reader->readBuf = palloc(XLOG_BLCKSZ); ctx->page_read = page_read; ctx->reorder = ReorderBufferAllocate(); diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c index 062093727d..8e95600ef7 100644 --- a/src/backend/replication/walsender.c +++ b/src/backend/replication/walsender.c @@ -579,8 +579,8 @@ StartReplication(StartReplicationCmd *cmd) errmsg("IDENTIFY_SYSTEM has not been run before START_REPLICATION"))); /* create xlogreader for physical replication */ - xlogreader = - XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close); + xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, + NULL, wal_segment_close); if (!xlogreader) ereport(ERROR, diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c index 712c85281c..8a6209b061 100644 --- a/src/bin/pg_rewind/parsexlog.c +++ b/src/bin/pg_rewind/parsexlog.c @@ -61,7 +61,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex, XLogReaderState *xlogreader; char *errormsg; - xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL); + xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL); if (xlogreader == NULL) pg_fatal("out of memory"); @@ -121,7 +121,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex, char *errormsg; XLogRecPtr endptr; - xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL); + xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL); if (xlogreader == NULL) pg_fatal("out of memory"); @@ -181,7 +181,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex, forkptr += SizeOfXLogShortPHD; } - xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL); + xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL); if (xlogreader == NULL) pg_fatal("out of memory"); diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c index f514de6b13..b635537659 100644 --- a/src/bin/pg_waldump/pg_waldump.c +++ b/src/bin/pg_waldump/pg_waldump.c @@ -1041,7 +1041,7 @@ main(int argc, char **argv) /* we have everything we need, start reading */ xlogreader_state = - XLogReaderAllocate(WalSegSz, waldir, WALDumpCloseSegment); + XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, waldir, WALDumpCloseSegment); if (!xlogreader_state) fatal_error("out of memory"); diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h index b9dad7ee51..1853516333 100644 --- a/src/include/access/xlogreader.h +++ b/src/include/access/xlogreader.h @@ -151,6 +151,7 @@ struct XLogReaderState * ---------------------------------------- */ /* variables to communicate with page reader */ + int pageSize; /* size of a page */ XLogRecPtr readPagePtr; /* page pointer to read */ int32 readLen; /* bytes requested to reader, or actual bytes * read by reader, which must be larger than @@ -243,7 +244,7 @@ struct XLogReaderState /* Get a new XLogReader */ extern XLogReaderState *XLogReaderAllocate(int wal_segment_size, - const char *waldir, + int page_size, const char *waldir, WALSegmentCleanupCB cleanup_cb); /* Free an XLogReader */ -- 2.27.0 ----Next_Part(Thu_Mar__4_11_28_53_2021_014)---- ^ permalink raw reply [nested|flat] 25+ messages in thread
* [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize. @ 2019-09-05 12:29 Kyotaro Horiguchi <[email protected]> 0 siblings, 0 replies; 25+ messages in thread From: Kyotaro Horiguchi @ 2019-09-05 12:29 UTC (permalink / raw) XLog reader facility and its callers currently relies on the same constant XLOG_BLCKSZ. This patch moves the responsibility to determine the block size to the callers of the facility. --- src/backend/access/transam/twophase.c | 3 +- src/backend/access/transam/xlog.c | 9 ++-- src/backend/access/transam/xlogreader.c | 57 ++++++++++++----------- src/backend/replication/logical/logical.c | 5 +- src/backend/replication/walsender.c | 4 +- src/bin/pg_rewind/parsexlog.c | 6 +-- src/bin/pg_waldump/pg_waldump.c | 2 +- src/include/access/xlogreader.h | 3 +- 8 files changed, 50 insertions(+), 39 deletions(-) diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c index 7d5a96a5f5..9f9e7de253 100644 --- a/src/backend/access/transam/twophase.c +++ b/src/backend/access/transam/twophase.c @@ -1325,7 +1325,8 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len) XLogReaderState *xlogreader; char *errormsg; - xlogreader = XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close); + xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, + NULL, wal_segment_close); if (!xlogreader) ereport(ERROR, diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index b517ca85ad..0822ac70cb 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -1222,7 +1222,10 @@ XLogInsertRecord(XLogRecData *rdata, appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len); if (!debug_reader) - debug_reader = XLogReaderAllocate(wal_segment_size, NULL, NULL); + { + debug_reader = + XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, NULL); + } if (!debug_reader) { @@ -6531,8 +6534,8 @@ StartupXLOG(void) OwnLatch(&XLogCtl->recoveryWakeupLatch); /* Set up XLOG reader facility */ - xlogreader = - XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close); + xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, + NULL, wal_segment_close); if (!xlogreader) ereport(ERROR, diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c index ba3e4868f7..e5baf13086 100644 --- a/src/backend/access/transam/xlogreader.c +++ b/src/backend/access/transam/xlogreader.c @@ -72,7 +72,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...) * Returns NULL if the xlogreader couldn't be allocated. */ XLogReaderState * -XLogReaderAllocate(int wal_segment_size, const char *waldir, +XLogReaderAllocate(int wal_segment_size, int page_size, const char *waldir, WALSegmentCleanupCB cleanup_cb) { XLogReaderState *state; @@ -87,6 +87,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir, state->cleanup_cb = cleanup_cb; state->max_block_id = -1; + state->pageSize = page_size; /* * Permanently allocate readBuf. We do it this way, rather than just @@ -95,7 +96,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir, * isn't guaranteed to have any particular alignment, whereas * palloc_extended() will provide MAXALIGN'd storage. */ - state->readBuf = (char *) palloc_extended(XLOG_BLCKSZ, + state->readBuf = (char *) palloc_extended(state->pageSize, MCXT_ALLOC_NO_OOM); if (!state->readBuf) { @@ -163,7 +164,7 @@ XLogReaderFree(XLogReaderState *state) * readRecordBufSize is set to the new buffer size. * * To avoid useless small increases, round its size to a multiple of - * XLOG_BLCKSZ, and make sure it's at least 5*Max(BLCKSZ, XLOG_BLCKSZ) to start + * page size, and make sure it's at least 5*Max(BLCKSZ, <page size>) to start * with. (That is enough for all "normal" records, but very large commit or * abort records might need more space.) */ @@ -172,8 +173,8 @@ allocate_recordbuf(XLogReaderState *state, uint32 reclength) { uint32 newSize = reclength; - newSize += XLOG_BLCKSZ - (newSize % XLOG_BLCKSZ); - newSize = Max(newSize, 5 * Max(BLCKSZ, XLOG_BLCKSZ)); + newSize += state->pageSize - (newSize % state->pageSize); + newSize = Max(newSize, 5 * Max(BLCKSZ, state->pageSize)); #ifndef FRONTEND @@ -238,6 +239,8 @@ void XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr) { Assert(!XLogRecPtrIsInvalid(RecPtr)); + Assert(state->readBuf != NULL && + state->readBuf == (void *) MAXALIGN(state->readBuf)); ResetDecoder(state); @@ -366,15 +369,15 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) XLogPageHeader pageHeader; targetPagePtr = - state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ); - targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ; + state->ReadRecPtr - (state->ReadRecPtr % state->pageSize); + targetRecOff = state->ReadRecPtr % state->pageSize; /* * Check if we have enough data. For the first record in the page, * the requesting length doesn't contain page header. */ if (XLogNeedData(state, targetPagePtr, - Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ), + Min(targetRecOff + SizeOfXLogRecord, state->pageSize), targetRecOff != 0)) return XLREAD_NEED_DATA; @@ -421,7 +424,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) * verified that we got the whole header. */ prec = (XLogRecord *) (state->readBuf + - state->ReadRecPtr % XLOG_BLCKSZ); + state->ReadRecPtr % state->pageSize); total_len = prec->xl_tot_len; /* @@ -432,7 +435,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) * ensure that we enter the XLREAD_CONTINUATION state below; * otherwise we might fail to apply ValidXLogRecordHeader at all. */ - if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord) + if (targetRecOff <= state->pageSize - SizeOfXLogRecord) { if (!ValidXLogRecordHeader(state, state->ReadRecPtr, state->PrevRecPtr, prec)) @@ -477,10 +480,10 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) * available */ targetPagePtr = - state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ); - targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ; + state->ReadRecPtr - (state->ReadRecPtr % state->pageSize); + targetRecOff = state->ReadRecPtr % state->pageSize; - request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ); + request_len = Min(targetRecOff + total_len, state->pageSize); record_len = request_len - targetRecOff; /* ReadRecPtr contains page header */ @@ -549,7 +552,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) /* Calculate pointer to beginning of next page */ state->recordContRecPtr = state->ReadRecPtr + record_len; - Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0); + Assert(state->recordContRecPtr % state->pageSize == 0); state->readRecordState = XLREAD_CONTINUATION; } @@ -576,7 +579,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) /* this request contains page header */ Assert (targetPagePtr != 0); if (XLogNeedData(state, targetPagePtr, - Min(state->recordRemainLen, XLOG_BLCKSZ), + Min(state->recordRemainLen, state->pageSize), false)) return XLREAD_NEED_DATA; @@ -628,7 +631,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) Assert(state->readLen >= pageHeaderSize); contdata = (char *) state->readBuf + pageHeaderSize; - record_len = XLOG_BLCKSZ - pageHeaderSize; + record_len = state->pageSize - pageHeaderSize; if (pageHeader->xlp_rem_len < record_len) record_len = pageHeader->xlp_rem_len; @@ -655,7 +658,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) } /* Calculate pointer to beginning of next page, and continue */ - state->recordContRecPtr += XLOG_BLCKSZ; + state->recordContRecPtr += state->pageSize; } /* targetPagePtr is pointing the last-read page here */ @@ -789,10 +792,10 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen, XLogPageHeaderSize((XLogPageHeader) state->readBuf); addLen = pageHeaderSize; - if (reqLen + pageHeaderSize <= XLOG_BLCKSZ) + if (reqLen + pageHeaderSize <= state->pageSize) addLen = pageHeaderSize; else - addLen = XLOG_BLCKSZ - reqLen; + addLen = state->pageSize - reqLen; Assert(addLen >= 0); } @@ -805,7 +808,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen, /* Data is not in our buffer, request the caller for it. */ XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize); targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize); - Assert((pageptr % XLOG_BLCKSZ) == 0); + Assert((pageptr % state->pageSize) == 0); /* * Every time we request to load new data of a page to the caller, even if @@ -829,7 +832,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen, * will not come back here, but will request the actual target page. */ state->readPagePtr = pageptr - targetPageOff; - state->readLen = XLOG_BLCKSZ; + state->readLen = state->pageSize; return true; } @@ -962,7 +965,7 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr, int32 offset; XLogPageHeader hdr = (XLogPageHeader) phdr; - Assert((recptr % XLOG_BLCKSZ) == 0); + Assert((recptr % state->pageSize) == 0); XLByteToSeg(recptr, segno, state->segcxt.ws_segsize); offset = XLogSegmentOffset(recptr, state->segcxt.ws_segsize); @@ -1016,10 +1019,10 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr, "WAL file is from different database system: incorrect segment size in page header"); return false; } - else if (longhdr->xlp_xlog_blcksz != XLOG_BLCKSZ) + else if (longhdr->xlp_xlog_blcksz != state->pageSize) { report_invalid_record(state, - "WAL file is from different database system: incorrect XLOG_BLCKSZ in page header"); + "WAL file is from different database system: incorrect block size in page header"); return false; } } @@ -1140,7 +1143,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr, * XLogNeedData() is prepared to handle that and will read at * least short page-header worth of data */ - targetRecOff = tmpRecPtr % XLOG_BLCKSZ; + targetRecOff = tmpRecPtr % state->pageSize; /* scroll back to page boundary */ targetPagePtr = tmpRecPtr - targetRecOff; @@ -1174,8 +1177,8 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr, * * Note that record headers are MAXALIGN'ed */ - if (MAXALIGN(header->xlp_rem_len) >= (XLOG_BLCKSZ - pageHeaderSize)) - tmpRecPtr = targetPagePtr + XLOG_BLCKSZ; + if (MAXALIGN(header->xlp_rem_len) >= (state->pageSize - pageHeaderSize)) + tmpRecPtr = targetPagePtr + state->pageSize; else { /* diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c index 9cbd8afa49..f2dd495310 100644 --- a/src/backend/replication/logical/logical.c +++ b/src/backend/replication/logical/logical.c @@ -199,11 +199,14 @@ StartupDecodingContext(List *output_plugin_options, ctx->slot = slot; - ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, cleanup_cb); + ctx->reader = + XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, cleanup_cb); + if (!ctx->reader) ereport(ERROR, (errcode(ERRCODE_OUT_OF_MEMORY), errmsg("out of memory"))); + ctx->reader->readBuf = palloc(XLOG_BLCKSZ); ctx->page_read = page_read; ctx->reorder = ReorderBufferAllocate(); diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c index 062093727d..8e95600ef7 100644 --- a/src/backend/replication/walsender.c +++ b/src/backend/replication/walsender.c @@ -579,8 +579,8 @@ StartReplication(StartReplicationCmd *cmd) errmsg("IDENTIFY_SYSTEM has not been run before START_REPLICATION"))); /* create xlogreader for physical replication */ - xlogreader = - XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close); + xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, + NULL, wal_segment_close); if (!xlogreader) ereport(ERROR, diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c index 712c85281c..8a6209b061 100644 --- a/src/bin/pg_rewind/parsexlog.c +++ b/src/bin/pg_rewind/parsexlog.c @@ -61,7 +61,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex, XLogReaderState *xlogreader; char *errormsg; - xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL); + xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL); if (xlogreader == NULL) pg_fatal("out of memory"); @@ -121,7 +121,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex, char *errormsg; XLogRecPtr endptr; - xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL); + xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL); if (xlogreader == NULL) pg_fatal("out of memory"); @@ -181,7 +181,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex, forkptr += SizeOfXLogShortPHD; } - xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL); + xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL); if (xlogreader == NULL) pg_fatal("out of memory"); diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c index f514de6b13..b635537659 100644 --- a/src/bin/pg_waldump/pg_waldump.c +++ b/src/bin/pg_waldump/pg_waldump.c @@ -1041,7 +1041,7 @@ main(int argc, char **argv) /* we have everything we need, start reading */ xlogreader_state = - XLogReaderAllocate(WalSegSz, waldir, WALDumpCloseSegment); + XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, waldir, WALDumpCloseSegment); if (!xlogreader_state) fatal_error("out of memory"); diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h index b9dad7ee51..1853516333 100644 --- a/src/include/access/xlogreader.h +++ b/src/include/access/xlogreader.h @@ -151,6 +151,7 @@ struct XLogReaderState * ---------------------------------------- */ /* variables to communicate with page reader */ + int pageSize; /* size of a page */ XLogRecPtr readPagePtr; /* page pointer to read */ int32 readLen; /* bytes requested to reader, or actual bytes * read by reader, which must be larger than @@ -243,7 +244,7 @@ struct XLogReaderState /* Get a new XLogReader */ extern XLogReaderState *XLogReaderAllocate(int wal_segment_size, - const char *waldir, + int page_size, const char *waldir, WALSegmentCleanupCB cleanup_cb); /* Free an XLogReader */ -- 2.27.0 ----Next_Part(Thu_Mar__4_11_28_53_2021_014)---- ^ permalink raw reply [nested|flat] 25+ messages in thread
* [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize. @ 2019-09-05 12:29 Kyotaro Horiguchi <[email protected]> 0 siblings, 0 replies; 25+ messages in thread From: Kyotaro Horiguchi @ 2019-09-05 12:29 UTC (permalink / raw) XLog reader facility and its callers currently relies on the same constant XLOG_BLCKSZ. This patch moves the responsibility to determine the block size to the callers of the facility. --- src/backend/access/transam/twophase.c | 3 +- src/backend/access/transam/xlog.c | 9 ++-- src/backend/access/transam/xlogreader.c | 57 ++++++++++++----------- src/backend/replication/logical/logical.c | 5 +- src/backend/replication/walsender.c | 4 +- src/bin/pg_rewind/parsexlog.c | 6 +-- src/bin/pg_waldump/pg_waldump.c | 2 +- src/include/access/xlogreader.h | 3 +- 8 files changed, 50 insertions(+), 39 deletions(-) diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c index 7d5a96a5f5..9f9e7de253 100644 --- a/src/backend/access/transam/twophase.c +++ b/src/backend/access/transam/twophase.c @@ -1325,7 +1325,8 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len) XLogReaderState *xlogreader; char *errormsg; - xlogreader = XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close); + xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, + NULL, wal_segment_close); if (!xlogreader) ereport(ERROR, diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index b517ca85ad..0822ac70cb 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -1222,7 +1222,10 @@ XLogInsertRecord(XLogRecData *rdata, appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len); if (!debug_reader) - debug_reader = XLogReaderAllocate(wal_segment_size, NULL, NULL); + { + debug_reader = + XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, NULL); + } if (!debug_reader) { @@ -6531,8 +6534,8 @@ StartupXLOG(void) OwnLatch(&XLogCtl->recoveryWakeupLatch); /* Set up XLOG reader facility */ - xlogreader = - XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close); + xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, + NULL, wal_segment_close); if (!xlogreader) ereport(ERROR, diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c index ba3e4868f7..e5baf13086 100644 --- a/src/backend/access/transam/xlogreader.c +++ b/src/backend/access/transam/xlogreader.c @@ -72,7 +72,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...) * Returns NULL if the xlogreader couldn't be allocated. */ XLogReaderState * -XLogReaderAllocate(int wal_segment_size, const char *waldir, +XLogReaderAllocate(int wal_segment_size, int page_size, const char *waldir, WALSegmentCleanupCB cleanup_cb) { XLogReaderState *state; @@ -87,6 +87,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir, state->cleanup_cb = cleanup_cb; state->max_block_id = -1; + state->pageSize = page_size; /* * Permanently allocate readBuf. We do it this way, rather than just @@ -95,7 +96,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir, * isn't guaranteed to have any particular alignment, whereas * palloc_extended() will provide MAXALIGN'd storage. */ - state->readBuf = (char *) palloc_extended(XLOG_BLCKSZ, + state->readBuf = (char *) palloc_extended(state->pageSize, MCXT_ALLOC_NO_OOM); if (!state->readBuf) { @@ -163,7 +164,7 @@ XLogReaderFree(XLogReaderState *state) * readRecordBufSize is set to the new buffer size. * * To avoid useless small increases, round its size to a multiple of - * XLOG_BLCKSZ, and make sure it's at least 5*Max(BLCKSZ, XLOG_BLCKSZ) to start + * page size, and make sure it's at least 5*Max(BLCKSZ, <page size>) to start * with. (That is enough for all "normal" records, but very large commit or * abort records might need more space.) */ @@ -172,8 +173,8 @@ allocate_recordbuf(XLogReaderState *state, uint32 reclength) { uint32 newSize = reclength; - newSize += XLOG_BLCKSZ - (newSize % XLOG_BLCKSZ); - newSize = Max(newSize, 5 * Max(BLCKSZ, XLOG_BLCKSZ)); + newSize += state->pageSize - (newSize % state->pageSize); + newSize = Max(newSize, 5 * Max(BLCKSZ, state->pageSize)); #ifndef FRONTEND @@ -238,6 +239,8 @@ void XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr) { Assert(!XLogRecPtrIsInvalid(RecPtr)); + Assert(state->readBuf != NULL && + state->readBuf == (void *) MAXALIGN(state->readBuf)); ResetDecoder(state); @@ -366,15 +369,15 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) XLogPageHeader pageHeader; targetPagePtr = - state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ); - targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ; + state->ReadRecPtr - (state->ReadRecPtr % state->pageSize); + targetRecOff = state->ReadRecPtr % state->pageSize; /* * Check if we have enough data. For the first record in the page, * the requesting length doesn't contain page header. */ if (XLogNeedData(state, targetPagePtr, - Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ), + Min(targetRecOff + SizeOfXLogRecord, state->pageSize), targetRecOff != 0)) return XLREAD_NEED_DATA; @@ -421,7 +424,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) * verified that we got the whole header. */ prec = (XLogRecord *) (state->readBuf + - state->ReadRecPtr % XLOG_BLCKSZ); + state->ReadRecPtr % state->pageSize); total_len = prec->xl_tot_len; /* @@ -432,7 +435,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) * ensure that we enter the XLREAD_CONTINUATION state below; * otherwise we might fail to apply ValidXLogRecordHeader at all. */ - if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord) + if (targetRecOff <= state->pageSize - SizeOfXLogRecord) { if (!ValidXLogRecordHeader(state, state->ReadRecPtr, state->PrevRecPtr, prec)) @@ -477,10 +480,10 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) * available */ targetPagePtr = - state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ); - targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ; + state->ReadRecPtr - (state->ReadRecPtr % state->pageSize); + targetRecOff = state->ReadRecPtr % state->pageSize; - request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ); + request_len = Min(targetRecOff + total_len, state->pageSize); record_len = request_len - targetRecOff; /* ReadRecPtr contains page header */ @@ -549,7 +552,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) /* Calculate pointer to beginning of next page */ state->recordContRecPtr = state->ReadRecPtr + record_len; - Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0); + Assert(state->recordContRecPtr % state->pageSize == 0); state->readRecordState = XLREAD_CONTINUATION; } @@ -576,7 +579,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) /* this request contains page header */ Assert (targetPagePtr != 0); if (XLogNeedData(state, targetPagePtr, - Min(state->recordRemainLen, XLOG_BLCKSZ), + Min(state->recordRemainLen, state->pageSize), false)) return XLREAD_NEED_DATA; @@ -628,7 +631,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) Assert(state->readLen >= pageHeaderSize); contdata = (char *) state->readBuf + pageHeaderSize; - record_len = XLOG_BLCKSZ - pageHeaderSize; + record_len = state->pageSize - pageHeaderSize; if (pageHeader->xlp_rem_len < record_len) record_len = pageHeader->xlp_rem_len; @@ -655,7 +658,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) } /* Calculate pointer to beginning of next page, and continue */ - state->recordContRecPtr += XLOG_BLCKSZ; + state->recordContRecPtr += state->pageSize; } /* targetPagePtr is pointing the last-read page here */ @@ -789,10 +792,10 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen, XLogPageHeaderSize((XLogPageHeader) state->readBuf); addLen = pageHeaderSize; - if (reqLen + pageHeaderSize <= XLOG_BLCKSZ) + if (reqLen + pageHeaderSize <= state->pageSize) addLen = pageHeaderSize; else - addLen = XLOG_BLCKSZ - reqLen; + addLen = state->pageSize - reqLen; Assert(addLen >= 0); } @@ -805,7 +808,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen, /* Data is not in our buffer, request the caller for it. */ XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize); targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize); - Assert((pageptr % XLOG_BLCKSZ) == 0); + Assert((pageptr % state->pageSize) == 0); /* * Every time we request to load new data of a page to the caller, even if @@ -829,7 +832,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen, * will not come back here, but will request the actual target page. */ state->readPagePtr = pageptr - targetPageOff; - state->readLen = XLOG_BLCKSZ; + state->readLen = state->pageSize; return true; } @@ -962,7 +965,7 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr, int32 offset; XLogPageHeader hdr = (XLogPageHeader) phdr; - Assert((recptr % XLOG_BLCKSZ) == 0); + Assert((recptr % state->pageSize) == 0); XLByteToSeg(recptr, segno, state->segcxt.ws_segsize); offset = XLogSegmentOffset(recptr, state->segcxt.ws_segsize); @@ -1016,10 +1019,10 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr, "WAL file is from different database system: incorrect segment size in page header"); return false; } - else if (longhdr->xlp_xlog_blcksz != XLOG_BLCKSZ) + else if (longhdr->xlp_xlog_blcksz != state->pageSize) { report_invalid_record(state, - "WAL file is from different database system: incorrect XLOG_BLCKSZ in page header"); + "WAL file is from different database system: incorrect block size in page header"); return false; } } @@ -1140,7 +1143,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr, * XLogNeedData() is prepared to handle that and will read at * least short page-header worth of data */ - targetRecOff = tmpRecPtr % XLOG_BLCKSZ; + targetRecOff = tmpRecPtr % state->pageSize; /* scroll back to page boundary */ targetPagePtr = tmpRecPtr - targetRecOff; @@ -1174,8 +1177,8 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr, * * Note that record headers are MAXALIGN'ed */ - if (MAXALIGN(header->xlp_rem_len) >= (XLOG_BLCKSZ - pageHeaderSize)) - tmpRecPtr = targetPagePtr + XLOG_BLCKSZ; + if (MAXALIGN(header->xlp_rem_len) >= (state->pageSize - pageHeaderSize)) + tmpRecPtr = targetPagePtr + state->pageSize; else { /* diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c index 9cbd8afa49..f2dd495310 100644 --- a/src/backend/replication/logical/logical.c +++ b/src/backend/replication/logical/logical.c @@ -199,11 +199,14 @@ StartupDecodingContext(List *output_plugin_options, ctx->slot = slot; - ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, cleanup_cb); + ctx->reader = + XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, cleanup_cb); + if (!ctx->reader) ereport(ERROR, (errcode(ERRCODE_OUT_OF_MEMORY), errmsg("out of memory"))); + ctx->reader->readBuf = palloc(XLOG_BLCKSZ); ctx->page_read = page_read; ctx->reorder = ReorderBufferAllocate(); diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c index 062093727d..8e95600ef7 100644 --- a/src/backend/replication/walsender.c +++ b/src/backend/replication/walsender.c @@ -579,8 +579,8 @@ StartReplication(StartReplicationCmd *cmd) errmsg("IDENTIFY_SYSTEM has not been run before START_REPLICATION"))); /* create xlogreader for physical replication */ - xlogreader = - XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close); + xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, + NULL, wal_segment_close); if (!xlogreader) ereport(ERROR, diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c index 712c85281c..8a6209b061 100644 --- a/src/bin/pg_rewind/parsexlog.c +++ b/src/bin/pg_rewind/parsexlog.c @@ -61,7 +61,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex, XLogReaderState *xlogreader; char *errormsg; - xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL); + xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL); if (xlogreader == NULL) pg_fatal("out of memory"); @@ -121,7 +121,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex, char *errormsg; XLogRecPtr endptr; - xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL); + xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL); if (xlogreader == NULL) pg_fatal("out of memory"); @@ -181,7 +181,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex, forkptr += SizeOfXLogShortPHD; } - xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL); + xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL); if (xlogreader == NULL) pg_fatal("out of memory"); diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c index f514de6b13..b635537659 100644 --- a/src/bin/pg_waldump/pg_waldump.c +++ b/src/bin/pg_waldump/pg_waldump.c @@ -1041,7 +1041,7 @@ main(int argc, char **argv) /* we have everything we need, start reading */ xlogreader_state = - XLogReaderAllocate(WalSegSz, waldir, WALDumpCloseSegment); + XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, waldir, WALDumpCloseSegment); if (!xlogreader_state) fatal_error("out of memory"); diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h index b9dad7ee51..1853516333 100644 --- a/src/include/access/xlogreader.h +++ b/src/include/access/xlogreader.h @@ -151,6 +151,7 @@ struct XLogReaderState * ---------------------------------------- */ /* variables to communicate with page reader */ + int pageSize; /* size of a page */ XLogRecPtr readPagePtr; /* page pointer to read */ int32 readLen; /* bytes requested to reader, or actual bytes * read by reader, which must be larger than @@ -243,7 +244,7 @@ struct XLogReaderState /* Get a new XLogReader */ extern XLogReaderState *XLogReaderAllocate(int wal_segment_size, - const char *waldir, + int page_size, const char *waldir, WALSegmentCleanupCB cleanup_cb); /* Free an XLogReader */ -- 2.27.0 ----Next_Part(Thu_Mar__4_11_28_53_2021_014)---- ^ permalink raw reply [nested|flat] 25+ messages in thread
* [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize. @ 2019-09-05 12:29 Kyotaro Horiguchi <[email protected]> 0 siblings, 0 replies; 25+ messages in thread From: Kyotaro Horiguchi @ 2019-09-05 12:29 UTC (permalink / raw) XLog reader facility and its callers currently relies on the same constant XLOG_BLCKSZ. This patch moves the responsibility to determine the block size to the callers of the facility. --- src/backend/access/transam/twophase.c | 3 +- src/backend/access/transam/xlog.c | 9 ++-- src/backend/access/transam/xlogreader.c | 57 ++++++++++++----------- src/backend/replication/logical/logical.c | 5 +- src/backend/replication/walsender.c | 4 +- src/bin/pg_rewind/parsexlog.c | 6 +-- src/bin/pg_waldump/pg_waldump.c | 2 +- src/include/access/xlogreader.h | 3 +- 8 files changed, 50 insertions(+), 39 deletions(-) diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c index 7d5a96a5f5..9f9e7de253 100644 --- a/src/backend/access/transam/twophase.c +++ b/src/backend/access/transam/twophase.c @@ -1325,7 +1325,8 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len) XLogReaderState *xlogreader; char *errormsg; - xlogreader = XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close); + xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, + NULL, wal_segment_close); if (!xlogreader) ereport(ERROR, diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index b517ca85ad..0822ac70cb 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -1222,7 +1222,10 @@ XLogInsertRecord(XLogRecData *rdata, appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len); if (!debug_reader) - debug_reader = XLogReaderAllocate(wal_segment_size, NULL, NULL); + { + debug_reader = + XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, NULL); + } if (!debug_reader) { @@ -6531,8 +6534,8 @@ StartupXLOG(void) OwnLatch(&XLogCtl->recoveryWakeupLatch); /* Set up XLOG reader facility */ - xlogreader = - XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close); + xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, + NULL, wal_segment_close); if (!xlogreader) ereport(ERROR, diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c index ba3e4868f7..e5baf13086 100644 --- a/src/backend/access/transam/xlogreader.c +++ b/src/backend/access/transam/xlogreader.c @@ -72,7 +72,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...) * Returns NULL if the xlogreader couldn't be allocated. */ XLogReaderState * -XLogReaderAllocate(int wal_segment_size, const char *waldir, +XLogReaderAllocate(int wal_segment_size, int page_size, const char *waldir, WALSegmentCleanupCB cleanup_cb) { XLogReaderState *state; @@ -87,6 +87,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir, state->cleanup_cb = cleanup_cb; state->max_block_id = -1; + state->pageSize = page_size; /* * Permanently allocate readBuf. We do it this way, rather than just @@ -95,7 +96,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir, * isn't guaranteed to have any particular alignment, whereas * palloc_extended() will provide MAXALIGN'd storage. */ - state->readBuf = (char *) palloc_extended(XLOG_BLCKSZ, + state->readBuf = (char *) palloc_extended(state->pageSize, MCXT_ALLOC_NO_OOM); if (!state->readBuf) { @@ -163,7 +164,7 @@ XLogReaderFree(XLogReaderState *state) * readRecordBufSize is set to the new buffer size. * * To avoid useless small increases, round its size to a multiple of - * XLOG_BLCKSZ, and make sure it's at least 5*Max(BLCKSZ, XLOG_BLCKSZ) to start + * page size, and make sure it's at least 5*Max(BLCKSZ, <page size>) to start * with. (That is enough for all "normal" records, but very large commit or * abort records might need more space.) */ @@ -172,8 +173,8 @@ allocate_recordbuf(XLogReaderState *state, uint32 reclength) { uint32 newSize = reclength; - newSize += XLOG_BLCKSZ - (newSize % XLOG_BLCKSZ); - newSize = Max(newSize, 5 * Max(BLCKSZ, XLOG_BLCKSZ)); + newSize += state->pageSize - (newSize % state->pageSize); + newSize = Max(newSize, 5 * Max(BLCKSZ, state->pageSize)); #ifndef FRONTEND @@ -238,6 +239,8 @@ void XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr) { Assert(!XLogRecPtrIsInvalid(RecPtr)); + Assert(state->readBuf != NULL && + state->readBuf == (void *) MAXALIGN(state->readBuf)); ResetDecoder(state); @@ -366,15 +369,15 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) XLogPageHeader pageHeader; targetPagePtr = - state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ); - targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ; + state->ReadRecPtr - (state->ReadRecPtr % state->pageSize); + targetRecOff = state->ReadRecPtr % state->pageSize; /* * Check if we have enough data. For the first record in the page, * the requesting length doesn't contain page header. */ if (XLogNeedData(state, targetPagePtr, - Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ), + Min(targetRecOff + SizeOfXLogRecord, state->pageSize), targetRecOff != 0)) return XLREAD_NEED_DATA; @@ -421,7 +424,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) * verified that we got the whole header. */ prec = (XLogRecord *) (state->readBuf + - state->ReadRecPtr % XLOG_BLCKSZ); + state->ReadRecPtr % state->pageSize); total_len = prec->xl_tot_len; /* @@ -432,7 +435,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) * ensure that we enter the XLREAD_CONTINUATION state below; * otherwise we might fail to apply ValidXLogRecordHeader at all. */ - if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord) + if (targetRecOff <= state->pageSize - SizeOfXLogRecord) { if (!ValidXLogRecordHeader(state, state->ReadRecPtr, state->PrevRecPtr, prec)) @@ -477,10 +480,10 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) * available */ targetPagePtr = - state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ); - targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ; + state->ReadRecPtr - (state->ReadRecPtr % state->pageSize); + targetRecOff = state->ReadRecPtr % state->pageSize; - request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ); + request_len = Min(targetRecOff + total_len, state->pageSize); record_len = request_len - targetRecOff; /* ReadRecPtr contains page header */ @@ -549,7 +552,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) /* Calculate pointer to beginning of next page */ state->recordContRecPtr = state->ReadRecPtr + record_len; - Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0); + Assert(state->recordContRecPtr % state->pageSize == 0); state->readRecordState = XLREAD_CONTINUATION; } @@ -576,7 +579,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) /* this request contains page header */ Assert (targetPagePtr != 0); if (XLogNeedData(state, targetPagePtr, - Min(state->recordRemainLen, XLOG_BLCKSZ), + Min(state->recordRemainLen, state->pageSize), false)) return XLREAD_NEED_DATA; @@ -628,7 +631,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) Assert(state->readLen >= pageHeaderSize); contdata = (char *) state->readBuf + pageHeaderSize; - record_len = XLOG_BLCKSZ - pageHeaderSize; + record_len = state->pageSize - pageHeaderSize; if (pageHeader->xlp_rem_len < record_len) record_len = pageHeader->xlp_rem_len; @@ -655,7 +658,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) } /* Calculate pointer to beginning of next page, and continue */ - state->recordContRecPtr += XLOG_BLCKSZ; + state->recordContRecPtr += state->pageSize; } /* targetPagePtr is pointing the last-read page here */ @@ -789,10 +792,10 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen, XLogPageHeaderSize((XLogPageHeader) state->readBuf); addLen = pageHeaderSize; - if (reqLen + pageHeaderSize <= XLOG_BLCKSZ) + if (reqLen + pageHeaderSize <= state->pageSize) addLen = pageHeaderSize; else - addLen = XLOG_BLCKSZ - reqLen; + addLen = state->pageSize - reqLen; Assert(addLen >= 0); } @@ -805,7 +808,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen, /* Data is not in our buffer, request the caller for it. */ XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize); targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize); - Assert((pageptr % XLOG_BLCKSZ) == 0); + Assert((pageptr % state->pageSize) == 0); /* * Every time we request to load new data of a page to the caller, even if @@ -829,7 +832,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen, * will not come back here, but will request the actual target page. */ state->readPagePtr = pageptr - targetPageOff; - state->readLen = XLOG_BLCKSZ; + state->readLen = state->pageSize; return true; } @@ -962,7 +965,7 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr, int32 offset; XLogPageHeader hdr = (XLogPageHeader) phdr; - Assert((recptr % XLOG_BLCKSZ) == 0); + Assert((recptr % state->pageSize) == 0); XLByteToSeg(recptr, segno, state->segcxt.ws_segsize); offset = XLogSegmentOffset(recptr, state->segcxt.ws_segsize); @@ -1016,10 +1019,10 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr, "WAL file is from different database system: incorrect segment size in page header"); return false; } - else if (longhdr->xlp_xlog_blcksz != XLOG_BLCKSZ) + else if (longhdr->xlp_xlog_blcksz != state->pageSize) { report_invalid_record(state, - "WAL file is from different database system: incorrect XLOG_BLCKSZ in page header"); + "WAL file is from different database system: incorrect block size in page header"); return false; } } @@ -1140,7 +1143,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr, * XLogNeedData() is prepared to handle that and will read at * least short page-header worth of data */ - targetRecOff = tmpRecPtr % XLOG_BLCKSZ; + targetRecOff = tmpRecPtr % state->pageSize; /* scroll back to page boundary */ targetPagePtr = tmpRecPtr - targetRecOff; @@ -1174,8 +1177,8 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr, * * Note that record headers are MAXALIGN'ed */ - if (MAXALIGN(header->xlp_rem_len) >= (XLOG_BLCKSZ - pageHeaderSize)) - tmpRecPtr = targetPagePtr + XLOG_BLCKSZ; + if (MAXALIGN(header->xlp_rem_len) >= (state->pageSize - pageHeaderSize)) + tmpRecPtr = targetPagePtr + state->pageSize; else { /* diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c index 9cbd8afa49..f2dd495310 100644 --- a/src/backend/replication/logical/logical.c +++ b/src/backend/replication/logical/logical.c @@ -199,11 +199,14 @@ StartupDecodingContext(List *output_plugin_options, ctx->slot = slot; - ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, cleanup_cb); + ctx->reader = + XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, cleanup_cb); + if (!ctx->reader) ereport(ERROR, (errcode(ERRCODE_OUT_OF_MEMORY), errmsg("out of memory"))); + ctx->reader->readBuf = palloc(XLOG_BLCKSZ); ctx->page_read = page_read; ctx->reorder = ReorderBufferAllocate(); diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c index 062093727d..8e95600ef7 100644 --- a/src/backend/replication/walsender.c +++ b/src/backend/replication/walsender.c @@ -579,8 +579,8 @@ StartReplication(StartReplicationCmd *cmd) errmsg("IDENTIFY_SYSTEM has not been run before START_REPLICATION"))); /* create xlogreader for physical replication */ - xlogreader = - XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close); + xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, + NULL, wal_segment_close); if (!xlogreader) ereport(ERROR, diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c index 712c85281c..8a6209b061 100644 --- a/src/bin/pg_rewind/parsexlog.c +++ b/src/bin/pg_rewind/parsexlog.c @@ -61,7 +61,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex, XLogReaderState *xlogreader; char *errormsg; - xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL); + xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL); if (xlogreader == NULL) pg_fatal("out of memory"); @@ -121,7 +121,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex, char *errormsg; XLogRecPtr endptr; - xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL); + xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL); if (xlogreader == NULL) pg_fatal("out of memory"); @@ -181,7 +181,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex, forkptr += SizeOfXLogShortPHD; } - xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL); + xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL); if (xlogreader == NULL) pg_fatal("out of memory"); diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c index f514de6b13..b635537659 100644 --- a/src/bin/pg_waldump/pg_waldump.c +++ b/src/bin/pg_waldump/pg_waldump.c @@ -1041,7 +1041,7 @@ main(int argc, char **argv) /* we have everything we need, start reading */ xlogreader_state = - XLogReaderAllocate(WalSegSz, waldir, WALDumpCloseSegment); + XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, waldir, WALDumpCloseSegment); if (!xlogreader_state) fatal_error("out of memory"); diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h index b9dad7ee51..1853516333 100644 --- a/src/include/access/xlogreader.h +++ b/src/include/access/xlogreader.h @@ -151,6 +151,7 @@ struct XLogReaderState * ---------------------------------------- */ /* variables to communicate with page reader */ + int pageSize; /* size of a page */ XLogRecPtr readPagePtr; /* page pointer to read */ int32 readLen; /* bytes requested to reader, or actual bytes * read by reader, which must be larger than @@ -243,7 +244,7 @@ struct XLogReaderState /* Get a new XLogReader */ extern XLogReaderState *XLogReaderAllocate(int wal_segment_size, - const char *waldir, + int page_size, const char *waldir, WALSegmentCleanupCB cleanup_cb); /* Free an XLogReader */ -- 2.27.0 ----Next_Part(Thu_Mar__4_11_28_53_2021_014)---- ^ permalink raw reply [nested|flat] 25+ messages in thread
* [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize. @ 2019-09-05 12:29 Kyotaro Horiguchi <[email protected]> 0 siblings, 0 replies; 25+ messages in thread From: Kyotaro Horiguchi @ 2019-09-05 12:29 UTC (permalink / raw) XLog reader facility and its callers currently relies on the same constant XLOG_BLCKSZ. This patch moves the responsibility to determine the block size to the callers of the facility. --- src/backend/access/transam/twophase.c | 3 +- src/backend/access/transam/xlog.c | 9 ++-- src/backend/access/transam/xlogreader.c | 57 ++++++++++++----------- src/backend/replication/logical/logical.c | 5 +- src/backend/replication/walsender.c | 4 +- src/bin/pg_rewind/parsexlog.c | 6 +-- src/bin/pg_waldump/pg_waldump.c | 2 +- src/include/access/xlogreader.h | 3 +- 8 files changed, 50 insertions(+), 39 deletions(-) diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c index 7d5a96a5f5..9f9e7de253 100644 --- a/src/backend/access/transam/twophase.c +++ b/src/backend/access/transam/twophase.c @@ -1325,7 +1325,8 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len) XLogReaderState *xlogreader; char *errormsg; - xlogreader = XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close); + xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, + NULL, wal_segment_close); if (!xlogreader) ereport(ERROR, diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index b517ca85ad..0822ac70cb 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -1222,7 +1222,10 @@ XLogInsertRecord(XLogRecData *rdata, appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len); if (!debug_reader) - debug_reader = XLogReaderAllocate(wal_segment_size, NULL, NULL); + { + debug_reader = + XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, NULL); + } if (!debug_reader) { @@ -6531,8 +6534,8 @@ StartupXLOG(void) OwnLatch(&XLogCtl->recoveryWakeupLatch); /* Set up XLOG reader facility */ - xlogreader = - XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close); + xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, + NULL, wal_segment_close); if (!xlogreader) ereport(ERROR, diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c index ba3e4868f7..e5baf13086 100644 --- a/src/backend/access/transam/xlogreader.c +++ b/src/backend/access/transam/xlogreader.c @@ -72,7 +72,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...) * Returns NULL if the xlogreader couldn't be allocated. */ XLogReaderState * -XLogReaderAllocate(int wal_segment_size, const char *waldir, +XLogReaderAllocate(int wal_segment_size, int page_size, const char *waldir, WALSegmentCleanupCB cleanup_cb) { XLogReaderState *state; @@ -87,6 +87,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir, state->cleanup_cb = cleanup_cb; state->max_block_id = -1; + state->pageSize = page_size; /* * Permanently allocate readBuf. We do it this way, rather than just @@ -95,7 +96,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir, * isn't guaranteed to have any particular alignment, whereas * palloc_extended() will provide MAXALIGN'd storage. */ - state->readBuf = (char *) palloc_extended(XLOG_BLCKSZ, + state->readBuf = (char *) palloc_extended(state->pageSize, MCXT_ALLOC_NO_OOM); if (!state->readBuf) { @@ -163,7 +164,7 @@ XLogReaderFree(XLogReaderState *state) * readRecordBufSize is set to the new buffer size. * * To avoid useless small increases, round its size to a multiple of - * XLOG_BLCKSZ, and make sure it's at least 5*Max(BLCKSZ, XLOG_BLCKSZ) to start + * page size, and make sure it's at least 5*Max(BLCKSZ, <page size>) to start * with. (That is enough for all "normal" records, but very large commit or * abort records might need more space.) */ @@ -172,8 +173,8 @@ allocate_recordbuf(XLogReaderState *state, uint32 reclength) { uint32 newSize = reclength; - newSize += XLOG_BLCKSZ - (newSize % XLOG_BLCKSZ); - newSize = Max(newSize, 5 * Max(BLCKSZ, XLOG_BLCKSZ)); + newSize += state->pageSize - (newSize % state->pageSize); + newSize = Max(newSize, 5 * Max(BLCKSZ, state->pageSize)); #ifndef FRONTEND @@ -238,6 +239,8 @@ void XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr) { Assert(!XLogRecPtrIsInvalid(RecPtr)); + Assert(state->readBuf != NULL && + state->readBuf == (void *) MAXALIGN(state->readBuf)); ResetDecoder(state); @@ -366,15 +369,15 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) XLogPageHeader pageHeader; targetPagePtr = - state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ); - targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ; + state->ReadRecPtr - (state->ReadRecPtr % state->pageSize); + targetRecOff = state->ReadRecPtr % state->pageSize; /* * Check if we have enough data. For the first record in the page, * the requesting length doesn't contain page header. */ if (XLogNeedData(state, targetPagePtr, - Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ), + Min(targetRecOff + SizeOfXLogRecord, state->pageSize), targetRecOff != 0)) return XLREAD_NEED_DATA; @@ -421,7 +424,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) * verified that we got the whole header. */ prec = (XLogRecord *) (state->readBuf + - state->ReadRecPtr % XLOG_BLCKSZ); + state->ReadRecPtr % state->pageSize); total_len = prec->xl_tot_len; /* @@ -432,7 +435,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) * ensure that we enter the XLREAD_CONTINUATION state below; * otherwise we might fail to apply ValidXLogRecordHeader at all. */ - if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord) + if (targetRecOff <= state->pageSize - SizeOfXLogRecord) { if (!ValidXLogRecordHeader(state, state->ReadRecPtr, state->PrevRecPtr, prec)) @@ -477,10 +480,10 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) * available */ targetPagePtr = - state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ); - targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ; + state->ReadRecPtr - (state->ReadRecPtr % state->pageSize); + targetRecOff = state->ReadRecPtr % state->pageSize; - request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ); + request_len = Min(targetRecOff + total_len, state->pageSize); record_len = request_len - targetRecOff; /* ReadRecPtr contains page header */ @@ -549,7 +552,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) /* Calculate pointer to beginning of next page */ state->recordContRecPtr = state->ReadRecPtr + record_len; - Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0); + Assert(state->recordContRecPtr % state->pageSize == 0); state->readRecordState = XLREAD_CONTINUATION; } @@ -576,7 +579,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) /* this request contains page header */ Assert (targetPagePtr != 0); if (XLogNeedData(state, targetPagePtr, - Min(state->recordRemainLen, XLOG_BLCKSZ), + Min(state->recordRemainLen, state->pageSize), false)) return XLREAD_NEED_DATA; @@ -628,7 +631,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) Assert(state->readLen >= pageHeaderSize); contdata = (char *) state->readBuf + pageHeaderSize; - record_len = XLOG_BLCKSZ - pageHeaderSize; + record_len = state->pageSize - pageHeaderSize; if (pageHeader->xlp_rem_len < record_len) record_len = pageHeader->xlp_rem_len; @@ -655,7 +658,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) } /* Calculate pointer to beginning of next page, and continue */ - state->recordContRecPtr += XLOG_BLCKSZ; + state->recordContRecPtr += state->pageSize; } /* targetPagePtr is pointing the last-read page here */ @@ -789,10 +792,10 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen, XLogPageHeaderSize((XLogPageHeader) state->readBuf); addLen = pageHeaderSize; - if (reqLen + pageHeaderSize <= XLOG_BLCKSZ) + if (reqLen + pageHeaderSize <= state->pageSize) addLen = pageHeaderSize; else - addLen = XLOG_BLCKSZ - reqLen; + addLen = state->pageSize - reqLen; Assert(addLen >= 0); } @@ -805,7 +808,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen, /* Data is not in our buffer, request the caller for it. */ XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize); targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize); - Assert((pageptr % XLOG_BLCKSZ) == 0); + Assert((pageptr % state->pageSize) == 0); /* * Every time we request to load new data of a page to the caller, even if @@ -829,7 +832,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen, * will not come back here, but will request the actual target page. */ state->readPagePtr = pageptr - targetPageOff; - state->readLen = XLOG_BLCKSZ; + state->readLen = state->pageSize; return true; } @@ -962,7 +965,7 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr, int32 offset; XLogPageHeader hdr = (XLogPageHeader) phdr; - Assert((recptr % XLOG_BLCKSZ) == 0); + Assert((recptr % state->pageSize) == 0); XLByteToSeg(recptr, segno, state->segcxt.ws_segsize); offset = XLogSegmentOffset(recptr, state->segcxt.ws_segsize); @@ -1016,10 +1019,10 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr, "WAL file is from different database system: incorrect segment size in page header"); return false; } - else if (longhdr->xlp_xlog_blcksz != XLOG_BLCKSZ) + else if (longhdr->xlp_xlog_blcksz != state->pageSize) { report_invalid_record(state, - "WAL file is from different database system: incorrect XLOG_BLCKSZ in page header"); + "WAL file is from different database system: incorrect block size in page header"); return false; } } @@ -1140,7 +1143,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr, * XLogNeedData() is prepared to handle that and will read at * least short page-header worth of data */ - targetRecOff = tmpRecPtr % XLOG_BLCKSZ; + targetRecOff = tmpRecPtr % state->pageSize; /* scroll back to page boundary */ targetPagePtr = tmpRecPtr - targetRecOff; @@ -1174,8 +1177,8 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr, * * Note that record headers are MAXALIGN'ed */ - if (MAXALIGN(header->xlp_rem_len) >= (XLOG_BLCKSZ - pageHeaderSize)) - tmpRecPtr = targetPagePtr + XLOG_BLCKSZ; + if (MAXALIGN(header->xlp_rem_len) >= (state->pageSize - pageHeaderSize)) + tmpRecPtr = targetPagePtr + state->pageSize; else { /* diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c index 9cbd8afa49..f2dd495310 100644 --- a/src/backend/replication/logical/logical.c +++ b/src/backend/replication/logical/logical.c @@ -199,11 +199,14 @@ StartupDecodingContext(List *output_plugin_options, ctx->slot = slot; - ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, cleanup_cb); + ctx->reader = + XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, cleanup_cb); + if (!ctx->reader) ereport(ERROR, (errcode(ERRCODE_OUT_OF_MEMORY), errmsg("out of memory"))); + ctx->reader->readBuf = palloc(XLOG_BLCKSZ); ctx->page_read = page_read; ctx->reorder = ReorderBufferAllocate(); diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c index 062093727d..8e95600ef7 100644 --- a/src/backend/replication/walsender.c +++ b/src/backend/replication/walsender.c @@ -579,8 +579,8 @@ StartReplication(StartReplicationCmd *cmd) errmsg("IDENTIFY_SYSTEM has not been run before START_REPLICATION"))); /* create xlogreader for physical replication */ - xlogreader = - XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close); + xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, + NULL, wal_segment_close); if (!xlogreader) ereport(ERROR, diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c index 712c85281c..8a6209b061 100644 --- a/src/bin/pg_rewind/parsexlog.c +++ b/src/bin/pg_rewind/parsexlog.c @@ -61,7 +61,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex, XLogReaderState *xlogreader; char *errormsg; - xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL); + xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL); if (xlogreader == NULL) pg_fatal("out of memory"); @@ -121,7 +121,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex, char *errormsg; XLogRecPtr endptr; - xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL); + xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL); if (xlogreader == NULL) pg_fatal("out of memory"); @@ -181,7 +181,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex, forkptr += SizeOfXLogShortPHD; } - xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL); + xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL); if (xlogreader == NULL) pg_fatal("out of memory"); diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c index f514de6b13..b635537659 100644 --- a/src/bin/pg_waldump/pg_waldump.c +++ b/src/bin/pg_waldump/pg_waldump.c @@ -1041,7 +1041,7 @@ main(int argc, char **argv) /* we have everything we need, start reading */ xlogreader_state = - XLogReaderAllocate(WalSegSz, waldir, WALDumpCloseSegment); + XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, waldir, WALDumpCloseSegment); if (!xlogreader_state) fatal_error("out of memory"); diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h index b9dad7ee51..1853516333 100644 --- a/src/include/access/xlogreader.h +++ b/src/include/access/xlogreader.h @@ -151,6 +151,7 @@ struct XLogReaderState * ---------------------------------------- */ /* variables to communicate with page reader */ + int pageSize; /* size of a page */ XLogRecPtr readPagePtr; /* page pointer to read */ int32 readLen; /* bytes requested to reader, or actual bytes * read by reader, which must be larger than @@ -243,7 +244,7 @@ struct XLogReaderState /* Get a new XLogReader */ extern XLogReaderState *XLogReaderAllocate(int wal_segment_size, - const char *waldir, + int page_size, const char *waldir, WALSegmentCleanupCB cleanup_cb); /* Free an XLogReader */ -- 2.27.0 ----Next_Part(Thu_Mar__4_11_28_53_2021_014)---- ^ permalink raw reply [nested|flat] 25+ messages in thread
* [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize. @ 2019-09-05 12:29 Kyotaro Horiguchi <[email protected]> 0 siblings, 0 replies; 25+ messages in thread From: Kyotaro Horiguchi @ 2019-09-05 12:29 UTC (permalink / raw) XLog reader facility and its callers currently relies on the same constant XLOG_BLCKSZ. This patch moves the responsibility to determine the block size to the callers of the facility. --- src/backend/access/transam/twophase.c | 3 +- src/backend/access/transam/xlog.c | 9 ++-- src/backend/access/transam/xlogreader.c | 57 ++++++++++++----------- src/backend/replication/logical/logical.c | 5 +- src/backend/replication/walsender.c | 4 +- src/bin/pg_rewind/parsexlog.c | 6 +-- src/bin/pg_waldump/pg_waldump.c | 2 +- src/include/access/xlogreader.h | 3 +- 8 files changed, 50 insertions(+), 39 deletions(-) diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c index 7d5a96a5f5..9f9e7de253 100644 --- a/src/backend/access/transam/twophase.c +++ b/src/backend/access/transam/twophase.c @@ -1325,7 +1325,8 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len) XLogReaderState *xlogreader; char *errormsg; - xlogreader = XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close); + xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, + NULL, wal_segment_close); if (!xlogreader) ereport(ERROR, diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index b517ca85ad..0822ac70cb 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -1222,7 +1222,10 @@ XLogInsertRecord(XLogRecData *rdata, appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len); if (!debug_reader) - debug_reader = XLogReaderAllocate(wal_segment_size, NULL, NULL); + { + debug_reader = + XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, NULL); + } if (!debug_reader) { @@ -6531,8 +6534,8 @@ StartupXLOG(void) OwnLatch(&XLogCtl->recoveryWakeupLatch); /* Set up XLOG reader facility */ - xlogreader = - XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close); + xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, + NULL, wal_segment_close); if (!xlogreader) ereport(ERROR, diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c index ba3e4868f7..e5baf13086 100644 --- a/src/backend/access/transam/xlogreader.c +++ b/src/backend/access/transam/xlogreader.c @@ -72,7 +72,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...) * Returns NULL if the xlogreader couldn't be allocated. */ XLogReaderState * -XLogReaderAllocate(int wal_segment_size, const char *waldir, +XLogReaderAllocate(int wal_segment_size, int page_size, const char *waldir, WALSegmentCleanupCB cleanup_cb) { XLogReaderState *state; @@ -87,6 +87,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir, state->cleanup_cb = cleanup_cb; state->max_block_id = -1; + state->pageSize = page_size; /* * Permanently allocate readBuf. We do it this way, rather than just @@ -95,7 +96,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir, * isn't guaranteed to have any particular alignment, whereas * palloc_extended() will provide MAXALIGN'd storage. */ - state->readBuf = (char *) palloc_extended(XLOG_BLCKSZ, + state->readBuf = (char *) palloc_extended(state->pageSize, MCXT_ALLOC_NO_OOM); if (!state->readBuf) { @@ -163,7 +164,7 @@ XLogReaderFree(XLogReaderState *state) * readRecordBufSize is set to the new buffer size. * * To avoid useless small increases, round its size to a multiple of - * XLOG_BLCKSZ, and make sure it's at least 5*Max(BLCKSZ, XLOG_BLCKSZ) to start + * page size, and make sure it's at least 5*Max(BLCKSZ, <page size>) to start * with. (That is enough for all "normal" records, but very large commit or * abort records might need more space.) */ @@ -172,8 +173,8 @@ allocate_recordbuf(XLogReaderState *state, uint32 reclength) { uint32 newSize = reclength; - newSize += XLOG_BLCKSZ - (newSize % XLOG_BLCKSZ); - newSize = Max(newSize, 5 * Max(BLCKSZ, XLOG_BLCKSZ)); + newSize += state->pageSize - (newSize % state->pageSize); + newSize = Max(newSize, 5 * Max(BLCKSZ, state->pageSize)); #ifndef FRONTEND @@ -238,6 +239,8 @@ void XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr) { Assert(!XLogRecPtrIsInvalid(RecPtr)); + Assert(state->readBuf != NULL && + state->readBuf == (void *) MAXALIGN(state->readBuf)); ResetDecoder(state); @@ -366,15 +369,15 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) XLogPageHeader pageHeader; targetPagePtr = - state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ); - targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ; + state->ReadRecPtr - (state->ReadRecPtr % state->pageSize); + targetRecOff = state->ReadRecPtr % state->pageSize; /* * Check if we have enough data. For the first record in the page, * the requesting length doesn't contain page header. */ if (XLogNeedData(state, targetPagePtr, - Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ), + Min(targetRecOff + SizeOfXLogRecord, state->pageSize), targetRecOff != 0)) return XLREAD_NEED_DATA; @@ -421,7 +424,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) * verified that we got the whole header. */ prec = (XLogRecord *) (state->readBuf + - state->ReadRecPtr % XLOG_BLCKSZ); + state->ReadRecPtr % state->pageSize); total_len = prec->xl_tot_len; /* @@ -432,7 +435,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) * ensure that we enter the XLREAD_CONTINUATION state below; * otherwise we might fail to apply ValidXLogRecordHeader at all. */ - if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord) + if (targetRecOff <= state->pageSize - SizeOfXLogRecord) { if (!ValidXLogRecordHeader(state, state->ReadRecPtr, state->PrevRecPtr, prec)) @@ -477,10 +480,10 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) * available */ targetPagePtr = - state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ); - targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ; + state->ReadRecPtr - (state->ReadRecPtr % state->pageSize); + targetRecOff = state->ReadRecPtr % state->pageSize; - request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ); + request_len = Min(targetRecOff + total_len, state->pageSize); record_len = request_len - targetRecOff; /* ReadRecPtr contains page header */ @@ -549,7 +552,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) /* Calculate pointer to beginning of next page */ state->recordContRecPtr = state->ReadRecPtr + record_len; - Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0); + Assert(state->recordContRecPtr % state->pageSize == 0); state->readRecordState = XLREAD_CONTINUATION; } @@ -576,7 +579,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) /* this request contains page header */ Assert (targetPagePtr != 0); if (XLogNeedData(state, targetPagePtr, - Min(state->recordRemainLen, XLOG_BLCKSZ), + Min(state->recordRemainLen, state->pageSize), false)) return XLREAD_NEED_DATA; @@ -628,7 +631,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) Assert(state->readLen >= pageHeaderSize); contdata = (char *) state->readBuf + pageHeaderSize; - record_len = XLOG_BLCKSZ - pageHeaderSize; + record_len = state->pageSize - pageHeaderSize; if (pageHeader->xlp_rem_len < record_len) record_len = pageHeader->xlp_rem_len; @@ -655,7 +658,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) } /* Calculate pointer to beginning of next page, and continue */ - state->recordContRecPtr += XLOG_BLCKSZ; + state->recordContRecPtr += state->pageSize; } /* targetPagePtr is pointing the last-read page here */ @@ -789,10 +792,10 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen, XLogPageHeaderSize((XLogPageHeader) state->readBuf); addLen = pageHeaderSize; - if (reqLen + pageHeaderSize <= XLOG_BLCKSZ) + if (reqLen + pageHeaderSize <= state->pageSize) addLen = pageHeaderSize; else - addLen = XLOG_BLCKSZ - reqLen; + addLen = state->pageSize - reqLen; Assert(addLen >= 0); } @@ -805,7 +808,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen, /* Data is not in our buffer, request the caller for it. */ XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize); targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize); - Assert((pageptr % XLOG_BLCKSZ) == 0); + Assert((pageptr % state->pageSize) == 0); /* * Every time we request to load new data of a page to the caller, even if @@ -829,7 +832,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen, * will not come back here, but will request the actual target page. */ state->readPagePtr = pageptr - targetPageOff; - state->readLen = XLOG_BLCKSZ; + state->readLen = state->pageSize; return true; } @@ -962,7 +965,7 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr, int32 offset; XLogPageHeader hdr = (XLogPageHeader) phdr; - Assert((recptr % XLOG_BLCKSZ) == 0); + Assert((recptr % state->pageSize) == 0); XLByteToSeg(recptr, segno, state->segcxt.ws_segsize); offset = XLogSegmentOffset(recptr, state->segcxt.ws_segsize); @@ -1016,10 +1019,10 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr, "WAL file is from different database system: incorrect segment size in page header"); return false; } - else if (longhdr->xlp_xlog_blcksz != XLOG_BLCKSZ) + else if (longhdr->xlp_xlog_blcksz != state->pageSize) { report_invalid_record(state, - "WAL file is from different database system: incorrect XLOG_BLCKSZ in page header"); + "WAL file is from different database system: incorrect block size in page header"); return false; } } @@ -1140,7 +1143,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr, * XLogNeedData() is prepared to handle that and will read at * least short page-header worth of data */ - targetRecOff = tmpRecPtr % XLOG_BLCKSZ; + targetRecOff = tmpRecPtr % state->pageSize; /* scroll back to page boundary */ targetPagePtr = tmpRecPtr - targetRecOff; @@ -1174,8 +1177,8 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr, * * Note that record headers are MAXALIGN'ed */ - if (MAXALIGN(header->xlp_rem_len) >= (XLOG_BLCKSZ - pageHeaderSize)) - tmpRecPtr = targetPagePtr + XLOG_BLCKSZ; + if (MAXALIGN(header->xlp_rem_len) >= (state->pageSize - pageHeaderSize)) + tmpRecPtr = targetPagePtr + state->pageSize; else { /* diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c index 9cbd8afa49..f2dd495310 100644 --- a/src/backend/replication/logical/logical.c +++ b/src/backend/replication/logical/logical.c @@ -199,11 +199,14 @@ StartupDecodingContext(List *output_plugin_options, ctx->slot = slot; - ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, cleanup_cb); + ctx->reader = + XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, cleanup_cb); + if (!ctx->reader) ereport(ERROR, (errcode(ERRCODE_OUT_OF_MEMORY), errmsg("out of memory"))); + ctx->reader->readBuf = palloc(XLOG_BLCKSZ); ctx->page_read = page_read; ctx->reorder = ReorderBufferAllocate(); diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c index 062093727d..8e95600ef7 100644 --- a/src/backend/replication/walsender.c +++ b/src/backend/replication/walsender.c @@ -579,8 +579,8 @@ StartReplication(StartReplicationCmd *cmd) errmsg("IDENTIFY_SYSTEM has not been run before START_REPLICATION"))); /* create xlogreader for physical replication */ - xlogreader = - XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close); + xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, + NULL, wal_segment_close); if (!xlogreader) ereport(ERROR, diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c index 712c85281c..8a6209b061 100644 --- a/src/bin/pg_rewind/parsexlog.c +++ b/src/bin/pg_rewind/parsexlog.c @@ -61,7 +61,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex, XLogReaderState *xlogreader; char *errormsg; - xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL); + xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL); if (xlogreader == NULL) pg_fatal("out of memory"); @@ -121,7 +121,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex, char *errormsg; XLogRecPtr endptr; - xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL); + xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL); if (xlogreader == NULL) pg_fatal("out of memory"); @@ -181,7 +181,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex, forkptr += SizeOfXLogShortPHD; } - xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL); + xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL); if (xlogreader == NULL) pg_fatal("out of memory"); diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c index f514de6b13..b635537659 100644 --- a/src/bin/pg_waldump/pg_waldump.c +++ b/src/bin/pg_waldump/pg_waldump.c @@ -1041,7 +1041,7 @@ main(int argc, char **argv) /* we have everything we need, start reading */ xlogreader_state = - XLogReaderAllocate(WalSegSz, waldir, WALDumpCloseSegment); + XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, waldir, WALDumpCloseSegment); if (!xlogreader_state) fatal_error("out of memory"); diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h index b9dad7ee51..1853516333 100644 --- a/src/include/access/xlogreader.h +++ b/src/include/access/xlogreader.h @@ -151,6 +151,7 @@ struct XLogReaderState * ---------------------------------------- */ /* variables to communicate with page reader */ + int pageSize; /* size of a page */ XLogRecPtr readPagePtr; /* page pointer to read */ int32 readLen; /* bytes requested to reader, or actual bytes * read by reader, which must be larger than @@ -243,7 +244,7 @@ struct XLogReaderState /* Get a new XLogReader */ extern XLogReaderState *XLogReaderAllocate(int wal_segment_size, - const char *waldir, + int page_size, const char *waldir, WALSegmentCleanupCB cleanup_cb); /* Free an XLogReader */ -- 2.27.0 ----Next_Part(Thu_Mar__4_11_28_53_2021_014)---- ^ permalink raw reply [nested|flat] 25+ messages in thread
* [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize. @ 2019-09-05 12:29 Kyotaro Horiguchi <[email protected]> 0 siblings, 0 replies; 25+ messages in thread From: Kyotaro Horiguchi @ 2019-09-05 12:29 UTC (permalink / raw) XLog reader facility and its callers currently relies on the same constant XLOG_BLCKSZ. This patch moves the responsibility to determine the block size to the callers of the facility. --- src/backend/access/transam/twophase.c | 3 +- src/backend/access/transam/xlog.c | 9 ++-- src/backend/access/transam/xlogreader.c | 57 ++++++++++++----------- src/backend/replication/logical/logical.c | 5 +- src/backend/replication/walsender.c | 4 +- src/bin/pg_rewind/parsexlog.c | 6 +-- src/bin/pg_waldump/pg_waldump.c | 2 +- src/include/access/xlogreader.h | 3 +- 8 files changed, 50 insertions(+), 39 deletions(-) diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c index 7d5a96a5f5..9f9e7de253 100644 --- a/src/backend/access/transam/twophase.c +++ b/src/backend/access/transam/twophase.c @@ -1325,7 +1325,8 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len) XLogReaderState *xlogreader; char *errormsg; - xlogreader = XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close); + xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, + NULL, wal_segment_close); if (!xlogreader) ereport(ERROR, diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index b517ca85ad..0822ac70cb 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -1222,7 +1222,10 @@ XLogInsertRecord(XLogRecData *rdata, appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len); if (!debug_reader) - debug_reader = XLogReaderAllocate(wal_segment_size, NULL, NULL); + { + debug_reader = + XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, NULL); + } if (!debug_reader) { @@ -6531,8 +6534,8 @@ StartupXLOG(void) OwnLatch(&XLogCtl->recoveryWakeupLatch); /* Set up XLOG reader facility */ - xlogreader = - XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close); + xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, + NULL, wal_segment_close); if (!xlogreader) ereport(ERROR, diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c index ba3e4868f7..e5baf13086 100644 --- a/src/backend/access/transam/xlogreader.c +++ b/src/backend/access/transam/xlogreader.c @@ -72,7 +72,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...) * Returns NULL if the xlogreader couldn't be allocated. */ XLogReaderState * -XLogReaderAllocate(int wal_segment_size, const char *waldir, +XLogReaderAllocate(int wal_segment_size, int page_size, const char *waldir, WALSegmentCleanupCB cleanup_cb) { XLogReaderState *state; @@ -87,6 +87,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir, state->cleanup_cb = cleanup_cb; state->max_block_id = -1; + state->pageSize = page_size; /* * Permanently allocate readBuf. We do it this way, rather than just @@ -95,7 +96,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir, * isn't guaranteed to have any particular alignment, whereas * palloc_extended() will provide MAXALIGN'd storage. */ - state->readBuf = (char *) palloc_extended(XLOG_BLCKSZ, + state->readBuf = (char *) palloc_extended(state->pageSize, MCXT_ALLOC_NO_OOM); if (!state->readBuf) { @@ -163,7 +164,7 @@ XLogReaderFree(XLogReaderState *state) * readRecordBufSize is set to the new buffer size. * * To avoid useless small increases, round its size to a multiple of - * XLOG_BLCKSZ, and make sure it's at least 5*Max(BLCKSZ, XLOG_BLCKSZ) to start + * page size, and make sure it's at least 5*Max(BLCKSZ, <page size>) to start * with. (That is enough for all "normal" records, but very large commit or * abort records might need more space.) */ @@ -172,8 +173,8 @@ allocate_recordbuf(XLogReaderState *state, uint32 reclength) { uint32 newSize = reclength; - newSize += XLOG_BLCKSZ - (newSize % XLOG_BLCKSZ); - newSize = Max(newSize, 5 * Max(BLCKSZ, XLOG_BLCKSZ)); + newSize += state->pageSize - (newSize % state->pageSize); + newSize = Max(newSize, 5 * Max(BLCKSZ, state->pageSize)); #ifndef FRONTEND @@ -238,6 +239,8 @@ void XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr) { Assert(!XLogRecPtrIsInvalid(RecPtr)); + Assert(state->readBuf != NULL && + state->readBuf == (void *) MAXALIGN(state->readBuf)); ResetDecoder(state); @@ -366,15 +369,15 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) XLogPageHeader pageHeader; targetPagePtr = - state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ); - targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ; + state->ReadRecPtr - (state->ReadRecPtr % state->pageSize); + targetRecOff = state->ReadRecPtr % state->pageSize; /* * Check if we have enough data. For the first record in the page, * the requesting length doesn't contain page header. */ if (XLogNeedData(state, targetPagePtr, - Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ), + Min(targetRecOff + SizeOfXLogRecord, state->pageSize), targetRecOff != 0)) return XLREAD_NEED_DATA; @@ -421,7 +424,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) * verified that we got the whole header. */ prec = (XLogRecord *) (state->readBuf + - state->ReadRecPtr % XLOG_BLCKSZ); + state->ReadRecPtr % state->pageSize); total_len = prec->xl_tot_len; /* @@ -432,7 +435,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) * ensure that we enter the XLREAD_CONTINUATION state below; * otherwise we might fail to apply ValidXLogRecordHeader at all. */ - if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord) + if (targetRecOff <= state->pageSize - SizeOfXLogRecord) { if (!ValidXLogRecordHeader(state, state->ReadRecPtr, state->PrevRecPtr, prec)) @@ -477,10 +480,10 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) * available */ targetPagePtr = - state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ); - targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ; + state->ReadRecPtr - (state->ReadRecPtr % state->pageSize); + targetRecOff = state->ReadRecPtr % state->pageSize; - request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ); + request_len = Min(targetRecOff + total_len, state->pageSize); record_len = request_len - targetRecOff; /* ReadRecPtr contains page header */ @@ -549,7 +552,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) /* Calculate pointer to beginning of next page */ state->recordContRecPtr = state->ReadRecPtr + record_len; - Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0); + Assert(state->recordContRecPtr % state->pageSize == 0); state->readRecordState = XLREAD_CONTINUATION; } @@ -576,7 +579,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) /* this request contains page header */ Assert (targetPagePtr != 0); if (XLogNeedData(state, targetPagePtr, - Min(state->recordRemainLen, XLOG_BLCKSZ), + Min(state->recordRemainLen, state->pageSize), false)) return XLREAD_NEED_DATA; @@ -628,7 +631,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) Assert(state->readLen >= pageHeaderSize); contdata = (char *) state->readBuf + pageHeaderSize; - record_len = XLOG_BLCKSZ - pageHeaderSize; + record_len = state->pageSize - pageHeaderSize; if (pageHeader->xlp_rem_len < record_len) record_len = pageHeader->xlp_rem_len; @@ -655,7 +658,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) } /* Calculate pointer to beginning of next page, and continue */ - state->recordContRecPtr += XLOG_BLCKSZ; + state->recordContRecPtr += state->pageSize; } /* targetPagePtr is pointing the last-read page here */ @@ -789,10 +792,10 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen, XLogPageHeaderSize((XLogPageHeader) state->readBuf); addLen = pageHeaderSize; - if (reqLen + pageHeaderSize <= XLOG_BLCKSZ) + if (reqLen + pageHeaderSize <= state->pageSize) addLen = pageHeaderSize; else - addLen = XLOG_BLCKSZ - reqLen; + addLen = state->pageSize - reqLen; Assert(addLen >= 0); } @@ -805,7 +808,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen, /* Data is not in our buffer, request the caller for it. */ XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize); targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize); - Assert((pageptr % XLOG_BLCKSZ) == 0); + Assert((pageptr % state->pageSize) == 0); /* * Every time we request to load new data of a page to the caller, even if @@ -829,7 +832,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen, * will not come back here, but will request the actual target page. */ state->readPagePtr = pageptr - targetPageOff; - state->readLen = XLOG_BLCKSZ; + state->readLen = state->pageSize; return true; } @@ -962,7 +965,7 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr, int32 offset; XLogPageHeader hdr = (XLogPageHeader) phdr; - Assert((recptr % XLOG_BLCKSZ) == 0); + Assert((recptr % state->pageSize) == 0); XLByteToSeg(recptr, segno, state->segcxt.ws_segsize); offset = XLogSegmentOffset(recptr, state->segcxt.ws_segsize); @@ -1016,10 +1019,10 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr, "WAL file is from different database system: incorrect segment size in page header"); return false; } - else if (longhdr->xlp_xlog_blcksz != XLOG_BLCKSZ) + else if (longhdr->xlp_xlog_blcksz != state->pageSize) { report_invalid_record(state, - "WAL file is from different database system: incorrect XLOG_BLCKSZ in page header"); + "WAL file is from different database system: incorrect block size in page header"); return false; } } @@ -1140,7 +1143,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr, * XLogNeedData() is prepared to handle that and will read at * least short page-header worth of data */ - targetRecOff = tmpRecPtr % XLOG_BLCKSZ; + targetRecOff = tmpRecPtr % state->pageSize; /* scroll back to page boundary */ targetPagePtr = tmpRecPtr - targetRecOff; @@ -1174,8 +1177,8 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr, * * Note that record headers are MAXALIGN'ed */ - if (MAXALIGN(header->xlp_rem_len) >= (XLOG_BLCKSZ - pageHeaderSize)) - tmpRecPtr = targetPagePtr + XLOG_BLCKSZ; + if (MAXALIGN(header->xlp_rem_len) >= (state->pageSize - pageHeaderSize)) + tmpRecPtr = targetPagePtr + state->pageSize; else { /* diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c index 9cbd8afa49..f2dd495310 100644 --- a/src/backend/replication/logical/logical.c +++ b/src/backend/replication/logical/logical.c @@ -199,11 +199,14 @@ StartupDecodingContext(List *output_plugin_options, ctx->slot = slot; - ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, cleanup_cb); + ctx->reader = + XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, cleanup_cb); + if (!ctx->reader) ereport(ERROR, (errcode(ERRCODE_OUT_OF_MEMORY), errmsg("out of memory"))); + ctx->reader->readBuf = palloc(XLOG_BLCKSZ); ctx->page_read = page_read; ctx->reorder = ReorderBufferAllocate(); diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c index 062093727d..8e95600ef7 100644 --- a/src/backend/replication/walsender.c +++ b/src/backend/replication/walsender.c @@ -579,8 +579,8 @@ StartReplication(StartReplicationCmd *cmd) errmsg("IDENTIFY_SYSTEM has not been run before START_REPLICATION"))); /* create xlogreader for physical replication */ - xlogreader = - XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close); + xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, + NULL, wal_segment_close); if (!xlogreader) ereport(ERROR, diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c index 712c85281c..8a6209b061 100644 --- a/src/bin/pg_rewind/parsexlog.c +++ b/src/bin/pg_rewind/parsexlog.c @@ -61,7 +61,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex, XLogReaderState *xlogreader; char *errormsg; - xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL); + xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL); if (xlogreader == NULL) pg_fatal("out of memory"); @@ -121,7 +121,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex, char *errormsg; XLogRecPtr endptr; - xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL); + xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL); if (xlogreader == NULL) pg_fatal("out of memory"); @@ -181,7 +181,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex, forkptr += SizeOfXLogShortPHD; } - xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL); + xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL); if (xlogreader == NULL) pg_fatal("out of memory"); diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c index f514de6b13..b635537659 100644 --- a/src/bin/pg_waldump/pg_waldump.c +++ b/src/bin/pg_waldump/pg_waldump.c @@ -1041,7 +1041,7 @@ main(int argc, char **argv) /* we have everything we need, start reading */ xlogreader_state = - XLogReaderAllocate(WalSegSz, waldir, WALDumpCloseSegment); + XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, waldir, WALDumpCloseSegment); if (!xlogreader_state) fatal_error("out of memory"); diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h index b9dad7ee51..1853516333 100644 --- a/src/include/access/xlogreader.h +++ b/src/include/access/xlogreader.h @@ -151,6 +151,7 @@ struct XLogReaderState * ---------------------------------------- */ /* variables to communicate with page reader */ + int pageSize; /* size of a page */ XLogRecPtr readPagePtr; /* page pointer to read */ int32 readLen; /* bytes requested to reader, or actual bytes * read by reader, which must be larger than @@ -243,7 +244,7 @@ struct XLogReaderState /* Get a new XLogReader */ extern XLogReaderState *XLogReaderAllocate(int wal_segment_size, - const char *waldir, + int page_size, const char *waldir, WALSegmentCleanupCB cleanup_cb); /* Free an XLogReader */ -- 2.27.0 ----Next_Part(Thu_Mar__4_11_28_53_2021_014)---- ^ permalink raw reply [nested|flat] 25+ messages in thread
* [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize. @ 2019-09-05 12:29 Kyotaro Horiguchi <[email protected]> 0 siblings, 0 replies; 25+ messages in thread From: Kyotaro Horiguchi @ 2019-09-05 12:29 UTC (permalink / raw) XLog reader facility and its callers currently relies on the same constant XLOG_BLCKSZ. This patch moves the responsibility to determine the block size to the callers of the facility. --- src/backend/access/transam/twophase.c | 3 +- src/backend/access/transam/xlog.c | 9 ++-- src/backend/access/transam/xlogreader.c | 57 ++++++++++++----------- src/backend/replication/logical/logical.c | 5 +- src/backend/replication/walsender.c | 4 +- src/bin/pg_rewind/parsexlog.c | 6 +-- src/bin/pg_waldump/pg_waldump.c | 2 +- src/include/access/xlogreader.h | 3 +- 8 files changed, 50 insertions(+), 39 deletions(-) diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c index 7d5a96a5f5..9f9e7de253 100644 --- a/src/backend/access/transam/twophase.c +++ b/src/backend/access/transam/twophase.c @@ -1325,7 +1325,8 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len) XLogReaderState *xlogreader; char *errormsg; - xlogreader = XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close); + xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, + NULL, wal_segment_close); if (!xlogreader) ereport(ERROR, diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index b517ca85ad..0822ac70cb 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -1222,7 +1222,10 @@ XLogInsertRecord(XLogRecData *rdata, appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len); if (!debug_reader) - debug_reader = XLogReaderAllocate(wal_segment_size, NULL, NULL); + { + debug_reader = + XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, NULL); + } if (!debug_reader) { @@ -6531,8 +6534,8 @@ StartupXLOG(void) OwnLatch(&XLogCtl->recoveryWakeupLatch); /* Set up XLOG reader facility */ - xlogreader = - XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close); + xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, + NULL, wal_segment_close); if (!xlogreader) ereport(ERROR, diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c index ba3e4868f7..e5baf13086 100644 --- a/src/backend/access/transam/xlogreader.c +++ b/src/backend/access/transam/xlogreader.c @@ -72,7 +72,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...) * Returns NULL if the xlogreader couldn't be allocated. */ XLogReaderState * -XLogReaderAllocate(int wal_segment_size, const char *waldir, +XLogReaderAllocate(int wal_segment_size, int page_size, const char *waldir, WALSegmentCleanupCB cleanup_cb) { XLogReaderState *state; @@ -87,6 +87,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir, state->cleanup_cb = cleanup_cb; state->max_block_id = -1; + state->pageSize = page_size; /* * Permanently allocate readBuf. We do it this way, rather than just @@ -95,7 +96,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir, * isn't guaranteed to have any particular alignment, whereas * palloc_extended() will provide MAXALIGN'd storage. */ - state->readBuf = (char *) palloc_extended(XLOG_BLCKSZ, + state->readBuf = (char *) palloc_extended(state->pageSize, MCXT_ALLOC_NO_OOM); if (!state->readBuf) { @@ -163,7 +164,7 @@ XLogReaderFree(XLogReaderState *state) * readRecordBufSize is set to the new buffer size. * * To avoid useless small increases, round its size to a multiple of - * XLOG_BLCKSZ, and make sure it's at least 5*Max(BLCKSZ, XLOG_BLCKSZ) to start + * page size, and make sure it's at least 5*Max(BLCKSZ, <page size>) to start * with. (That is enough for all "normal" records, but very large commit or * abort records might need more space.) */ @@ -172,8 +173,8 @@ allocate_recordbuf(XLogReaderState *state, uint32 reclength) { uint32 newSize = reclength; - newSize += XLOG_BLCKSZ - (newSize % XLOG_BLCKSZ); - newSize = Max(newSize, 5 * Max(BLCKSZ, XLOG_BLCKSZ)); + newSize += state->pageSize - (newSize % state->pageSize); + newSize = Max(newSize, 5 * Max(BLCKSZ, state->pageSize)); #ifndef FRONTEND @@ -238,6 +239,8 @@ void XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr) { Assert(!XLogRecPtrIsInvalid(RecPtr)); + Assert(state->readBuf != NULL && + state->readBuf == (void *) MAXALIGN(state->readBuf)); ResetDecoder(state); @@ -366,15 +369,15 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) XLogPageHeader pageHeader; targetPagePtr = - state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ); - targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ; + state->ReadRecPtr - (state->ReadRecPtr % state->pageSize); + targetRecOff = state->ReadRecPtr % state->pageSize; /* * Check if we have enough data. For the first record in the page, * the requesting length doesn't contain page header. */ if (XLogNeedData(state, targetPagePtr, - Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ), + Min(targetRecOff + SizeOfXLogRecord, state->pageSize), targetRecOff != 0)) return XLREAD_NEED_DATA; @@ -421,7 +424,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) * verified that we got the whole header. */ prec = (XLogRecord *) (state->readBuf + - state->ReadRecPtr % XLOG_BLCKSZ); + state->ReadRecPtr % state->pageSize); total_len = prec->xl_tot_len; /* @@ -432,7 +435,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) * ensure that we enter the XLREAD_CONTINUATION state below; * otherwise we might fail to apply ValidXLogRecordHeader at all. */ - if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord) + if (targetRecOff <= state->pageSize - SizeOfXLogRecord) { if (!ValidXLogRecordHeader(state, state->ReadRecPtr, state->PrevRecPtr, prec)) @@ -477,10 +480,10 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) * available */ targetPagePtr = - state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ); - targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ; + state->ReadRecPtr - (state->ReadRecPtr % state->pageSize); + targetRecOff = state->ReadRecPtr % state->pageSize; - request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ); + request_len = Min(targetRecOff + total_len, state->pageSize); record_len = request_len - targetRecOff; /* ReadRecPtr contains page header */ @@ -549,7 +552,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) /* Calculate pointer to beginning of next page */ state->recordContRecPtr = state->ReadRecPtr + record_len; - Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0); + Assert(state->recordContRecPtr % state->pageSize == 0); state->readRecordState = XLREAD_CONTINUATION; } @@ -576,7 +579,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) /* this request contains page header */ Assert (targetPagePtr != 0); if (XLogNeedData(state, targetPagePtr, - Min(state->recordRemainLen, XLOG_BLCKSZ), + Min(state->recordRemainLen, state->pageSize), false)) return XLREAD_NEED_DATA; @@ -628,7 +631,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) Assert(state->readLen >= pageHeaderSize); contdata = (char *) state->readBuf + pageHeaderSize; - record_len = XLOG_BLCKSZ - pageHeaderSize; + record_len = state->pageSize - pageHeaderSize; if (pageHeader->xlp_rem_len < record_len) record_len = pageHeader->xlp_rem_len; @@ -655,7 +658,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) } /* Calculate pointer to beginning of next page, and continue */ - state->recordContRecPtr += XLOG_BLCKSZ; + state->recordContRecPtr += state->pageSize; } /* targetPagePtr is pointing the last-read page here */ @@ -789,10 +792,10 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen, XLogPageHeaderSize((XLogPageHeader) state->readBuf); addLen = pageHeaderSize; - if (reqLen + pageHeaderSize <= XLOG_BLCKSZ) + if (reqLen + pageHeaderSize <= state->pageSize) addLen = pageHeaderSize; else - addLen = XLOG_BLCKSZ - reqLen; + addLen = state->pageSize - reqLen; Assert(addLen >= 0); } @@ -805,7 +808,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen, /* Data is not in our buffer, request the caller for it. */ XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize); targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize); - Assert((pageptr % XLOG_BLCKSZ) == 0); + Assert((pageptr % state->pageSize) == 0); /* * Every time we request to load new data of a page to the caller, even if @@ -829,7 +832,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen, * will not come back here, but will request the actual target page. */ state->readPagePtr = pageptr - targetPageOff; - state->readLen = XLOG_BLCKSZ; + state->readLen = state->pageSize; return true; } @@ -962,7 +965,7 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr, int32 offset; XLogPageHeader hdr = (XLogPageHeader) phdr; - Assert((recptr % XLOG_BLCKSZ) == 0); + Assert((recptr % state->pageSize) == 0); XLByteToSeg(recptr, segno, state->segcxt.ws_segsize); offset = XLogSegmentOffset(recptr, state->segcxt.ws_segsize); @@ -1016,10 +1019,10 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr, "WAL file is from different database system: incorrect segment size in page header"); return false; } - else if (longhdr->xlp_xlog_blcksz != XLOG_BLCKSZ) + else if (longhdr->xlp_xlog_blcksz != state->pageSize) { report_invalid_record(state, - "WAL file is from different database system: incorrect XLOG_BLCKSZ in page header"); + "WAL file is from different database system: incorrect block size in page header"); return false; } } @@ -1140,7 +1143,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr, * XLogNeedData() is prepared to handle that and will read at * least short page-header worth of data */ - targetRecOff = tmpRecPtr % XLOG_BLCKSZ; + targetRecOff = tmpRecPtr % state->pageSize; /* scroll back to page boundary */ targetPagePtr = tmpRecPtr - targetRecOff; @@ -1174,8 +1177,8 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr, * * Note that record headers are MAXALIGN'ed */ - if (MAXALIGN(header->xlp_rem_len) >= (XLOG_BLCKSZ - pageHeaderSize)) - tmpRecPtr = targetPagePtr + XLOG_BLCKSZ; + if (MAXALIGN(header->xlp_rem_len) >= (state->pageSize - pageHeaderSize)) + tmpRecPtr = targetPagePtr + state->pageSize; else { /* diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c index 9cbd8afa49..f2dd495310 100644 --- a/src/backend/replication/logical/logical.c +++ b/src/backend/replication/logical/logical.c @@ -199,11 +199,14 @@ StartupDecodingContext(List *output_plugin_options, ctx->slot = slot; - ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, cleanup_cb); + ctx->reader = + XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, cleanup_cb); + if (!ctx->reader) ereport(ERROR, (errcode(ERRCODE_OUT_OF_MEMORY), errmsg("out of memory"))); + ctx->reader->readBuf = palloc(XLOG_BLCKSZ); ctx->page_read = page_read; ctx->reorder = ReorderBufferAllocate(); diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c index 062093727d..8e95600ef7 100644 --- a/src/backend/replication/walsender.c +++ b/src/backend/replication/walsender.c @@ -579,8 +579,8 @@ StartReplication(StartReplicationCmd *cmd) errmsg("IDENTIFY_SYSTEM has not been run before START_REPLICATION"))); /* create xlogreader for physical replication */ - xlogreader = - XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close); + xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, + NULL, wal_segment_close); if (!xlogreader) ereport(ERROR, diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c index 712c85281c..8a6209b061 100644 --- a/src/bin/pg_rewind/parsexlog.c +++ b/src/bin/pg_rewind/parsexlog.c @@ -61,7 +61,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex, XLogReaderState *xlogreader; char *errormsg; - xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL); + xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL); if (xlogreader == NULL) pg_fatal("out of memory"); @@ -121,7 +121,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex, char *errormsg; XLogRecPtr endptr; - xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL); + xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL); if (xlogreader == NULL) pg_fatal("out of memory"); @@ -181,7 +181,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex, forkptr += SizeOfXLogShortPHD; } - xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL); + xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL); if (xlogreader == NULL) pg_fatal("out of memory"); diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c index f514de6b13..b635537659 100644 --- a/src/bin/pg_waldump/pg_waldump.c +++ b/src/bin/pg_waldump/pg_waldump.c @@ -1041,7 +1041,7 @@ main(int argc, char **argv) /* we have everything we need, start reading */ xlogreader_state = - XLogReaderAllocate(WalSegSz, waldir, WALDumpCloseSegment); + XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, waldir, WALDumpCloseSegment); if (!xlogreader_state) fatal_error("out of memory"); diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h index b9dad7ee51..1853516333 100644 --- a/src/include/access/xlogreader.h +++ b/src/include/access/xlogreader.h @@ -151,6 +151,7 @@ struct XLogReaderState * ---------------------------------------- */ /* variables to communicate with page reader */ + int pageSize; /* size of a page */ XLogRecPtr readPagePtr; /* page pointer to read */ int32 readLen; /* bytes requested to reader, or actual bytes * read by reader, which must be larger than @@ -243,7 +244,7 @@ struct XLogReaderState /* Get a new XLogReader */ extern XLogReaderState *XLogReaderAllocate(int wal_segment_size, - const char *waldir, + int page_size, const char *waldir, WALSegmentCleanupCB cleanup_cb); /* Free an XLogReader */ -- 2.27.0 ----Next_Part(Thu_Mar__4_11_28_53_2021_014)---- ^ permalink raw reply [nested|flat] 25+ messages in thread
* [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize. @ 2019-09-05 12:29 Kyotaro Horiguchi <[email protected]> 0 siblings, 0 replies; 25+ messages in thread From: Kyotaro Horiguchi @ 2019-09-05 12:29 UTC (permalink / raw) XLog reader facility and its callers currently relies on the same constant XLOG_BLCKSZ. This patch moves the responsibility to determine the block size to the callers of the facility. --- src/backend/access/transam/twophase.c | 3 +- src/backend/access/transam/xlog.c | 9 ++-- src/backend/access/transam/xlogreader.c | 57 ++++++++++++----------- src/backend/replication/logical/logical.c | 5 +- src/backend/replication/walsender.c | 4 +- src/bin/pg_rewind/parsexlog.c | 6 +-- src/bin/pg_waldump/pg_waldump.c | 2 +- src/include/access/xlogreader.h | 3 +- 8 files changed, 50 insertions(+), 39 deletions(-) diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c index 7d5a96a5f5..9f9e7de253 100644 --- a/src/backend/access/transam/twophase.c +++ b/src/backend/access/transam/twophase.c @@ -1325,7 +1325,8 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len) XLogReaderState *xlogreader; char *errormsg; - xlogreader = XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close); + xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, + NULL, wal_segment_close); if (!xlogreader) ereport(ERROR, diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index b517ca85ad..0822ac70cb 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -1222,7 +1222,10 @@ XLogInsertRecord(XLogRecData *rdata, appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len); if (!debug_reader) - debug_reader = XLogReaderAllocate(wal_segment_size, NULL, NULL); + { + debug_reader = + XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, NULL); + } if (!debug_reader) { @@ -6531,8 +6534,8 @@ StartupXLOG(void) OwnLatch(&XLogCtl->recoveryWakeupLatch); /* Set up XLOG reader facility */ - xlogreader = - XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close); + xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, + NULL, wal_segment_close); if (!xlogreader) ereport(ERROR, diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c index ba3e4868f7..e5baf13086 100644 --- a/src/backend/access/transam/xlogreader.c +++ b/src/backend/access/transam/xlogreader.c @@ -72,7 +72,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...) * Returns NULL if the xlogreader couldn't be allocated. */ XLogReaderState * -XLogReaderAllocate(int wal_segment_size, const char *waldir, +XLogReaderAllocate(int wal_segment_size, int page_size, const char *waldir, WALSegmentCleanupCB cleanup_cb) { XLogReaderState *state; @@ -87,6 +87,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir, state->cleanup_cb = cleanup_cb; state->max_block_id = -1; + state->pageSize = page_size; /* * Permanently allocate readBuf. We do it this way, rather than just @@ -95,7 +96,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir, * isn't guaranteed to have any particular alignment, whereas * palloc_extended() will provide MAXALIGN'd storage. */ - state->readBuf = (char *) palloc_extended(XLOG_BLCKSZ, + state->readBuf = (char *) palloc_extended(state->pageSize, MCXT_ALLOC_NO_OOM); if (!state->readBuf) { @@ -163,7 +164,7 @@ XLogReaderFree(XLogReaderState *state) * readRecordBufSize is set to the new buffer size. * * To avoid useless small increases, round its size to a multiple of - * XLOG_BLCKSZ, and make sure it's at least 5*Max(BLCKSZ, XLOG_BLCKSZ) to start + * page size, and make sure it's at least 5*Max(BLCKSZ, <page size>) to start * with. (That is enough for all "normal" records, but very large commit or * abort records might need more space.) */ @@ -172,8 +173,8 @@ allocate_recordbuf(XLogReaderState *state, uint32 reclength) { uint32 newSize = reclength; - newSize += XLOG_BLCKSZ - (newSize % XLOG_BLCKSZ); - newSize = Max(newSize, 5 * Max(BLCKSZ, XLOG_BLCKSZ)); + newSize += state->pageSize - (newSize % state->pageSize); + newSize = Max(newSize, 5 * Max(BLCKSZ, state->pageSize)); #ifndef FRONTEND @@ -238,6 +239,8 @@ void XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr) { Assert(!XLogRecPtrIsInvalid(RecPtr)); + Assert(state->readBuf != NULL && + state->readBuf == (void *) MAXALIGN(state->readBuf)); ResetDecoder(state); @@ -366,15 +369,15 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) XLogPageHeader pageHeader; targetPagePtr = - state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ); - targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ; + state->ReadRecPtr - (state->ReadRecPtr % state->pageSize); + targetRecOff = state->ReadRecPtr % state->pageSize; /* * Check if we have enough data. For the first record in the page, * the requesting length doesn't contain page header. */ if (XLogNeedData(state, targetPagePtr, - Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ), + Min(targetRecOff + SizeOfXLogRecord, state->pageSize), targetRecOff != 0)) return XLREAD_NEED_DATA; @@ -421,7 +424,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) * verified that we got the whole header. */ prec = (XLogRecord *) (state->readBuf + - state->ReadRecPtr % XLOG_BLCKSZ); + state->ReadRecPtr % state->pageSize); total_len = prec->xl_tot_len; /* @@ -432,7 +435,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) * ensure that we enter the XLREAD_CONTINUATION state below; * otherwise we might fail to apply ValidXLogRecordHeader at all. */ - if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord) + if (targetRecOff <= state->pageSize - SizeOfXLogRecord) { if (!ValidXLogRecordHeader(state, state->ReadRecPtr, state->PrevRecPtr, prec)) @@ -477,10 +480,10 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) * available */ targetPagePtr = - state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ); - targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ; + state->ReadRecPtr - (state->ReadRecPtr % state->pageSize); + targetRecOff = state->ReadRecPtr % state->pageSize; - request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ); + request_len = Min(targetRecOff + total_len, state->pageSize); record_len = request_len - targetRecOff; /* ReadRecPtr contains page header */ @@ -549,7 +552,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) /* Calculate pointer to beginning of next page */ state->recordContRecPtr = state->ReadRecPtr + record_len; - Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0); + Assert(state->recordContRecPtr % state->pageSize == 0); state->readRecordState = XLREAD_CONTINUATION; } @@ -576,7 +579,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) /* this request contains page header */ Assert (targetPagePtr != 0); if (XLogNeedData(state, targetPagePtr, - Min(state->recordRemainLen, XLOG_BLCKSZ), + Min(state->recordRemainLen, state->pageSize), false)) return XLREAD_NEED_DATA; @@ -628,7 +631,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) Assert(state->readLen >= pageHeaderSize); contdata = (char *) state->readBuf + pageHeaderSize; - record_len = XLOG_BLCKSZ - pageHeaderSize; + record_len = state->pageSize - pageHeaderSize; if (pageHeader->xlp_rem_len < record_len) record_len = pageHeader->xlp_rem_len; @@ -655,7 +658,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) } /* Calculate pointer to beginning of next page, and continue */ - state->recordContRecPtr += XLOG_BLCKSZ; + state->recordContRecPtr += state->pageSize; } /* targetPagePtr is pointing the last-read page here */ @@ -789,10 +792,10 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen, XLogPageHeaderSize((XLogPageHeader) state->readBuf); addLen = pageHeaderSize; - if (reqLen + pageHeaderSize <= XLOG_BLCKSZ) + if (reqLen + pageHeaderSize <= state->pageSize) addLen = pageHeaderSize; else - addLen = XLOG_BLCKSZ - reqLen; + addLen = state->pageSize - reqLen; Assert(addLen >= 0); } @@ -805,7 +808,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen, /* Data is not in our buffer, request the caller for it. */ XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize); targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize); - Assert((pageptr % XLOG_BLCKSZ) == 0); + Assert((pageptr % state->pageSize) == 0); /* * Every time we request to load new data of a page to the caller, even if @@ -829,7 +832,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen, * will not come back here, but will request the actual target page. */ state->readPagePtr = pageptr - targetPageOff; - state->readLen = XLOG_BLCKSZ; + state->readLen = state->pageSize; return true; } @@ -962,7 +965,7 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr, int32 offset; XLogPageHeader hdr = (XLogPageHeader) phdr; - Assert((recptr % XLOG_BLCKSZ) == 0); + Assert((recptr % state->pageSize) == 0); XLByteToSeg(recptr, segno, state->segcxt.ws_segsize); offset = XLogSegmentOffset(recptr, state->segcxt.ws_segsize); @@ -1016,10 +1019,10 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr, "WAL file is from different database system: incorrect segment size in page header"); return false; } - else if (longhdr->xlp_xlog_blcksz != XLOG_BLCKSZ) + else if (longhdr->xlp_xlog_blcksz != state->pageSize) { report_invalid_record(state, - "WAL file is from different database system: incorrect XLOG_BLCKSZ in page header"); + "WAL file is from different database system: incorrect block size in page header"); return false; } } @@ -1140,7 +1143,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr, * XLogNeedData() is prepared to handle that and will read at * least short page-header worth of data */ - targetRecOff = tmpRecPtr % XLOG_BLCKSZ; + targetRecOff = tmpRecPtr % state->pageSize; /* scroll back to page boundary */ targetPagePtr = tmpRecPtr - targetRecOff; @@ -1174,8 +1177,8 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr, * * Note that record headers are MAXALIGN'ed */ - if (MAXALIGN(header->xlp_rem_len) >= (XLOG_BLCKSZ - pageHeaderSize)) - tmpRecPtr = targetPagePtr + XLOG_BLCKSZ; + if (MAXALIGN(header->xlp_rem_len) >= (state->pageSize - pageHeaderSize)) + tmpRecPtr = targetPagePtr + state->pageSize; else { /* diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c index 9cbd8afa49..f2dd495310 100644 --- a/src/backend/replication/logical/logical.c +++ b/src/backend/replication/logical/logical.c @@ -199,11 +199,14 @@ StartupDecodingContext(List *output_plugin_options, ctx->slot = slot; - ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, cleanup_cb); + ctx->reader = + XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, cleanup_cb); + if (!ctx->reader) ereport(ERROR, (errcode(ERRCODE_OUT_OF_MEMORY), errmsg("out of memory"))); + ctx->reader->readBuf = palloc(XLOG_BLCKSZ); ctx->page_read = page_read; ctx->reorder = ReorderBufferAllocate(); diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c index 062093727d..8e95600ef7 100644 --- a/src/backend/replication/walsender.c +++ b/src/backend/replication/walsender.c @@ -579,8 +579,8 @@ StartReplication(StartReplicationCmd *cmd) errmsg("IDENTIFY_SYSTEM has not been run before START_REPLICATION"))); /* create xlogreader for physical replication */ - xlogreader = - XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close); + xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, + NULL, wal_segment_close); if (!xlogreader) ereport(ERROR, diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c index 712c85281c..8a6209b061 100644 --- a/src/bin/pg_rewind/parsexlog.c +++ b/src/bin/pg_rewind/parsexlog.c @@ -61,7 +61,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex, XLogReaderState *xlogreader; char *errormsg; - xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL); + xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL); if (xlogreader == NULL) pg_fatal("out of memory"); @@ -121,7 +121,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex, char *errormsg; XLogRecPtr endptr; - xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL); + xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL); if (xlogreader == NULL) pg_fatal("out of memory"); @@ -181,7 +181,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex, forkptr += SizeOfXLogShortPHD; } - xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL); + xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL); if (xlogreader == NULL) pg_fatal("out of memory"); diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c index f514de6b13..b635537659 100644 --- a/src/bin/pg_waldump/pg_waldump.c +++ b/src/bin/pg_waldump/pg_waldump.c @@ -1041,7 +1041,7 @@ main(int argc, char **argv) /* we have everything we need, start reading */ xlogreader_state = - XLogReaderAllocate(WalSegSz, waldir, WALDumpCloseSegment); + XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, waldir, WALDumpCloseSegment); if (!xlogreader_state) fatal_error("out of memory"); diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h index b9dad7ee51..1853516333 100644 --- a/src/include/access/xlogreader.h +++ b/src/include/access/xlogreader.h @@ -151,6 +151,7 @@ struct XLogReaderState * ---------------------------------------- */ /* variables to communicate with page reader */ + int pageSize; /* size of a page */ XLogRecPtr readPagePtr; /* page pointer to read */ int32 readLen; /* bytes requested to reader, or actual bytes * read by reader, which must be larger than @@ -243,7 +244,7 @@ struct XLogReaderState /* Get a new XLogReader */ extern XLogReaderState *XLogReaderAllocate(int wal_segment_size, - const char *waldir, + int page_size, const char *waldir, WALSegmentCleanupCB cleanup_cb); /* Free an XLogReader */ -- 2.27.0 ----Next_Part(Thu_Mar__4_11_28_53_2021_014)---- ^ permalink raw reply [nested|flat] 25+ messages in thread
* [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize. @ 2019-09-05 12:29 Kyotaro Horiguchi <[email protected]> 0 siblings, 0 replies; 25+ messages in thread From: Kyotaro Horiguchi @ 2019-09-05 12:29 UTC (permalink / raw) XLog reader facility and its callers currently relies on the same constant XLOG_BLCKSZ. This patch moves the responsibility to determine the block size to the callers of the facility. --- src/backend/access/transam/twophase.c | 3 +- src/backend/access/transam/xlog.c | 9 ++-- src/backend/access/transam/xlogreader.c | 57 ++++++++++++----------- src/backend/replication/logical/logical.c | 5 +- src/backend/replication/walsender.c | 4 +- src/bin/pg_rewind/parsexlog.c | 6 +-- src/bin/pg_waldump/pg_waldump.c | 2 +- src/include/access/xlogreader.h | 3 +- 8 files changed, 50 insertions(+), 39 deletions(-) diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c index 7d5a96a5f5..9f9e7de253 100644 --- a/src/backend/access/transam/twophase.c +++ b/src/backend/access/transam/twophase.c @@ -1325,7 +1325,8 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len) XLogReaderState *xlogreader; char *errormsg; - xlogreader = XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close); + xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, + NULL, wal_segment_close); if (!xlogreader) ereport(ERROR, diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index b517ca85ad..0822ac70cb 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -1222,7 +1222,10 @@ XLogInsertRecord(XLogRecData *rdata, appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len); if (!debug_reader) - debug_reader = XLogReaderAllocate(wal_segment_size, NULL, NULL); + { + debug_reader = + XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, NULL); + } if (!debug_reader) { @@ -6531,8 +6534,8 @@ StartupXLOG(void) OwnLatch(&XLogCtl->recoveryWakeupLatch); /* Set up XLOG reader facility */ - xlogreader = - XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close); + xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, + NULL, wal_segment_close); if (!xlogreader) ereport(ERROR, diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c index ba3e4868f7..e5baf13086 100644 --- a/src/backend/access/transam/xlogreader.c +++ b/src/backend/access/transam/xlogreader.c @@ -72,7 +72,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...) * Returns NULL if the xlogreader couldn't be allocated. */ XLogReaderState * -XLogReaderAllocate(int wal_segment_size, const char *waldir, +XLogReaderAllocate(int wal_segment_size, int page_size, const char *waldir, WALSegmentCleanupCB cleanup_cb) { XLogReaderState *state; @@ -87,6 +87,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir, state->cleanup_cb = cleanup_cb; state->max_block_id = -1; + state->pageSize = page_size; /* * Permanently allocate readBuf. We do it this way, rather than just @@ -95,7 +96,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir, * isn't guaranteed to have any particular alignment, whereas * palloc_extended() will provide MAXALIGN'd storage. */ - state->readBuf = (char *) palloc_extended(XLOG_BLCKSZ, + state->readBuf = (char *) palloc_extended(state->pageSize, MCXT_ALLOC_NO_OOM); if (!state->readBuf) { @@ -163,7 +164,7 @@ XLogReaderFree(XLogReaderState *state) * readRecordBufSize is set to the new buffer size. * * To avoid useless small increases, round its size to a multiple of - * XLOG_BLCKSZ, and make sure it's at least 5*Max(BLCKSZ, XLOG_BLCKSZ) to start + * page size, and make sure it's at least 5*Max(BLCKSZ, <page size>) to start * with. (That is enough for all "normal" records, but very large commit or * abort records might need more space.) */ @@ -172,8 +173,8 @@ allocate_recordbuf(XLogReaderState *state, uint32 reclength) { uint32 newSize = reclength; - newSize += XLOG_BLCKSZ - (newSize % XLOG_BLCKSZ); - newSize = Max(newSize, 5 * Max(BLCKSZ, XLOG_BLCKSZ)); + newSize += state->pageSize - (newSize % state->pageSize); + newSize = Max(newSize, 5 * Max(BLCKSZ, state->pageSize)); #ifndef FRONTEND @@ -238,6 +239,8 @@ void XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr) { Assert(!XLogRecPtrIsInvalid(RecPtr)); + Assert(state->readBuf != NULL && + state->readBuf == (void *) MAXALIGN(state->readBuf)); ResetDecoder(state); @@ -366,15 +369,15 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) XLogPageHeader pageHeader; targetPagePtr = - state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ); - targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ; + state->ReadRecPtr - (state->ReadRecPtr % state->pageSize); + targetRecOff = state->ReadRecPtr % state->pageSize; /* * Check if we have enough data. For the first record in the page, * the requesting length doesn't contain page header. */ if (XLogNeedData(state, targetPagePtr, - Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ), + Min(targetRecOff + SizeOfXLogRecord, state->pageSize), targetRecOff != 0)) return XLREAD_NEED_DATA; @@ -421,7 +424,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) * verified that we got the whole header. */ prec = (XLogRecord *) (state->readBuf + - state->ReadRecPtr % XLOG_BLCKSZ); + state->ReadRecPtr % state->pageSize); total_len = prec->xl_tot_len; /* @@ -432,7 +435,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) * ensure that we enter the XLREAD_CONTINUATION state below; * otherwise we might fail to apply ValidXLogRecordHeader at all. */ - if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord) + if (targetRecOff <= state->pageSize - SizeOfXLogRecord) { if (!ValidXLogRecordHeader(state, state->ReadRecPtr, state->PrevRecPtr, prec)) @@ -477,10 +480,10 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) * available */ targetPagePtr = - state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ); - targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ; + state->ReadRecPtr - (state->ReadRecPtr % state->pageSize); + targetRecOff = state->ReadRecPtr % state->pageSize; - request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ); + request_len = Min(targetRecOff + total_len, state->pageSize); record_len = request_len - targetRecOff; /* ReadRecPtr contains page header */ @@ -549,7 +552,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) /* Calculate pointer to beginning of next page */ state->recordContRecPtr = state->ReadRecPtr + record_len; - Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0); + Assert(state->recordContRecPtr % state->pageSize == 0); state->readRecordState = XLREAD_CONTINUATION; } @@ -576,7 +579,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) /* this request contains page header */ Assert (targetPagePtr != 0); if (XLogNeedData(state, targetPagePtr, - Min(state->recordRemainLen, XLOG_BLCKSZ), + Min(state->recordRemainLen, state->pageSize), false)) return XLREAD_NEED_DATA; @@ -628,7 +631,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) Assert(state->readLen >= pageHeaderSize); contdata = (char *) state->readBuf + pageHeaderSize; - record_len = XLOG_BLCKSZ - pageHeaderSize; + record_len = state->pageSize - pageHeaderSize; if (pageHeader->xlp_rem_len < record_len) record_len = pageHeader->xlp_rem_len; @@ -655,7 +658,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) } /* Calculate pointer to beginning of next page, and continue */ - state->recordContRecPtr += XLOG_BLCKSZ; + state->recordContRecPtr += state->pageSize; } /* targetPagePtr is pointing the last-read page here */ @@ -789,10 +792,10 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen, XLogPageHeaderSize((XLogPageHeader) state->readBuf); addLen = pageHeaderSize; - if (reqLen + pageHeaderSize <= XLOG_BLCKSZ) + if (reqLen + pageHeaderSize <= state->pageSize) addLen = pageHeaderSize; else - addLen = XLOG_BLCKSZ - reqLen; + addLen = state->pageSize - reqLen; Assert(addLen >= 0); } @@ -805,7 +808,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen, /* Data is not in our buffer, request the caller for it. */ XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize); targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize); - Assert((pageptr % XLOG_BLCKSZ) == 0); + Assert((pageptr % state->pageSize) == 0); /* * Every time we request to load new data of a page to the caller, even if @@ -829,7 +832,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen, * will not come back here, but will request the actual target page. */ state->readPagePtr = pageptr - targetPageOff; - state->readLen = XLOG_BLCKSZ; + state->readLen = state->pageSize; return true; } @@ -962,7 +965,7 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr, int32 offset; XLogPageHeader hdr = (XLogPageHeader) phdr; - Assert((recptr % XLOG_BLCKSZ) == 0); + Assert((recptr % state->pageSize) == 0); XLByteToSeg(recptr, segno, state->segcxt.ws_segsize); offset = XLogSegmentOffset(recptr, state->segcxt.ws_segsize); @@ -1016,10 +1019,10 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr, "WAL file is from different database system: incorrect segment size in page header"); return false; } - else if (longhdr->xlp_xlog_blcksz != XLOG_BLCKSZ) + else if (longhdr->xlp_xlog_blcksz != state->pageSize) { report_invalid_record(state, - "WAL file is from different database system: incorrect XLOG_BLCKSZ in page header"); + "WAL file is from different database system: incorrect block size in page header"); return false; } } @@ -1140,7 +1143,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr, * XLogNeedData() is prepared to handle that and will read at * least short page-header worth of data */ - targetRecOff = tmpRecPtr % XLOG_BLCKSZ; + targetRecOff = tmpRecPtr % state->pageSize; /* scroll back to page boundary */ targetPagePtr = tmpRecPtr - targetRecOff; @@ -1174,8 +1177,8 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr, * * Note that record headers are MAXALIGN'ed */ - if (MAXALIGN(header->xlp_rem_len) >= (XLOG_BLCKSZ - pageHeaderSize)) - tmpRecPtr = targetPagePtr + XLOG_BLCKSZ; + if (MAXALIGN(header->xlp_rem_len) >= (state->pageSize - pageHeaderSize)) + tmpRecPtr = targetPagePtr + state->pageSize; else { /* diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c index 9cbd8afa49..f2dd495310 100644 --- a/src/backend/replication/logical/logical.c +++ b/src/backend/replication/logical/logical.c @@ -199,11 +199,14 @@ StartupDecodingContext(List *output_plugin_options, ctx->slot = slot; - ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, cleanup_cb); + ctx->reader = + XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, cleanup_cb); + if (!ctx->reader) ereport(ERROR, (errcode(ERRCODE_OUT_OF_MEMORY), errmsg("out of memory"))); + ctx->reader->readBuf = palloc(XLOG_BLCKSZ); ctx->page_read = page_read; ctx->reorder = ReorderBufferAllocate(); diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c index 062093727d..8e95600ef7 100644 --- a/src/backend/replication/walsender.c +++ b/src/backend/replication/walsender.c @@ -579,8 +579,8 @@ StartReplication(StartReplicationCmd *cmd) errmsg("IDENTIFY_SYSTEM has not been run before START_REPLICATION"))); /* create xlogreader for physical replication */ - xlogreader = - XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close); + xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, + NULL, wal_segment_close); if (!xlogreader) ereport(ERROR, diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c index 712c85281c..8a6209b061 100644 --- a/src/bin/pg_rewind/parsexlog.c +++ b/src/bin/pg_rewind/parsexlog.c @@ -61,7 +61,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex, XLogReaderState *xlogreader; char *errormsg; - xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL); + xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL); if (xlogreader == NULL) pg_fatal("out of memory"); @@ -121,7 +121,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex, char *errormsg; XLogRecPtr endptr; - xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL); + xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL); if (xlogreader == NULL) pg_fatal("out of memory"); @@ -181,7 +181,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex, forkptr += SizeOfXLogShortPHD; } - xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL); + xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL); if (xlogreader == NULL) pg_fatal("out of memory"); diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c index f514de6b13..b635537659 100644 --- a/src/bin/pg_waldump/pg_waldump.c +++ b/src/bin/pg_waldump/pg_waldump.c @@ -1041,7 +1041,7 @@ main(int argc, char **argv) /* we have everything we need, start reading */ xlogreader_state = - XLogReaderAllocate(WalSegSz, waldir, WALDumpCloseSegment); + XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, waldir, WALDumpCloseSegment); if (!xlogreader_state) fatal_error("out of memory"); diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h index b9dad7ee51..1853516333 100644 --- a/src/include/access/xlogreader.h +++ b/src/include/access/xlogreader.h @@ -151,6 +151,7 @@ struct XLogReaderState * ---------------------------------------- */ /* variables to communicate with page reader */ + int pageSize; /* size of a page */ XLogRecPtr readPagePtr; /* page pointer to read */ int32 readLen; /* bytes requested to reader, or actual bytes * read by reader, which must be larger than @@ -243,7 +244,7 @@ struct XLogReaderState /* Get a new XLogReader */ extern XLogReaderState *XLogReaderAllocate(int wal_segment_size, - const char *waldir, + int page_size, const char *waldir, WALSegmentCleanupCB cleanup_cb); /* Free an XLogReader */ -- 2.27.0 ----Next_Part(Thu_Mar__4_11_28_53_2021_014)---- ^ permalink raw reply [nested|flat] 25+ messages in thread
* [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize. @ 2019-09-05 12:29 Kyotaro Horiguchi <[email protected]> 0 siblings, 0 replies; 25+ messages in thread From: Kyotaro Horiguchi @ 2019-09-05 12:29 UTC (permalink / raw) XLog reader facility and its callers currently relies on the same constant XLOG_BLCKSZ. This patch moves the responsibility to determine the block size to the callers of the facility. --- src/backend/access/transam/twophase.c | 3 +- src/backend/access/transam/xlog.c | 9 ++-- src/backend/access/transam/xlogreader.c | 57 ++++++++++++----------- src/backend/replication/logical/logical.c | 5 +- src/backend/replication/walsender.c | 4 +- src/bin/pg_rewind/parsexlog.c | 6 +-- src/bin/pg_waldump/pg_waldump.c | 2 +- src/include/access/xlogreader.h | 3 +- 8 files changed, 50 insertions(+), 39 deletions(-) diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c index 7d5a96a5f5..9f9e7de253 100644 --- a/src/backend/access/transam/twophase.c +++ b/src/backend/access/transam/twophase.c @@ -1325,7 +1325,8 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len) XLogReaderState *xlogreader; char *errormsg; - xlogreader = XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close); + xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, + NULL, wal_segment_close); if (!xlogreader) ereport(ERROR, diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index b517ca85ad..0822ac70cb 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -1222,7 +1222,10 @@ XLogInsertRecord(XLogRecData *rdata, appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len); if (!debug_reader) - debug_reader = XLogReaderAllocate(wal_segment_size, NULL, NULL); + { + debug_reader = + XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, NULL); + } if (!debug_reader) { @@ -6531,8 +6534,8 @@ StartupXLOG(void) OwnLatch(&XLogCtl->recoveryWakeupLatch); /* Set up XLOG reader facility */ - xlogreader = - XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close); + xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, + NULL, wal_segment_close); if (!xlogreader) ereport(ERROR, diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c index ba3e4868f7..e5baf13086 100644 --- a/src/backend/access/transam/xlogreader.c +++ b/src/backend/access/transam/xlogreader.c @@ -72,7 +72,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...) * Returns NULL if the xlogreader couldn't be allocated. */ XLogReaderState * -XLogReaderAllocate(int wal_segment_size, const char *waldir, +XLogReaderAllocate(int wal_segment_size, int page_size, const char *waldir, WALSegmentCleanupCB cleanup_cb) { XLogReaderState *state; @@ -87,6 +87,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir, state->cleanup_cb = cleanup_cb; state->max_block_id = -1; + state->pageSize = page_size; /* * Permanently allocate readBuf. We do it this way, rather than just @@ -95,7 +96,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir, * isn't guaranteed to have any particular alignment, whereas * palloc_extended() will provide MAXALIGN'd storage. */ - state->readBuf = (char *) palloc_extended(XLOG_BLCKSZ, + state->readBuf = (char *) palloc_extended(state->pageSize, MCXT_ALLOC_NO_OOM); if (!state->readBuf) { @@ -163,7 +164,7 @@ XLogReaderFree(XLogReaderState *state) * readRecordBufSize is set to the new buffer size. * * To avoid useless small increases, round its size to a multiple of - * XLOG_BLCKSZ, and make sure it's at least 5*Max(BLCKSZ, XLOG_BLCKSZ) to start + * page size, and make sure it's at least 5*Max(BLCKSZ, <page size>) to start * with. (That is enough for all "normal" records, but very large commit or * abort records might need more space.) */ @@ -172,8 +173,8 @@ allocate_recordbuf(XLogReaderState *state, uint32 reclength) { uint32 newSize = reclength; - newSize += XLOG_BLCKSZ - (newSize % XLOG_BLCKSZ); - newSize = Max(newSize, 5 * Max(BLCKSZ, XLOG_BLCKSZ)); + newSize += state->pageSize - (newSize % state->pageSize); + newSize = Max(newSize, 5 * Max(BLCKSZ, state->pageSize)); #ifndef FRONTEND @@ -238,6 +239,8 @@ void XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr) { Assert(!XLogRecPtrIsInvalid(RecPtr)); + Assert(state->readBuf != NULL && + state->readBuf == (void *) MAXALIGN(state->readBuf)); ResetDecoder(state); @@ -366,15 +369,15 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) XLogPageHeader pageHeader; targetPagePtr = - state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ); - targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ; + state->ReadRecPtr - (state->ReadRecPtr % state->pageSize); + targetRecOff = state->ReadRecPtr % state->pageSize; /* * Check if we have enough data. For the first record in the page, * the requesting length doesn't contain page header. */ if (XLogNeedData(state, targetPagePtr, - Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ), + Min(targetRecOff + SizeOfXLogRecord, state->pageSize), targetRecOff != 0)) return XLREAD_NEED_DATA; @@ -421,7 +424,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) * verified that we got the whole header. */ prec = (XLogRecord *) (state->readBuf + - state->ReadRecPtr % XLOG_BLCKSZ); + state->ReadRecPtr % state->pageSize); total_len = prec->xl_tot_len; /* @@ -432,7 +435,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) * ensure that we enter the XLREAD_CONTINUATION state below; * otherwise we might fail to apply ValidXLogRecordHeader at all. */ - if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord) + if (targetRecOff <= state->pageSize - SizeOfXLogRecord) { if (!ValidXLogRecordHeader(state, state->ReadRecPtr, state->PrevRecPtr, prec)) @@ -477,10 +480,10 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) * available */ targetPagePtr = - state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ); - targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ; + state->ReadRecPtr - (state->ReadRecPtr % state->pageSize); + targetRecOff = state->ReadRecPtr % state->pageSize; - request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ); + request_len = Min(targetRecOff + total_len, state->pageSize); record_len = request_len - targetRecOff; /* ReadRecPtr contains page header */ @@ -549,7 +552,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) /* Calculate pointer to beginning of next page */ state->recordContRecPtr = state->ReadRecPtr + record_len; - Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0); + Assert(state->recordContRecPtr % state->pageSize == 0); state->readRecordState = XLREAD_CONTINUATION; } @@ -576,7 +579,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) /* this request contains page header */ Assert (targetPagePtr != 0); if (XLogNeedData(state, targetPagePtr, - Min(state->recordRemainLen, XLOG_BLCKSZ), + Min(state->recordRemainLen, state->pageSize), false)) return XLREAD_NEED_DATA; @@ -628,7 +631,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) Assert(state->readLen >= pageHeaderSize); contdata = (char *) state->readBuf + pageHeaderSize; - record_len = XLOG_BLCKSZ - pageHeaderSize; + record_len = state->pageSize - pageHeaderSize; if (pageHeader->xlp_rem_len < record_len) record_len = pageHeader->xlp_rem_len; @@ -655,7 +658,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) } /* Calculate pointer to beginning of next page, and continue */ - state->recordContRecPtr += XLOG_BLCKSZ; + state->recordContRecPtr += state->pageSize; } /* targetPagePtr is pointing the last-read page here */ @@ -789,10 +792,10 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen, XLogPageHeaderSize((XLogPageHeader) state->readBuf); addLen = pageHeaderSize; - if (reqLen + pageHeaderSize <= XLOG_BLCKSZ) + if (reqLen + pageHeaderSize <= state->pageSize) addLen = pageHeaderSize; else - addLen = XLOG_BLCKSZ - reqLen; + addLen = state->pageSize - reqLen; Assert(addLen >= 0); } @@ -805,7 +808,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen, /* Data is not in our buffer, request the caller for it. */ XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize); targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize); - Assert((pageptr % XLOG_BLCKSZ) == 0); + Assert((pageptr % state->pageSize) == 0); /* * Every time we request to load new data of a page to the caller, even if @@ -829,7 +832,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen, * will not come back here, but will request the actual target page. */ state->readPagePtr = pageptr - targetPageOff; - state->readLen = XLOG_BLCKSZ; + state->readLen = state->pageSize; return true; } @@ -962,7 +965,7 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr, int32 offset; XLogPageHeader hdr = (XLogPageHeader) phdr; - Assert((recptr % XLOG_BLCKSZ) == 0); + Assert((recptr % state->pageSize) == 0); XLByteToSeg(recptr, segno, state->segcxt.ws_segsize); offset = XLogSegmentOffset(recptr, state->segcxt.ws_segsize); @@ -1016,10 +1019,10 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr, "WAL file is from different database system: incorrect segment size in page header"); return false; } - else if (longhdr->xlp_xlog_blcksz != XLOG_BLCKSZ) + else if (longhdr->xlp_xlog_blcksz != state->pageSize) { report_invalid_record(state, - "WAL file is from different database system: incorrect XLOG_BLCKSZ in page header"); + "WAL file is from different database system: incorrect block size in page header"); return false; } } @@ -1140,7 +1143,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr, * XLogNeedData() is prepared to handle that and will read at * least short page-header worth of data */ - targetRecOff = tmpRecPtr % XLOG_BLCKSZ; + targetRecOff = tmpRecPtr % state->pageSize; /* scroll back to page boundary */ targetPagePtr = tmpRecPtr - targetRecOff; @@ -1174,8 +1177,8 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr, * * Note that record headers are MAXALIGN'ed */ - if (MAXALIGN(header->xlp_rem_len) >= (XLOG_BLCKSZ - pageHeaderSize)) - tmpRecPtr = targetPagePtr + XLOG_BLCKSZ; + if (MAXALIGN(header->xlp_rem_len) >= (state->pageSize - pageHeaderSize)) + tmpRecPtr = targetPagePtr + state->pageSize; else { /* diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c index 9cbd8afa49..f2dd495310 100644 --- a/src/backend/replication/logical/logical.c +++ b/src/backend/replication/logical/logical.c @@ -199,11 +199,14 @@ StartupDecodingContext(List *output_plugin_options, ctx->slot = slot; - ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, cleanup_cb); + ctx->reader = + XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, cleanup_cb); + if (!ctx->reader) ereport(ERROR, (errcode(ERRCODE_OUT_OF_MEMORY), errmsg("out of memory"))); + ctx->reader->readBuf = palloc(XLOG_BLCKSZ); ctx->page_read = page_read; ctx->reorder = ReorderBufferAllocate(); diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c index 062093727d..8e95600ef7 100644 --- a/src/backend/replication/walsender.c +++ b/src/backend/replication/walsender.c @@ -579,8 +579,8 @@ StartReplication(StartReplicationCmd *cmd) errmsg("IDENTIFY_SYSTEM has not been run before START_REPLICATION"))); /* create xlogreader for physical replication */ - xlogreader = - XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close); + xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, + NULL, wal_segment_close); if (!xlogreader) ereport(ERROR, diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c index 712c85281c..8a6209b061 100644 --- a/src/bin/pg_rewind/parsexlog.c +++ b/src/bin/pg_rewind/parsexlog.c @@ -61,7 +61,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex, XLogReaderState *xlogreader; char *errormsg; - xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL); + xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL); if (xlogreader == NULL) pg_fatal("out of memory"); @@ -121,7 +121,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex, char *errormsg; XLogRecPtr endptr; - xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL); + xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL); if (xlogreader == NULL) pg_fatal("out of memory"); @@ -181,7 +181,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex, forkptr += SizeOfXLogShortPHD; } - xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL); + xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL); if (xlogreader == NULL) pg_fatal("out of memory"); diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c index f514de6b13..b635537659 100644 --- a/src/bin/pg_waldump/pg_waldump.c +++ b/src/bin/pg_waldump/pg_waldump.c @@ -1041,7 +1041,7 @@ main(int argc, char **argv) /* we have everything we need, start reading */ xlogreader_state = - XLogReaderAllocate(WalSegSz, waldir, WALDumpCloseSegment); + XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, waldir, WALDumpCloseSegment); if (!xlogreader_state) fatal_error("out of memory"); diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h index b9dad7ee51..1853516333 100644 --- a/src/include/access/xlogreader.h +++ b/src/include/access/xlogreader.h @@ -151,6 +151,7 @@ struct XLogReaderState * ---------------------------------------- */ /* variables to communicate with page reader */ + int pageSize; /* size of a page */ XLogRecPtr readPagePtr; /* page pointer to read */ int32 readLen; /* bytes requested to reader, or actual bytes * read by reader, which must be larger than @@ -243,7 +244,7 @@ struct XLogReaderState /* Get a new XLogReader */ extern XLogReaderState *XLogReaderAllocate(int wal_segment_size, - const char *waldir, + int page_size, const char *waldir, WALSegmentCleanupCB cleanup_cb); /* Free an XLogReader */ -- 2.27.0 ----Next_Part(Thu_Mar__4_11_28_53_2021_014)---- ^ permalink raw reply [nested|flat] 25+ messages in thread
* [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize. @ 2019-09-05 12:29 Kyotaro Horiguchi <[email protected]> 0 siblings, 0 replies; 25+ messages in thread From: Kyotaro Horiguchi @ 2019-09-05 12:29 UTC (permalink / raw) XLog reader facility and its callers currently relies on the same constant XLOG_BLCKSZ. This patch moves the responsibility to determine the block size to the callers of the facility. --- src/backend/access/transam/twophase.c | 3 +- src/backend/access/transam/xlog.c | 9 ++-- src/backend/access/transam/xlogreader.c | 57 ++++++++++++----------- src/backend/replication/logical/logical.c | 5 +- src/backend/replication/walsender.c | 4 +- src/bin/pg_rewind/parsexlog.c | 6 +-- src/bin/pg_waldump/pg_waldump.c | 2 +- src/include/access/xlogreader.h | 3 +- 8 files changed, 50 insertions(+), 39 deletions(-) diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c index 7d5a96a5f5..9f9e7de253 100644 --- a/src/backend/access/transam/twophase.c +++ b/src/backend/access/transam/twophase.c @@ -1325,7 +1325,8 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len) XLogReaderState *xlogreader; char *errormsg; - xlogreader = XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close); + xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, + NULL, wal_segment_close); if (!xlogreader) ereport(ERROR, diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index b517ca85ad..0822ac70cb 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -1222,7 +1222,10 @@ XLogInsertRecord(XLogRecData *rdata, appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len); if (!debug_reader) - debug_reader = XLogReaderAllocate(wal_segment_size, NULL, NULL); + { + debug_reader = + XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, NULL); + } if (!debug_reader) { @@ -6531,8 +6534,8 @@ StartupXLOG(void) OwnLatch(&XLogCtl->recoveryWakeupLatch); /* Set up XLOG reader facility */ - xlogreader = - XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close); + xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, + NULL, wal_segment_close); if (!xlogreader) ereport(ERROR, diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c index ba3e4868f7..e5baf13086 100644 --- a/src/backend/access/transam/xlogreader.c +++ b/src/backend/access/transam/xlogreader.c @@ -72,7 +72,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...) * Returns NULL if the xlogreader couldn't be allocated. */ XLogReaderState * -XLogReaderAllocate(int wal_segment_size, const char *waldir, +XLogReaderAllocate(int wal_segment_size, int page_size, const char *waldir, WALSegmentCleanupCB cleanup_cb) { XLogReaderState *state; @@ -87,6 +87,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir, state->cleanup_cb = cleanup_cb; state->max_block_id = -1; + state->pageSize = page_size; /* * Permanently allocate readBuf. We do it this way, rather than just @@ -95,7 +96,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir, * isn't guaranteed to have any particular alignment, whereas * palloc_extended() will provide MAXALIGN'd storage. */ - state->readBuf = (char *) palloc_extended(XLOG_BLCKSZ, + state->readBuf = (char *) palloc_extended(state->pageSize, MCXT_ALLOC_NO_OOM); if (!state->readBuf) { @@ -163,7 +164,7 @@ XLogReaderFree(XLogReaderState *state) * readRecordBufSize is set to the new buffer size. * * To avoid useless small increases, round its size to a multiple of - * XLOG_BLCKSZ, and make sure it's at least 5*Max(BLCKSZ, XLOG_BLCKSZ) to start + * page size, and make sure it's at least 5*Max(BLCKSZ, <page size>) to start * with. (That is enough for all "normal" records, but very large commit or * abort records might need more space.) */ @@ -172,8 +173,8 @@ allocate_recordbuf(XLogReaderState *state, uint32 reclength) { uint32 newSize = reclength; - newSize += XLOG_BLCKSZ - (newSize % XLOG_BLCKSZ); - newSize = Max(newSize, 5 * Max(BLCKSZ, XLOG_BLCKSZ)); + newSize += state->pageSize - (newSize % state->pageSize); + newSize = Max(newSize, 5 * Max(BLCKSZ, state->pageSize)); #ifndef FRONTEND @@ -238,6 +239,8 @@ void XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr) { Assert(!XLogRecPtrIsInvalid(RecPtr)); + Assert(state->readBuf != NULL && + state->readBuf == (void *) MAXALIGN(state->readBuf)); ResetDecoder(state); @@ -366,15 +369,15 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) XLogPageHeader pageHeader; targetPagePtr = - state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ); - targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ; + state->ReadRecPtr - (state->ReadRecPtr % state->pageSize); + targetRecOff = state->ReadRecPtr % state->pageSize; /* * Check if we have enough data. For the first record in the page, * the requesting length doesn't contain page header. */ if (XLogNeedData(state, targetPagePtr, - Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ), + Min(targetRecOff + SizeOfXLogRecord, state->pageSize), targetRecOff != 0)) return XLREAD_NEED_DATA; @@ -421,7 +424,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) * verified that we got the whole header. */ prec = (XLogRecord *) (state->readBuf + - state->ReadRecPtr % XLOG_BLCKSZ); + state->ReadRecPtr % state->pageSize); total_len = prec->xl_tot_len; /* @@ -432,7 +435,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) * ensure that we enter the XLREAD_CONTINUATION state below; * otherwise we might fail to apply ValidXLogRecordHeader at all. */ - if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord) + if (targetRecOff <= state->pageSize - SizeOfXLogRecord) { if (!ValidXLogRecordHeader(state, state->ReadRecPtr, state->PrevRecPtr, prec)) @@ -477,10 +480,10 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) * available */ targetPagePtr = - state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ); - targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ; + state->ReadRecPtr - (state->ReadRecPtr % state->pageSize); + targetRecOff = state->ReadRecPtr % state->pageSize; - request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ); + request_len = Min(targetRecOff + total_len, state->pageSize); record_len = request_len - targetRecOff; /* ReadRecPtr contains page header */ @@ -549,7 +552,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) /* Calculate pointer to beginning of next page */ state->recordContRecPtr = state->ReadRecPtr + record_len; - Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0); + Assert(state->recordContRecPtr % state->pageSize == 0); state->readRecordState = XLREAD_CONTINUATION; } @@ -576,7 +579,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) /* this request contains page header */ Assert (targetPagePtr != 0); if (XLogNeedData(state, targetPagePtr, - Min(state->recordRemainLen, XLOG_BLCKSZ), + Min(state->recordRemainLen, state->pageSize), false)) return XLREAD_NEED_DATA; @@ -628,7 +631,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) Assert(state->readLen >= pageHeaderSize); contdata = (char *) state->readBuf + pageHeaderSize; - record_len = XLOG_BLCKSZ - pageHeaderSize; + record_len = state->pageSize - pageHeaderSize; if (pageHeader->xlp_rem_len < record_len) record_len = pageHeader->xlp_rem_len; @@ -655,7 +658,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) } /* Calculate pointer to beginning of next page, and continue */ - state->recordContRecPtr += XLOG_BLCKSZ; + state->recordContRecPtr += state->pageSize; } /* targetPagePtr is pointing the last-read page here */ @@ -789,10 +792,10 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen, XLogPageHeaderSize((XLogPageHeader) state->readBuf); addLen = pageHeaderSize; - if (reqLen + pageHeaderSize <= XLOG_BLCKSZ) + if (reqLen + pageHeaderSize <= state->pageSize) addLen = pageHeaderSize; else - addLen = XLOG_BLCKSZ - reqLen; + addLen = state->pageSize - reqLen; Assert(addLen >= 0); } @@ -805,7 +808,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen, /* Data is not in our buffer, request the caller for it. */ XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize); targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize); - Assert((pageptr % XLOG_BLCKSZ) == 0); + Assert((pageptr % state->pageSize) == 0); /* * Every time we request to load new data of a page to the caller, even if @@ -829,7 +832,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen, * will not come back here, but will request the actual target page. */ state->readPagePtr = pageptr - targetPageOff; - state->readLen = XLOG_BLCKSZ; + state->readLen = state->pageSize; return true; } @@ -962,7 +965,7 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr, int32 offset; XLogPageHeader hdr = (XLogPageHeader) phdr; - Assert((recptr % XLOG_BLCKSZ) == 0); + Assert((recptr % state->pageSize) == 0); XLByteToSeg(recptr, segno, state->segcxt.ws_segsize); offset = XLogSegmentOffset(recptr, state->segcxt.ws_segsize); @@ -1016,10 +1019,10 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr, "WAL file is from different database system: incorrect segment size in page header"); return false; } - else if (longhdr->xlp_xlog_blcksz != XLOG_BLCKSZ) + else if (longhdr->xlp_xlog_blcksz != state->pageSize) { report_invalid_record(state, - "WAL file is from different database system: incorrect XLOG_BLCKSZ in page header"); + "WAL file is from different database system: incorrect block size in page header"); return false; } } @@ -1140,7 +1143,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr, * XLogNeedData() is prepared to handle that and will read at * least short page-header worth of data */ - targetRecOff = tmpRecPtr % XLOG_BLCKSZ; + targetRecOff = tmpRecPtr % state->pageSize; /* scroll back to page boundary */ targetPagePtr = tmpRecPtr - targetRecOff; @@ -1174,8 +1177,8 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr, * * Note that record headers are MAXALIGN'ed */ - if (MAXALIGN(header->xlp_rem_len) >= (XLOG_BLCKSZ - pageHeaderSize)) - tmpRecPtr = targetPagePtr + XLOG_BLCKSZ; + if (MAXALIGN(header->xlp_rem_len) >= (state->pageSize - pageHeaderSize)) + tmpRecPtr = targetPagePtr + state->pageSize; else { /* diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c index 9cbd8afa49..f2dd495310 100644 --- a/src/backend/replication/logical/logical.c +++ b/src/backend/replication/logical/logical.c @@ -199,11 +199,14 @@ StartupDecodingContext(List *output_plugin_options, ctx->slot = slot; - ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, cleanup_cb); + ctx->reader = + XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, cleanup_cb); + if (!ctx->reader) ereport(ERROR, (errcode(ERRCODE_OUT_OF_MEMORY), errmsg("out of memory"))); + ctx->reader->readBuf = palloc(XLOG_BLCKSZ); ctx->page_read = page_read; ctx->reorder = ReorderBufferAllocate(); diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c index 062093727d..8e95600ef7 100644 --- a/src/backend/replication/walsender.c +++ b/src/backend/replication/walsender.c @@ -579,8 +579,8 @@ StartReplication(StartReplicationCmd *cmd) errmsg("IDENTIFY_SYSTEM has not been run before START_REPLICATION"))); /* create xlogreader for physical replication */ - xlogreader = - XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close); + xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, + NULL, wal_segment_close); if (!xlogreader) ereport(ERROR, diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c index 712c85281c..8a6209b061 100644 --- a/src/bin/pg_rewind/parsexlog.c +++ b/src/bin/pg_rewind/parsexlog.c @@ -61,7 +61,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex, XLogReaderState *xlogreader; char *errormsg; - xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL); + xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL); if (xlogreader == NULL) pg_fatal("out of memory"); @@ -121,7 +121,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex, char *errormsg; XLogRecPtr endptr; - xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL); + xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL); if (xlogreader == NULL) pg_fatal("out of memory"); @@ -181,7 +181,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex, forkptr += SizeOfXLogShortPHD; } - xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL); + xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL); if (xlogreader == NULL) pg_fatal("out of memory"); diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c index f514de6b13..b635537659 100644 --- a/src/bin/pg_waldump/pg_waldump.c +++ b/src/bin/pg_waldump/pg_waldump.c @@ -1041,7 +1041,7 @@ main(int argc, char **argv) /* we have everything we need, start reading */ xlogreader_state = - XLogReaderAllocate(WalSegSz, waldir, WALDumpCloseSegment); + XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, waldir, WALDumpCloseSegment); if (!xlogreader_state) fatal_error("out of memory"); diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h index b9dad7ee51..1853516333 100644 --- a/src/include/access/xlogreader.h +++ b/src/include/access/xlogreader.h @@ -151,6 +151,7 @@ struct XLogReaderState * ---------------------------------------- */ /* variables to communicate with page reader */ + int pageSize; /* size of a page */ XLogRecPtr readPagePtr; /* page pointer to read */ int32 readLen; /* bytes requested to reader, or actual bytes * read by reader, which must be larger than @@ -243,7 +244,7 @@ struct XLogReaderState /* Get a new XLogReader */ extern XLogReaderState *XLogReaderAllocate(int wal_segment_size, - const char *waldir, + int page_size, const char *waldir, WALSegmentCleanupCB cleanup_cb); /* Free an XLogReader */ -- 2.27.0 ----Next_Part(Thu_Mar__4_11_28_53_2021_014)---- ^ permalink raw reply [nested|flat] 25+ messages in thread
* [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize. @ 2019-09-05 12:29 Kyotaro Horiguchi <[email protected]> 0 siblings, 0 replies; 25+ messages in thread From: Kyotaro Horiguchi @ 2019-09-05 12:29 UTC (permalink / raw) XLog reader facility and its callers currently relies on the same constant XLOG_BLCKSZ. This patch moves the responsibility to determine the block size to the callers of the facility. --- src/backend/access/transam/twophase.c | 3 +- src/backend/access/transam/xlog.c | 9 ++-- src/backend/access/transam/xlogreader.c | 57 ++++++++++++----------- src/backend/replication/logical/logical.c | 5 +- src/backend/replication/walsender.c | 4 +- src/bin/pg_rewind/parsexlog.c | 6 +-- src/bin/pg_waldump/pg_waldump.c | 2 +- src/include/access/xlogreader.h | 3 +- 8 files changed, 50 insertions(+), 39 deletions(-) diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c index 7d5a96a5f5..9f9e7de253 100644 --- a/src/backend/access/transam/twophase.c +++ b/src/backend/access/transam/twophase.c @@ -1325,7 +1325,8 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len) XLogReaderState *xlogreader; char *errormsg; - xlogreader = XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close); + xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, + NULL, wal_segment_close); if (!xlogreader) ereport(ERROR, diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index b517ca85ad..0822ac70cb 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -1222,7 +1222,10 @@ XLogInsertRecord(XLogRecData *rdata, appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len); if (!debug_reader) - debug_reader = XLogReaderAllocate(wal_segment_size, NULL, NULL); + { + debug_reader = + XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, NULL); + } if (!debug_reader) { @@ -6531,8 +6534,8 @@ StartupXLOG(void) OwnLatch(&XLogCtl->recoveryWakeupLatch); /* Set up XLOG reader facility */ - xlogreader = - XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close); + xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, + NULL, wal_segment_close); if (!xlogreader) ereport(ERROR, diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c index ba3e4868f7..e5baf13086 100644 --- a/src/backend/access/transam/xlogreader.c +++ b/src/backend/access/transam/xlogreader.c @@ -72,7 +72,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...) * Returns NULL if the xlogreader couldn't be allocated. */ XLogReaderState * -XLogReaderAllocate(int wal_segment_size, const char *waldir, +XLogReaderAllocate(int wal_segment_size, int page_size, const char *waldir, WALSegmentCleanupCB cleanup_cb) { XLogReaderState *state; @@ -87,6 +87,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir, state->cleanup_cb = cleanup_cb; state->max_block_id = -1; + state->pageSize = page_size; /* * Permanently allocate readBuf. We do it this way, rather than just @@ -95,7 +96,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir, * isn't guaranteed to have any particular alignment, whereas * palloc_extended() will provide MAXALIGN'd storage. */ - state->readBuf = (char *) palloc_extended(XLOG_BLCKSZ, + state->readBuf = (char *) palloc_extended(state->pageSize, MCXT_ALLOC_NO_OOM); if (!state->readBuf) { @@ -163,7 +164,7 @@ XLogReaderFree(XLogReaderState *state) * readRecordBufSize is set to the new buffer size. * * To avoid useless small increases, round its size to a multiple of - * XLOG_BLCKSZ, and make sure it's at least 5*Max(BLCKSZ, XLOG_BLCKSZ) to start + * page size, and make sure it's at least 5*Max(BLCKSZ, <page size>) to start * with. (That is enough for all "normal" records, but very large commit or * abort records might need more space.) */ @@ -172,8 +173,8 @@ allocate_recordbuf(XLogReaderState *state, uint32 reclength) { uint32 newSize = reclength; - newSize += XLOG_BLCKSZ - (newSize % XLOG_BLCKSZ); - newSize = Max(newSize, 5 * Max(BLCKSZ, XLOG_BLCKSZ)); + newSize += state->pageSize - (newSize % state->pageSize); + newSize = Max(newSize, 5 * Max(BLCKSZ, state->pageSize)); #ifndef FRONTEND @@ -238,6 +239,8 @@ void XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr) { Assert(!XLogRecPtrIsInvalid(RecPtr)); + Assert(state->readBuf != NULL && + state->readBuf == (void *) MAXALIGN(state->readBuf)); ResetDecoder(state); @@ -366,15 +369,15 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) XLogPageHeader pageHeader; targetPagePtr = - state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ); - targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ; + state->ReadRecPtr - (state->ReadRecPtr % state->pageSize); + targetRecOff = state->ReadRecPtr % state->pageSize; /* * Check if we have enough data. For the first record in the page, * the requesting length doesn't contain page header. */ if (XLogNeedData(state, targetPagePtr, - Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ), + Min(targetRecOff + SizeOfXLogRecord, state->pageSize), targetRecOff != 0)) return XLREAD_NEED_DATA; @@ -421,7 +424,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) * verified that we got the whole header. */ prec = (XLogRecord *) (state->readBuf + - state->ReadRecPtr % XLOG_BLCKSZ); + state->ReadRecPtr % state->pageSize); total_len = prec->xl_tot_len; /* @@ -432,7 +435,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) * ensure that we enter the XLREAD_CONTINUATION state below; * otherwise we might fail to apply ValidXLogRecordHeader at all. */ - if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord) + if (targetRecOff <= state->pageSize - SizeOfXLogRecord) { if (!ValidXLogRecordHeader(state, state->ReadRecPtr, state->PrevRecPtr, prec)) @@ -477,10 +480,10 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) * available */ targetPagePtr = - state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ); - targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ; + state->ReadRecPtr - (state->ReadRecPtr % state->pageSize); + targetRecOff = state->ReadRecPtr % state->pageSize; - request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ); + request_len = Min(targetRecOff + total_len, state->pageSize); record_len = request_len - targetRecOff; /* ReadRecPtr contains page header */ @@ -549,7 +552,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) /* Calculate pointer to beginning of next page */ state->recordContRecPtr = state->ReadRecPtr + record_len; - Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0); + Assert(state->recordContRecPtr % state->pageSize == 0); state->readRecordState = XLREAD_CONTINUATION; } @@ -576,7 +579,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) /* this request contains page header */ Assert (targetPagePtr != 0); if (XLogNeedData(state, targetPagePtr, - Min(state->recordRemainLen, XLOG_BLCKSZ), + Min(state->recordRemainLen, state->pageSize), false)) return XLREAD_NEED_DATA; @@ -628,7 +631,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) Assert(state->readLen >= pageHeaderSize); contdata = (char *) state->readBuf + pageHeaderSize; - record_len = XLOG_BLCKSZ - pageHeaderSize; + record_len = state->pageSize - pageHeaderSize; if (pageHeader->xlp_rem_len < record_len) record_len = pageHeader->xlp_rem_len; @@ -655,7 +658,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) } /* Calculate pointer to beginning of next page, and continue */ - state->recordContRecPtr += XLOG_BLCKSZ; + state->recordContRecPtr += state->pageSize; } /* targetPagePtr is pointing the last-read page here */ @@ -789,10 +792,10 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen, XLogPageHeaderSize((XLogPageHeader) state->readBuf); addLen = pageHeaderSize; - if (reqLen + pageHeaderSize <= XLOG_BLCKSZ) + if (reqLen + pageHeaderSize <= state->pageSize) addLen = pageHeaderSize; else - addLen = XLOG_BLCKSZ - reqLen; + addLen = state->pageSize - reqLen; Assert(addLen >= 0); } @@ -805,7 +808,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen, /* Data is not in our buffer, request the caller for it. */ XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize); targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize); - Assert((pageptr % XLOG_BLCKSZ) == 0); + Assert((pageptr % state->pageSize) == 0); /* * Every time we request to load new data of a page to the caller, even if @@ -829,7 +832,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen, * will not come back here, but will request the actual target page. */ state->readPagePtr = pageptr - targetPageOff; - state->readLen = XLOG_BLCKSZ; + state->readLen = state->pageSize; return true; } @@ -962,7 +965,7 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr, int32 offset; XLogPageHeader hdr = (XLogPageHeader) phdr; - Assert((recptr % XLOG_BLCKSZ) == 0); + Assert((recptr % state->pageSize) == 0); XLByteToSeg(recptr, segno, state->segcxt.ws_segsize); offset = XLogSegmentOffset(recptr, state->segcxt.ws_segsize); @@ -1016,10 +1019,10 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr, "WAL file is from different database system: incorrect segment size in page header"); return false; } - else if (longhdr->xlp_xlog_blcksz != XLOG_BLCKSZ) + else if (longhdr->xlp_xlog_blcksz != state->pageSize) { report_invalid_record(state, - "WAL file is from different database system: incorrect XLOG_BLCKSZ in page header"); + "WAL file is from different database system: incorrect block size in page header"); return false; } } @@ -1140,7 +1143,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr, * XLogNeedData() is prepared to handle that and will read at * least short page-header worth of data */ - targetRecOff = tmpRecPtr % XLOG_BLCKSZ; + targetRecOff = tmpRecPtr % state->pageSize; /* scroll back to page boundary */ targetPagePtr = tmpRecPtr - targetRecOff; @@ -1174,8 +1177,8 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr, * * Note that record headers are MAXALIGN'ed */ - if (MAXALIGN(header->xlp_rem_len) >= (XLOG_BLCKSZ - pageHeaderSize)) - tmpRecPtr = targetPagePtr + XLOG_BLCKSZ; + if (MAXALIGN(header->xlp_rem_len) >= (state->pageSize - pageHeaderSize)) + tmpRecPtr = targetPagePtr + state->pageSize; else { /* diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c index 9cbd8afa49..f2dd495310 100644 --- a/src/backend/replication/logical/logical.c +++ b/src/backend/replication/logical/logical.c @@ -199,11 +199,14 @@ StartupDecodingContext(List *output_plugin_options, ctx->slot = slot; - ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, cleanup_cb); + ctx->reader = + XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, cleanup_cb); + if (!ctx->reader) ereport(ERROR, (errcode(ERRCODE_OUT_OF_MEMORY), errmsg("out of memory"))); + ctx->reader->readBuf = palloc(XLOG_BLCKSZ); ctx->page_read = page_read; ctx->reorder = ReorderBufferAllocate(); diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c index 062093727d..8e95600ef7 100644 --- a/src/backend/replication/walsender.c +++ b/src/backend/replication/walsender.c @@ -579,8 +579,8 @@ StartReplication(StartReplicationCmd *cmd) errmsg("IDENTIFY_SYSTEM has not been run before START_REPLICATION"))); /* create xlogreader for physical replication */ - xlogreader = - XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close); + xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, + NULL, wal_segment_close); if (!xlogreader) ereport(ERROR, diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c index 712c85281c..8a6209b061 100644 --- a/src/bin/pg_rewind/parsexlog.c +++ b/src/bin/pg_rewind/parsexlog.c @@ -61,7 +61,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex, XLogReaderState *xlogreader; char *errormsg; - xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL); + xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL); if (xlogreader == NULL) pg_fatal("out of memory"); @@ -121,7 +121,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex, char *errormsg; XLogRecPtr endptr; - xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL); + xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL); if (xlogreader == NULL) pg_fatal("out of memory"); @@ -181,7 +181,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex, forkptr += SizeOfXLogShortPHD; } - xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL); + xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL); if (xlogreader == NULL) pg_fatal("out of memory"); diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c index f514de6b13..b635537659 100644 --- a/src/bin/pg_waldump/pg_waldump.c +++ b/src/bin/pg_waldump/pg_waldump.c @@ -1041,7 +1041,7 @@ main(int argc, char **argv) /* we have everything we need, start reading */ xlogreader_state = - XLogReaderAllocate(WalSegSz, waldir, WALDumpCloseSegment); + XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, waldir, WALDumpCloseSegment); if (!xlogreader_state) fatal_error("out of memory"); diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h index b9dad7ee51..1853516333 100644 --- a/src/include/access/xlogreader.h +++ b/src/include/access/xlogreader.h @@ -151,6 +151,7 @@ struct XLogReaderState * ---------------------------------------- */ /* variables to communicate with page reader */ + int pageSize; /* size of a page */ XLogRecPtr readPagePtr; /* page pointer to read */ int32 readLen; /* bytes requested to reader, or actual bytes * read by reader, which must be larger than @@ -243,7 +244,7 @@ struct XLogReaderState /* Get a new XLogReader */ extern XLogReaderState *XLogReaderAllocate(int wal_segment_size, - const char *waldir, + int page_size, const char *waldir, WALSegmentCleanupCB cleanup_cb); /* Free an XLogReader */ -- 2.27.0 ----Next_Part(Thu_Mar__4_11_28_53_2021_014)---- ^ permalink raw reply [nested|flat] 25+ messages in thread
* [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize. @ 2019-09-05 12:29 Kyotaro Horiguchi <[email protected]> 0 siblings, 0 replies; 25+ messages in thread From: Kyotaro Horiguchi @ 2019-09-05 12:29 UTC (permalink / raw) XLog reader facility and its callers currently relies on the same constant XLOG_BLCKSZ. This patch moves the responsibility to determine the block size to the callers of the facility. --- src/backend/access/transam/twophase.c | 3 +- src/backend/access/transam/xlog.c | 9 ++-- src/backend/access/transam/xlogreader.c | 57 ++++++++++++----------- src/backend/replication/logical/logical.c | 5 +- src/backend/replication/walsender.c | 4 +- src/bin/pg_rewind/parsexlog.c | 6 +-- src/bin/pg_waldump/pg_waldump.c | 2 +- src/include/access/xlogreader.h | 3 +- 8 files changed, 50 insertions(+), 39 deletions(-) diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c index 7d5a96a5f5..9f9e7de253 100644 --- a/src/backend/access/transam/twophase.c +++ b/src/backend/access/transam/twophase.c @@ -1325,7 +1325,8 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len) XLogReaderState *xlogreader; char *errormsg; - xlogreader = XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close); + xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, + NULL, wal_segment_close); if (!xlogreader) ereport(ERROR, diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index b517ca85ad..0822ac70cb 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -1222,7 +1222,10 @@ XLogInsertRecord(XLogRecData *rdata, appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len); if (!debug_reader) - debug_reader = XLogReaderAllocate(wal_segment_size, NULL, NULL); + { + debug_reader = + XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, NULL); + } if (!debug_reader) { @@ -6531,8 +6534,8 @@ StartupXLOG(void) OwnLatch(&XLogCtl->recoveryWakeupLatch); /* Set up XLOG reader facility */ - xlogreader = - XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close); + xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, + NULL, wal_segment_close); if (!xlogreader) ereport(ERROR, diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c index ba3e4868f7..e5baf13086 100644 --- a/src/backend/access/transam/xlogreader.c +++ b/src/backend/access/transam/xlogreader.c @@ -72,7 +72,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...) * Returns NULL if the xlogreader couldn't be allocated. */ XLogReaderState * -XLogReaderAllocate(int wal_segment_size, const char *waldir, +XLogReaderAllocate(int wal_segment_size, int page_size, const char *waldir, WALSegmentCleanupCB cleanup_cb) { XLogReaderState *state; @@ -87,6 +87,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir, state->cleanup_cb = cleanup_cb; state->max_block_id = -1; + state->pageSize = page_size; /* * Permanently allocate readBuf. We do it this way, rather than just @@ -95,7 +96,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir, * isn't guaranteed to have any particular alignment, whereas * palloc_extended() will provide MAXALIGN'd storage. */ - state->readBuf = (char *) palloc_extended(XLOG_BLCKSZ, + state->readBuf = (char *) palloc_extended(state->pageSize, MCXT_ALLOC_NO_OOM); if (!state->readBuf) { @@ -163,7 +164,7 @@ XLogReaderFree(XLogReaderState *state) * readRecordBufSize is set to the new buffer size. * * To avoid useless small increases, round its size to a multiple of - * XLOG_BLCKSZ, and make sure it's at least 5*Max(BLCKSZ, XLOG_BLCKSZ) to start + * page size, and make sure it's at least 5*Max(BLCKSZ, <page size>) to start * with. (That is enough for all "normal" records, but very large commit or * abort records might need more space.) */ @@ -172,8 +173,8 @@ allocate_recordbuf(XLogReaderState *state, uint32 reclength) { uint32 newSize = reclength; - newSize += XLOG_BLCKSZ - (newSize % XLOG_BLCKSZ); - newSize = Max(newSize, 5 * Max(BLCKSZ, XLOG_BLCKSZ)); + newSize += state->pageSize - (newSize % state->pageSize); + newSize = Max(newSize, 5 * Max(BLCKSZ, state->pageSize)); #ifndef FRONTEND @@ -238,6 +239,8 @@ void XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr) { Assert(!XLogRecPtrIsInvalid(RecPtr)); + Assert(state->readBuf != NULL && + state->readBuf == (void *) MAXALIGN(state->readBuf)); ResetDecoder(state); @@ -366,15 +369,15 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) XLogPageHeader pageHeader; targetPagePtr = - state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ); - targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ; + state->ReadRecPtr - (state->ReadRecPtr % state->pageSize); + targetRecOff = state->ReadRecPtr % state->pageSize; /* * Check if we have enough data. For the first record in the page, * the requesting length doesn't contain page header. */ if (XLogNeedData(state, targetPagePtr, - Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ), + Min(targetRecOff + SizeOfXLogRecord, state->pageSize), targetRecOff != 0)) return XLREAD_NEED_DATA; @@ -421,7 +424,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) * verified that we got the whole header. */ prec = (XLogRecord *) (state->readBuf + - state->ReadRecPtr % XLOG_BLCKSZ); + state->ReadRecPtr % state->pageSize); total_len = prec->xl_tot_len; /* @@ -432,7 +435,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) * ensure that we enter the XLREAD_CONTINUATION state below; * otherwise we might fail to apply ValidXLogRecordHeader at all. */ - if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord) + if (targetRecOff <= state->pageSize - SizeOfXLogRecord) { if (!ValidXLogRecordHeader(state, state->ReadRecPtr, state->PrevRecPtr, prec)) @@ -477,10 +480,10 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) * available */ targetPagePtr = - state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ); - targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ; + state->ReadRecPtr - (state->ReadRecPtr % state->pageSize); + targetRecOff = state->ReadRecPtr % state->pageSize; - request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ); + request_len = Min(targetRecOff + total_len, state->pageSize); record_len = request_len - targetRecOff; /* ReadRecPtr contains page header */ @@ -549,7 +552,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) /* Calculate pointer to beginning of next page */ state->recordContRecPtr = state->ReadRecPtr + record_len; - Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0); + Assert(state->recordContRecPtr % state->pageSize == 0); state->readRecordState = XLREAD_CONTINUATION; } @@ -576,7 +579,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) /* this request contains page header */ Assert (targetPagePtr != 0); if (XLogNeedData(state, targetPagePtr, - Min(state->recordRemainLen, XLOG_BLCKSZ), + Min(state->recordRemainLen, state->pageSize), false)) return XLREAD_NEED_DATA; @@ -628,7 +631,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) Assert(state->readLen >= pageHeaderSize); contdata = (char *) state->readBuf + pageHeaderSize; - record_len = XLOG_BLCKSZ - pageHeaderSize; + record_len = state->pageSize - pageHeaderSize; if (pageHeader->xlp_rem_len < record_len) record_len = pageHeader->xlp_rem_len; @@ -655,7 +658,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) } /* Calculate pointer to beginning of next page, and continue */ - state->recordContRecPtr += XLOG_BLCKSZ; + state->recordContRecPtr += state->pageSize; } /* targetPagePtr is pointing the last-read page here */ @@ -789,10 +792,10 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen, XLogPageHeaderSize((XLogPageHeader) state->readBuf); addLen = pageHeaderSize; - if (reqLen + pageHeaderSize <= XLOG_BLCKSZ) + if (reqLen + pageHeaderSize <= state->pageSize) addLen = pageHeaderSize; else - addLen = XLOG_BLCKSZ - reqLen; + addLen = state->pageSize - reqLen; Assert(addLen >= 0); } @@ -805,7 +808,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen, /* Data is not in our buffer, request the caller for it. */ XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize); targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize); - Assert((pageptr % XLOG_BLCKSZ) == 0); + Assert((pageptr % state->pageSize) == 0); /* * Every time we request to load new data of a page to the caller, even if @@ -829,7 +832,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen, * will not come back here, but will request the actual target page. */ state->readPagePtr = pageptr - targetPageOff; - state->readLen = XLOG_BLCKSZ; + state->readLen = state->pageSize; return true; } @@ -962,7 +965,7 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr, int32 offset; XLogPageHeader hdr = (XLogPageHeader) phdr; - Assert((recptr % XLOG_BLCKSZ) == 0); + Assert((recptr % state->pageSize) == 0); XLByteToSeg(recptr, segno, state->segcxt.ws_segsize); offset = XLogSegmentOffset(recptr, state->segcxt.ws_segsize); @@ -1016,10 +1019,10 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr, "WAL file is from different database system: incorrect segment size in page header"); return false; } - else if (longhdr->xlp_xlog_blcksz != XLOG_BLCKSZ) + else if (longhdr->xlp_xlog_blcksz != state->pageSize) { report_invalid_record(state, - "WAL file is from different database system: incorrect XLOG_BLCKSZ in page header"); + "WAL file is from different database system: incorrect block size in page header"); return false; } } @@ -1140,7 +1143,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr, * XLogNeedData() is prepared to handle that and will read at * least short page-header worth of data */ - targetRecOff = tmpRecPtr % XLOG_BLCKSZ; + targetRecOff = tmpRecPtr % state->pageSize; /* scroll back to page boundary */ targetPagePtr = tmpRecPtr - targetRecOff; @@ -1174,8 +1177,8 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr, * * Note that record headers are MAXALIGN'ed */ - if (MAXALIGN(header->xlp_rem_len) >= (XLOG_BLCKSZ - pageHeaderSize)) - tmpRecPtr = targetPagePtr + XLOG_BLCKSZ; + if (MAXALIGN(header->xlp_rem_len) >= (state->pageSize - pageHeaderSize)) + tmpRecPtr = targetPagePtr + state->pageSize; else { /* diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c index 9cbd8afa49..f2dd495310 100644 --- a/src/backend/replication/logical/logical.c +++ b/src/backend/replication/logical/logical.c @@ -199,11 +199,14 @@ StartupDecodingContext(List *output_plugin_options, ctx->slot = slot; - ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, cleanup_cb); + ctx->reader = + XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, cleanup_cb); + if (!ctx->reader) ereport(ERROR, (errcode(ERRCODE_OUT_OF_MEMORY), errmsg("out of memory"))); + ctx->reader->readBuf = palloc(XLOG_BLCKSZ); ctx->page_read = page_read; ctx->reorder = ReorderBufferAllocate(); diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c index 062093727d..8e95600ef7 100644 --- a/src/backend/replication/walsender.c +++ b/src/backend/replication/walsender.c @@ -579,8 +579,8 @@ StartReplication(StartReplicationCmd *cmd) errmsg("IDENTIFY_SYSTEM has not been run before START_REPLICATION"))); /* create xlogreader for physical replication */ - xlogreader = - XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close); + xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, + NULL, wal_segment_close); if (!xlogreader) ereport(ERROR, diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c index 712c85281c..8a6209b061 100644 --- a/src/bin/pg_rewind/parsexlog.c +++ b/src/bin/pg_rewind/parsexlog.c @@ -61,7 +61,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex, XLogReaderState *xlogreader; char *errormsg; - xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL); + xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL); if (xlogreader == NULL) pg_fatal("out of memory"); @@ -121,7 +121,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex, char *errormsg; XLogRecPtr endptr; - xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL); + xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL); if (xlogreader == NULL) pg_fatal("out of memory"); @@ -181,7 +181,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex, forkptr += SizeOfXLogShortPHD; } - xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL); + xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL); if (xlogreader == NULL) pg_fatal("out of memory"); diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c index f514de6b13..b635537659 100644 --- a/src/bin/pg_waldump/pg_waldump.c +++ b/src/bin/pg_waldump/pg_waldump.c @@ -1041,7 +1041,7 @@ main(int argc, char **argv) /* we have everything we need, start reading */ xlogreader_state = - XLogReaderAllocate(WalSegSz, waldir, WALDumpCloseSegment); + XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, waldir, WALDumpCloseSegment); if (!xlogreader_state) fatal_error("out of memory"); diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h index b9dad7ee51..1853516333 100644 --- a/src/include/access/xlogreader.h +++ b/src/include/access/xlogreader.h @@ -151,6 +151,7 @@ struct XLogReaderState * ---------------------------------------- */ /* variables to communicate with page reader */ + int pageSize; /* size of a page */ XLogRecPtr readPagePtr; /* page pointer to read */ int32 readLen; /* bytes requested to reader, or actual bytes * read by reader, which must be larger than @@ -243,7 +244,7 @@ struct XLogReaderState /* Get a new XLogReader */ extern XLogReaderState *XLogReaderAllocate(int wal_segment_size, - const char *waldir, + int page_size, const char *waldir, WALSegmentCleanupCB cleanup_cb); /* Free an XLogReader */ -- 2.27.0 ----Next_Part(Thu_Mar__4_11_28_53_2021_014)---- ^ permalink raw reply [nested|flat] 25+ messages in thread
* [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize. @ 2019-09-05 12:29 Kyotaro Horiguchi <[email protected]> 0 siblings, 0 replies; 25+ messages in thread From: Kyotaro Horiguchi @ 2019-09-05 12:29 UTC (permalink / raw) XLog reader facility and its callers currently relies on the same constant XLOG_BLCKSZ. This patch moves the responsibility to determine the block size to the callers of the facility. --- src/backend/access/transam/twophase.c | 3 +- src/backend/access/transam/xlog.c | 9 ++-- src/backend/access/transam/xlogreader.c | 57 ++++++++++++----------- src/backend/replication/logical/logical.c | 5 +- src/backend/replication/walsender.c | 4 +- src/bin/pg_rewind/parsexlog.c | 6 +-- src/bin/pg_waldump/pg_waldump.c | 2 +- src/include/access/xlogreader.h | 3 +- 8 files changed, 50 insertions(+), 39 deletions(-) diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c index 7d5a96a5f5..9f9e7de253 100644 --- a/src/backend/access/transam/twophase.c +++ b/src/backend/access/transam/twophase.c @@ -1325,7 +1325,8 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len) XLogReaderState *xlogreader; char *errormsg; - xlogreader = XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close); + xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, + NULL, wal_segment_close); if (!xlogreader) ereport(ERROR, diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index b517ca85ad..0822ac70cb 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -1222,7 +1222,10 @@ XLogInsertRecord(XLogRecData *rdata, appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len); if (!debug_reader) - debug_reader = XLogReaderAllocate(wal_segment_size, NULL, NULL); + { + debug_reader = + XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, NULL); + } if (!debug_reader) { @@ -6531,8 +6534,8 @@ StartupXLOG(void) OwnLatch(&XLogCtl->recoveryWakeupLatch); /* Set up XLOG reader facility */ - xlogreader = - XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close); + xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, + NULL, wal_segment_close); if (!xlogreader) ereport(ERROR, diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c index ba3e4868f7..e5baf13086 100644 --- a/src/backend/access/transam/xlogreader.c +++ b/src/backend/access/transam/xlogreader.c @@ -72,7 +72,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...) * Returns NULL if the xlogreader couldn't be allocated. */ XLogReaderState * -XLogReaderAllocate(int wal_segment_size, const char *waldir, +XLogReaderAllocate(int wal_segment_size, int page_size, const char *waldir, WALSegmentCleanupCB cleanup_cb) { XLogReaderState *state; @@ -87,6 +87,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir, state->cleanup_cb = cleanup_cb; state->max_block_id = -1; + state->pageSize = page_size; /* * Permanently allocate readBuf. We do it this way, rather than just @@ -95,7 +96,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir, * isn't guaranteed to have any particular alignment, whereas * palloc_extended() will provide MAXALIGN'd storage. */ - state->readBuf = (char *) palloc_extended(XLOG_BLCKSZ, + state->readBuf = (char *) palloc_extended(state->pageSize, MCXT_ALLOC_NO_OOM); if (!state->readBuf) { @@ -163,7 +164,7 @@ XLogReaderFree(XLogReaderState *state) * readRecordBufSize is set to the new buffer size. * * To avoid useless small increases, round its size to a multiple of - * XLOG_BLCKSZ, and make sure it's at least 5*Max(BLCKSZ, XLOG_BLCKSZ) to start + * page size, and make sure it's at least 5*Max(BLCKSZ, <page size>) to start * with. (That is enough for all "normal" records, but very large commit or * abort records might need more space.) */ @@ -172,8 +173,8 @@ allocate_recordbuf(XLogReaderState *state, uint32 reclength) { uint32 newSize = reclength; - newSize += XLOG_BLCKSZ - (newSize % XLOG_BLCKSZ); - newSize = Max(newSize, 5 * Max(BLCKSZ, XLOG_BLCKSZ)); + newSize += state->pageSize - (newSize % state->pageSize); + newSize = Max(newSize, 5 * Max(BLCKSZ, state->pageSize)); #ifndef FRONTEND @@ -238,6 +239,8 @@ void XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr) { Assert(!XLogRecPtrIsInvalid(RecPtr)); + Assert(state->readBuf != NULL && + state->readBuf == (void *) MAXALIGN(state->readBuf)); ResetDecoder(state); @@ -366,15 +369,15 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) XLogPageHeader pageHeader; targetPagePtr = - state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ); - targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ; + state->ReadRecPtr - (state->ReadRecPtr % state->pageSize); + targetRecOff = state->ReadRecPtr % state->pageSize; /* * Check if we have enough data. For the first record in the page, * the requesting length doesn't contain page header. */ if (XLogNeedData(state, targetPagePtr, - Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ), + Min(targetRecOff + SizeOfXLogRecord, state->pageSize), targetRecOff != 0)) return XLREAD_NEED_DATA; @@ -421,7 +424,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) * verified that we got the whole header. */ prec = (XLogRecord *) (state->readBuf + - state->ReadRecPtr % XLOG_BLCKSZ); + state->ReadRecPtr % state->pageSize); total_len = prec->xl_tot_len; /* @@ -432,7 +435,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) * ensure that we enter the XLREAD_CONTINUATION state below; * otherwise we might fail to apply ValidXLogRecordHeader at all. */ - if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord) + if (targetRecOff <= state->pageSize - SizeOfXLogRecord) { if (!ValidXLogRecordHeader(state, state->ReadRecPtr, state->PrevRecPtr, prec)) @@ -477,10 +480,10 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) * available */ targetPagePtr = - state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ); - targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ; + state->ReadRecPtr - (state->ReadRecPtr % state->pageSize); + targetRecOff = state->ReadRecPtr % state->pageSize; - request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ); + request_len = Min(targetRecOff + total_len, state->pageSize); record_len = request_len - targetRecOff; /* ReadRecPtr contains page header */ @@ -549,7 +552,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) /* Calculate pointer to beginning of next page */ state->recordContRecPtr = state->ReadRecPtr + record_len; - Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0); + Assert(state->recordContRecPtr % state->pageSize == 0); state->readRecordState = XLREAD_CONTINUATION; } @@ -576,7 +579,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) /* this request contains page header */ Assert (targetPagePtr != 0); if (XLogNeedData(state, targetPagePtr, - Min(state->recordRemainLen, XLOG_BLCKSZ), + Min(state->recordRemainLen, state->pageSize), false)) return XLREAD_NEED_DATA; @@ -628,7 +631,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) Assert(state->readLen >= pageHeaderSize); contdata = (char *) state->readBuf + pageHeaderSize; - record_len = XLOG_BLCKSZ - pageHeaderSize; + record_len = state->pageSize - pageHeaderSize; if (pageHeader->xlp_rem_len < record_len) record_len = pageHeader->xlp_rem_len; @@ -655,7 +658,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) } /* Calculate pointer to beginning of next page, and continue */ - state->recordContRecPtr += XLOG_BLCKSZ; + state->recordContRecPtr += state->pageSize; } /* targetPagePtr is pointing the last-read page here */ @@ -789,10 +792,10 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen, XLogPageHeaderSize((XLogPageHeader) state->readBuf); addLen = pageHeaderSize; - if (reqLen + pageHeaderSize <= XLOG_BLCKSZ) + if (reqLen + pageHeaderSize <= state->pageSize) addLen = pageHeaderSize; else - addLen = XLOG_BLCKSZ - reqLen; + addLen = state->pageSize - reqLen; Assert(addLen >= 0); } @@ -805,7 +808,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen, /* Data is not in our buffer, request the caller for it. */ XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize); targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize); - Assert((pageptr % XLOG_BLCKSZ) == 0); + Assert((pageptr % state->pageSize) == 0); /* * Every time we request to load new data of a page to the caller, even if @@ -829,7 +832,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen, * will not come back here, but will request the actual target page. */ state->readPagePtr = pageptr - targetPageOff; - state->readLen = XLOG_BLCKSZ; + state->readLen = state->pageSize; return true; } @@ -962,7 +965,7 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr, int32 offset; XLogPageHeader hdr = (XLogPageHeader) phdr; - Assert((recptr % XLOG_BLCKSZ) == 0); + Assert((recptr % state->pageSize) == 0); XLByteToSeg(recptr, segno, state->segcxt.ws_segsize); offset = XLogSegmentOffset(recptr, state->segcxt.ws_segsize); @@ -1016,10 +1019,10 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr, "WAL file is from different database system: incorrect segment size in page header"); return false; } - else if (longhdr->xlp_xlog_blcksz != XLOG_BLCKSZ) + else if (longhdr->xlp_xlog_blcksz != state->pageSize) { report_invalid_record(state, - "WAL file is from different database system: incorrect XLOG_BLCKSZ in page header"); + "WAL file is from different database system: incorrect block size in page header"); return false; } } @@ -1140,7 +1143,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr, * XLogNeedData() is prepared to handle that and will read at * least short page-header worth of data */ - targetRecOff = tmpRecPtr % XLOG_BLCKSZ; + targetRecOff = tmpRecPtr % state->pageSize; /* scroll back to page boundary */ targetPagePtr = tmpRecPtr - targetRecOff; @@ -1174,8 +1177,8 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr, * * Note that record headers are MAXALIGN'ed */ - if (MAXALIGN(header->xlp_rem_len) >= (XLOG_BLCKSZ - pageHeaderSize)) - tmpRecPtr = targetPagePtr + XLOG_BLCKSZ; + if (MAXALIGN(header->xlp_rem_len) >= (state->pageSize - pageHeaderSize)) + tmpRecPtr = targetPagePtr + state->pageSize; else { /* diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c index 9cbd8afa49..f2dd495310 100644 --- a/src/backend/replication/logical/logical.c +++ b/src/backend/replication/logical/logical.c @@ -199,11 +199,14 @@ StartupDecodingContext(List *output_plugin_options, ctx->slot = slot; - ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, cleanup_cb); + ctx->reader = + XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, cleanup_cb); + if (!ctx->reader) ereport(ERROR, (errcode(ERRCODE_OUT_OF_MEMORY), errmsg("out of memory"))); + ctx->reader->readBuf = palloc(XLOG_BLCKSZ); ctx->page_read = page_read; ctx->reorder = ReorderBufferAllocate(); diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c index 062093727d..8e95600ef7 100644 --- a/src/backend/replication/walsender.c +++ b/src/backend/replication/walsender.c @@ -579,8 +579,8 @@ StartReplication(StartReplicationCmd *cmd) errmsg("IDENTIFY_SYSTEM has not been run before START_REPLICATION"))); /* create xlogreader for physical replication */ - xlogreader = - XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close); + xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, + NULL, wal_segment_close); if (!xlogreader) ereport(ERROR, diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c index 712c85281c..8a6209b061 100644 --- a/src/bin/pg_rewind/parsexlog.c +++ b/src/bin/pg_rewind/parsexlog.c @@ -61,7 +61,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex, XLogReaderState *xlogreader; char *errormsg; - xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL); + xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL); if (xlogreader == NULL) pg_fatal("out of memory"); @@ -121,7 +121,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex, char *errormsg; XLogRecPtr endptr; - xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL); + xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL); if (xlogreader == NULL) pg_fatal("out of memory"); @@ -181,7 +181,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex, forkptr += SizeOfXLogShortPHD; } - xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL); + xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL); if (xlogreader == NULL) pg_fatal("out of memory"); diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c index f514de6b13..b635537659 100644 --- a/src/bin/pg_waldump/pg_waldump.c +++ b/src/bin/pg_waldump/pg_waldump.c @@ -1041,7 +1041,7 @@ main(int argc, char **argv) /* we have everything we need, start reading */ xlogreader_state = - XLogReaderAllocate(WalSegSz, waldir, WALDumpCloseSegment); + XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, waldir, WALDumpCloseSegment); if (!xlogreader_state) fatal_error("out of memory"); diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h index b9dad7ee51..1853516333 100644 --- a/src/include/access/xlogreader.h +++ b/src/include/access/xlogreader.h @@ -151,6 +151,7 @@ struct XLogReaderState * ---------------------------------------- */ /* variables to communicate with page reader */ + int pageSize; /* size of a page */ XLogRecPtr readPagePtr; /* page pointer to read */ int32 readLen; /* bytes requested to reader, or actual bytes * read by reader, which must be larger than @@ -243,7 +244,7 @@ struct XLogReaderState /* Get a new XLogReader */ extern XLogReaderState *XLogReaderAllocate(int wal_segment_size, - const char *waldir, + int page_size, const char *waldir, WALSegmentCleanupCB cleanup_cb); /* Free an XLogReader */ -- 2.27.0 ----Next_Part(Thu_Mar__4_11_28_53_2021_014)---- ^ permalink raw reply [nested|flat] 25+ messages in thread
* [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize. @ 2019-09-05 12:29 Kyotaro Horiguchi <[email protected]> 0 siblings, 0 replies; 25+ messages in thread From: Kyotaro Horiguchi @ 2019-09-05 12:29 UTC (permalink / raw) XLog reader facility and its callers currently relies on the same constant XLOG_BLCKSZ. This patch moves the responsibility to determine the block size to the callers of the facility. --- src/backend/access/transam/twophase.c | 3 +- src/backend/access/transam/xlog.c | 9 ++-- src/backend/access/transam/xlogreader.c | 57 ++++++++++++----------- src/backend/replication/logical/logical.c | 5 +- src/backend/replication/walsender.c | 4 +- src/bin/pg_rewind/parsexlog.c | 6 +-- src/bin/pg_waldump/pg_waldump.c | 2 +- src/include/access/xlogreader.h | 3 +- 8 files changed, 50 insertions(+), 39 deletions(-) diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c index 7d5a96a5f5..9f9e7de253 100644 --- a/src/backend/access/transam/twophase.c +++ b/src/backend/access/transam/twophase.c @@ -1325,7 +1325,8 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len) XLogReaderState *xlogreader; char *errormsg; - xlogreader = XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close); + xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, + NULL, wal_segment_close); if (!xlogreader) ereport(ERROR, diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index b517ca85ad..0822ac70cb 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -1222,7 +1222,10 @@ XLogInsertRecord(XLogRecData *rdata, appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len); if (!debug_reader) - debug_reader = XLogReaderAllocate(wal_segment_size, NULL, NULL); + { + debug_reader = + XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, NULL); + } if (!debug_reader) { @@ -6531,8 +6534,8 @@ StartupXLOG(void) OwnLatch(&XLogCtl->recoveryWakeupLatch); /* Set up XLOG reader facility */ - xlogreader = - XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close); + xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, + NULL, wal_segment_close); if (!xlogreader) ereport(ERROR, diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c index ba3e4868f7..e5baf13086 100644 --- a/src/backend/access/transam/xlogreader.c +++ b/src/backend/access/transam/xlogreader.c @@ -72,7 +72,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...) * Returns NULL if the xlogreader couldn't be allocated. */ XLogReaderState * -XLogReaderAllocate(int wal_segment_size, const char *waldir, +XLogReaderAllocate(int wal_segment_size, int page_size, const char *waldir, WALSegmentCleanupCB cleanup_cb) { XLogReaderState *state; @@ -87,6 +87,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir, state->cleanup_cb = cleanup_cb; state->max_block_id = -1; + state->pageSize = page_size; /* * Permanently allocate readBuf. We do it this way, rather than just @@ -95,7 +96,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir, * isn't guaranteed to have any particular alignment, whereas * palloc_extended() will provide MAXALIGN'd storage. */ - state->readBuf = (char *) palloc_extended(XLOG_BLCKSZ, + state->readBuf = (char *) palloc_extended(state->pageSize, MCXT_ALLOC_NO_OOM); if (!state->readBuf) { @@ -163,7 +164,7 @@ XLogReaderFree(XLogReaderState *state) * readRecordBufSize is set to the new buffer size. * * To avoid useless small increases, round its size to a multiple of - * XLOG_BLCKSZ, and make sure it's at least 5*Max(BLCKSZ, XLOG_BLCKSZ) to start + * page size, and make sure it's at least 5*Max(BLCKSZ, <page size>) to start * with. (That is enough for all "normal" records, but very large commit or * abort records might need more space.) */ @@ -172,8 +173,8 @@ allocate_recordbuf(XLogReaderState *state, uint32 reclength) { uint32 newSize = reclength; - newSize += XLOG_BLCKSZ - (newSize % XLOG_BLCKSZ); - newSize = Max(newSize, 5 * Max(BLCKSZ, XLOG_BLCKSZ)); + newSize += state->pageSize - (newSize % state->pageSize); + newSize = Max(newSize, 5 * Max(BLCKSZ, state->pageSize)); #ifndef FRONTEND @@ -238,6 +239,8 @@ void XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr) { Assert(!XLogRecPtrIsInvalid(RecPtr)); + Assert(state->readBuf != NULL && + state->readBuf == (void *) MAXALIGN(state->readBuf)); ResetDecoder(state); @@ -366,15 +369,15 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) XLogPageHeader pageHeader; targetPagePtr = - state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ); - targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ; + state->ReadRecPtr - (state->ReadRecPtr % state->pageSize); + targetRecOff = state->ReadRecPtr % state->pageSize; /* * Check if we have enough data. For the first record in the page, * the requesting length doesn't contain page header. */ if (XLogNeedData(state, targetPagePtr, - Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ), + Min(targetRecOff + SizeOfXLogRecord, state->pageSize), targetRecOff != 0)) return XLREAD_NEED_DATA; @@ -421,7 +424,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) * verified that we got the whole header. */ prec = (XLogRecord *) (state->readBuf + - state->ReadRecPtr % XLOG_BLCKSZ); + state->ReadRecPtr % state->pageSize); total_len = prec->xl_tot_len; /* @@ -432,7 +435,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) * ensure that we enter the XLREAD_CONTINUATION state below; * otherwise we might fail to apply ValidXLogRecordHeader at all. */ - if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord) + if (targetRecOff <= state->pageSize - SizeOfXLogRecord) { if (!ValidXLogRecordHeader(state, state->ReadRecPtr, state->PrevRecPtr, prec)) @@ -477,10 +480,10 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) * available */ targetPagePtr = - state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ); - targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ; + state->ReadRecPtr - (state->ReadRecPtr % state->pageSize); + targetRecOff = state->ReadRecPtr % state->pageSize; - request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ); + request_len = Min(targetRecOff + total_len, state->pageSize); record_len = request_len - targetRecOff; /* ReadRecPtr contains page header */ @@ -549,7 +552,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) /* Calculate pointer to beginning of next page */ state->recordContRecPtr = state->ReadRecPtr + record_len; - Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0); + Assert(state->recordContRecPtr % state->pageSize == 0); state->readRecordState = XLREAD_CONTINUATION; } @@ -576,7 +579,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) /* this request contains page header */ Assert (targetPagePtr != 0); if (XLogNeedData(state, targetPagePtr, - Min(state->recordRemainLen, XLOG_BLCKSZ), + Min(state->recordRemainLen, state->pageSize), false)) return XLREAD_NEED_DATA; @@ -628,7 +631,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) Assert(state->readLen >= pageHeaderSize); contdata = (char *) state->readBuf + pageHeaderSize; - record_len = XLOG_BLCKSZ - pageHeaderSize; + record_len = state->pageSize - pageHeaderSize; if (pageHeader->xlp_rem_len < record_len) record_len = pageHeader->xlp_rem_len; @@ -655,7 +658,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) } /* Calculate pointer to beginning of next page, and continue */ - state->recordContRecPtr += XLOG_BLCKSZ; + state->recordContRecPtr += state->pageSize; } /* targetPagePtr is pointing the last-read page here */ @@ -789,10 +792,10 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen, XLogPageHeaderSize((XLogPageHeader) state->readBuf); addLen = pageHeaderSize; - if (reqLen + pageHeaderSize <= XLOG_BLCKSZ) + if (reqLen + pageHeaderSize <= state->pageSize) addLen = pageHeaderSize; else - addLen = XLOG_BLCKSZ - reqLen; + addLen = state->pageSize - reqLen; Assert(addLen >= 0); } @@ -805,7 +808,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen, /* Data is not in our buffer, request the caller for it. */ XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize); targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize); - Assert((pageptr % XLOG_BLCKSZ) == 0); + Assert((pageptr % state->pageSize) == 0); /* * Every time we request to load new data of a page to the caller, even if @@ -829,7 +832,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen, * will not come back here, but will request the actual target page. */ state->readPagePtr = pageptr - targetPageOff; - state->readLen = XLOG_BLCKSZ; + state->readLen = state->pageSize; return true; } @@ -962,7 +965,7 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr, int32 offset; XLogPageHeader hdr = (XLogPageHeader) phdr; - Assert((recptr % XLOG_BLCKSZ) == 0); + Assert((recptr % state->pageSize) == 0); XLByteToSeg(recptr, segno, state->segcxt.ws_segsize); offset = XLogSegmentOffset(recptr, state->segcxt.ws_segsize); @@ -1016,10 +1019,10 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr, "WAL file is from different database system: incorrect segment size in page header"); return false; } - else if (longhdr->xlp_xlog_blcksz != XLOG_BLCKSZ) + else if (longhdr->xlp_xlog_blcksz != state->pageSize) { report_invalid_record(state, - "WAL file is from different database system: incorrect XLOG_BLCKSZ in page header"); + "WAL file is from different database system: incorrect block size in page header"); return false; } } @@ -1140,7 +1143,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr, * XLogNeedData() is prepared to handle that and will read at * least short page-header worth of data */ - targetRecOff = tmpRecPtr % XLOG_BLCKSZ; + targetRecOff = tmpRecPtr % state->pageSize; /* scroll back to page boundary */ targetPagePtr = tmpRecPtr - targetRecOff; @@ -1174,8 +1177,8 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr, * * Note that record headers are MAXALIGN'ed */ - if (MAXALIGN(header->xlp_rem_len) >= (XLOG_BLCKSZ - pageHeaderSize)) - tmpRecPtr = targetPagePtr + XLOG_BLCKSZ; + if (MAXALIGN(header->xlp_rem_len) >= (state->pageSize - pageHeaderSize)) + tmpRecPtr = targetPagePtr + state->pageSize; else { /* diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c index 9cbd8afa49..f2dd495310 100644 --- a/src/backend/replication/logical/logical.c +++ b/src/backend/replication/logical/logical.c @@ -199,11 +199,14 @@ StartupDecodingContext(List *output_plugin_options, ctx->slot = slot; - ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, cleanup_cb); + ctx->reader = + XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, cleanup_cb); + if (!ctx->reader) ereport(ERROR, (errcode(ERRCODE_OUT_OF_MEMORY), errmsg("out of memory"))); + ctx->reader->readBuf = palloc(XLOG_BLCKSZ); ctx->page_read = page_read; ctx->reorder = ReorderBufferAllocate(); diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c index 062093727d..8e95600ef7 100644 --- a/src/backend/replication/walsender.c +++ b/src/backend/replication/walsender.c @@ -579,8 +579,8 @@ StartReplication(StartReplicationCmd *cmd) errmsg("IDENTIFY_SYSTEM has not been run before START_REPLICATION"))); /* create xlogreader for physical replication */ - xlogreader = - XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close); + xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, + NULL, wal_segment_close); if (!xlogreader) ereport(ERROR, diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c index 712c85281c..8a6209b061 100644 --- a/src/bin/pg_rewind/parsexlog.c +++ b/src/bin/pg_rewind/parsexlog.c @@ -61,7 +61,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex, XLogReaderState *xlogreader; char *errormsg; - xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL); + xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL); if (xlogreader == NULL) pg_fatal("out of memory"); @@ -121,7 +121,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex, char *errormsg; XLogRecPtr endptr; - xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL); + xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL); if (xlogreader == NULL) pg_fatal("out of memory"); @@ -181,7 +181,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex, forkptr += SizeOfXLogShortPHD; } - xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL); + xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL); if (xlogreader == NULL) pg_fatal("out of memory"); diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c index f514de6b13..b635537659 100644 --- a/src/bin/pg_waldump/pg_waldump.c +++ b/src/bin/pg_waldump/pg_waldump.c @@ -1041,7 +1041,7 @@ main(int argc, char **argv) /* we have everything we need, start reading */ xlogreader_state = - XLogReaderAllocate(WalSegSz, waldir, WALDumpCloseSegment); + XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, waldir, WALDumpCloseSegment); if (!xlogreader_state) fatal_error("out of memory"); diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h index b9dad7ee51..1853516333 100644 --- a/src/include/access/xlogreader.h +++ b/src/include/access/xlogreader.h @@ -151,6 +151,7 @@ struct XLogReaderState * ---------------------------------------- */ /* variables to communicate with page reader */ + int pageSize; /* size of a page */ XLogRecPtr readPagePtr; /* page pointer to read */ int32 readLen; /* bytes requested to reader, or actual bytes * read by reader, which must be larger than @@ -243,7 +244,7 @@ struct XLogReaderState /* Get a new XLogReader */ extern XLogReaderState *XLogReaderAllocate(int wal_segment_size, - const char *waldir, + int page_size, const char *waldir, WALSegmentCleanupCB cleanup_cb); /* Free an XLogReader */ -- 2.27.0 ----Next_Part(Thu_Mar__4_11_28_53_2021_014)---- ^ permalink raw reply [nested|flat] 25+ messages in thread
* [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize. @ 2019-09-05 12:29 Kyotaro Horiguchi <[email protected]> 0 siblings, 0 replies; 25+ messages in thread From: Kyotaro Horiguchi @ 2019-09-05 12:29 UTC (permalink / raw) XLog reader facility and its callers currently relies on the same constant XLOG_BLCKSZ. This patch moves the responsibility to determine the block size to the callers of the facility. --- src/backend/access/transam/twophase.c | 3 +- src/backend/access/transam/xlog.c | 9 ++-- src/backend/access/transam/xlogreader.c | 57 ++++++++++++----------- src/backend/replication/logical/logical.c | 5 +- src/backend/replication/walsender.c | 4 +- src/bin/pg_rewind/parsexlog.c | 6 +-- src/bin/pg_waldump/pg_waldump.c | 2 +- src/include/access/xlogreader.h | 3 +- 8 files changed, 50 insertions(+), 39 deletions(-) diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c index 7d5a96a5f5..9f9e7de253 100644 --- a/src/backend/access/transam/twophase.c +++ b/src/backend/access/transam/twophase.c @@ -1325,7 +1325,8 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len) XLogReaderState *xlogreader; char *errormsg; - xlogreader = XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close); + xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, + NULL, wal_segment_close); if (!xlogreader) ereport(ERROR, diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index b517ca85ad..0822ac70cb 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -1222,7 +1222,10 @@ XLogInsertRecord(XLogRecData *rdata, appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len); if (!debug_reader) - debug_reader = XLogReaderAllocate(wal_segment_size, NULL, NULL); + { + debug_reader = + XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, NULL); + } if (!debug_reader) { @@ -6531,8 +6534,8 @@ StartupXLOG(void) OwnLatch(&XLogCtl->recoveryWakeupLatch); /* Set up XLOG reader facility */ - xlogreader = - XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close); + xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, + NULL, wal_segment_close); if (!xlogreader) ereport(ERROR, diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c index ba3e4868f7..e5baf13086 100644 --- a/src/backend/access/transam/xlogreader.c +++ b/src/backend/access/transam/xlogreader.c @@ -72,7 +72,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...) * Returns NULL if the xlogreader couldn't be allocated. */ XLogReaderState * -XLogReaderAllocate(int wal_segment_size, const char *waldir, +XLogReaderAllocate(int wal_segment_size, int page_size, const char *waldir, WALSegmentCleanupCB cleanup_cb) { XLogReaderState *state; @@ -87,6 +87,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir, state->cleanup_cb = cleanup_cb; state->max_block_id = -1; + state->pageSize = page_size; /* * Permanently allocate readBuf. We do it this way, rather than just @@ -95,7 +96,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir, * isn't guaranteed to have any particular alignment, whereas * palloc_extended() will provide MAXALIGN'd storage. */ - state->readBuf = (char *) palloc_extended(XLOG_BLCKSZ, + state->readBuf = (char *) palloc_extended(state->pageSize, MCXT_ALLOC_NO_OOM); if (!state->readBuf) { @@ -163,7 +164,7 @@ XLogReaderFree(XLogReaderState *state) * readRecordBufSize is set to the new buffer size. * * To avoid useless small increases, round its size to a multiple of - * XLOG_BLCKSZ, and make sure it's at least 5*Max(BLCKSZ, XLOG_BLCKSZ) to start + * page size, and make sure it's at least 5*Max(BLCKSZ, <page size>) to start * with. (That is enough for all "normal" records, but very large commit or * abort records might need more space.) */ @@ -172,8 +173,8 @@ allocate_recordbuf(XLogReaderState *state, uint32 reclength) { uint32 newSize = reclength; - newSize += XLOG_BLCKSZ - (newSize % XLOG_BLCKSZ); - newSize = Max(newSize, 5 * Max(BLCKSZ, XLOG_BLCKSZ)); + newSize += state->pageSize - (newSize % state->pageSize); + newSize = Max(newSize, 5 * Max(BLCKSZ, state->pageSize)); #ifndef FRONTEND @@ -238,6 +239,8 @@ void XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr) { Assert(!XLogRecPtrIsInvalid(RecPtr)); + Assert(state->readBuf != NULL && + state->readBuf == (void *) MAXALIGN(state->readBuf)); ResetDecoder(state); @@ -366,15 +369,15 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) XLogPageHeader pageHeader; targetPagePtr = - state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ); - targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ; + state->ReadRecPtr - (state->ReadRecPtr % state->pageSize); + targetRecOff = state->ReadRecPtr % state->pageSize; /* * Check if we have enough data. For the first record in the page, * the requesting length doesn't contain page header. */ if (XLogNeedData(state, targetPagePtr, - Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ), + Min(targetRecOff + SizeOfXLogRecord, state->pageSize), targetRecOff != 0)) return XLREAD_NEED_DATA; @@ -421,7 +424,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) * verified that we got the whole header. */ prec = (XLogRecord *) (state->readBuf + - state->ReadRecPtr % XLOG_BLCKSZ); + state->ReadRecPtr % state->pageSize); total_len = prec->xl_tot_len; /* @@ -432,7 +435,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) * ensure that we enter the XLREAD_CONTINUATION state below; * otherwise we might fail to apply ValidXLogRecordHeader at all. */ - if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord) + if (targetRecOff <= state->pageSize - SizeOfXLogRecord) { if (!ValidXLogRecordHeader(state, state->ReadRecPtr, state->PrevRecPtr, prec)) @@ -477,10 +480,10 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) * available */ targetPagePtr = - state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ); - targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ; + state->ReadRecPtr - (state->ReadRecPtr % state->pageSize); + targetRecOff = state->ReadRecPtr % state->pageSize; - request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ); + request_len = Min(targetRecOff + total_len, state->pageSize); record_len = request_len - targetRecOff; /* ReadRecPtr contains page header */ @@ -549,7 +552,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) /* Calculate pointer to beginning of next page */ state->recordContRecPtr = state->ReadRecPtr + record_len; - Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0); + Assert(state->recordContRecPtr % state->pageSize == 0); state->readRecordState = XLREAD_CONTINUATION; } @@ -576,7 +579,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) /* this request contains page header */ Assert (targetPagePtr != 0); if (XLogNeedData(state, targetPagePtr, - Min(state->recordRemainLen, XLOG_BLCKSZ), + Min(state->recordRemainLen, state->pageSize), false)) return XLREAD_NEED_DATA; @@ -628,7 +631,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) Assert(state->readLen >= pageHeaderSize); contdata = (char *) state->readBuf + pageHeaderSize; - record_len = XLOG_BLCKSZ - pageHeaderSize; + record_len = state->pageSize - pageHeaderSize; if (pageHeader->xlp_rem_len < record_len) record_len = pageHeader->xlp_rem_len; @@ -655,7 +658,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) } /* Calculate pointer to beginning of next page, and continue */ - state->recordContRecPtr += XLOG_BLCKSZ; + state->recordContRecPtr += state->pageSize; } /* targetPagePtr is pointing the last-read page here */ @@ -789,10 +792,10 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen, XLogPageHeaderSize((XLogPageHeader) state->readBuf); addLen = pageHeaderSize; - if (reqLen + pageHeaderSize <= XLOG_BLCKSZ) + if (reqLen + pageHeaderSize <= state->pageSize) addLen = pageHeaderSize; else - addLen = XLOG_BLCKSZ - reqLen; + addLen = state->pageSize - reqLen; Assert(addLen >= 0); } @@ -805,7 +808,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen, /* Data is not in our buffer, request the caller for it. */ XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize); targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize); - Assert((pageptr % XLOG_BLCKSZ) == 0); + Assert((pageptr % state->pageSize) == 0); /* * Every time we request to load new data of a page to the caller, even if @@ -829,7 +832,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen, * will not come back here, but will request the actual target page. */ state->readPagePtr = pageptr - targetPageOff; - state->readLen = XLOG_BLCKSZ; + state->readLen = state->pageSize; return true; } @@ -962,7 +965,7 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr, int32 offset; XLogPageHeader hdr = (XLogPageHeader) phdr; - Assert((recptr % XLOG_BLCKSZ) == 0); + Assert((recptr % state->pageSize) == 0); XLByteToSeg(recptr, segno, state->segcxt.ws_segsize); offset = XLogSegmentOffset(recptr, state->segcxt.ws_segsize); @@ -1016,10 +1019,10 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr, "WAL file is from different database system: incorrect segment size in page header"); return false; } - else if (longhdr->xlp_xlog_blcksz != XLOG_BLCKSZ) + else if (longhdr->xlp_xlog_blcksz != state->pageSize) { report_invalid_record(state, - "WAL file is from different database system: incorrect XLOG_BLCKSZ in page header"); + "WAL file is from different database system: incorrect block size in page header"); return false; } } @@ -1140,7 +1143,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr, * XLogNeedData() is prepared to handle that and will read at * least short page-header worth of data */ - targetRecOff = tmpRecPtr % XLOG_BLCKSZ; + targetRecOff = tmpRecPtr % state->pageSize; /* scroll back to page boundary */ targetPagePtr = tmpRecPtr - targetRecOff; @@ -1174,8 +1177,8 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr, * * Note that record headers are MAXALIGN'ed */ - if (MAXALIGN(header->xlp_rem_len) >= (XLOG_BLCKSZ - pageHeaderSize)) - tmpRecPtr = targetPagePtr + XLOG_BLCKSZ; + if (MAXALIGN(header->xlp_rem_len) >= (state->pageSize - pageHeaderSize)) + tmpRecPtr = targetPagePtr + state->pageSize; else { /* diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c index 9cbd8afa49..f2dd495310 100644 --- a/src/backend/replication/logical/logical.c +++ b/src/backend/replication/logical/logical.c @@ -199,11 +199,14 @@ StartupDecodingContext(List *output_plugin_options, ctx->slot = slot; - ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, cleanup_cb); + ctx->reader = + XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, cleanup_cb); + if (!ctx->reader) ereport(ERROR, (errcode(ERRCODE_OUT_OF_MEMORY), errmsg("out of memory"))); + ctx->reader->readBuf = palloc(XLOG_BLCKSZ); ctx->page_read = page_read; ctx->reorder = ReorderBufferAllocate(); diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c index 062093727d..8e95600ef7 100644 --- a/src/backend/replication/walsender.c +++ b/src/backend/replication/walsender.c @@ -579,8 +579,8 @@ StartReplication(StartReplicationCmd *cmd) errmsg("IDENTIFY_SYSTEM has not been run before START_REPLICATION"))); /* create xlogreader for physical replication */ - xlogreader = - XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close); + xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, + NULL, wal_segment_close); if (!xlogreader) ereport(ERROR, diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c index 712c85281c..8a6209b061 100644 --- a/src/bin/pg_rewind/parsexlog.c +++ b/src/bin/pg_rewind/parsexlog.c @@ -61,7 +61,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex, XLogReaderState *xlogreader; char *errormsg; - xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL); + xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL); if (xlogreader == NULL) pg_fatal("out of memory"); @@ -121,7 +121,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex, char *errormsg; XLogRecPtr endptr; - xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL); + xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL); if (xlogreader == NULL) pg_fatal("out of memory"); @@ -181,7 +181,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex, forkptr += SizeOfXLogShortPHD; } - xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL); + xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL); if (xlogreader == NULL) pg_fatal("out of memory"); diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c index f514de6b13..b635537659 100644 --- a/src/bin/pg_waldump/pg_waldump.c +++ b/src/bin/pg_waldump/pg_waldump.c @@ -1041,7 +1041,7 @@ main(int argc, char **argv) /* we have everything we need, start reading */ xlogreader_state = - XLogReaderAllocate(WalSegSz, waldir, WALDumpCloseSegment); + XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, waldir, WALDumpCloseSegment); if (!xlogreader_state) fatal_error("out of memory"); diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h index b9dad7ee51..1853516333 100644 --- a/src/include/access/xlogreader.h +++ b/src/include/access/xlogreader.h @@ -151,6 +151,7 @@ struct XLogReaderState * ---------------------------------------- */ /* variables to communicate with page reader */ + int pageSize; /* size of a page */ XLogRecPtr readPagePtr; /* page pointer to read */ int32 readLen; /* bytes requested to reader, or actual bytes * read by reader, which must be larger than @@ -243,7 +244,7 @@ struct XLogReaderState /* Get a new XLogReader */ extern XLogReaderState *XLogReaderAllocate(int wal_segment_size, - const char *waldir, + int page_size, const char *waldir, WALSegmentCleanupCB cleanup_cb); /* Free an XLogReader */ -- 2.27.0 ----Next_Part(Thu_Mar__4_11_28_53_2021_014)---- ^ permalink raw reply [nested|flat] 25+ messages in thread
* [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize. @ 2019-09-05 12:29 Kyotaro Horiguchi <[email protected]> 0 siblings, 0 replies; 25+ messages in thread From: Kyotaro Horiguchi @ 2019-09-05 12:29 UTC (permalink / raw) XLog reader facility and its callers currently relies on the same constant XLOG_BLCKSZ. This patch moves the responsibility to determine the block size to the callers of the facility. --- src/backend/access/transam/twophase.c | 3 +- src/backend/access/transam/xlog.c | 9 ++-- src/backend/access/transam/xlogreader.c | 57 ++++++++++++----------- src/backend/replication/logical/logical.c | 5 +- src/backend/replication/walsender.c | 4 +- src/bin/pg_rewind/parsexlog.c | 6 +-- src/bin/pg_waldump/pg_waldump.c | 2 +- src/include/access/xlogreader.h | 3 +- 8 files changed, 50 insertions(+), 39 deletions(-) diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c index 7d5a96a5f5..9f9e7de253 100644 --- a/src/backend/access/transam/twophase.c +++ b/src/backend/access/transam/twophase.c @@ -1325,7 +1325,8 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len) XLogReaderState *xlogreader; char *errormsg; - xlogreader = XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close); + xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, + NULL, wal_segment_close); if (!xlogreader) ereport(ERROR, diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index b517ca85ad..0822ac70cb 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -1222,7 +1222,10 @@ XLogInsertRecord(XLogRecData *rdata, appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len); if (!debug_reader) - debug_reader = XLogReaderAllocate(wal_segment_size, NULL, NULL); + { + debug_reader = + XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, NULL); + } if (!debug_reader) { @@ -6531,8 +6534,8 @@ StartupXLOG(void) OwnLatch(&XLogCtl->recoveryWakeupLatch); /* Set up XLOG reader facility */ - xlogreader = - XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close); + xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, + NULL, wal_segment_close); if (!xlogreader) ereport(ERROR, diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c index ba3e4868f7..e5baf13086 100644 --- a/src/backend/access/transam/xlogreader.c +++ b/src/backend/access/transam/xlogreader.c @@ -72,7 +72,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...) * Returns NULL if the xlogreader couldn't be allocated. */ XLogReaderState * -XLogReaderAllocate(int wal_segment_size, const char *waldir, +XLogReaderAllocate(int wal_segment_size, int page_size, const char *waldir, WALSegmentCleanupCB cleanup_cb) { XLogReaderState *state; @@ -87,6 +87,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir, state->cleanup_cb = cleanup_cb; state->max_block_id = -1; + state->pageSize = page_size; /* * Permanently allocate readBuf. We do it this way, rather than just @@ -95,7 +96,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir, * isn't guaranteed to have any particular alignment, whereas * palloc_extended() will provide MAXALIGN'd storage. */ - state->readBuf = (char *) palloc_extended(XLOG_BLCKSZ, + state->readBuf = (char *) palloc_extended(state->pageSize, MCXT_ALLOC_NO_OOM); if (!state->readBuf) { @@ -163,7 +164,7 @@ XLogReaderFree(XLogReaderState *state) * readRecordBufSize is set to the new buffer size. * * To avoid useless small increases, round its size to a multiple of - * XLOG_BLCKSZ, and make sure it's at least 5*Max(BLCKSZ, XLOG_BLCKSZ) to start + * page size, and make sure it's at least 5*Max(BLCKSZ, <page size>) to start * with. (That is enough for all "normal" records, but very large commit or * abort records might need more space.) */ @@ -172,8 +173,8 @@ allocate_recordbuf(XLogReaderState *state, uint32 reclength) { uint32 newSize = reclength; - newSize += XLOG_BLCKSZ - (newSize % XLOG_BLCKSZ); - newSize = Max(newSize, 5 * Max(BLCKSZ, XLOG_BLCKSZ)); + newSize += state->pageSize - (newSize % state->pageSize); + newSize = Max(newSize, 5 * Max(BLCKSZ, state->pageSize)); #ifndef FRONTEND @@ -238,6 +239,8 @@ void XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr) { Assert(!XLogRecPtrIsInvalid(RecPtr)); + Assert(state->readBuf != NULL && + state->readBuf == (void *) MAXALIGN(state->readBuf)); ResetDecoder(state); @@ -366,15 +369,15 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) XLogPageHeader pageHeader; targetPagePtr = - state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ); - targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ; + state->ReadRecPtr - (state->ReadRecPtr % state->pageSize); + targetRecOff = state->ReadRecPtr % state->pageSize; /* * Check if we have enough data. For the first record in the page, * the requesting length doesn't contain page header. */ if (XLogNeedData(state, targetPagePtr, - Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ), + Min(targetRecOff + SizeOfXLogRecord, state->pageSize), targetRecOff != 0)) return XLREAD_NEED_DATA; @@ -421,7 +424,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) * verified that we got the whole header. */ prec = (XLogRecord *) (state->readBuf + - state->ReadRecPtr % XLOG_BLCKSZ); + state->ReadRecPtr % state->pageSize); total_len = prec->xl_tot_len; /* @@ -432,7 +435,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) * ensure that we enter the XLREAD_CONTINUATION state below; * otherwise we might fail to apply ValidXLogRecordHeader at all. */ - if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord) + if (targetRecOff <= state->pageSize - SizeOfXLogRecord) { if (!ValidXLogRecordHeader(state, state->ReadRecPtr, state->PrevRecPtr, prec)) @@ -477,10 +480,10 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) * available */ targetPagePtr = - state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ); - targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ; + state->ReadRecPtr - (state->ReadRecPtr % state->pageSize); + targetRecOff = state->ReadRecPtr % state->pageSize; - request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ); + request_len = Min(targetRecOff + total_len, state->pageSize); record_len = request_len - targetRecOff; /* ReadRecPtr contains page header */ @@ -549,7 +552,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) /* Calculate pointer to beginning of next page */ state->recordContRecPtr = state->ReadRecPtr + record_len; - Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0); + Assert(state->recordContRecPtr % state->pageSize == 0); state->readRecordState = XLREAD_CONTINUATION; } @@ -576,7 +579,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) /* this request contains page header */ Assert (targetPagePtr != 0); if (XLogNeedData(state, targetPagePtr, - Min(state->recordRemainLen, XLOG_BLCKSZ), + Min(state->recordRemainLen, state->pageSize), false)) return XLREAD_NEED_DATA; @@ -628,7 +631,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) Assert(state->readLen >= pageHeaderSize); contdata = (char *) state->readBuf + pageHeaderSize; - record_len = XLOG_BLCKSZ - pageHeaderSize; + record_len = state->pageSize - pageHeaderSize; if (pageHeader->xlp_rem_len < record_len) record_len = pageHeader->xlp_rem_len; @@ -655,7 +658,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) } /* Calculate pointer to beginning of next page, and continue */ - state->recordContRecPtr += XLOG_BLCKSZ; + state->recordContRecPtr += state->pageSize; } /* targetPagePtr is pointing the last-read page here */ @@ -789,10 +792,10 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen, XLogPageHeaderSize((XLogPageHeader) state->readBuf); addLen = pageHeaderSize; - if (reqLen + pageHeaderSize <= XLOG_BLCKSZ) + if (reqLen + pageHeaderSize <= state->pageSize) addLen = pageHeaderSize; else - addLen = XLOG_BLCKSZ - reqLen; + addLen = state->pageSize - reqLen; Assert(addLen >= 0); } @@ -805,7 +808,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen, /* Data is not in our buffer, request the caller for it. */ XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize); targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize); - Assert((pageptr % XLOG_BLCKSZ) == 0); + Assert((pageptr % state->pageSize) == 0); /* * Every time we request to load new data of a page to the caller, even if @@ -829,7 +832,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen, * will not come back here, but will request the actual target page. */ state->readPagePtr = pageptr - targetPageOff; - state->readLen = XLOG_BLCKSZ; + state->readLen = state->pageSize; return true; } @@ -962,7 +965,7 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr, int32 offset; XLogPageHeader hdr = (XLogPageHeader) phdr; - Assert((recptr % XLOG_BLCKSZ) == 0); + Assert((recptr % state->pageSize) == 0); XLByteToSeg(recptr, segno, state->segcxt.ws_segsize); offset = XLogSegmentOffset(recptr, state->segcxt.ws_segsize); @@ -1016,10 +1019,10 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr, "WAL file is from different database system: incorrect segment size in page header"); return false; } - else if (longhdr->xlp_xlog_blcksz != XLOG_BLCKSZ) + else if (longhdr->xlp_xlog_blcksz != state->pageSize) { report_invalid_record(state, - "WAL file is from different database system: incorrect XLOG_BLCKSZ in page header"); + "WAL file is from different database system: incorrect block size in page header"); return false; } } @@ -1140,7 +1143,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr, * XLogNeedData() is prepared to handle that and will read at * least short page-header worth of data */ - targetRecOff = tmpRecPtr % XLOG_BLCKSZ; + targetRecOff = tmpRecPtr % state->pageSize; /* scroll back to page boundary */ targetPagePtr = tmpRecPtr - targetRecOff; @@ -1174,8 +1177,8 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr, * * Note that record headers are MAXALIGN'ed */ - if (MAXALIGN(header->xlp_rem_len) >= (XLOG_BLCKSZ - pageHeaderSize)) - tmpRecPtr = targetPagePtr + XLOG_BLCKSZ; + if (MAXALIGN(header->xlp_rem_len) >= (state->pageSize - pageHeaderSize)) + tmpRecPtr = targetPagePtr + state->pageSize; else { /* diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c index 9cbd8afa49..f2dd495310 100644 --- a/src/backend/replication/logical/logical.c +++ b/src/backend/replication/logical/logical.c @@ -199,11 +199,14 @@ StartupDecodingContext(List *output_plugin_options, ctx->slot = slot; - ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, cleanup_cb); + ctx->reader = + XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, cleanup_cb); + if (!ctx->reader) ereport(ERROR, (errcode(ERRCODE_OUT_OF_MEMORY), errmsg("out of memory"))); + ctx->reader->readBuf = palloc(XLOG_BLCKSZ); ctx->page_read = page_read; ctx->reorder = ReorderBufferAllocate(); diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c index 062093727d..8e95600ef7 100644 --- a/src/backend/replication/walsender.c +++ b/src/backend/replication/walsender.c @@ -579,8 +579,8 @@ StartReplication(StartReplicationCmd *cmd) errmsg("IDENTIFY_SYSTEM has not been run before START_REPLICATION"))); /* create xlogreader for physical replication */ - xlogreader = - XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close); + xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, + NULL, wal_segment_close); if (!xlogreader) ereport(ERROR, diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c index 712c85281c..8a6209b061 100644 --- a/src/bin/pg_rewind/parsexlog.c +++ b/src/bin/pg_rewind/parsexlog.c @@ -61,7 +61,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex, XLogReaderState *xlogreader; char *errormsg; - xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL); + xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL); if (xlogreader == NULL) pg_fatal("out of memory"); @@ -121,7 +121,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex, char *errormsg; XLogRecPtr endptr; - xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL); + xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL); if (xlogreader == NULL) pg_fatal("out of memory"); @@ -181,7 +181,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex, forkptr += SizeOfXLogShortPHD; } - xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL); + xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL); if (xlogreader == NULL) pg_fatal("out of memory"); diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c index f514de6b13..b635537659 100644 --- a/src/bin/pg_waldump/pg_waldump.c +++ b/src/bin/pg_waldump/pg_waldump.c @@ -1041,7 +1041,7 @@ main(int argc, char **argv) /* we have everything we need, start reading */ xlogreader_state = - XLogReaderAllocate(WalSegSz, waldir, WALDumpCloseSegment); + XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, waldir, WALDumpCloseSegment); if (!xlogreader_state) fatal_error("out of memory"); diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h index b9dad7ee51..1853516333 100644 --- a/src/include/access/xlogreader.h +++ b/src/include/access/xlogreader.h @@ -151,6 +151,7 @@ struct XLogReaderState * ---------------------------------------- */ /* variables to communicate with page reader */ + int pageSize; /* size of a page */ XLogRecPtr readPagePtr; /* page pointer to read */ int32 readLen; /* bytes requested to reader, or actual bytes * read by reader, which must be larger than @@ -243,7 +244,7 @@ struct XLogReaderState /* Get a new XLogReader */ extern XLogReaderState *XLogReaderAllocate(int wal_segment_size, - const char *waldir, + int page_size, const char *waldir, WALSegmentCleanupCB cleanup_cb); /* Free an XLogReader */ -- 2.27.0 ----Next_Part(Thu_Mar__4_11_28_53_2021_014)---- ^ permalink raw reply [nested|flat] 25+ messages in thread
* [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize. @ 2019-09-05 12:29 Kyotaro Horiguchi <[email protected]> 0 siblings, 0 replies; 25+ messages in thread From: Kyotaro Horiguchi @ 2019-09-05 12:29 UTC (permalink / raw) XLog reader facility and its callers currently relies on the same constant XLOG_BLCKSZ. This patch moves the responsibility to determine the block size to the callers of the facility. --- src/backend/access/transam/twophase.c | 3 +- src/backend/access/transam/xlog.c | 9 ++-- src/backend/access/transam/xlogreader.c | 57 ++++++++++++----------- src/backend/replication/logical/logical.c | 5 +- src/backend/replication/walsender.c | 4 +- src/bin/pg_rewind/parsexlog.c | 6 +-- src/bin/pg_waldump/pg_waldump.c | 2 +- src/include/access/xlogreader.h | 3 +- 8 files changed, 50 insertions(+), 39 deletions(-) diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c index 7d5a96a5f5..9f9e7de253 100644 --- a/src/backend/access/transam/twophase.c +++ b/src/backend/access/transam/twophase.c @@ -1325,7 +1325,8 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len) XLogReaderState *xlogreader; char *errormsg; - xlogreader = XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close); + xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, + NULL, wal_segment_close); if (!xlogreader) ereport(ERROR, diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index b517ca85ad..0822ac70cb 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -1222,7 +1222,10 @@ XLogInsertRecord(XLogRecData *rdata, appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len); if (!debug_reader) - debug_reader = XLogReaderAllocate(wal_segment_size, NULL, NULL); + { + debug_reader = + XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, NULL); + } if (!debug_reader) { @@ -6531,8 +6534,8 @@ StartupXLOG(void) OwnLatch(&XLogCtl->recoveryWakeupLatch); /* Set up XLOG reader facility */ - xlogreader = - XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close); + xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, + NULL, wal_segment_close); if (!xlogreader) ereport(ERROR, diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c index ba3e4868f7..e5baf13086 100644 --- a/src/backend/access/transam/xlogreader.c +++ b/src/backend/access/transam/xlogreader.c @@ -72,7 +72,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...) * Returns NULL if the xlogreader couldn't be allocated. */ XLogReaderState * -XLogReaderAllocate(int wal_segment_size, const char *waldir, +XLogReaderAllocate(int wal_segment_size, int page_size, const char *waldir, WALSegmentCleanupCB cleanup_cb) { XLogReaderState *state; @@ -87,6 +87,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir, state->cleanup_cb = cleanup_cb; state->max_block_id = -1; + state->pageSize = page_size; /* * Permanently allocate readBuf. We do it this way, rather than just @@ -95,7 +96,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir, * isn't guaranteed to have any particular alignment, whereas * palloc_extended() will provide MAXALIGN'd storage. */ - state->readBuf = (char *) palloc_extended(XLOG_BLCKSZ, + state->readBuf = (char *) palloc_extended(state->pageSize, MCXT_ALLOC_NO_OOM); if (!state->readBuf) { @@ -163,7 +164,7 @@ XLogReaderFree(XLogReaderState *state) * readRecordBufSize is set to the new buffer size. * * To avoid useless small increases, round its size to a multiple of - * XLOG_BLCKSZ, and make sure it's at least 5*Max(BLCKSZ, XLOG_BLCKSZ) to start + * page size, and make sure it's at least 5*Max(BLCKSZ, <page size>) to start * with. (That is enough for all "normal" records, but very large commit or * abort records might need more space.) */ @@ -172,8 +173,8 @@ allocate_recordbuf(XLogReaderState *state, uint32 reclength) { uint32 newSize = reclength; - newSize += XLOG_BLCKSZ - (newSize % XLOG_BLCKSZ); - newSize = Max(newSize, 5 * Max(BLCKSZ, XLOG_BLCKSZ)); + newSize += state->pageSize - (newSize % state->pageSize); + newSize = Max(newSize, 5 * Max(BLCKSZ, state->pageSize)); #ifndef FRONTEND @@ -238,6 +239,8 @@ void XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr) { Assert(!XLogRecPtrIsInvalid(RecPtr)); + Assert(state->readBuf != NULL && + state->readBuf == (void *) MAXALIGN(state->readBuf)); ResetDecoder(state); @@ -366,15 +369,15 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) XLogPageHeader pageHeader; targetPagePtr = - state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ); - targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ; + state->ReadRecPtr - (state->ReadRecPtr % state->pageSize); + targetRecOff = state->ReadRecPtr % state->pageSize; /* * Check if we have enough data. For the first record in the page, * the requesting length doesn't contain page header. */ if (XLogNeedData(state, targetPagePtr, - Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ), + Min(targetRecOff + SizeOfXLogRecord, state->pageSize), targetRecOff != 0)) return XLREAD_NEED_DATA; @@ -421,7 +424,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) * verified that we got the whole header. */ prec = (XLogRecord *) (state->readBuf + - state->ReadRecPtr % XLOG_BLCKSZ); + state->ReadRecPtr % state->pageSize); total_len = prec->xl_tot_len; /* @@ -432,7 +435,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) * ensure that we enter the XLREAD_CONTINUATION state below; * otherwise we might fail to apply ValidXLogRecordHeader at all. */ - if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord) + if (targetRecOff <= state->pageSize - SizeOfXLogRecord) { if (!ValidXLogRecordHeader(state, state->ReadRecPtr, state->PrevRecPtr, prec)) @@ -477,10 +480,10 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) * available */ targetPagePtr = - state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ); - targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ; + state->ReadRecPtr - (state->ReadRecPtr % state->pageSize); + targetRecOff = state->ReadRecPtr % state->pageSize; - request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ); + request_len = Min(targetRecOff + total_len, state->pageSize); record_len = request_len - targetRecOff; /* ReadRecPtr contains page header */ @@ -549,7 +552,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) /* Calculate pointer to beginning of next page */ state->recordContRecPtr = state->ReadRecPtr + record_len; - Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0); + Assert(state->recordContRecPtr % state->pageSize == 0); state->readRecordState = XLREAD_CONTINUATION; } @@ -576,7 +579,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) /* this request contains page header */ Assert (targetPagePtr != 0); if (XLogNeedData(state, targetPagePtr, - Min(state->recordRemainLen, XLOG_BLCKSZ), + Min(state->recordRemainLen, state->pageSize), false)) return XLREAD_NEED_DATA; @@ -628,7 +631,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) Assert(state->readLen >= pageHeaderSize); contdata = (char *) state->readBuf + pageHeaderSize; - record_len = XLOG_BLCKSZ - pageHeaderSize; + record_len = state->pageSize - pageHeaderSize; if (pageHeader->xlp_rem_len < record_len) record_len = pageHeader->xlp_rem_len; @@ -655,7 +658,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) } /* Calculate pointer to beginning of next page, and continue */ - state->recordContRecPtr += XLOG_BLCKSZ; + state->recordContRecPtr += state->pageSize; } /* targetPagePtr is pointing the last-read page here */ @@ -789,10 +792,10 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen, XLogPageHeaderSize((XLogPageHeader) state->readBuf); addLen = pageHeaderSize; - if (reqLen + pageHeaderSize <= XLOG_BLCKSZ) + if (reqLen + pageHeaderSize <= state->pageSize) addLen = pageHeaderSize; else - addLen = XLOG_BLCKSZ - reqLen; + addLen = state->pageSize - reqLen; Assert(addLen >= 0); } @@ -805,7 +808,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen, /* Data is not in our buffer, request the caller for it. */ XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize); targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize); - Assert((pageptr % XLOG_BLCKSZ) == 0); + Assert((pageptr % state->pageSize) == 0); /* * Every time we request to load new data of a page to the caller, even if @@ -829,7 +832,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen, * will not come back here, but will request the actual target page. */ state->readPagePtr = pageptr - targetPageOff; - state->readLen = XLOG_BLCKSZ; + state->readLen = state->pageSize; return true; } @@ -962,7 +965,7 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr, int32 offset; XLogPageHeader hdr = (XLogPageHeader) phdr; - Assert((recptr % XLOG_BLCKSZ) == 0); + Assert((recptr % state->pageSize) == 0); XLByteToSeg(recptr, segno, state->segcxt.ws_segsize); offset = XLogSegmentOffset(recptr, state->segcxt.ws_segsize); @@ -1016,10 +1019,10 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr, "WAL file is from different database system: incorrect segment size in page header"); return false; } - else if (longhdr->xlp_xlog_blcksz != XLOG_BLCKSZ) + else if (longhdr->xlp_xlog_blcksz != state->pageSize) { report_invalid_record(state, - "WAL file is from different database system: incorrect XLOG_BLCKSZ in page header"); + "WAL file is from different database system: incorrect block size in page header"); return false; } } @@ -1140,7 +1143,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr, * XLogNeedData() is prepared to handle that and will read at * least short page-header worth of data */ - targetRecOff = tmpRecPtr % XLOG_BLCKSZ; + targetRecOff = tmpRecPtr % state->pageSize; /* scroll back to page boundary */ targetPagePtr = tmpRecPtr - targetRecOff; @@ -1174,8 +1177,8 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr, * * Note that record headers are MAXALIGN'ed */ - if (MAXALIGN(header->xlp_rem_len) >= (XLOG_BLCKSZ - pageHeaderSize)) - tmpRecPtr = targetPagePtr + XLOG_BLCKSZ; + if (MAXALIGN(header->xlp_rem_len) >= (state->pageSize - pageHeaderSize)) + tmpRecPtr = targetPagePtr + state->pageSize; else { /* diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c index 9cbd8afa49..f2dd495310 100644 --- a/src/backend/replication/logical/logical.c +++ b/src/backend/replication/logical/logical.c @@ -199,11 +199,14 @@ StartupDecodingContext(List *output_plugin_options, ctx->slot = slot; - ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, cleanup_cb); + ctx->reader = + XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, cleanup_cb); + if (!ctx->reader) ereport(ERROR, (errcode(ERRCODE_OUT_OF_MEMORY), errmsg("out of memory"))); + ctx->reader->readBuf = palloc(XLOG_BLCKSZ); ctx->page_read = page_read; ctx->reorder = ReorderBufferAllocate(); diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c index 062093727d..8e95600ef7 100644 --- a/src/backend/replication/walsender.c +++ b/src/backend/replication/walsender.c @@ -579,8 +579,8 @@ StartReplication(StartReplicationCmd *cmd) errmsg("IDENTIFY_SYSTEM has not been run before START_REPLICATION"))); /* create xlogreader for physical replication */ - xlogreader = - XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close); + xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, + NULL, wal_segment_close); if (!xlogreader) ereport(ERROR, diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c index 712c85281c..8a6209b061 100644 --- a/src/bin/pg_rewind/parsexlog.c +++ b/src/bin/pg_rewind/parsexlog.c @@ -61,7 +61,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex, XLogReaderState *xlogreader; char *errormsg; - xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL); + xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL); if (xlogreader == NULL) pg_fatal("out of memory"); @@ -121,7 +121,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex, char *errormsg; XLogRecPtr endptr; - xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL); + xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL); if (xlogreader == NULL) pg_fatal("out of memory"); @@ -181,7 +181,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex, forkptr += SizeOfXLogShortPHD; } - xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL); + xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL); if (xlogreader == NULL) pg_fatal("out of memory"); diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c index f514de6b13..b635537659 100644 --- a/src/bin/pg_waldump/pg_waldump.c +++ b/src/bin/pg_waldump/pg_waldump.c @@ -1041,7 +1041,7 @@ main(int argc, char **argv) /* we have everything we need, start reading */ xlogreader_state = - XLogReaderAllocate(WalSegSz, waldir, WALDumpCloseSegment); + XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, waldir, WALDumpCloseSegment); if (!xlogreader_state) fatal_error("out of memory"); diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h index b9dad7ee51..1853516333 100644 --- a/src/include/access/xlogreader.h +++ b/src/include/access/xlogreader.h @@ -151,6 +151,7 @@ struct XLogReaderState * ---------------------------------------- */ /* variables to communicate with page reader */ + int pageSize; /* size of a page */ XLogRecPtr readPagePtr; /* page pointer to read */ int32 readLen; /* bytes requested to reader, or actual bytes * read by reader, which must be larger than @@ -243,7 +244,7 @@ struct XLogReaderState /* Get a new XLogReader */ extern XLogReaderState *XLogReaderAllocate(int wal_segment_size, - const char *waldir, + int page_size, const char *waldir, WALSegmentCleanupCB cleanup_cb); /* Free an XLogReader */ -- 2.27.0 ----Next_Part(Thu_Mar__4_11_28_53_2021_014)---- ^ permalink raw reply [nested|flat] 25+ messages in thread
* [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize. @ 2019-09-05 12:29 Kyotaro Horiguchi <[email protected]> 0 siblings, 0 replies; 25+ messages in thread From: Kyotaro Horiguchi @ 2019-09-05 12:29 UTC (permalink / raw) XLog reader facility and its callers currently relies on the same constant XLOG_BLCKSZ. This patch moves the responsibility to determine the block size to the callers of the facility. --- src/backend/access/transam/twophase.c | 3 +- src/backend/access/transam/xlog.c | 9 ++-- src/backend/access/transam/xlogreader.c | 57 ++++++++++++----------- src/backend/replication/logical/logical.c | 5 +- src/backend/replication/walsender.c | 4 +- src/bin/pg_rewind/parsexlog.c | 6 +-- src/bin/pg_waldump/pg_waldump.c | 2 +- src/include/access/xlogreader.h | 3 +- 8 files changed, 50 insertions(+), 39 deletions(-) diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c index 7d5a96a5f5..9f9e7de253 100644 --- a/src/backend/access/transam/twophase.c +++ b/src/backend/access/transam/twophase.c @@ -1325,7 +1325,8 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len) XLogReaderState *xlogreader; char *errormsg; - xlogreader = XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close); + xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, + NULL, wal_segment_close); if (!xlogreader) ereport(ERROR, diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index b517ca85ad..0822ac70cb 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -1222,7 +1222,10 @@ XLogInsertRecord(XLogRecData *rdata, appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len); if (!debug_reader) - debug_reader = XLogReaderAllocate(wal_segment_size, NULL, NULL); + { + debug_reader = + XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, NULL); + } if (!debug_reader) { @@ -6531,8 +6534,8 @@ StartupXLOG(void) OwnLatch(&XLogCtl->recoveryWakeupLatch); /* Set up XLOG reader facility */ - xlogreader = - XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close); + xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, + NULL, wal_segment_close); if (!xlogreader) ereport(ERROR, diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c index ba3e4868f7..e5baf13086 100644 --- a/src/backend/access/transam/xlogreader.c +++ b/src/backend/access/transam/xlogreader.c @@ -72,7 +72,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...) * Returns NULL if the xlogreader couldn't be allocated. */ XLogReaderState * -XLogReaderAllocate(int wal_segment_size, const char *waldir, +XLogReaderAllocate(int wal_segment_size, int page_size, const char *waldir, WALSegmentCleanupCB cleanup_cb) { XLogReaderState *state; @@ -87,6 +87,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir, state->cleanup_cb = cleanup_cb; state->max_block_id = -1; + state->pageSize = page_size; /* * Permanently allocate readBuf. We do it this way, rather than just @@ -95,7 +96,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir, * isn't guaranteed to have any particular alignment, whereas * palloc_extended() will provide MAXALIGN'd storage. */ - state->readBuf = (char *) palloc_extended(XLOG_BLCKSZ, + state->readBuf = (char *) palloc_extended(state->pageSize, MCXT_ALLOC_NO_OOM); if (!state->readBuf) { @@ -163,7 +164,7 @@ XLogReaderFree(XLogReaderState *state) * readRecordBufSize is set to the new buffer size. * * To avoid useless small increases, round its size to a multiple of - * XLOG_BLCKSZ, and make sure it's at least 5*Max(BLCKSZ, XLOG_BLCKSZ) to start + * page size, and make sure it's at least 5*Max(BLCKSZ, <page size>) to start * with. (That is enough for all "normal" records, but very large commit or * abort records might need more space.) */ @@ -172,8 +173,8 @@ allocate_recordbuf(XLogReaderState *state, uint32 reclength) { uint32 newSize = reclength; - newSize += XLOG_BLCKSZ - (newSize % XLOG_BLCKSZ); - newSize = Max(newSize, 5 * Max(BLCKSZ, XLOG_BLCKSZ)); + newSize += state->pageSize - (newSize % state->pageSize); + newSize = Max(newSize, 5 * Max(BLCKSZ, state->pageSize)); #ifndef FRONTEND @@ -238,6 +239,8 @@ void XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr) { Assert(!XLogRecPtrIsInvalid(RecPtr)); + Assert(state->readBuf != NULL && + state->readBuf == (void *) MAXALIGN(state->readBuf)); ResetDecoder(state); @@ -366,15 +369,15 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) XLogPageHeader pageHeader; targetPagePtr = - state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ); - targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ; + state->ReadRecPtr - (state->ReadRecPtr % state->pageSize); + targetRecOff = state->ReadRecPtr % state->pageSize; /* * Check if we have enough data. For the first record in the page, * the requesting length doesn't contain page header. */ if (XLogNeedData(state, targetPagePtr, - Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ), + Min(targetRecOff + SizeOfXLogRecord, state->pageSize), targetRecOff != 0)) return XLREAD_NEED_DATA; @@ -421,7 +424,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) * verified that we got the whole header. */ prec = (XLogRecord *) (state->readBuf + - state->ReadRecPtr % XLOG_BLCKSZ); + state->ReadRecPtr % state->pageSize); total_len = prec->xl_tot_len; /* @@ -432,7 +435,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) * ensure that we enter the XLREAD_CONTINUATION state below; * otherwise we might fail to apply ValidXLogRecordHeader at all. */ - if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord) + if (targetRecOff <= state->pageSize - SizeOfXLogRecord) { if (!ValidXLogRecordHeader(state, state->ReadRecPtr, state->PrevRecPtr, prec)) @@ -477,10 +480,10 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) * available */ targetPagePtr = - state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ); - targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ; + state->ReadRecPtr - (state->ReadRecPtr % state->pageSize); + targetRecOff = state->ReadRecPtr % state->pageSize; - request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ); + request_len = Min(targetRecOff + total_len, state->pageSize); record_len = request_len - targetRecOff; /* ReadRecPtr contains page header */ @@ -549,7 +552,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) /* Calculate pointer to beginning of next page */ state->recordContRecPtr = state->ReadRecPtr + record_len; - Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0); + Assert(state->recordContRecPtr % state->pageSize == 0); state->readRecordState = XLREAD_CONTINUATION; } @@ -576,7 +579,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) /* this request contains page header */ Assert (targetPagePtr != 0); if (XLogNeedData(state, targetPagePtr, - Min(state->recordRemainLen, XLOG_BLCKSZ), + Min(state->recordRemainLen, state->pageSize), false)) return XLREAD_NEED_DATA; @@ -628,7 +631,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) Assert(state->readLen >= pageHeaderSize); contdata = (char *) state->readBuf + pageHeaderSize; - record_len = XLOG_BLCKSZ - pageHeaderSize; + record_len = state->pageSize - pageHeaderSize; if (pageHeader->xlp_rem_len < record_len) record_len = pageHeader->xlp_rem_len; @@ -655,7 +658,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) } /* Calculate pointer to beginning of next page, and continue */ - state->recordContRecPtr += XLOG_BLCKSZ; + state->recordContRecPtr += state->pageSize; } /* targetPagePtr is pointing the last-read page here */ @@ -789,10 +792,10 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen, XLogPageHeaderSize((XLogPageHeader) state->readBuf); addLen = pageHeaderSize; - if (reqLen + pageHeaderSize <= XLOG_BLCKSZ) + if (reqLen + pageHeaderSize <= state->pageSize) addLen = pageHeaderSize; else - addLen = XLOG_BLCKSZ - reqLen; + addLen = state->pageSize - reqLen; Assert(addLen >= 0); } @@ -805,7 +808,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen, /* Data is not in our buffer, request the caller for it. */ XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize); targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize); - Assert((pageptr % XLOG_BLCKSZ) == 0); + Assert((pageptr % state->pageSize) == 0); /* * Every time we request to load new data of a page to the caller, even if @@ -829,7 +832,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen, * will not come back here, but will request the actual target page. */ state->readPagePtr = pageptr - targetPageOff; - state->readLen = XLOG_BLCKSZ; + state->readLen = state->pageSize; return true; } @@ -962,7 +965,7 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr, int32 offset; XLogPageHeader hdr = (XLogPageHeader) phdr; - Assert((recptr % XLOG_BLCKSZ) == 0); + Assert((recptr % state->pageSize) == 0); XLByteToSeg(recptr, segno, state->segcxt.ws_segsize); offset = XLogSegmentOffset(recptr, state->segcxt.ws_segsize); @@ -1016,10 +1019,10 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr, "WAL file is from different database system: incorrect segment size in page header"); return false; } - else if (longhdr->xlp_xlog_blcksz != XLOG_BLCKSZ) + else if (longhdr->xlp_xlog_blcksz != state->pageSize) { report_invalid_record(state, - "WAL file is from different database system: incorrect XLOG_BLCKSZ in page header"); + "WAL file is from different database system: incorrect block size in page header"); return false; } } @@ -1140,7 +1143,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr, * XLogNeedData() is prepared to handle that and will read at * least short page-header worth of data */ - targetRecOff = tmpRecPtr % XLOG_BLCKSZ; + targetRecOff = tmpRecPtr % state->pageSize; /* scroll back to page boundary */ targetPagePtr = tmpRecPtr - targetRecOff; @@ -1174,8 +1177,8 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr, * * Note that record headers are MAXALIGN'ed */ - if (MAXALIGN(header->xlp_rem_len) >= (XLOG_BLCKSZ - pageHeaderSize)) - tmpRecPtr = targetPagePtr + XLOG_BLCKSZ; + if (MAXALIGN(header->xlp_rem_len) >= (state->pageSize - pageHeaderSize)) + tmpRecPtr = targetPagePtr + state->pageSize; else { /* diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c index 9cbd8afa49..f2dd495310 100644 --- a/src/backend/replication/logical/logical.c +++ b/src/backend/replication/logical/logical.c @@ -199,11 +199,14 @@ StartupDecodingContext(List *output_plugin_options, ctx->slot = slot; - ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, cleanup_cb); + ctx->reader = + XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, cleanup_cb); + if (!ctx->reader) ereport(ERROR, (errcode(ERRCODE_OUT_OF_MEMORY), errmsg("out of memory"))); + ctx->reader->readBuf = palloc(XLOG_BLCKSZ); ctx->page_read = page_read; ctx->reorder = ReorderBufferAllocate(); diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c index 062093727d..8e95600ef7 100644 --- a/src/backend/replication/walsender.c +++ b/src/backend/replication/walsender.c @@ -579,8 +579,8 @@ StartReplication(StartReplicationCmd *cmd) errmsg("IDENTIFY_SYSTEM has not been run before START_REPLICATION"))); /* create xlogreader for physical replication */ - xlogreader = - XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close); + xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, + NULL, wal_segment_close); if (!xlogreader) ereport(ERROR, diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c index 712c85281c..8a6209b061 100644 --- a/src/bin/pg_rewind/parsexlog.c +++ b/src/bin/pg_rewind/parsexlog.c @@ -61,7 +61,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex, XLogReaderState *xlogreader; char *errormsg; - xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL); + xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL); if (xlogreader == NULL) pg_fatal("out of memory"); @@ -121,7 +121,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex, char *errormsg; XLogRecPtr endptr; - xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL); + xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL); if (xlogreader == NULL) pg_fatal("out of memory"); @@ -181,7 +181,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex, forkptr += SizeOfXLogShortPHD; } - xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL); + xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL); if (xlogreader == NULL) pg_fatal("out of memory"); diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c index f514de6b13..b635537659 100644 --- a/src/bin/pg_waldump/pg_waldump.c +++ b/src/bin/pg_waldump/pg_waldump.c @@ -1041,7 +1041,7 @@ main(int argc, char **argv) /* we have everything we need, start reading */ xlogreader_state = - XLogReaderAllocate(WalSegSz, waldir, WALDumpCloseSegment); + XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, waldir, WALDumpCloseSegment); if (!xlogreader_state) fatal_error("out of memory"); diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h index b9dad7ee51..1853516333 100644 --- a/src/include/access/xlogreader.h +++ b/src/include/access/xlogreader.h @@ -151,6 +151,7 @@ struct XLogReaderState * ---------------------------------------- */ /* variables to communicate with page reader */ + int pageSize; /* size of a page */ XLogRecPtr readPagePtr; /* page pointer to read */ int32 readLen; /* bytes requested to reader, or actual bytes * read by reader, which must be larger than @@ -243,7 +244,7 @@ struct XLogReaderState /* Get a new XLogReader */ extern XLogReaderState *XLogReaderAllocate(int wal_segment_size, - const char *waldir, + int page_size, const char *waldir, WALSegmentCleanupCB cleanup_cb); /* Free an XLogReader */ -- 2.27.0 ----Next_Part(Thu_Mar__4_11_28_53_2021_014)---- ^ permalink raw reply [nested|flat] 25+ messages in thread
* [PATCH v16 4/4] Allow xlogreader to use different xlog blocksize. @ 2019-09-05 12:29 Kyotaro Horiguchi <[email protected]> 0 siblings, 0 replies; 25+ messages in thread From: Kyotaro Horiguchi @ 2019-09-05 12:29 UTC (permalink / raw) XLog reader facility and its callers currently relies on the same constant XLOG_BLCKSZ. This patch moves the responsibility to determine the block size to the callers of the facility. --- src/backend/access/transam/twophase.c | 3 +- src/backend/access/transam/xlog.c | 9 ++-- src/backend/access/transam/xlogreader.c | 57 ++++++++++++----------- src/backend/replication/logical/logical.c | 5 +- src/backend/replication/walsender.c | 4 +- src/bin/pg_rewind/parsexlog.c | 6 +-- src/bin/pg_waldump/pg_waldump.c | 2 +- src/include/access/xlogreader.h | 3 +- 8 files changed, 50 insertions(+), 39 deletions(-) diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c index 9a56b3487e..b64c61f742 100644 --- a/src/backend/access/transam/twophase.c +++ b/src/backend/access/transam/twophase.c @@ -1324,7 +1324,8 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len) XLogReaderState *xlogreader; char *errormsg; - xlogreader = XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close); + xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, + NULL, wal_segment_close); if (!xlogreader) ereport(ERROR, diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index eccb88b611..ec78de5dd9 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -1213,7 +1213,10 @@ XLogInsertRecord(XLogRecData *rdata, appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len); if (!debug_reader) - debug_reader = XLogReaderAllocate(wal_segment_size, NULL, NULL); + { + debug_reader = + XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, NULL); + } if (!debug_reader) { @@ -6469,8 +6472,8 @@ StartupXLOG(void) OwnLatch(&XLogCtl->recoveryWakeupLatch); /* Set up XLOG reader facility */ - xlogreader = - XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close); + xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, + NULL, wal_segment_close); if (!xlogreader) ereport(ERROR, diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c index d829e57baa..582b1dd11b 100644 --- a/src/backend/access/transam/xlogreader.c +++ b/src/backend/access/transam/xlogreader.c @@ -72,7 +72,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...) * Returns NULL if the xlogreader couldn't be allocated. */ XLogReaderState * -XLogReaderAllocate(int wal_segment_size, const char *waldir, +XLogReaderAllocate(int wal_segment_size, int page_size, const char *waldir, WALSegmentCleanupCB cleanup_cb) { XLogReaderState *state; @@ -87,6 +87,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir, state->cleanup_cb = cleanup_cb; state->max_block_id = -1; + state->pageSize = page_size; /* * Permanently allocate readBuf. We do it this way, rather than just @@ -95,7 +96,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir, * isn't guaranteed to have any particular alignment, whereas * palloc_extended() will provide MAXALIGN'd storage. */ - state->readBuf = (char *) palloc_extended(XLOG_BLCKSZ, + state->readBuf = (char *) palloc_extended(state->pageSize, MCXT_ALLOC_NO_OOM); if (!state->readBuf) { @@ -163,7 +164,7 @@ XLogReaderFree(XLogReaderState *state) * readRecordBufSize is set to the new buffer size. * * To avoid useless small increases, round its size to a multiple of - * XLOG_BLCKSZ, and make sure it's at least 5*Max(BLCKSZ, XLOG_BLCKSZ) to start + * page size, and make sure it's at least 5*Max(BLCKSZ, <page size>) to start * with. (That is enough for all "normal" records, but very large commit or * abort records might need more space.) */ @@ -172,8 +173,8 @@ allocate_recordbuf(XLogReaderState *state, uint32 reclength) { uint32 newSize = reclength; - newSize += XLOG_BLCKSZ - (newSize % XLOG_BLCKSZ); - newSize = Max(newSize, 5 * Max(BLCKSZ, XLOG_BLCKSZ)); + newSize += state->pageSize - (newSize % state->pageSize); + newSize = Max(newSize, 5 * Max(BLCKSZ, state->pageSize)); #ifndef FRONTEND @@ -238,6 +239,8 @@ void XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr) { Assert(!XLogRecPtrIsInvalid(RecPtr)); + Assert(state->readBuf != NULL && + state->readBuf == (void *) MAXALIGN(state->readBuf)); ResetDecoder(state); @@ -366,15 +369,15 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) XLogPageHeader pageHeader; targetPagePtr = - state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ); - targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ; + state->ReadRecPtr - (state->ReadRecPtr % state->pageSize); + targetRecOff = state->ReadRecPtr % state->pageSize; /* * Check if we have enough data. For the first record in the page, * the requesting length doesn't contain page header. */ if (XLogNeedData(state, targetPagePtr, - Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ), + Min(targetRecOff + SizeOfXLogRecord, state->pageSize), targetRecOff != 0)) return XLREAD_NEED_DATA; @@ -422,7 +425,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) * verified that we got the whole header. */ prec = (XLogRecord *) (state->readBuf + - state->ReadRecPtr % XLOG_BLCKSZ); + state->ReadRecPtr % state->pageSize); total_len = prec->xl_tot_len; /* @@ -433,7 +436,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) * ensure that we enter the XLREAD_CONTINUATION state below; * otherwise we might fail to apply ValidXLogRecordHeader at all. */ - if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord) + if (targetRecOff <= state->pageSize - SizeOfXLogRecord) { if (!ValidXLogRecordHeader(state, state->ReadRecPtr, state->PrevRecPtr, prec)) @@ -479,10 +482,10 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) * available */ targetPagePtr = - state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ); - targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ; + state->ReadRecPtr - (state->ReadRecPtr % state->pageSize); + targetRecOff = state->ReadRecPtr % state->pageSize; - request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ); + request_len = Min(targetRecOff + total_len, state->pageSize); record_len = request_len - targetRecOff; /* ReadRecPtr contains page header */ @@ -552,7 +555,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) /* Calculate pointer to beginning of next page */ state->recordContRecPtr = state->ReadRecPtr + record_len; - Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0); + Assert(state->recordContRecPtr % state->pageSize == 0); state->readRecordState = XLREAD_CONTINUATION; } @@ -579,7 +582,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) /* this request contains page header */ Assert (targetPagePtr != 0); if (XLogNeedData(state, targetPagePtr, - Min(state->recordRemainLen, XLOG_BLCKSZ), + Min(state->recordRemainLen, state->pageSize), false)) return XLREAD_NEED_DATA; @@ -631,7 +634,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) Assert(state->readLen >= pageHeaderSize); contdata = (char *) state->readBuf + pageHeaderSize; - record_len = XLOG_BLCKSZ - pageHeaderSize; + record_len = state->pageSize - pageHeaderSize; if (pageHeader->xlp_rem_len < record_len) record_len = pageHeader->xlp_rem_len; @@ -658,7 +661,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) } /* Calculate pointer to beginning of next page, and continue */ - state->recordContRecPtr += XLOG_BLCKSZ; + state->recordContRecPtr += state->pageSize; } /* targetPagePtr is pointing the last-read page here */ @@ -792,10 +795,10 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen, XLogPageHeaderSize((XLogPageHeader) state->readBuf); addLen = pageHeaderSize; - if (reqLen + pageHeaderSize <= XLOG_BLCKSZ) + if (reqLen + pageHeaderSize <= state->pageSize) addLen = pageHeaderSize; else - addLen = XLOG_BLCKSZ - reqLen; + addLen = state->pageSize - reqLen; Assert(addLen >= 0); } @@ -808,7 +811,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen, /* Data is not in our buffer, request the caller for it. */ XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize); targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize); - Assert((pageptr % XLOG_BLCKSZ) == 0); + Assert((pageptr % state->pageSize) == 0); /* * Every time we request to load new data of a page to the caller, even if @@ -832,7 +835,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen, * will not come back here, but will request the actual target page. */ state->readPagePtr = pageptr - targetPageOff; - state->readLen = XLOG_BLCKSZ; + state->readLen = state->pageSize; return true; } @@ -968,7 +971,7 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr, int32 offset; XLogPageHeader hdr = (XLogPageHeader) phdr; - Assert((recptr % XLOG_BLCKSZ) == 0); + Assert((recptr % state->pageSize) == 0); XLByteToSeg(recptr, segno, state->segcxt.ws_segsize); offset = XLogSegmentOffset(recptr, state->segcxt.ws_segsize); @@ -1022,10 +1025,10 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr, "WAL file is from different database system: incorrect segment size in page header"); return false; } - else if (longhdr->xlp_xlog_blcksz != XLOG_BLCKSZ) + else if (longhdr->xlp_xlog_blcksz != state->pageSize) { report_invalid_record(state, - "WAL file is from different database system: incorrect XLOG_BLCKSZ in page header"); + "WAL file is from different database system: incorrect block size in page header"); return false; } } @@ -1146,7 +1149,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr, * XLogNeedData() is prepared to handle that and will read at * least short page-header worth of data */ - targetRecOff = tmpRecPtr % XLOG_BLCKSZ; + targetRecOff = tmpRecPtr % state->pageSize; /* scroll back to page boundary */ targetPagePtr = tmpRecPtr - targetRecOff; @@ -1180,8 +1183,8 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr, * * Note that record headers are MAXALIGN'ed */ - if (MAXALIGN(header->xlp_rem_len) >= (XLOG_BLCKSZ - pageHeaderSize)) - tmpRecPtr = targetPagePtr + XLOG_BLCKSZ; + if (MAXALIGN(header->xlp_rem_len) >= (state->pageSize - pageHeaderSize)) + tmpRecPtr = targetPagePtr + state->pageSize; else { /* diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c index 97def6f781..ee322ba796 100644 --- a/src/backend/replication/logical/logical.c +++ b/src/backend/replication/logical/logical.c @@ -189,11 +189,14 @@ StartupDecodingContext(List *output_plugin_options, ctx->slot = slot; - ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, cleanup_cb); + ctx->reader = + XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, cleanup_cb); + if (!ctx->reader) ereport(ERROR, (errcode(ERRCODE_OUT_OF_MEMORY), errmsg("out of memory"))); + ctx->reader->readBuf = palloc(XLOG_BLCKSZ); ctx->page_read = page_read; ctx->reorder = ReorderBufferAllocate(); diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c index 33d952f9fd..a0914ddb25 100644 --- a/src/backend/replication/walsender.c +++ b/src/backend/replication/walsender.c @@ -578,8 +578,8 @@ StartReplication(StartReplicationCmd *cmd) errmsg("IDENTIFY_SYSTEM has not been run before START_REPLICATION"))); /* create xlogreader for physical replication */ - xlogreader = - XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close); + xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, + NULL, wal_segment_close); if (!xlogreader) ereport(ERROR, diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c index 7344a7c692..c5881594b7 100644 --- a/src/bin/pg_rewind/parsexlog.c +++ b/src/bin/pg_rewind/parsexlog.c @@ -58,7 +58,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex, XLogReaderState *xlogreader; char *errormsg; - xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL); + xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL); if (xlogreader == NULL) pg_fatal("out of memory"); @@ -112,7 +112,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex, char *errormsg; XLogRecPtr endptr; - xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL); + xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL); if (xlogreader == NULL) pg_fatal("out of memory"); @@ -172,7 +172,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex, forkptr += SizeOfXLogShortPHD; } - xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL); + xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL); if (xlogreader == NULL) pg_fatal("out of memory"); diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c index 9329a6a6d6..025479431d 100644 --- a/src/bin/pg_waldump/pg_waldump.c +++ b/src/bin/pg_waldump/pg_waldump.c @@ -1043,7 +1043,7 @@ main(int argc, char **argv) /* we have everything we need, start reading */ xlogreader_state = - XLogReaderAllocate(WalSegSz, waldir, WALDumpCloseSegment); + XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, waldir, WALDumpCloseSegment); if (!xlogreader_state) fatal_error("out of memory"); diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h index 732d463a49..e9d7d70db7 100644 --- a/src/include/access/xlogreader.h +++ b/src/include/access/xlogreader.h @@ -151,6 +151,7 @@ struct XLogReaderState * ---------------------------------------- */ /* variables to communicate with page reader */ + int pageSize; /* size of a page */ XLogRecPtr readPagePtr; /* page pointer to read */ int32 readLen; /* bytes requested to reader, or actual bytes * read by reader, which must be larger than @@ -243,7 +244,7 @@ struct XLogReaderState /* Get a new XLogReader */ extern XLogReaderState *XLogReaderAllocate(int wal_segment_size, - const char *waldir, + int page_size, const char *waldir, WALSegmentCleanupCB cleanup_cb); /* Free an XLogReader */ -- 2.18.4 ----Next_Part(Tue_Sep__8_16_35_16_2020_701)---- ^ permalink raw reply [nested|flat] 25+ messages in thread
* [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize. @ 2019-09-05 12:29 Kyotaro Horiguchi <[email protected]> 0 siblings, 0 replies; 25+ messages in thread From: Kyotaro Horiguchi @ 2019-09-05 12:29 UTC (permalink / raw) XLog reader facility and its callers currently relies on the same constant XLOG_BLCKSZ. This patch moves the responsibility to determine the block size to the callers of the facility. --- src/backend/access/transam/twophase.c | 3 +- src/backend/access/transam/xlog.c | 9 ++-- src/backend/access/transam/xlogreader.c | 57 ++++++++++++----------- src/backend/replication/logical/logical.c | 5 +- src/backend/replication/walsender.c | 4 +- src/bin/pg_rewind/parsexlog.c | 6 +-- src/bin/pg_waldump/pg_waldump.c | 2 +- src/include/access/xlogreader.h | 3 +- 8 files changed, 50 insertions(+), 39 deletions(-) diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c index 7d5a96a5f5..9f9e7de253 100644 --- a/src/backend/access/transam/twophase.c +++ b/src/backend/access/transam/twophase.c @@ -1325,7 +1325,8 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len) XLogReaderState *xlogreader; char *errormsg; - xlogreader = XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close); + xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, + NULL, wal_segment_close); if (!xlogreader) ereport(ERROR, diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index b517ca85ad..0822ac70cb 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -1222,7 +1222,10 @@ XLogInsertRecord(XLogRecData *rdata, appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len); if (!debug_reader) - debug_reader = XLogReaderAllocate(wal_segment_size, NULL, NULL); + { + debug_reader = + XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, NULL); + } if (!debug_reader) { @@ -6531,8 +6534,8 @@ StartupXLOG(void) OwnLatch(&XLogCtl->recoveryWakeupLatch); /* Set up XLOG reader facility */ - xlogreader = - XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close); + xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, + NULL, wal_segment_close); if (!xlogreader) ereport(ERROR, diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c index ba3e4868f7..e5baf13086 100644 --- a/src/backend/access/transam/xlogreader.c +++ b/src/backend/access/transam/xlogreader.c @@ -72,7 +72,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...) * Returns NULL if the xlogreader couldn't be allocated. */ XLogReaderState * -XLogReaderAllocate(int wal_segment_size, const char *waldir, +XLogReaderAllocate(int wal_segment_size, int page_size, const char *waldir, WALSegmentCleanupCB cleanup_cb) { XLogReaderState *state; @@ -87,6 +87,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir, state->cleanup_cb = cleanup_cb; state->max_block_id = -1; + state->pageSize = page_size; /* * Permanently allocate readBuf. We do it this way, rather than just @@ -95,7 +96,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir, * isn't guaranteed to have any particular alignment, whereas * palloc_extended() will provide MAXALIGN'd storage. */ - state->readBuf = (char *) palloc_extended(XLOG_BLCKSZ, + state->readBuf = (char *) palloc_extended(state->pageSize, MCXT_ALLOC_NO_OOM); if (!state->readBuf) { @@ -163,7 +164,7 @@ XLogReaderFree(XLogReaderState *state) * readRecordBufSize is set to the new buffer size. * * To avoid useless small increases, round its size to a multiple of - * XLOG_BLCKSZ, and make sure it's at least 5*Max(BLCKSZ, XLOG_BLCKSZ) to start + * page size, and make sure it's at least 5*Max(BLCKSZ, <page size>) to start * with. (That is enough for all "normal" records, but very large commit or * abort records might need more space.) */ @@ -172,8 +173,8 @@ allocate_recordbuf(XLogReaderState *state, uint32 reclength) { uint32 newSize = reclength; - newSize += XLOG_BLCKSZ - (newSize % XLOG_BLCKSZ); - newSize = Max(newSize, 5 * Max(BLCKSZ, XLOG_BLCKSZ)); + newSize += state->pageSize - (newSize % state->pageSize); + newSize = Max(newSize, 5 * Max(BLCKSZ, state->pageSize)); #ifndef FRONTEND @@ -238,6 +239,8 @@ void XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr) { Assert(!XLogRecPtrIsInvalid(RecPtr)); + Assert(state->readBuf != NULL && + state->readBuf == (void *) MAXALIGN(state->readBuf)); ResetDecoder(state); @@ -366,15 +369,15 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) XLogPageHeader pageHeader; targetPagePtr = - state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ); - targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ; + state->ReadRecPtr - (state->ReadRecPtr % state->pageSize); + targetRecOff = state->ReadRecPtr % state->pageSize; /* * Check if we have enough data. For the first record in the page, * the requesting length doesn't contain page header. */ if (XLogNeedData(state, targetPagePtr, - Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ), + Min(targetRecOff + SizeOfXLogRecord, state->pageSize), targetRecOff != 0)) return XLREAD_NEED_DATA; @@ -421,7 +424,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) * verified that we got the whole header. */ prec = (XLogRecord *) (state->readBuf + - state->ReadRecPtr % XLOG_BLCKSZ); + state->ReadRecPtr % state->pageSize); total_len = prec->xl_tot_len; /* @@ -432,7 +435,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) * ensure that we enter the XLREAD_CONTINUATION state below; * otherwise we might fail to apply ValidXLogRecordHeader at all. */ - if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord) + if (targetRecOff <= state->pageSize - SizeOfXLogRecord) { if (!ValidXLogRecordHeader(state, state->ReadRecPtr, state->PrevRecPtr, prec)) @@ -477,10 +480,10 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) * available */ targetPagePtr = - state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ); - targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ; + state->ReadRecPtr - (state->ReadRecPtr % state->pageSize); + targetRecOff = state->ReadRecPtr % state->pageSize; - request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ); + request_len = Min(targetRecOff + total_len, state->pageSize); record_len = request_len - targetRecOff; /* ReadRecPtr contains page header */ @@ -549,7 +552,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) /* Calculate pointer to beginning of next page */ state->recordContRecPtr = state->ReadRecPtr + record_len; - Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0); + Assert(state->recordContRecPtr % state->pageSize == 0); state->readRecordState = XLREAD_CONTINUATION; } @@ -576,7 +579,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) /* this request contains page header */ Assert (targetPagePtr != 0); if (XLogNeedData(state, targetPagePtr, - Min(state->recordRemainLen, XLOG_BLCKSZ), + Min(state->recordRemainLen, state->pageSize), false)) return XLREAD_NEED_DATA; @@ -628,7 +631,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) Assert(state->readLen >= pageHeaderSize); contdata = (char *) state->readBuf + pageHeaderSize; - record_len = XLOG_BLCKSZ - pageHeaderSize; + record_len = state->pageSize - pageHeaderSize; if (pageHeader->xlp_rem_len < record_len) record_len = pageHeader->xlp_rem_len; @@ -655,7 +658,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) } /* Calculate pointer to beginning of next page, and continue */ - state->recordContRecPtr += XLOG_BLCKSZ; + state->recordContRecPtr += state->pageSize; } /* targetPagePtr is pointing the last-read page here */ @@ -789,10 +792,10 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen, XLogPageHeaderSize((XLogPageHeader) state->readBuf); addLen = pageHeaderSize; - if (reqLen + pageHeaderSize <= XLOG_BLCKSZ) + if (reqLen + pageHeaderSize <= state->pageSize) addLen = pageHeaderSize; else - addLen = XLOG_BLCKSZ - reqLen; + addLen = state->pageSize - reqLen; Assert(addLen >= 0); } @@ -805,7 +808,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen, /* Data is not in our buffer, request the caller for it. */ XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize); targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize); - Assert((pageptr % XLOG_BLCKSZ) == 0); + Assert((pageptr % state->pageSize) == 0); /* * Every time we request to load new data of a page to the caller, even if @@ -829,7 +832,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen, * will not come back here, but will request the actual target page. */ state->readPagePtr = pageptr - targetPageOff; - state->readLen = XLOG_BLCKSZ; + state->readLen = state->pageSize; return true; } @@ -962,7 +965,7 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr, int32 offset; XLogPageHeader hdr = (XLogPageHeader) phdr; - Assert((recptr % XLOG_BLCKSZ) == 0); + Assert((recptr % state->pageSize) == 0); XLByteToSeg(recptr, segno, state->segcxt.ws_segsize); offset = XLogSegmentOffset(recptr, state->segcxt.ws_segsize); @@ -1016,10 +1019,10 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr, "WAL file is from different database system: incorrect segment size in page header"); return false; } - else if (longhdr->xlp_xlog_blcksz != XLOG_BLCKSZ) + else if (longhdr->xlp_xlog_blcksz != state->pageSize) { report_invalid_record(state, - "WAL file is from different database system: incorrect XLOG_BLCKSZ in page header"); + "WAL file is from different database system: incorrect block size in page header"); return false; } } @@ -1140,7 +1143,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr, * XLogNeedData() is prepared to handle that and will read at * least short page-header worth of data */ - targetRecOff = tmpRecPtr % XLOG_BLCKSZ; + targetRecOff = tmpRecPtr % state->pageSize; /* scroll back to page boundary */ targetPagePtr = tmpRecPtr - targetRecOff; @@ -1174,8 +1177,8 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr, * * Note that record headers are MAXALIGN'ed */ - if (MAXALIGN(header->xlp_rem_len) >= (XLOG_BLCKSZ - pageHeaderSize)) - tmpRecPtr = targetPagePtr + XLOG_BLCKSZ; + if (MAXALIGN(header->xlp_rem_len) >= (state->pageSize - pageHeaderSize)) + tmpRecPtr = targetPagePtr + state->pageSize; else { /* diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c index 9cbd8afa49..f2dd495310 100644 --- a/src/backend/replication/logical/logical.c +++ b/src/backend/replication/logical/logical.c @@ -199,11 +199,14 @@ StartupDecodingContext(List *output_plugin_options, ctx->slot = slot; - ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, cleanup_cb); + ctx->reader = + XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, cleanup_cb); + if (!ctx->reader) ereport(ERROR, (errcode(ERRCODE_OUT_OF_MEMORY), errmsg("out of memory"))); + ctx->reader->readBuf = palloc(XLOG_BLCKSZ); ctx->page_read = page_read; ctx->reorder = ReorderBufferAllocate(); diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c index 062093727d..8e95600ef7 100644 --- a/src/backend/replication/walsender.c +++ b/src/backend/replication/walsender.c @@ -579,8 +579,8 @@ StartReplication(StartReplicationCmd *cmd) errmsg("IDENTIFY_SYSTEM has not been run before START_REPLICATION"))); /* create xlogreader for physical replication */ - xlogreader = - XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close); + xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, + NULL, wal_segment_close); if (!xlogreader) ereport(ERROR, diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c index 712c85281c..8a6209b061 100644 --- a/src/bin/pg_rewind/parsexlog.c +++ b/src/bin/pg_rewind/parsexlog.c @@ -61,7 +61,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex, XLogReaderState *xlogreader; char *errormsg; - xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL); + xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL); if (xlogreader == NULL) pg_fatal("out of memory"); @@ -121,7 +121,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex, char *errormsg; XLogRecPtr endptr; - xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL); + xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL); if (xlogreader == NULL) pg_fatal("out of memory"); @@ -181,7 +181,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex, forkptr += SizeOfXLogShortPHD; } - xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL); + xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL); if (xlogreader == NULL) pg_fatal("out of memory"); diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c index f514de6b13..b635537659 100644 --- a/src/bin/pg_waldump/pg_waldump.c +++ b/src/bin/pg_waldump/pg_waldump.c @@ -1041,7 +1041,7 @@ main(int argc, char **argv) /* we have everything we need, start reading */ xlogreader_state = - XLogReaderAllocate(WalSegSz, waldir, WALDumpCloseSegment); + XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, waldir, WALDumpCloseSegment); if (!xlogreader_state) fatal_error("out of memory"); diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h index b9dad7ee51..1853516333 100644 --- a/src/include/access/xlogreader.h +++ b/src/include/access/xlogreader.h @@ -151,6 +151,7 @@ struct XLogReaderState * ---------------------------------------- */ /* variables to communicate with page reader */ + int pageSize; /* size of a page */ XLogRecPtr readPagePtr; /* page pointer to read */ int32 readLen; /* bytes requested to reader, or actual bytes * read by reader, which must be larger than @@ -243,7 +244,7 @@ struct XLogReaderState /* Get a new XLogReader */ extern XLogReaderState *XLogReaderAllocate(int wal_segment_size, - const char *waldir, + int page_size, const char *waldir, WALSegmentCleanupCB cleanup_cb); /* Free an XLogReader */ -- 2.27.0 ----Next_Part(Thu_Mar__4_11_28_53_2021_014)---- ^ permalink raw reply [nested|flat] 25+ messages in thread
* [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize. @ 2019-09-05 12:29 Kyotaro Horiguchi <[email protected]> 0 siblings, 0 replies; 25+ messages in thread From: Kyotaro Horiguchi @ 2019-09-05 12:29 UTC (permalink / raw) XLog reader facility and its callers currently relies on the same constant XLOG_BLCKSZ. This patch moves the responsibility to determine the block size to the callers of the facility. --- src/backend/access/transam/twophase.c | 3 +- src/backend/access/transam/xlog.c | 9 ++-- src/backend/access/transam/xlogreader.c | 57 ++++++++++++----------- src/backend/replication/logical/logical.c | 5 +- src/backend/replication/walsender.c | 4 +- src/bin/pg_rewind/parsexlog.c | 6 +-- src/bin/pg_waldump/pg_waldump.c | 2 +- src/include/access/xlogreader.h | 3 +- 8 files changed, 50 insertions(+), 39 deletions(-) diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c index 7d5a96a5f5..9f9e7de253 100644 --- a/src/backend/access/transam/twophase.c +++ b/src/backend/access/transam/twophase.c @@ -1325,7 +1325,8 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len) XLogReaderState *xlogreader; char *errormsg; - xlogreader = XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close); + xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, + NULL, wal_segment_close); if (!xlogreader) ereport(ERROR, diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index b517ca85ad..0822ac70cb 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -1222,7 +1222,10 @@ XLogInsertRecord(XLogRecData *rdata, appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len); if (!debug_reader) - debug_reader = XLogReaderAllocate(wal_segment_size, NULL, NULL); + { + debug_reader = + XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, NULL); + } if (!debug_reader) { @@ -6531,8 +6534,8 @@ StartupXLOG(void) OwnLatch(&XLogCtl->recoveryWakeupLatch); /* Set up XLOG reader facility */ - xlogreader = - XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close); + xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, + NULL, wal_segment_close); if (!xlogreader) ereport(ERROR, diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c index ba3e4868f7..e5baf13086 100644 --- a/src/backend/access/transam/xlogreader.c +++ b/src/backend/access/transam/xlogreader.c @@ -72,7 +72,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...) * Returns NULL if the xlogreader couldn't be allocated. */ XLogReaderState * -XLogReaderAllocate(int wal_segment_size, const char *waldir, +XLogReaderAllocate(int wal_segment_size, int page_size, const char *waldir, WALSegmentCleanupCB cleanup_cb) { XLogReaderState *state; @@ -87,6 +87,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir, state->cleanup_cb = cleanup_cb; state->max_block_id = -1; + state->pageSize = page_size; /* * Permanently allocate readBuf. We do it this way, rather than just @@ -95,7 +96,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir, * isn't guaranteed to have any particular alignment, whereas * palloc_extended() will provide MAXALIGN'd storage. */ - state->readBuf = (char *) palloc_extended(XLOG_BLCKSZ, + state->readBuf = (char *) palloc_extended(state->pageSize, MCXT_ALLOC_NO_OOM); if (!state->readBuf) { @@ -163,7 +164,7 @@ XLogReaderFree(XLogReaderState *state) * readRecordBufSize is set to the new buffer size. * * To avoid useless small increases, round its size to a multiple of - * XLOG_BLCKSZ, and make sure it's at least 5*Max(BLCKSZ, XLOG_BLCKSZ) to start + * page size, and make sure it's at least 5*Max(BLCKSZ, <page size>) to start * with. (That is enough for all "normal" records, but very large commit or * abort records might need more space.) */ @@ -172,8 +173,8 @@ allocate_recordbuf(XLogReaderState *state, uint32 reclength) { uint32 newSize = reclength; - newSize += XLOG_BLCKSZ - (newSize % XLOG_BLCKSZ); - newSize = Max(newSize, 5 * Max(BLCKSZ, XLOG_BLCKSZ)); + newSize += state->pageSize - (newSize % state->pageSize); + newSize = Max(newSize, 5 * Max(BLCKSZ, state->pageSize)); #ifndef FRONTEND @@ -238,6 +239,8 @@ void XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr) { Assert(!XLogRecPtrIsInvalid(RecPtr)); + Assert(state->readBuf != NULL && + state->readBuf == (void *) MAXALIGN(state->readBuf)); ResetDecoder(state); @@ -366,15 +369,15 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) XLogPageHeader pageHeader; targetPagePtr = - state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ); - targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ; + state->ReadRecPtr - (state->ReadRecPtr % state->pageSize); + targetRecOff = state->ReadRecPtr % state->pageSize; /* * Check if we have enough data. For the first record in the page, * the requesting length doesn't contain page header. */ if (XLogNeedData(state, targetPagePtr, - Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ), + Min(targetRecOff + SizeOfXLogRecord, state->pageSize), targetRecOff != 0)) return XLREAD_NEED_DATA; @@ -421,7 +424,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) * verified that we got the whole header. */ prec = (XLogRecord *) (state->readBuf + - state->ReadRecPtr % XLOG_BLCKSZ); + state->ReadRecPtr % state->pageSize); total_len = prec->xl_tot_len; /* @@ -432,7 +435,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) * ensure that we enter the XLREAD_CONTINUATION state below; * otherwise we might fail to apply ValidXLogRecordHeader at all. */ - if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord) + if (targetRecOff <= state->pageSize - SizeOfXLogRecord) { if (!ValidXLogRecordHeader(state, state->ReadRecPtr, state->PrevRecPtr, prec)) @@ -477,10 +480,10 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) * available */ targetPagePtr = - state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ); - targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ; + state->ReadRecPtr - (state->ReadRecPtr % state->pageSize); + targetRecOff = state->ReadRecPtr % state->pageSize; - request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ); + request_len = Min(targetRecOff + total_len, state->pageSize); record_len = request_len - targetRecOff; /* ReadRecPtr contains page header */ @@ -549,7 +552,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) /* Calculate pointer to beginning of next page */ state->recordContRecPtr = state->ReadRecPtr + record_len; - Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0); + Assert(state->recordContRecPtr % state->pageSize == 0); state->readRecordState = XLREAD_CONTINUATION; } @@ -576,7 +579,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) /* this request contains page header */ Assert (targetPagePtr != 0); if (XLogNeedData(state, targetPagePtr, - Min(state->recordRemainLen, XLOG_BLCKSZ), + Min(state->recordRemainLen, state->pageSize), false)) return XLREAD_NEED_DATA; @@ -628,7 +631,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) Assert(state->readLen >= pageHeaderSize); contdata = (char *) state->readBuf + pageHeaderSize; - record_len = XLOG_BLCKSZ - pageHeaderSize; + record_len = state->pageSize - pageHeaderSize; if (pageHeader->xlp_rem_len < record_len) record_len = pageHeader->xlp_rem_len; @@ -655,7 +658,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) } /* Calculate pointer to beginning of next page, and continue */ - state->recordContRecPtr += XLOG_BLCKSZ; + state->recordContRecPtr += state->pageSize; } /* targetPagePtr is pointing the last-read page here */ @@ -789,10 +792,10 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen, XLogPageHeaderSize((XLogPageHeader) state->readBuf); addLen = pageHeaderSize; - if (reqLen + pageHeaderSize <= XLOG_BLCKSZ) + if (reqLen + pageHeaderSize <= state->pageSize) addLen = pageHeaderSize; else - addLen = XLOG_BLCKSZ - reqLen; + addLen = state->pageSize - reqLen; Assert(addLen >= 0); } @@ -805,7 +808,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen, /* Data is not in our buffer, request the caller for it. */ XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize); targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize); - Assert((pageptr % XLOG_BLCKSZ) == 0); + Assert((pageptr % state->pageSize) == 0); /* * Every time we request to load new data of a page to the caller, even if @@ -829,7 +832,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen, * will not come back here, but will request the actual target page. */ state->readPagePtr = pageptr - targetPageOff; - state->readLen = XLOG_BLCKSZ; + state->readLen = state->pageSize; return true; } @@ -962,7 +965,7 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr, int32 offset; XLogPageHeader hdr = (XLogPageHeader) phdr; - Assert((recptr % XLOG_BLCKSZ) == 0); + Assert((recptr % state->pageSize) == 0); XLByteToSeg(recptr, segno, state->segcxt.ws_segsize); offset = XLogSegmentOffset(recptr, state->segcxt.ws_segsize); @@ -1016,10 +1019,10 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr, "WAL file is from different database system: incorrect segment size in page header"); return false; } - else if (longhdr->xlp_xlog_blcksz != XLOG_BLCKSZ) + else if (longhdr->xlp_xlog_blcksz != state->pageSize) { report_invalid_record(state, - "WAL file is from different database system: incorrect XLOG_BLCKSZ in page header"); + "WAL file is from different database system: incorrect block size in page header"); return false; } } @@ -1140,7 +1143,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr, * XLogNeedData() is prepared to handle that and will read at * least short page-header worth of data */ - targetRecOff = tmpRecPtr % XLOG_BLCKSZ; + targetRecOff = tmpRecPtr % state->pageSize; /* scroll back to page boundary */ targetPagePtr = tmpRecPtr - targetRecOff; @@ -1174,8 +1177,8 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr, * * Note that record headers are MAXALIGN'ed */ - if (MAXALIGN(header->xlp_rem_len) >= (XLOG_BLCKSZ - pageHeaderSize)) - tmpRecPtr = targetPagePtr + XLOG_BLCKSZ; + if (MAXALIGN(header->xlp_rem_len) >= (state->pageSize - pageHeaderSize)) + tmpRecPtr = targetPagePtr + state->pageSize; else { /* diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c index 9cbd8afa49..f2dd495310 100644 --- a/src/backend/replication/logical/logical.c +++ b/src/backend/replication/logical/logical.c @@ -199,11 +199,14 @@ StartupDecodingContext(List *output_plugin_options, ctx->slot = slot; - ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, cleanup_cb); + ctx->reader = + XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, cleanup_cb); + if (!ctx->reader) ereport(ERROR, (errcode(ERRCODE_OUT_OF_MEMORY), errmsg("out of memory"))); + ctx->reader->readBuf = palloc(XLOG_BLCKSZ); ctx->page_read = page_read; ctx->reorder = ReorderBufferAllocate(); diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c index 062093727d..8e95600ef7 100644 --- a/src/backend/replication/walsender.c +++ b/src/backend/replication/walsender.c @@ -579,8 +579,8 @@ StartReplication(StartReplicationCmd *cmd) errmsg("IDENTIFY_SYSTEM has not been run before START_REPLICATION"))); /* create xlogreader for physical replication */ - xlogreader = - XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close); + xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, + NULL, wal_segment_close); if (!xlogreader) ereport(ERROR, diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c index 712c85281c..8a6209b061 100644 --- a/src/bin/pg_rewind/parsexlog.c +++ b/src/bin/pg_rewind/parsexlog.c @@ -61,7 +61,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex, XLogReaderState *xlogreader; char *errormsg; - xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL); + xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL); if (xlogreader == NULL) pg_fatal("out of memory"); @@ -121,7 +121,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex, char *errormsg; XLogRecPtr endptr; - xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL); + xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL); if (xlogreader == NULL) pg_fatal("out of memory"); @@ -181,7 +181,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex, forkptr += SizeOfXLogShortPHD; } - xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL); + xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL); if (xlogreader == NULL) pg_fatal("out of memory"); diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c index f514de6b13..b635537659 100644 --- a/src/bin/pg_waldump/pg_waldump.c +++ b/src/bin/pg_waldump/pg_waldump.c @@ -1041,7 +1041,7 @@ main(int argc, char **argv) /* we have everything we need, start reading */ xlogreader_state = - XLogReaderAllocate(WalSegSz, waldir, WALDumpCloseSegment); + XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, waldir, WALDumpCloseSegment); if (!xlogreader_state) fatal_error("out of memory"); diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h index b9dad7ee51..1853516333 100644 --- a/src/include/access/xlogreader.h +++ b/src/include/access/xlogreader.h @@ -151,6 +151,7 @@ struct XLogReaderState * ---------------------------------------- */ /* variables to communicate with page reader */ + int pageSize; /* size of a page */ XLogRecPtr readPagePtr; /* page pointer to read */ int32 readLen; /* bytes requested to reader, or actual bytes * read by reader, which must be larger than @@ -243,7 +244,7 @@ struct XLogReaderState /* Get a new XLogReader */ extern XLogReaderState *XLogReaderAllocate(int wal_segment_size, - const char *waldir, + int page_size, const char *waldir, WALSegmentCleanupCB cleanup_cb); /* Free an XLogReader */ -- 2.27.0 ----Next_Part(Thu_Mar__4_11_28_53_2021_014)---- ^ permalink raw reply [nested|flat] 25+ messages in thread
* [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize. @ 2019-09-05 12:29 Kyotaro Horiguchi <[email protected]> 0 siblings, 0 replies; 25+ messages in thread From: Kyotaro Horiguchi @ 2019-09-05 12:29 UTC (permalink / raw) XLog reader facility and its callers currently relies on the same constant XLOG_BLCKSZ. This patch moves the responsibility to determine the block size to the callers of the facility. --- src/backend/access/transam/twophase.c | 3 +- src/backend/access/transam/xlog.c | 9 ++-- src/backend/access/transam/xlogreader.c | 57 ++++++++++++----------- src/backend/replication/logical/logical.c | 5 +- src/backend/replication/walsender.c | 4 +- src/bin/pg_rewind/parsexlog.c | 6 +-- src/bin/pg_waldump/pg_waldump.c | 2 +- src/include/access/xlogreader.h | 3 +- 8 files changed, 50 insertions(+), 39 deletions(-) diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c index 7d5a96a5f5..9f9e7de253 100644 --- a/src/backend/access/transam/twophase.c +++ b/src/backend/access/transam/twophase.c @@ -1325,7 +1325,8 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len) XLogReaderState *xlogreader; char *errormsg; - xlogreader = XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close); + xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, + NULL, wal_segment_close); if (!xlogreader) ereport(ERROR, diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index b517ca85ad..0822ac70cb 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -1222,7 +1222,10 @@ XLogInsertRecord(XLogRecData *rdata, appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len); if (!debug_reader) - debug_reader = XLogReaderAllocate(wal_segment_size, NULL, NULL); + { + debug_reader = + XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, NULL); + } if (!debug_reader) { @@ -6531,8 +6534,8 @@ StartupXLOG(void) OwnLatch(&XLogCtl->recoveryWakeupLatch); /* Set up XLOG reader facility */ - xlogreader = - XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close); + xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, + NULL, wal_segment_close); if (!xlogreader) ereport(ERROR, diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c index ba3e4868f7..e5baf13086 100644 --- a/src/backend/access/transam/xlogreader.c +++ b/src/backend/access/transam/xlogreader.c @@ -72,7 +72,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...) * Returns NULL if the xlogreader couldn't be allocated. */ XLogReaderState * -XLogReaderAllocate(int wal_segment_size, const char *waldir, +XLogReaderAllocate(int wal_segment_size, int page_size, const char *waldir, WALSegmentCleanupCB cleanup_cb) { XLogReaderState *state; @@ -87,6 +87,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir, state->cleanup_cb = cleanup_cb; state->max_block_id = -1; + state->pageSize = page_size; /* * Permanently allocate readBuf. We do it this way, rather than just @@ -95,7 +96,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir, * isn't guaranteed to have any particular alignment, whereas * palloc_extended() will provide MAXALIGN'd storage. */ - state->readBuf = (char *) palloc_extended(XLOG_BLCKSZ, + state->readBuf = (char *) palloc_extended(state->pageSize, MCXT_ALLOC_NO_OOM); if (!state->readBuf) { @@ -163,7 +164,7 @@ XLogReaderFree(XLogReaderState *state) * readRecordBufSize is set to the new buffer size. * * To avoid useless small increases, round its size to a multiple of - * XLOG_BLCKSZ, and make sure it's at least 5*Max(BLCKSZ, XLOG_BLCKSZ) to start + * page size, and make sure it's at least 5*Max(BLCKSZ, <page size>) to start * with. (That is enough for all "normal" records, but very large commit or * abort records might need more space.) */ @@ -172,8 +173,8 @@ allocate_recordbuf(XLogReaderState *state, uint32 reclength) { uint32 newSize = reclength; - newSize += XLOG_BLCKSZ - (newSize % XLOG_BLCKSZ); - newSize = Max(newSize, 5 * Max(BLCKSZ, XLOG_BLCKSZ)); + newSize += state->pageSize - (newSize % state->pageSize); + newSize = Max(newSize, 5 * Max(BLCKSZ, state->pageSize)); #ifndef FRONTEND @@ -238,6 +239,8 @@ void XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr) { Assert(!XLogRecPtrIsInvalid(RecPtr)); + Assert(state->readBuf != NULL && + state->readBuf == (void *) MAXALIGN(state->readBuf)); ResetDecoder(state); @@ -366,15 +369,15 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) XLogPageHeader pageHeader; targetPagePtr = - state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ); - targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ; + state->ReadRecPtr - (state->ReadRecPtr % state->pageSize); + targetRecOff = state->ReadRecPtr % state->pageSize; /* * Check if we have enough data. For the first record in the page, * the requesting length doesn't contain page header. */ if (XLogNeedData(state, targetPagePtr, - Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ), + Min(targetRecOff + SizeOfXLogRecord, state->pageSize), targetRecOff != 0)) return XLREAD_NEED_DATA; @@ -421,7 +424,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) * verified that we got the whole header. */ prec = (XLogRecord *) (state->readBuf + - state->ReadRecPtr % XLOG_BLCKSZ); + state->ReadRecPtr % state->pageSize); total_len = prec->xl_tot_len; /* @@ -432,7 +435,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) * ensure that we enter the XLREAD_CONTINUATION state below; * otherwise we might fail to apply ValidXLogRecordHeader at all. */ - if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord) + if (targetRecOff <= state->pageSize - SizeOfXLogRecord) { if (!ValidXLogRecordHeader(state, state->ReadRecPtr, state->PrevRecPtr, prec)) @@ -477,10 +480,10 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) * available */ targetPagePtr = - state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ); - targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ; + state->ReadRecPtr - (state->ReadRecPtr % state->pageSize); + targetRecOff = state->ReadRecPtr % state->pageSize; - request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ); + request_len = Min(targetRecOff + total_len, state->pageSize); record_len = request_len - targetRecOff; /* ReadRecPtr contains page header */ @@ -549,7 +552,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) /* Calculate pointer to beginning of next page */ state->recordContRecPtr = state->ReadRecPtr + record_len; - Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0); + Assert(state->recordContRecPtr % state->pageSize == 0); state->readRecordState = XLREAD_CONTINUATION; } @@ -576,7 +579,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) /* this request contains page header */ Assert (targetPagePtr != 0); if (XLogNeedData(state, targetPagePtr, - Min(state->recordRemainLen, XLOG_BLCKSZ), + Min(state->recordRemainLen, state->pageSize), false)) return XLREAD_NEED_DATA; @@ -628,7 +631,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) Assert(state->readLen >= pageHeaderSize); contdata = (char *) state->readBuf + pageHeaderSize; - record_len = XLOG_BLCKSZ - pageHeaderSize; + record_len = state->pageSize - pageHeaderSize; if (pageHeader->xlp_rem_len < record_len) record_len = pageHeader->xlp_rem_len; @@ -655,7 +658,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) } /* Calculate pointer to beginning of next page, and continue */ - state->recordContRecPtr += XLOG_BLCKSZ; + state->recordContRecPtr += state->pageSize; } /* targetPagePtr is pointing the last-read page here */ @@ -789,10 +792,10 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen, XLogPageHeaderSize((XLogPageHeader) state->readBuf); addLen = pageHeaderSize; - if (reqLen + pageHeaderSize <= XLOG_BLCKSZ) + if (reqLen + pageHeaderSize <= state->pageSize) addLen = pageHeaderSize; else - addLen = XLOG_BLCKSZ - reqLen; + addLen = state->pageSize - reqLen; Assert(addLen >= 0); } @@ -805,7 +808,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen, /* Data is not in our buffer, request the caller for it. */ XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize); targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize); - Assert((pageptr % XLOG_BLCKSZ) == 0); + Assert((pageptr % state->pageSize) == 0); /* * Every time we request to load new data of a page to the caller, even if @@ -829,7 +832,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen, * will not come back here, but will request the actual target page. */ state->readPagePtr = pageptr - targetPageOff; - state->readLen = XLOG_BLCKSZ; + state->readLen = state->pageSize; return true; } @@ -962,7 +965,7 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr, int32 offset; XLogPageHeader hdr = (XLogPageHeader) phdr; - Assert((recptr % XLOG_BLCKSZ) == 0); + Assert((recptr % state->pageSize) == 0); XLByteToSeg(recptr, segno, state->segcxt.ws_segsize); offset = XLogSegmentOffset(recptr, state->segcxt.ws_segsize); @@ -1016,10 +1019,10 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr, "WAL file is from different database system: incorrect segment size in page header"); return false; } - else if (longhdr->xlp_xlog_blcksz != XLOG_BLCKSZ) + else if (longhdr->xlp_xlog_blcksz != state->pageSize) { report_invalid_record(state, - "WAL file is from different database system: incorrect XLOG_BLCKSZ in page header"); + "WAL file is from different database system: incorrect block size in page header"); return false; } } @@ -1140,7 +1143,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr, * XLogNeedData() is prepared to handle that and will read at * least short page-header worth of data */ - targetRecOff = tmpRecPtr % XLOG_BLCKSZ; + targetRecOff = tmpRecPtr % state->pageSize; /* scroll back to page boundary */ targetPagePtr = tmpRecPtr - targetRecOff; @@ -1174,8 +1177,8 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr, * * Note that record headers are MAXALIGN'ed */ - if (MAXALIGN(header->xlp_rem_len) >= (XLOG_BLCKSZ - pageHeaderSize)) - tmpRecPtr = targetPagePtr + XLOG_BLCKSZ; + if (MAXALIGN(header->xlp_rem_len) >= (state->pageSize - pageHeaderSize)) + tmpRecPtr = targetPagePtr + state->pageSize; else { /* diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c index 9cbd8afa49..f2dd495310 100644 --- a/src/backend/replication/logical/logical.c +++ b/src/backend/replication/logical/logical.c @@ -199,11 +199,14 @@ StartupDecodingContext(List *output_plugin_options, ctx->slot = slot; - ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, cleanup_cb); + ctx->reader = + XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, cleanup_cb); + if (!ctx->reader) ereport(ERROR, (errcode(ERRCODE_OUT_OF_MEMORY), errmsg("out of memory"))); + ctx->reader->readBuf = palloc(XLOG_BLCKSZ); ctx->page_read = page_read; ctx->reorder = ReorderBufferAllocate(); diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c index 062093727d..8e95600ef7 100644 --- a/src/backend/replication/walsender.c +++ b/src/backend/replication/walsender.c @@ -579,8 +579,8 @@ StartReplication(StartReplicationCmd *cmd) errmsg("IDENTIFY_SYSTEM has not been run before START_REPLICATION"))); /* create xlogreader for physical replication */ - xlogreader = - XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close); + xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, + NULL, wal_segment_close); if (!xlogreader) ereport(ERROR, diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c index 712c85281c..8a6209b061 100644 --- a/src/bin/pg_rewind/parsexlog.c +++ b/src/bin/pg_rewind/parsexlog.c @@ -61,7 +61,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex, XLogReaderState *xlogreader; char *errormsg; - xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL); + xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL); if (xlogreader == NULL) pg_fatal("out of memory"); @@ -121,7 +121,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex, char *errormsg; XLogRecPtr endptr; - xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL); + xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL); if (xlogreader == NULL) pg_fatal("out of memory"); @@ -181,7 +181,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex, forkptr += SizeOfXLogShortPHD; } - xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL); + xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL); if (xlogreader == NULL) pg_fatal("out of memory"); diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c index f514de6b13..b635537659 100644 --- a/src/bin/pg_waldump/pg_waldump.c +++ b/src/bin/pg_waldump/pg_waldump.c @@ -1041,7 +1041,7 @@ main(int argc, char **argv) /* we have everything we need, start reading */ xlogreader_state = - XLogReaderAllocate(WalSegSz, waldir, WALDumpCloseSegment); + XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, waldir, WALDumpCloseSegment); if (!xlogreader_state) fatal_error("out of memory"); diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h index b9dad7ee51..1853516333 100644 --- a/src/include/access/xlogreader.h +++ b/src/include/access/xlogreader.h @@ -151,6 +151,7 @@ struct XLogReaderState * ---------------------------------------- */ /* variables to communicate with page reader */ + int pageSize; /* size of a page */ XLogRecPtr readPagePtr; /* page pointer to read */ int32 readLen; /* bytes requested to reader, or actual bytes * read by reader, which must be larger than @@ -243,7 +244,7 @@ struct XLogReaderState /* Get a new XLogReader */ extern XLogReaderState *XLogReaderAllocate(int wal_segment_size, - const char *waldir, + int page_size, const char *waldir, WALSegmentCleanupCB cleanup_cb); /* Free an XLogReader */ -- 2.27.0 ----Next_Part(Thu_Mar__4_11_28_53_2021_014)---- ^ permalink raw reply [nested|flat] 25+ messages in thread
end of thread, other threads:[~2019-09-05 12:29 UTC | newest] Thread overview: 25+ messages (download: mbox mbox.gz follow: Atom feed) -- links below jump to the message on this page -- 2019-09-05 12:29 [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize. Kyotaro Horiguchi <[email protected]> 2019-09-05 12:29 [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize. Kyotaro Horiguchi <[email protected]> 2019-09-05 12:29 [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize. Kyotaro Horiguchi <[email protected]> 2019-09-05 12:29 [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize. Kyotaro Horiguchi <[email protected]> 2019-09-05 12:29 [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize. Kyotaro Horiguchi <[email protected]> 2019-09-05 12:29 [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize. Kyotaro Horiguchi <[email protected]> 2019-09-05 12:29 [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize. Kyotaro Horiguchi <[email protected]> 2019-09-05 12:29 [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize. Kyotaro Horiguchi <[email protected]> 2019-09-05 12:29 [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize. Kyotaro Horiguchi <[email protected]> 2019-09-05 12:29 [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize. Kyotaro Horiguchi <[email protected]> 2019-09-05 12:29 [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize. Kyotaro Horiguchi <[email protected]> 2019-09-05 12:29 [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize. Kyotaro Horiguchi <[email protected]> 2019-09-05 12:29 [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize. Kyotaro Horiguchi <[email protected]> 2019-09-05 12:29 [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize. Kyotaro Horiguchi <[email protected]> 2019-09-05 12:29 [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize. Kyotaro Horiguchi <[email protected]> 2019-09-05 12:29 [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize. Kyotaro Horiguchi <[email protected]> 2019-09-05 12:29 [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize. Kyotaro Horiguchi <[email protected]> 2019-09-05 12:29 [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize. Kyotaro Horiguchi <[email protected]> 2019-09-05 12:29 [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize. Kyotaro Horiguchi <[email protected]> 2019-09-05 12:29 [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize. Kyotaro Horiguchi <[email protected]> 2019-09-05 12:29 [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize. Kyotaro Horiguchi <[email protected]> 2019-09-05 12:29 [PATCH v16 4/4] Allow xlogreader to use different xlog blocksize. Kyotaro Horiguchi <[email protected]> 2019-09-05 12:29 [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize. Kyotaro Horiguchi <[email protected]> 2019-09-05 12:29 [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize. Kyotaro Horiguchi <[email protected]> 2019-09-05 12:29 [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize. Kyotaro Horiguchi <[email protected]>
This inbox is served by agora; see mirroring instructions for how to clone and mirror all data and code used for this inbox