public inbox for [email protected]
help / color / mirror / Atom feed[PATCH v9 2/4] Move page-reader out of XLogReadRecord
70+ messages / 7 participants
[nested] [flat]
* [PATCH v17 2/4] Move page-reader out of XLogReadRecord
@ 2019-09-10 03:58 Kyotaro Horiguchi <[email protected]>
0 siblings, 0 replies; 70+ messages in thread
From: Kyotaro Horiguchi @ 2019-09-10 03:58 UTC (permalink / raw)
This is the second step of removing callbacks from WAL record reader.
Since it is essential to take in additional data while reading a
record, the function have to ask caller for new data while keeping
working state. Thus the function is turned into a state machine.
---
src/backend/access/transam/twophase.c | 15 +-
src/backend/access/transam/xlog.c | 58 +-
src/backend/access/transam/xlogreader.c | 697 +++++++++++-------
src/backend/access/transam/xlogutils.c | 17 +-
src/backend/replication/logical/logical.c | 26 +-
.../replication/logical/logicalfuncs.c | 13 +-
src/backend/replication/slotfuncs.c | 18 +-
src/backend/replication/walsender.c | 32 +-
src/bin/pg_rewind/parsexlog.c | 84 ++-
src/bin/pg_waldump/pg_waldump.c | 36 +-
src/include/access/xlogreader.h | 121 ++-
src/include/access/xlogutils.h | 4 +-
src/include/pg_config_manual.h | 2 +-
src/include/replication/logical.h | 11 +-
14 files changed, 646 insertions(+), 488 deletions(-)
diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index 80d2d20d6c..7d5a96a5f5 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -1325,11 +1325,8 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
XLogReaderState *xlogreader;
char *errormsg;
- xlogreader = XLogReaderAllocate(wal_segment_size, NULL,
- XL_ROUTINE(.page_read = &read_local_xlog_page,
- .segment_open = &wal_segment_open,
- .segment_close = &wal_segment_close),
- NULL);
+ xlogreader = XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+
if (!xlogreader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
@@ -1337,7 +1334,13 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
errdetail("Failed while allocating a WAL reading processor.")));
XLogBeginRead(xlogreader, lsn);
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!read_local_xlog_page(xlogreader))
+ break;
+ }
+
if (record == NULL)
ereport(ERROR,
(errcode_for_file_access(),
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 606328a65a..670996ae67 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -836,13 +836,6 @@ static XLogSource currentSource = XLOG_FROM_ANY;
static bool lastSourceFailed = false;
static bool pendingWalRcvRestart = false;
-typedef struct XLogPageReadPrivate
-{
- int emode;
- bool fetching_ckpt; /* are we fetching a checkpoint record? */
- bool randAccess;
-} XLogPageReadPrivate;
-
/*
* These variables track when we last obtained some WAL data to process,
* and where we got it from. (XLogReceiptSource is initially the same as
@@ -917,8 +910,8 @@ static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath,
static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli,
XLogSource source, bool notfoundOk);
static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, XLogSource source);
-static bool XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
+static bool XLogPageRead(XLogReaderState *xlogreader,
+ bool fetching_ckpt, int emode, bool randAccess);
static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
bool fetching_ckpt, XLogRecPtr tliRecPtr);
static int emode_for_corrupt_record(int emode, XLogRecPtr RecPtr);
@@ -1231,8 +1224,7 @@ XLogInsertRecord(XLogRecData *rdata,
appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len);
if (!debug_reader)
- debug_reader = XLogReaderAllocate(wal_segment_size, NULL,
- XL_ROUTINE(), NULL);
+ debug_reader = XLogReaderAllocate(wal_segment_size, NULL, NULL);
if (!debug_reader)
{
@@ -4344,15 +4336,10 @@ CleanupBackupHistory(void)
* record is available.
*/
static XLogRecord *
-ReadRecord(XLogReaderState *xlogreader, int emode,
- bool fetching_ckpt)
+ReadRecord(XLogReaderState *xlogreader, int emode, bool fetching_ckpt)
{
XLogRecord *record;
- XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
-
- private->fetching_ckpt = fetching_ckpt;
- private->emode = emode;
- private->randAccess = (xlogreader->ReadRecPtr == InvalidXLogRecPtr);
+ bool randAccess = (xlogreader->ReadRecPtr == InvalidXLogRecPtr);
/* This is the first attempt to read this page. */
lastSourceFailed = false;
@@ -4360,8 +4347,16 @@ ReadRecord(XLogReaderState *xlogreader, int emode,
for (;;)
{
char *errormsg;
+ XLogReadRecordResult result;
+
+ while ((result = XLogReadRecord(xlogreader, &record, &errormsg))
+ == XLREAD_NEED_DATA)
+ {
+ if (!XLogPageRead(xlogreader, fetching_ckpt, emode, randAccess))
+ break;
+
+ }
- record = XLogReadRecord(xlogreader, &errormsg);
ReadRecPtr = xlogreader->ReadRecPtr;
EndRecPtr = xlogreader->EndRecPtr;
if (record == NULL)
@@ -6380,7 +6375,6 @@ StartupXLOG(void)
bool backupFromStandby = false;
DBState dbstate_at_startup;
XLogReaderState *xlogreader;
- XLogPageReadPrivate private;
bool promoted = false;
struct stat st;
@@ -6539,13 +6533,9 @@ StartupXLOG(void)
OwnLatch(&XLogCtl->recoveryWakeupLatch);
/* Set up XLOG reader facility */
- MemSet(&private, 0, sizeof(XLogPageReadPrivate));
xlogreader =
- XLogReaderAllocate(wal_segment_size, NULL,
- XL_ROUTINE(.page_read = &XLogPageRead,
- .segment_open = NULL,
- .segment_close = wal_segment_close),
- &private);
+ XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+
if (!xlogreader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
@@ -11987,12 +11977,13 @@ CancelBackup(void)
* sleep and retry.
*/
static bool
-XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetRecPtr, char *readBuf)
+XLogPageRead(XLogReaderState *xlogreader,
+ bool fetching_ckpt, int emode, bool randAccess)
{
- XLogPageReadPrivate *private =
- (XLogPageReadPrivate *) xlogreader->private_data;
- int emode = private->emode;
+ char *readBuf = xlogreader->readBuf;
+ XLogRecPtr targetPagePtr = xlogreader->readPagePtr;
+ int reqLen = xlogreader->readLen;
+ XLogRecPtr targetRecPtr = xlogreader->ReadRecPtr;
uint32 targetPageOff;
XLogSegNo targetSegNo PG_USED_FOR_ASSERTS_ONLY;
int r;
@@ -12035,8 +12026,8 @@ retry:
flushedUpto < targetPagePtr + reqLen))
{
if (!WaitForWALToBecomeAvailable(targetPagePtr + reqLen,
- private->randAccess,
- private->fetching_ckpt,
+ randAccess,
+ fetching_ckpt,
targetRecPtr))
{
if (readFile >= 0)
@@ -12141,6 +12132,7 @@ retry:
goto next_record_is_invalid;
}
+ Assert(xlogreader->readPagePtr == targetPagePtr);
xlogreader->readLen = readLen;
return true;
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index 7d04995b7b..ba3e4868f7 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -40,7 +40,7 @@ static bool XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr,
int reqLen, bool header_inclusive);
static void XLogReaderInvalReadState(XLogReaderState *state);
static bool ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
- XLogRecPtr PrevRecPtr, XLogRecord *record, bool randAccess);
+ XLogRecPtr PrevRecPtr, XLogRecord *record);
static bool ValidXLogRecord(XLogReaderState *state, XLogRecord *record,
XLogRecPtr recptr);
static void ResetDecoder(XLogReaderState *state);
@@ -73,7 +73,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...)
*/
XLogReaderState *
XLogReaderAllocate(int wal_segment_size, const char *waldir,
- XLogReaderRoutine *routine, void *private_data)
+ WALSegmentCleanupCB cleanup_cb)
{
XLogReaderState *state;
@@ -84,7 +84,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
return NULL;
/* initialize caller-provided support functions */
- state->routine = *routine;
+ state->cleanup_cb = cleanup_cb;
state->max_block_id = -1;
@@ -107,8 +107,6 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
WALOpenSegmentInit(&state->seg, &state->segcxt, wal_segment_size,
waldir);
- /* system_identifier initialized to zeroes above */
- state->private_data = private_data;
/* ReadRecPtr, EndRecPtr and readLen initialized to zeroes above */
state->errormsg_buf = palloc_extended(MAX_ERRORMSG_LEN + 1,
MCXT_ALLOC_NO_OOM);
@@ -140,8 +138,8 @@ XLogReaderFree(XLogReaderState *state)
{
int block_id;
- if (state->seg.ws_file != -1)
- state->routine.segment_close(state);
+ if (state->seg.ws_file >= 0)
+ state->cleanup_cb(state);
for (block_id = 0; block_id <= XLR_MAX_BLOCK_ID; block_id++)
{
@@ -246,6 +244,7 @@ XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
/* Begin at the passed-in record pointer. */
state->EndRecPtr = RecPtr;
state->ReadRecPtr = InvalidXLogRecPtr;
+ state->readRecordState = XLREAD_NEXT_RECORD;
}
/*
@@ -254,317 +253,449 @@ XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
* XLogBeginRead() or XLogFindNextRecord() must be called before the first call
* to XLogReadRecord().
*
- * If the page_read callback fails to read the requested data, NULL is
- * returned. The callback is expected to have reported the error; errormsg
- * is set to NULL.
+ * This function may return XLREAD_NEED_DATA several times before returning a
+ * result record. The caller shall read in some new data then call this
+ * function again with the same parameters.
*
- * If the reading fails for some other reason, NULL is also returned, and
- * *errormsg is set to a string with details of the failure.
+ * When a record is successfully read, returns XLREAD_SUCCESS with result
+ * record being stored in *record. Otherwise *record is NULL.
*
- * The returned pointer (or *errormsg) points to an internal buffer that's
- * valid until the next call to XLogReadRecord.
+ * Returns XLREAD_NEED_DATA if more data is needed to finish reading the
+ * current record. In that case, state->readPagePtr and state->readLen inform
+ * the desired position and minimum length of data needed. The caller shall
+ * read in the requested data and set state->readBuf to point to a buffer
+ * containing it. The caller must also set state->readPageTLI and
+ * state->readLen to indicate the timeline that it was read from, and the
+ * length of data that is now available (which must be >= given readLen),
+ * respectively.
+ *
+ * If invalid data is encountered, returns XLREAD_FAIL with *record being set to
+ * NULL. *errormsg is set to a string with details of the failure.
+ * The returned pointer (or *errormsg) points to an internal buffer that's valid
+ * until the next call to XLogReadRecord.
+ *
+ *
+ * This function runs a state machine consists of the following states.
+ *
+ * XLREAD_NEXT_RECORD :
+ * The initial state, if called with valid RecPtr, try to read a record at
+ * that position. If invalid RecPtr is given try to read a record just after
+ * the last one previously read.
+ * This state ens after setting ReadRecPtr. Then goes to XLREAD_TOT_LEN.
+ *
+ * XLREAD_TOT_LEN:
+ * Examining record header. Ends after reading record total
+ * length. recordRemainLen and recordGotLen are initialized.
+ *
+ * XLREAD_FIRST_FRAGMENT:
+ * Reading the first fragment. Ends with finishing reading a single
+ * record. Goes to XLREAD_NEXT_RECORD if that's all or
+ * XLREAD_CONTINUATION if we have continuation.
+
+ * XLREAD_CONTINUATION:
+ * Reading continuation of record. Ends with finishing the whole record then
+ * goes to XLREAD_NEXT_RECORD. During this state, recordRemainLen indicates
+ * how much is left and readRecordBuf holds the partially assert
+ * record.recordContRecPtr points to the beginning of the next page where to
+ * continue.
+ *
+ * If wrong data found in any state, the state machine stays at the current
+ * state. This behavior allows to continue reading a reacord switching among
+ * different souces, while streaming replication.
*/
-XLogRecord *
-XLogReadRecord(XLogReaderState *state, char **errormsg)
+XLogReadRecordResult
+XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
{
- XLogRecPtr RecPtr;
- XLogRecord *record;
- XLogRecPtr targetPagePtr;
- bool randAccess;
- uint32 len,
- total_len;
- uint32 targetRecOff;
- uint32 pageHeaderSize;
- bool gotheader;
+ XLogRecord *prec;
- /*
- * randAccess indicates whether to verify the previous-record pointer of
- * the record we're reading. We only do this if we're reading
- * sequentially, which is what we initially assume.
- */
- randAccess = false;
+ *record = NULL;
/* reset error state */
*errormsg = NULL;
state->errormsg_buf[0] = '\0';
- ResetDecoder(state);
-
- RecPtr = state->EndRecPtr;
-
- if (state->ReadRecPtr != InvalidXLogRecPtr)
- {
- /* read the record after the one we just read */
-
- /*
- * EndRecPtr is pointing to end+1 of the previous WAL record. If
- * we're at a page boundary, no more records can fit on the current
- * page. We must skip over the page header, but we can't do that until
- * we've read in the page, since the header size is variable.
- */
- }
- else
- {
- /*
- * Caller supplied a position to start at.
- *
- * In this case, EndRecPtr should already be pointing to a valid
- * record starting position.
- */
- Assert(XRecOffIsValid(RecPtr));
- randAccess = true;
- }
-
- state->currRecPtr = RecPtr;
-
- targetPagePtr = RecPtr - (RecPtr % XLOG_BLCKSZ);
- targetRecOff = RecPtr % XLOG_BLCKSZ;
-
- /*
- * Read the page containing the record into state->readBuf. Request enough
- * byte to cover the whole record header, or at least the part of it that
- * fits on the same page.
- */
- while (XLogNeedData(state, targetPagePtr,
- Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
- targetRecOff != 0))
- {
- if (!state->routine.page_read(state, state->readPagePtr, state->readLen,
- RecPtr, state->readBuf))
- break;
- }
-
- if (!state->page_verified)
- goto err;
-
- /*
- * We have at least the page header, so we can examine it now.
- */
- pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
- if (targetRecOff == 0)
- {
- /*
- * At page start, so skip over page header.
- */
- RecPtr += pageHeaderSize;
- targetRecOff = pageHeaderSize;
- }
- else if (targetRecOff < pageHeaderSize)
- {
- report_invalid_record(state, "invalid record offset at %X/%X",
- LSN_FORMAT_ARGS(RecPtr));
- goto err;
- }
-
- if ((((XLogPageHeader) state->readBuf)->xlp_info & XLP_FIRST_IS_CONTRECORD) &&
- targetRecOff == pageHeaderSize)
- {
- report_invalid_record(state, "contrecord is requested by %X/%X",
- LSN_FORMAT_ARGS(RecPtr));
- goto err;
- }
-
- /* XLogNeedData has verified the page header */
- Assert(pageHeaderSize <= state->readLen);
-
- /*
- * Read the record length.
- *
- * NB: Even though we use an XLogRecord pointer here, the whole record
- * header might not fit on this page. xl_tot_len is the first field of the
- * struct, so it must be on this page (the records are MAXALIGNed), but we
- * cannot access any other fields until we've verified that we got the
- * whole header.
- */
- record = (XLogRecord *) (state->readBuf + RecPtr % XLOG_BLCKSZ);
- total_len = record->xl_tot_len;
-
- /*
- * If the whole record header is on this page, validate it immediately.
- * Otherwise do just a basic sanity check on xl_tot_len, and validate the
- * rest of the header after reading it from the next page. The xl_tot_len
- * check is necessary here to ensure that we enter the "Need to reassemble
- * record" code path below; otherwise we might fail to apply
- * ValidXLogRecordHeader at all.
- */
- if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
- {
- if (!ValidXLogRecordHeader(state, RecPtr, state->ReadRecPtr, record,
- randAccess))
- goto err;
- gotheader = true;
- }
- else
- {
- /* XXX: more validation should be done here */
- if (total_len < SizeOfXLogRecord)
- {
- report_invalid_record(state,
- "invalid record length at %X/%X: wanted %u, got %u",
- LSN_FORMAT_ARGS(RecPtr),
- (uint32) SizeOfXLogRecord, total_len);
- goto err;
- }
- gotheader = false;
- }
-
- len = XLOG_BLCKSZ - RecPtr % XLOG_BLCKSZ;
- if (total_len > len)
+ switch (state->readRecordState)
{
- /* Need to reassemble record */
- char *contdata;
- XLogPageHeader pageHeader;
- char *buffer;
- uint32 gotlen;
-
- /*
- * Enlarge readRecordBuf as needed.
- */
- if (total_len > state->readRecordBufSize &&
- !allocate_recordbuf(state, total_len))
- {
- /* We treat this as a "bogus data" condition */
- report_invalid_record(state, "record length %u at %X/%X too long",
- total_len, LSN_FORMAT_ARGS(RecPtr));
- goto err;
- }
+ case XLREAD_NEXT_RECORD:
+ ResetDecoder(state);
- /* Copy the first fragment of the record from the first page. */
- memcpy(state->readRecordBuf,
- state->readBuf + RecPtr % XLOG_BLCKSZ, len);
- buffer = state->readRecordBuf + len;
- gotlen = len;
-
- do
- {
- int rest_len = total_len - gotlen;
-
- /* Calculate pointer to beginning of next page */
- targetPagePtr += XLOG_BLCKSZ;
-
- /* Wait for the next page to become available */
- while (XLogNeedData(state, targetPagePtr,
- Min(rest_len, XLOG_BLCKSZ),
- false))
+ if (state->ReadRecPtr != InvalidXLogRecPtr)
{
- if (!state->routine.page_read(state, state->readPagePtr,
- state->readLen,
- state->ReadRecPtr,
- state->readBuf))
- break;
+ /* read the record after the one we just read */
+
+ /*
+ * EndRecPtr is pointing to end+1 of the previous WAL record.
+ * If we're at a page boundary, no more records can fit on the
+ * current page. We must skip over the page header, but we
+ * can't do that until we've read in the page, since the header
+ * size is variable.
+ */
+ state->PrevRecPtr = state->ReadRecPtr;
+ state->ReadRecPtr = state->EndRecPtr;
+ }
+ else
+ {
+ /*
+ * Caller supplied a position to start at.
+ *
+ * In this case, EndRecPtr should already be pointing to a
+ * valid record starting position.
+ */
+ Assert(XRecOffIsValid(state->EndRecPtr));
+ state->ReadRecPtr = state->EndRecPtr;
+
+ /*
+ * We cannot verify the previous-record pointer when we're
+ * seeking to a particular record. Reset PrevRecPtr so that we
+ * won't try doing that.
+ */
+ state->PrevRecPtr = InvalidXLogRecPtr;
+ state->EndRecPtr = InvalidXLogRecPtr; /* to be tidy */
}
+ state->record_verified = false;
+ state->readRecordState = XLREAD_TOT_LEN;
+ /* fall through */
+
+ case XLREAD_TOT_LEN:
+ {
+ uint32 total_len;
+ uint32 pageHeaderSize;
+ XLogRecPtr targetPagePtr;
+ uint32 targetRecOff;
+ XLogPageHeader pageHeader;
+
+ targetPagePtr =
+ state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
+ targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+
+ /*
+ * Check if we have enough data. For the first record in the page,
+ * the requesting length doesn't contain page header.
+ */
+ if (XLogNeedData(state, targetPagePtr,
+ Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+ targetRecOff != 0))
+ return XLREAD_NEED_DATA;
+
+ /* error out if caller supplied bogus page */
if (!state->page_verified)
goto err;
- Assert(SizeOfXLogShortPHD <= state->readLen);
+ /* examine page header now. */
+ pageHeaderSize =
+ XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+ if (targetRecOff == 0)
+ {
+ /* At page start, so skip over page header. */
+ state->ReadRecPtr += pageHeaderSize;
+ targetRecOff = pageHeaderSize;
+ }
+ else if (targetRecOff < pageHeaderSize)
+ {
+ report_invalid_record(state, "invalid record offset at %X/%X",
+ LSN_FORMAT_ARGS(state->ReadRecPtr));
+ goto err;
+ }
- /* Check that the continuation on next page looks valid */
pageHeader = (XLogPageHeader) state->readBuf;
- if (!(pageHeader->xlp_info & XLP_FIRST_IS_CONTRECORD))
+ if ((pageHeader->xlp_info & XLP_FIRST_IS_CONTRECORD) &&
+ targetRecOff == pageHeaderSize)
{
- report_invalid_record(state,
- "there is no contrecord flag at %X/%X",
- LSN_FORMAT_ARGS(RecPtr));
+ report_invalid_record(state, "contrecord is requested by %X/%X",
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr);
goto err;
}
- /*
- * Cross-check that xlp_rem_len agrees with how much of the record
- * we expect there to be left.
- */
- if (pageHeader->xlp_rem_len == 0 ||
- total_len != (pageHeader->xlp_rem_len + gotlen))
- {
- report_invalid_record(state,
- "invalid contrecord length %u (expected %lld) at %X/%X",
- pageHeader->xlp_rem_len,
- ((long long) total_len) - gotlen,
- LSN_FORMAT_ARGS(RecPtr));
- goto err;
- }
-
- /* Append the continuation from this page to the buffer */
- pageHeaderSize = XLogPageHeaderSize(pageHeader);
-
+ /* XLogNeedData has verified the page header */
Assert(pageHeaderSize <= state->readLen);
- contdata = (char *) state->readBuf + pageHeaderSize;
- len = XLOG_BLCKSZ - pageHeaderSize;
- if (pageHeader->xlp_rem_len < len)
- len = pageHeader->xlp_rem_len;
+ /*
+ * Read the record length.
+ *
+ * NB: Even though we use an XLogRecord pointer here, the whole
+ * record header might not fit on this page. xl_tot_len is the first
+ * field of the struct, so it must be on this page (the records are
+ * MAXALIGNed), but we cannot access any other fields until we've
+ * verified that we got the whole header.
+ */
+ prec = (XLogRecord *) (state->readBuf +
+ state->ReadRecPtr % XLOG_BLCKSZ);
+ total_len = prec->xl_tot_len;
- Assert (pageHeaderSize + len <= state->readLen);
- memcpy(buffer, (char *) contdata, len);
- buffer += len;
- gotlen += len;
+ /*
+ * If the whole record header is on this page, validate it
+ * immediately. Otherwise do just a basic sanity check on
+ * xl_tot_len, and validate the rest of the header after reading it
+ * from the next page. The xl_tot_len check is necessary here to
+ * ensure that we enter the XLREAD_CONTINUATION state below;
+ * otherwise we might fail to apply ValidXLogRecordHeader at all.
+ */
+ if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
+ {
+ if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
+ state->PrevRecPtr, prec))
+ goto err;
- /* If we just reassembled the record header, validate it. */
- if (!gotheader)
+ state->record_verified = true;
+ }
+ else
{
- record = (XLogRecord *) state->readRecordBuf;
- if (!ValidXLogRecordHeader(state, RecPtr, state->ReadRecPtr,
- record, randAccess))
+ /* XXX: more validation should be done here */
+ if (total_len < SizeOfXLogRecord)
+ {
+ report_invalid_record(state,
+ "invalid record length at %X/%X: wanted %u, got %u",
+ LSN_FORMAT_ARGS(state->ReadRecPtr),
+ (uint32) SizeOfXLogRecord, total_len);
goto err;
- gotheader = true;
+ }
}
- } while (gotlen < total_len);
-
- Assert(gotheader);
- record = (XLogRecord *) state->readRecordBuf;
- if (!ValidXLogRecord(state, record, RecPtr))
- goto err;
+ /*
+ * Wait for the rest of the record, or the part of the record that
+ * fit on the first page if crossed a page boundary, to become
+ * available.
+ */
+ state->recordGotLen = 0;
+ state->recordRemainLen = total_len;
+ state->readRecordState = XLREAD_FIRST_FRAGMENT;
+ }
+ /* fall through */
- pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
- state->ReadRecPtr = RecPtr;
- state->EndRecPtr = targetPagePtr + pageHeaderSize
- + MAXALIGN(pageHeader->xlp_rem_len);
- }
- else
- {
- /* Wait for the record data to become available */
- while (XLogNeedData(state, targetPagePtr,
- Min(targetRecOff + total_len, XLOG_BLCKSZ), true))
+ case XLREAD_FIRST_FRAGMENT:
{
- if (!state->routine.page_read(state, state->readPagePtr,
- state->readLen,
- state->ReadRecPtr, state->readBuf))
+ uint32 total_len = state->recordRemainLen;
+ uint32 request_len;
+ uint32 record_len;
+ XLogRecPtr targetPagePtr;
+ uint32 targetRecOff;
+
+ /*
+ * Wait for the rest of the record on the first page to become
+ * available
+ */
+ targetPagePtr =
+ state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
+ targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+
+ request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ);
+ record_len = request_len - targetRecOff;
+
+ /* ReadRecPtr contains page header */
+ Assert (targetRecOff != 0);
+ if (XLogNeedData(state, targetPagePtr, request_len, true))
+ return XLREAD_NEED_DATA;
+
+ /* error out if caller supplied bogus page */
+ if (!state->page_verified)
+ goto err;
+
+ prec = (XLogRecord *) (state->readBuf + targetRecOff);
+
+ /* validate record header if not yet */
+ if (!state->record_verified && record_len >= SizeOfXLogRecord)
+ {
+ if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
+ state->PrevRecPtr, prec))
+ goto err;
+
+ state->record_verified = true;
+ }
+
+
+ if (total_len == record_len)
+ {
+ /* Record does not cross a page boundary */
+ Assert(state->record_verified);
+
+ if (!ValidXLogRecord(state, prec, state->ReadRecPtr))
+ goto err;
+
+ state->record_verified = true; /* to be tidy */
+
+ /* We already checked the header earlier */
+ state->EndRecPtr = state->ReadRecPtr + MAXALIGN(record_len);
+
+ *record = prec;
+ state->readRecordState = XLREAD_NEXT_RECORD;
break;
+ }
+
+ /*
+ * The record continues on the next page. Need to reassemble
+ * record
+ */
+ Assert(total_len > record_len);
+
+ /* Enlarge readRecordBuf as needed. */
+ if (total_len > state->readRecordBufSize &&
+ !allocate_recordbuf(state, total_len))
+ {
+ /* We treat this as a "bogus data" condition */
+ report_invalid_record(state,
+ "record length %u at %X/%X too long",
+ total_len,
+ LSN_FORMAT_ARGS(state->ReadRecPtr));
+ goto err;
+ }
+
+ /* Copy the first fragment of the record from the first page. */
+ memcpy(state->readRecordBuf, state->readBuf + targetRecOff,
+ record_len);
+ state->recordGotLen += record_len;
+ state->recordRemainLen -= record_len;
+
+ /* Calculate pointer to beginning of next page */
+ state->recordContRecPtr = state->ReadRecPtr + record_len;
+ Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0);
+
+ state->readRecordState = XLREAD_CONTINUATION;
}
+ /* fall through */
+
+ case XLREAD_CONTINUATION:
+ {
+ XLogPageHeader pageHeader;
+ uint32 pageHeaderSize;
+ XLogRecPtr targetPagePtr;
+
+ /* we enter this state only if we haven't read the whole record. */
+ Assert (state->recordRemainLen > 0);
+
+ while(state->recordRemainLen > 0)
+ {
+ char *contdata;
+ uint32 request_len;
+ uint32 record_len;
+
+ /* Wait for the next page to become available */
+ targetPagePtr = state->recordContRecPtr;
+
+ /* this request contains page header */
+ Assert (targetPagePtr != 0);
+ if (XLogNeedData(state, targetPagePtr,
+ Min(state->recordRemainLen, XLOG_BLCKSZ),
+ false))
+ return XLREAD_NEED_DATA;
+
+ if (!state->page_verified)
+ goto err;
+
+ Assert(SizeOfXLogShortPHD <= state->readLen);
- if (!state->page_verified)
- goto err;
+ /* Check that the continuation on next page looks valid */
+ pageHeader = (XLogPageHeader) state->readBuf;
+ if (!(pageHeader->xlp_info & XLP_FIRST_IS_CONTRECORD))
+ {
+ report_invalid_record(
+ state,
+ "there is no contrecord flag at %X/%X reading %X/%X",
+ (uint32) (state->recordContRecPtr >> 32),
+ (uint32) state->recordContRecPtr,
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr);
+ goto err;
+ }
- /* Record does not cross a page boundary */
- if (!ValidXLogRecord(state, record, RecPtr))
- goto err;
+ /*
+ * Cross-check that xlp_rem_len agrees with how much of the
+ * record we expect there to be left.
+ */
+ if (pageHeader->xlp_rem_len == 0 ||
+ pageHeader->xlp_rem_len != state->recordRemainLen)
+ {
+ report_invalid_record(
+ state,
+ "invalid contrecord length %u at %X/%X reading %X/%X, expected %u",
+ pageHeader->xlp_rem_len,
+ (uint32) (state->recordContRecPtr >> 32),
+ (uint32) state->recordContRecPtr,
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr,
+ state->recordRemainLen);
+ goto err;
+ }
- state->EndRecPtr = RecPtr + MAXALIGN(total_len);
+ /* Append the continuation from this page to the buffer */
+ pageHeaderSize = XLogPageHeaderSize(pageHeader);
- state->ReadRecPtr = RecPtr;
+ /*
+ * XLogNeedData should have ensured that the whole page header
+ * was read
+ */
+ Assert(state->readLen >= pageHeaderSize);
+
+ contdata = (char *) state->readBuf + pageHeaderSize;
+ record_len = XLOG_BLCKSZ - pageHeaderSize;
+ if (pageHeader->xlp_rem_len < record_len)
+ record_len = pageHeader->xlp_rem_len;
+
+ request_len = record_len + pageHeaderSize;
+
+ /* XLogNeedData should have ensured all needed data was read */
+ Assert (state->readLen >= request_len);
+
+ memcpy(state->readRecordBuf + state->recordGotLen,
+ (char *) contdata, record_len);
+ state->recordGotLen += record_len;
+ state->recordRemainLen -= record_len;
+
+ /* If we just reassembled the record header, validate it. */
+ if (!state->record_verified)
+ {
+ Assert(state->recordGotLen >= SizeOfXLogRecord);
+ if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
+ state->PrevRecPtr,
+ (XLogRecord *) state->readRecordBuf))
+ goto err;
+
+ state->record_verified = true;
+ }
+
+ /* Calculate pointer to beginning of next page, and continue */
+ state->recordContRecPtr += XLOG_BLCKSZ;
+ }
+
+ /* targetPagePtr is pointing the last-read page here */
+ prec = (XLogRecord *) state->readRecordBuf;
+ if (!ValidXLogRecord(state, prec, state->ReadRecPtr))
+ goto err;
+
+ pageHeaderSize =
+ XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+ state->EndRecPtr = targetPagePtr + pageHeaderSize
+ + MAXALIGN(pageHeader->xlp_rem_len);
+
+ *record = prec;
+ state->readRecordState = XLREAD_NEXT_RECORD;
+ break;
+ }
}
/*
* Special processing if it's an XLOG SWITCH record
*/
- if (record->xl_rmid == RM_XLOG_ID &&
- (record->xl_info & ~XLR_INFO_MASK) == XLOG_SWITCH)
+ if ((*record)->xl_rmid == RM_XLOG_ID &&
+ ((*record)->xl_info & ~XLR_INFO_MASK) == XLOG_SWITCH)
{
/* Pretend it extends to end of segment */
state->EndRecPtr += state->segcxt.ws_segsize - 1;
state->EndRecPtr -= XLogSegmentOffset(state->EndRecPtr, state->segcxt.ws_segsize);
}
- if (DecodeXLogRecord(state, record, errormsg))
- return record;
- else
- return NULL;
+ Assert (!*record || state->readLen >= 0);
+ if (DecodeXLogRecord(state, *record, errormsg))
+ return XLREAD_SUCCESS;
+
+ *record = NULL;
+ return XLREAD_FAIL;
err:
/*
- * Invalidate the read state. We might read from a different source after
+ * Invalidate the read page. We might read from a different source after
* failure.
*/
XLogReaderInvalReadState(state);
@@ -572,7 +703,8 @@ err:
if (state->errormsg_buf[0] != '\0')
*errormsg = state->errormsg_buf;
- return NULL;
+ *record = NULL;
+ return XLREAD_FAIL;
}
/*
@@ -724,11 +856,12 @@ XLogReaderInvalReadState(XLogReaderState *state)
*
* This is just a convenience subroutine to avoid duplicated code in
* XLogReadRecord. It's not intended for use from anywhere else.
+ *
+ * If PrevRecPtr is valid, the xl_prev is is cross-checked with it.
*/
static bool
ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
- XLogRecPtr PrevRecPtr, XLogRecord *record,
- bool randAccess)
+ XLogRecPtr PrevRecPtr, XLogRecord *record)
{
if (record->xl_tot_len < SizeOfXLogRecord)
{
@@ -745,7 +878,7 @@ ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
record->xl_rmid, LSN_FORMAT_ARGS(RecPtr));
return false;
}
- if (randAccess)
+ if (PrevRecPtr == InvalidXLogRecPtr)
{
/*
* We can't exactly verify the prev-link, but surely it should be less
@@ -975,11 +1108,14 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
* XLogReadRecord() will read the next valid record.
*/
XLogRecPtr
-XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
+XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
+ XLogFindNextRecordCB read_page, void *private)
{
XLogRecPtr tmpRecPtr;
XLogRecPtr found = InvalidXLogRecPtr;
XLogPageHeader header;
+ XLogRecord *record;
+ XLogReadRecordResult result;
char *errormsg;
Assert(!XLogRecPtrIsInvalid(RecPtr));
@@ -1012,9 +1148,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
while(XLogNeedData(state, targetPagePtr, targetRecOff,
targetRecOff != 0))
{
- if (!state->routine.page_read(state, state->readPagePtr,
- state->readLen,
- state->ReadRecPtr, state->readBuf))
+ if (!read_page(state, private))
break;
}
@@ -1066,8 +1200,16 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
* or we just jumped over the remaining data of a continuation.
*/
XLogBeginRead(state, tmpRecPtr);
- while (XLogReadRecord(state, &errormsg) != NULL)
+ while ((result = XLogReadRecord(state, &record, &errormsg)) !=
+ XLREAD_FAIL)
{
+ if (result == XLREAD_NEED_DATA)
+ {
+ if (!read_page(state, private))
+ goto err;
+ continue;
+ }
+
/* past the record we've found, break out */
if (RecPtr <= state->ReadRecPtr)
{
@@ -1087,9 +1229,9 @@ err:
#endif /* FRONTEND */
/*
- * Helper function to ease writing of XLogRoutine->page_read callbacks.
- * If this function is used, caller must supply a segment_open callback in
- * 'state', as that is used here.
+ * Helper function to ease writing of page_read callback.
+ * If this function is used, caller must supply a segment_open callback and
+ * segment_close callback as that is used here.
*
* Read 'count' bytes into 'buf', starting at location 'startptr', from WAL
* fetched from timeline 'tli'.
@@ -1102,6 +1244,7 @@ err:
*/
bool
WALRead(XLogReaderState *state,
+ WALSegmentOpenCB segopenfn, WALSegmentCloseCB segclosefn,
char *buf, XLogRecPtr startptr, Size count, TimeLineID tli,
WALReadError *errinfo)
{
@@ -1133,10 +1276,10 @@ WALRead(XLogReaderState *state,
XLogSegNo nextSegNo;
if (state->seg.ws_file >= 0)
- state->routine.segment_close(state);
+ segclosefn(state);
XLByteToSeg(recptr, nextSegNo, state->segcxt.ws_segsize);
- state->routine.segment_open(state, nextSegNo, &tli);
+ segopenfn(state, nextSegNo, &tli);
/* This shouldn't happen -- indicates a bug in segment_open */
Assert(state->seg.ws_file >= 0);
diff --git a/src/backend/access/transam/xlogutils.c b/src/backend/access/transam/xlogutils.c
index 46eda33f25..b003990745 100644
--- a/src/backend/access/transam/xlogutils.c
+++ b/src/backend/access/transam/xlogutils.c
@@ -686,8 +686,7 @@ XLogTruncateRelation(RelFileNode rnode, ForkNumber forkNum,
void
XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wantLength)
{
- const XLogRecPtr lastReadPage = state->seg.ws_segno *
- state->segcxt.ws_segsize + state->readLen;
+ const XLogRecPtr lastReadPage = state->readPagePtr;
Assert(wantPage != InvalidXLogRecPtr && wantPage % XLOG_BLCKSZ == 0);
Assert(wantLength <= XLOG_BLCKSZ);
@@ -702,7 +701,7 @@ XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wa
* current TLI has since become historical.
*/
if (lastReadPage == wantPage &&
- state->readLen != 0 &&
+ state->page_verified &&
lastReadPage + state->readLen >= wantPage + Min(wantLength, XLOG_BLCKSZ - 1))
return;
@@ -788,6 +787,7 @@ wal_segment_open(XLogReaderState *state, XLogSegNo nextSegNo,
char path[MAXPGPATH];
XLogFilePath(path, tli, nextSegNo, state->segcxt.ws_segsize);
+ elog(LOG, "HOGE: %lu, %d => %s", nextSegNo, tli, path);
state->seg.ws_file = BasicOpenFile(path, O_RDONLY | PG_BINARY);
if (state->seg.ws_file >= 0)
return;
@@ -825,9 +825,11 @@ wal_segment_close(XLogReaderState *state)
* loop for now.
*/
bool
-read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *cur_page)
+read_local_xlog_page(XLogReaderState *state)
{
+ XLogRecPtr targetPagePtr = state->readPagePtr;
+ int reqLen = state->readLen;
+ char *cur_page = state->readBuf;
XLogRecPtr read_upto,
loc;
TimeLineID tli;
@@ -940,11 +942,12 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
* as 'count', read the whole page anyway. It's guaranteed to be
* zero-padded up to the page boundary if it's incomplete.
*/
- if (!WALRead(state, cur_page, targetPagePtr, XLOG_BLCKSZ, tli,
- &errinfo))
+ if (!WALRead(state, wal_segment_open, wal_segment_close,
+ cur_page, targetPagePtr, XLOG_BLCKSZ, tli, &errinfo))
WALReadRaiseError(&errinfo);
/* number of valid bytes in the buffer */
+ state->readPagePtr = targetPagePtr;
state->readLen = count;
return true;
}
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index 37b75deb72..9cbd8afa49 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -148,7 +148,8 @@ StartupDecodingContext(List *output_plugin_options,
TransactionId xmin_horizon,
bool need_full_snapshot,
bool fast_forward,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress)
@@ -198,11 +199,12 @@ StartupDecodingContext(List *output_plugin_options,
ctx->slot = slot;
- ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, xl_routine, ctx);
+ ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, cleanup_cb);
if (!ctx->reader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
errmsg("out of memory")));
+ ctx->page_read = page_read;
ctx->reorder = ReorderBufferAllocate();
ctx->snapshot_builder =
@@ -319,7 +321,8 @@ CreateInitDecodingContext(const char *plugin,
List *output_plugin_options,
bool need_full_snapshot,
XLogRecPtr restart_lsn,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress)
@@ -422,7 +425,7 @@ CreateInitDecodingContext(const char *plugin,
ctx = StartupDecodingContext(NIL, restart_lsn, xmin_horizon,
need_full_snapshot, false,
- xl_routine, prepare_write, do_write,
+ page_read, cleanup_cb, prepare_write, do_write,
update_progress);
/* call output plugin initialization callback */
@@ -476,7 +479,8 @@ LogicalDecodingContext *
CreateDecodingContext(XLogRecPtr start_lsn,
List *output_plugin_options,
bool fast_forward,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress)
@@ -528,8 +532,8 @@ CreateDecodingContext(XLogRecPtr start_lsn,
ctx = StartupDecodingContext(output_plugin_options,
start_lsn, InvalidTransactionId, false,
- fast_forward, xl_routine, prepare_write,
- do_write, update_progress);
+ fast_forward, page_read, cleanup_cb,
+ prepare_write, do_write, update_progress);
/* call output plugin initialization callback */
old_context = MemoryContextSwitchTo(ctx->context);
@@ -585,7 +589,13 @@ DecodingContextFindStartpoint(LogicalDecodingContext *ctx)
char *err = NULL;
/* the read_page callback waits for new WAL */
- record = XLogReadRecord(ctx->reader, &err);
+ while (XLogReadRecord(ctx->reader, &record, &err) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!ctx->page_read(ctx->reader))
+ break;
+ }
+
if (err)
elog(ERROR, "%s", err);
if (!record)
diff --git a/src/backend/replication/logical/logicalfuncs.c b/src/backend/replication/logical/logicalfuncs.c
index f7e055874e..b69f6911c5 100644
--- a/src/backend/replication/logical/logicalfuncs.c
+++ b/src/backend/replication/logical/logicalfuncs.c
@@ -233,9 +233,8 @@ pg_logical_slot_get_changes_guts(FunctionCallInfo fcinfo, bool confirm, bool bin
ctx = CreateDecodingContext(InvalidXLogRecPtr,
options,
false,
- XL_ROUTINE(.page_read = read_local_xlog_page,
- .segment_open = wal_segment_open,
- .segment_close = wal_segment_close),
+ read_local_xlog_page,
+ wal_segment_close,
LogicalOutputPrepareWrite,
LogicalOutputWrite, NULL);
@@ -284,7 +283,13 @@ pg_logical_slot_get_changes_guts(FunctionCallInfo fcinfo, bool confirm, bool bin
XLogRecord *record;
char *errm = NULL;
- record = XLogReadRecord(ctx->reader, &errm);
+ while (XLogReadRecord(ctx->reader, &record, &errm) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!ctx->page_read(ctx->reader))
+ break;
+ }
+
if (errm)
elog(ERROR, "%s", errm);
diff --git a/src/backend/replication/slotfuncs.c b/src/backend/replication/slotfuncs.c
index 9817b44113..deb7cf15eb 100644
--- a/src/backend/replication/slotfuncs.c
+++ b/src/backend/replication/slotfuncs.c
@@ -153,9 +153,8 @@ create_logical_replication_slot(char *name, char *plugin,
ctx = CreateInitDecodingContext(plugin, NIL,
false, /* just catalogs is OK */
restart_lsn,
- XL_ROUTINE(.page_read = read_local_xlog_page,
- .segment_open = wal_segment_open,
- .segment_close = wal_segment_close),
+ read_local_xlog_page,
+ wal_segment_close,
NULL, NULL, NULL);
/*
@@ -512,9 +511,8 @@ pg_logical_replication_slot_advance(XLogRecPtr moveto)
ctx = CreateDecodingContext(InvalidXLogRecPtr,
NIL,
true, /* fast_forward */
- XL_ROUTINE(.page_read = read_local_xlog_page,
- .segment_open = wal_segment_open,
- .segment_close = wal_segment_close),
+ read_local_xlog_page,
+ wal_segment_close,
NULL, NULL, NULL);
/*
@@ -536,7 +534,13 @@ pg_logical_replication_slot_advance(XLogRecPtr moveto)
* Read records. No changes are generated in fast_forward mode,
* but snapbuilder/slot statuses are updated properly.
*/
- record = XLogReadRecord(ctx->reader, &errm);
+ while (XLogReadRecord(ctx->reader, &record, &errm) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!ctx->page_read(ctx->reader))
+ break;
+ }
+
if (errm)
elog(ERROR, "%s", errm);
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index c36b91ae5e..062093727d 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -580,10 +580,7 @@ StartReplication(StartReplicationCmd *cmd)
/* create xlogreader for physical replication */
xlogreader =
- XLogReaderAllocate(wal_segment_size, NULL,
- XL_ROUTINE(.segment_open = WalSndSegmentOpen,
- .segment_close = wal_segment_close),
- NULL);
+ XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
if (!xlogreader)
ereport(ERROR,
@@ -807,9 +804,11 @@ StartReplication(StartReplicationCmd *cmd)
* set every time WAL is flushed.
*/
static bool
-logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetRecPtr, char *cur_page)
+logical_read_xlog_page(XLogReaderState *state)
{
+ XLogRecPtr targetPagePtr = state->readPagePtr;
+ int reqLen = state->readLen;
+ char *cur_page = state->readBuf;
XLogRecPtr flushptr;
int count;
WALReadError errinfo;
@@ -837,7 +836,7 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
count = flushptr - targetPagePtr; /* part of the page available */
/* now actually read the data, we know it's there */
- if (!WALRead(state,
+ if (!WALRead(state, WalSndSegmentOpen, wal_segment_close,
cur_page,
targetPagePtr,
XLOG_BLCKSZ,
@@ -1011,9 +1010,8 @@ CreateReplicationSlot(CreateReplicationSlotCmd *cmd)
ctx = CreateInitDecodingContext(cmd->plugin, NIL, need_full_snapshot,
InvalidXLogRecPtr,
- XL_ROUTINE(.page_read = logical_read_xlog_page,
- .segment_open = WalSndSegmentOpen,
- .segment_close = wal_segment_close),
+ logical_read_xlog_page,
+ wal_segment_close,
WalSndPrepareWrite, WalSndWriteData,
WalSndUpdateProgress);
@@ -1171,9 +1169,8 @@ StartLogicalReplication(StartReplicationCmd *cmd)
*/
logical_decoding_ctx =
CreateDecodingContext(cmd->startpoint, cmd->options, false,
- XL_ROUTINE(.page_read = logical_read_xlog_page,
- .segment_open = WalSndSegmentOpen,
- .segment_close = wal_segment_close),
+ logical_read_xlog_page,
+ wal_segment_close,
WalSndPrepareWrite, WalSndWriteData,
WalSndUpdateProgress);
xlogreader = logical_decoding_ctx->reader;
@@ -2749,7 +2746,7 @@ XLogSendPhysical(void)
enlargeStringInfo(&output_message, nbytes);
retry:
- if (!WALRead(xlogreader,
+ if (!WALRead(xlogreader, WalSndSegmentOpen, wal_segment_close,
&output_message.data[output_message.len],
startptr,
nbytes,
@@ -2847,7 +2844,12 @@ XLogSendLogical(void)
*/
WalSndCaughtUp = false;
- record = XLogReadRecord(logical_decoding_ctx->reader, &errm);
+ while (XLogReadRecord(logical_decoding_ctx->reader, &record, &errm) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!logical_decoding_ctx->page_read(logical_decoding_ctx->reader))
+ break;
+ }
/* xlog record was invalid */
if (errm != NULL)
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index cf119848b0..712c85281c 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -41,15 +41,9 @@ static int xlogreadfd = -1;
static XLogSegNo xlogreadsegno = -1;
static char xlogfpath[MAXPGPATH];
-typedef struct XLogPageReadPrivate
-{
- const char *restoreCommand;
- int tliIndex;
-} XLogPageReadPrivate;
-
-static bool SimpleXLogPageRead(XLogReaderState *xlogreader,
- XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
+static bool SimpleXLogPageRead(XLogReaderState *xlogreader,
+ const char *datadir, int *tliIndex,
+ const char *restoreCommand);
/*
* Read WAL from the datadir/pg_wal, starting from 'startpoint' on timeline
@@ -66,20 +60,22 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
XLogRecord *record;
XLogReaderState *xlogreader;
char *errormsg;
- XLogPageReadPrivate private;
- private.tliIndex = tliIndex;
- private.restoreCommand = restoreCommand;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir,
- XL_ROUTINE(.page_read = &SimpleXLogPageRead),
- &private);
+ xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+
if (xlogreader == NULL)
pg_fatal("out of memory");
XLogBeginRead(xlogreader, startpoint);
do
{
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!SimpleXLogPageRead(xlogreader, datadir,
+ &tliIndex, restoreCommand))
+ break;
+ }
if (record == NULL)
{
@@ -123,19 +119,19 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex,
XLogRecord *record;
XLogReaderState *xlogreader;
char *errormsg;
- XLogPageReadPrivate private;
XLogRecPtr endptr;
- private.tliIndex = tliIndex;
- private.restoreCommand = restoreCommand;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir,
- XL_ROUTINE(.page_read = &SimpleXLogPageRead),
- &private);
+ xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
XLogBeginRead(xlogreader, ptr);
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!SimpleXLogPageRead(xlogreader, datadir, &tliIndex, restoreCommand))
+ break;
+ }
if (record == NULL)
{
if (errormsg)
@@ -170,7 +166,6 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
XLogRecPtr searchptr;
XLogReaderState *xlogreader;
char *errormsg;
- XLogPageReadPrivate private;
/*
* The given fork pointer points to the end of the last common record,
@@ -186,11 +181,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
forkptr += SizeOfXLogShortPHD;
}
- private.tliIndex = tliIndex;
- private.restoreCommand = restoreCommand;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir,
- XL_ROUTINE(.page_read = &SimpleXLogPageRead),
- &private);
+ xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
@@ -200,7 +191,13 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
uint8 info;
XLogBeginRead(xlogreader, searchptr);
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!SimpleXLogPageRead(xlogreader, datadir,
+ &tliIndex, restoreCommand))
+ break;
+ }
if (record == NULL)
{
@@ -247,10 +244,11 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
/* XLogReader callback function, to read a WAL page */
static bool
-SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *readBuf)
+SimpleXLogPageRead(XLogReaderState *xlogreader, const char *datadir,
+ int *tliIndex, const char *restoreCommand)
{
- XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
+ XLogRecPtr targetPagePtr = xlogreader->readPagePtr;
+ char *readBuf = xlogreader->readBuf;
uint32 targetPageOff;
XLogRecPtr targetSegEnd;
XLogSegNo targetSegNo;
@@ -283,14 +281,14 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
* be done both forward and backward, consider also switching timeline
* accordingly.
*/
- while (private->tliIndex < targetNentries - 1 &&
- targetHistory[private->tliIndex].end < targetSegEnd)
- private->tliIndex++;
- while (private->tliIndex > 0 &&
- targetHistory[private->tliIndex].begin >= targetSegEnd)
- private->tliIndex--;
+ while (*tliIndex < targetNentries - 1 &&
+ targetHistory[*tliIndex].end < targetSegEnd)
+ (*tliIndex)++;
+ while (*tliIndex > 0 &&
+ targetHistory[*tliIndex].begin >= targetSegEnd)
+ (*tliIndex)--;
- XLogFileName(xlogfname, targetHistory[private->tliIndex].tli,
+ XLogFileName(xlogfname, targetHistory[*tliIndex].tli,
xlogreadsegno, WalSegSz);
snprintf(xlogfpath, MAXPGPATH, "%s/" XLOGDIR "/%s",
@@ -303,7 +301,7 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
/*
* If we have no restore_command to execute, then exit.
*/
- if (private->restoreCommand == NULL)
+ if (restoreCommand == NULL)
{
pg_log_error("could not open file \"%s\": %m", xlogfpath);
xlogreader->readLen = -1;
@@ -317,7 +315,7 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
xlogreadfd = RestoreArchivedFile(xlogreader->segcxt.ws_dir,
xlogfname,
WalSegSz,
- private->restoreCommand);
+ restoreCommand);
if (xlogreadfd < 0)
{
@@ -359,7 +357,7 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
Assert(targetSegNo == xlogreadsegno);
- xlogreader->seg.ws_tli = targetHistory[private->tliIndex].tli;
+ xlogreader->seg.ws_tli = targetHistory[*tliIndex].tli;
xlogreader->readLen = XLOG_BLCKSZ;
return true;
}
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index 0f0e0723a3..f514de6b13 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -330,12 +330,17 @@ WALDumpCloseSegment(XLogReaderState *state)
state->seg.ws_file = -1;
}
-/* pg_waldump's XLogReaderRoutine->page_read callback */
+/*
+ * pg_waldump's WAL page rader, also used as page_read callback for
+ * XLogFindNextRecord
+ */
static bool
-WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetPtr, char *readBuff)
+WALDumpReadPage(XLogReaderState *state, void *priv)
{
- XLogDumpPrivate *private = state->private_data;
+ XLogRecPtr targetPagePtr = state->readPagePtr;
+ int reqLen = state->readLen;
+ char *readBuff = state->readBuf;
+ XLogDumpPrivate *private = (XLogDumpPrivate *) priv;
int count = XLOG_BLCKSZ;
WALReadError errinfo;
@@ -353,8 +358,8 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
}
}
- if (!WALRead(state, readBuff, targetPagePtr, count, private->timeline,
- &errinfo))
+ if (!WALRead(state, WALDumpOpenSegment, WALDumpCloseSegment,
+ readBuff, targetPagePtr, count, private->timeline, &errinfo))
{
WALOpenSegment *seg = &errinfo.wre_seg;
char fname[MAXPGPATH];
@@ -374,6 +379,7 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
(Size) errinfo.wre_req);
}
+ Assert(count >= state->readLen);
state->readLen = count;
return true;
}
@@ -1035,16 +1041,14 @@ main(int argc, char **argv)
/* we have everything we need, start reading */
xlogreader_state =
- XLogReaderAllocate(WalSegSz, waldir,
- XL_ROUTINE(.page_read = WALDumpReadPage,
- .segment_open = WALDumpOpenSegment,
- .segment_close = WALDumpCloseSegment),
- &private);
+ XLogReaderAllocate(WalSegSz, waldir, WALDumpCloseSegment);
+
if (!xlogreader_state)
fatal_error("out of memory");
/* first find a valid recptr to start from */
- first_record = XLogFindNextRecord(xlogreader_state, private.startptr);
+ first_record = XLogFindNextRecord(xlogreader_state, private.startptr,
+ &WALDumpReadPage, (void*) &private);
if (first_record == InvalidXLogRecPtr)
fatal_error("could not find a valid record after %X/%X",
@@ -1067,7 +1071,13 @@ main(int argc, char **argv)
for (;;)
{
/* try to read the next record */
- record = XLogReadRecord(xlogreader_state, &errormsg);
+ while (XLogReadRecord(xlogreader_state, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!WALDumpReadPage(xlogreader_state, (void *) &private))
+ break;
+ }
+
if (!record)
{
if (!config.follow || private.endptr_reached)
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index 23d9e70a59..4e9268b553 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -57,64 +57,15 @@ typedef struct WALSegmentContext
typedef struct XLogReaderState XLogReaderState;
-/* Function type definition for the read_page callback */
-typedef bool (*XLogPageReadCB) (XLogReaderState *xlogreader,
- XLogRecPtr targetPagePtr,
- int reqLen,
- XLogRecPtr targetRecPtr,
- char *readBuf);
+/* Function type definition for the segment cleanup callback */
+typedef void (*WALSegmentCleanupCB) (XLogReaderState *xlogreader);
+
+/* Function type definition for the open/close callbacks for WALRead() */
typedef void (*WALSegmentOpenCB) (XLogReaderState *xlogreader,
XLogSegNo nextSegNo,
TimeLineID *tli_p);
typedef void (*WALSegmentCloseCB) (XLogReaderState *xlogreader);
-typedef struct XLogReaderRoutine
-{
- /*
- * Data input callback
- *
- * This callback shall read at least reqLen valid bytes of the xlog page
- * starting at targetPagePtr, and store them in readBuf. The callback
- * shall return the number of bytes read (never more than XLOG_BLCKSZ), or
- * -1 on failure. The callback shall sleep, if necessary, to wait for the
- * requested bytes to become available. The callback will not be invoked
- * again for the same page unless more than the returned number of bytes
- * are needed.
- *
- * targetRecPtr is the position of the WAL record we're reading. Usually
- * it is equal to targetPagePtr + reqLen, but sometimes xlogreader needs
- * to read and verify the page or segment header, before it reads the
- * actual WAL record it's interested in. In that case, targetRecPtr can
- * be used to determine which timeline to read the page from.
- *
- * The callback shall set ->seg.ws_tli to the TLI of the file the page was
- * read from.
- */
- XLogPageReadCB page_read;
-
- /*
- * Callback to open the specified WAL segment for reading. ->seg.ws_file
- * shall be set to the file descriptor of the opened segment. In case of
- * failure, an error shall be raised by the callback and it shall not
- * return.
- *
- * "nextSegNo" is the number of the segment to be opened.
- *
- * "tli_p" is an input/output argument. WALRead() uses it to pass the
- * timeline in which the new segment should be found, but the callback can
- * use it to return the TLI that it actually opened.
- */
- WALSegmentOpenCB segment_open;
-
- /*
- * WAL segment close callback. ->seg.ws_file shall be set to a negative
- * number.
- */
- WALSegmentCloseCB segment_close;
-} XLogReaderRoutine;
-
-#define XL_ROUTINE(...) &(XLogReaderRoutine){__VA_ARGS__}
-
typedef struct
{
/* Is this block ref in use? */
@@ -144,12 +95,35 @@ typedef struct
uint16 data_bufsz;
} DecodedBkpBlock;
+/* Return code from XLogReadRecord */
+typedef enum XLogReadRecordResult
+{
+ XLREAD_SUCCESS, /* record is successfully read */
+ XLREAD_NEED_DATA, /* need more data. see XLogReadRecord. */
+ XLREAD_FAIL /* failed during reading a record */
+} XLogReadRecordResult;
+
+/*
+ * internal state of XLogReadRecord
+ *
+ * XLogReadState runs a state machine while reading a record. Theses states
+ * are not seen outside the function. Each state may repeat several times
+ * exiting requesting caller for new data. See the comment of XLogReadRecrod
+ * for details.
+ */
+typedef enum XLogReadRecordState {
+ XLREAD_NEXT_RECORD,
+ XLREAD_TOT_LEN,
+ XLREAD_FIRST_FRAGMENT,
+ XLREAD_CONTINUATION
+} XLogReadRecordState;
+
struct XLogReaderState
{
/*
* Operational callbacks
*/
- XLogReaderRoutine routine;
+ WALSegmentCleanupCB cleanup_cb;
/* ----------------------------------------
* Public parameters
@@ -162,18 +136,14 @@ struct XLogReaderState
*/
uint64 system_identifier;
- /*
- * Opaque data for callbacks to use. Not used by XLogReader.
- */
- void *private_data;
-
/*
* Start and end point of last record read. EndRecPtr is also used as the
* position to read next. Calling XLogBeginRead() sets EndRecPtr to the
* starting position and ReadRecPtr to invalid.
*/
- XLogRecPtr ReadRecPtr; /* start of last record read */
+ XLogRecPtr ReadRecPtr; /* start of last record read or being read */
XLogRecPtr EndRecPtr; /* end+1 of last record read */
+ XLogRecPtr PrevRecPtr; /* start of previous record read */
/* ----------------------------------------
* Communication with page reader
@@ -187,7 +157,9 @@ struct XLogReaderState
* the request, or -1 on error */
TimeLineID readPageTLI; /* TLI for data currently in readBuf */
char *readBuf; /* buffer to store data */
- bool page_verified; /* is the page on the buffer verified? */
+ bool page_verified; /* is the page header on the buffer verified? */
+ bool record_verified;/* is the current record header verified? */
+
/* ----------------------------------------
@@ -229,8 +201,6 @@ struct XLogReaderState
XLogRecPtr latestPagePtr;
TimeLineID latestPageTLI;
- /* beginning of the WAL record being read. */
- XLogRecPtr currRecPtr;
/* timeline to read it from, 0 if a lookup is required */
TimeLineID currTLI;
@@ -257,6 +227,15 @@ struct XLogReaderState
char *readRecordBuf;
uint32 readRecordBufSize;
+ /*
+ * XLogReadRecord() state
+ */
+ XLogReadRecordState readRecordState;/* state machine state */
+ int recordGotLen; /* amount of current record that has
+ * already been read */
+ int recordRemainLen; /* length of current record that remains */
+ XLogRecPtr recordContRecPtr; /* where the current record continues */
+
/* Buffer to hold error message */
char *errormsg_buf;
};
@@ -264,9 +243,7 @@ struct XLogReaderState
/* Get a new XLogReader */
extern XLogReaderState *XLogReaderAllocate(int wal_segment_size,
const char *waldir,
- XLogReaderRoutine *routine,
- void *private_data);
-extern XLogReaderRoutine *LocalXLogReaderRoutine(void);
+ WALSegmentCleanupCB cleanup_cb);
/* Free an XLogReader */
extern void XLogReaderFree(XLogReaderState *state);
@@ -274,12 +251,17 @@ extern void XLogReaderFree(XLogReaderState *state);
/* Position the XLogReader to given record */
extern void XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr);
#ifdef FRONTEND
-extern XLogRecPtr XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr);
+/* Function type definition for the read_page callback */
+typedef bool (*XLogFindNextRecordCB) (XLogReaderState *xlogreader,
+ void *private);
+extern XLogRecPtr XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
+ XLogFindNextRecordCB read_page, void *private);
#endif /* FRONTEND */
/* Read the next XLog record. Returns NULL on end-of-WAL or failure */
-extern struct XLogRecord *XLogReadRecord(XLogReaderState *state,
- char **errormsg);
+extern XLogReadRecordResult XLogReadRecord(XLogReaderState *state,
+ XLogRecord **record,
+ char **errormsg);
/* Validate a page */
extern bool XLogReaderValidatePageHeader(XLogReaderState *state,
@@ -299,6 +281,7 @@ typedef struct WALReadError
} WALReadError;
extern bool WALRead(XLogReaderState *state,
+ WALSegmentOpenCB segopenfn, WALSegmentCloseCB sgclosefn,
char *buf, XLogRecPtr startptr, Size count,
TimeLineID tli, WALReadError *errinfo);
diff --git a/src/include/access/xlogutils.h b/src/include/access/xlogutils.h
index 364a21c4ea..397fb27fc2 100644
--- a/src/include/access/xlogutils.h
+++ b/src/include/access/xlogutils.h
@@ -47,9 +47,7 @@ extern Buffer XLogReadBufferExtended(RelFileNode rnode, ForkNumber forknum,
extern Relation CreateFakeRelcacheEntry(RelFileNode rnode);
extern void FreeFakeRelcacheEntry(Relation fakerel);
-extern bool read_local_xlog_page(XLogReaderState *state,
- XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetRecPtr, char *cur_page);
+extern bool read_local_xlog_page(XLogReaderState *state);
extern void wal_segment_open(XLogReaderState *state,
XLogSegNo nextSegNo,
TimeLineID *tli_p);
diff --git a/src/include/pg_config_manual.h b/src/include/pg_config_manual.h
index f10ad0acd6..94749609b4 100644
--- a/src/include/pg_config_manual.h
+++ b/src/include/pg_config_manual.h
@@ -377,7 +377,7 @@
* Enable debugging print statements for WAL-related operations; see
* also the wal_debug GUC var.
*/
-/* #define WAL_DEBUG */
+#define WAL_DEBUG
/*
* Enable tracing of resource consumption during sort operations;
diff --git a/src/include/replication/logical.h b/src/include/replication/logical.h
index c253403372..b936f31393 100644
--- a/src/include/replication/logical.h
+++ b/src/include/replication/logical.h
@@ -29,6 +29,10 @@ typedef void (*LogicalOutputPluginWriterUpdateProgress) (struct LogicalDecodingC
TransactionId xid
);
+typedef struct LogicalDecodingContext LogicalDecodingContext;
+
+typedef bool (*LogicalDecodingXLogPageReadCB)(XLogReaderState *ctx);
+
typedef struct LogicalDecodingContext
{
/* memory context this is all allocated in */
@@ -39,6 +43,7 @@ typedef struct LogicalDecodingContext
/* infrastructure pieces for decoding */
XLogReaderState *reader;
+ LogicalDecodingXLogPageReadCB page_read;
struct ReorderBuffer *reorder;
struct SnapBuild *snapshot_builder;
@@ -105,14 +110,16 @@ extern LogicalDecodingContext *CreateInitDecodingContext(const char *plugin,
List *output_plugin_options,
bool need_full_snapshot,
XLogRecPtr restart_lsn,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress);
extern LogicalDecodingContext *CreateDecodingContext(XLogRecPtr start_lsn,
List *output_plugin_options,
bool fast_forward,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress);
--
2.27.0
----Next_Part(Thu_Mar__4_11_28_53_2021_014)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
filename="v17-0003-Remove-globals-readOff-readLen-and-readSegNo.patch"
^ permalink raw reply [nested|flat] 70+ messages in thread
* [PATCH v9 2/4] Move page-reader out of XLogReadRecord
@ 2019-09-10 03:58 Kyotaro Horiguchi <[email protected]>
0 siblings, 0 replies; 70+ messages in thread
From: Kyotaro Horiguchi @ 2019-09-10 03:58 UTC (permalink / raw)
This is the second step of removing callbacks from WAL record reader.
Since it is essential to take in additional data while reading a
record, the function have to ask caller for new data while keeping
working state. Thus the function is turned into a state machine.
---
src/backend/access/transam/twophase.c | 11 +-
src/backend/access/transam/xlog.c | 54 +-
src/backend/access/transam/xlogreader.c | 681 +++++++++++-------
src/backend/access/transam/xlogutils.c | 12 +-
src/backend/replication/logical/logical.c | 17 +-
.../replication/logical/logicalfuncs.c | 8 +-
src/backend/replication/slotfuncs.c | 8 +-
src/backend/replication/walsender.c | 13 +-
src/bin/pg_rewind/parsexlog.c | 81 ++-
src/bin/pg_waldump/pg_waldump.c | 24 +-
src/include/access/xlogreader.h | 90 +--
src/include/access/xlogutils.h | 4 +-
src/include/replication/logical.h | 9 +-
13 files changed, 601 insertions(+), 411 deletions(-)
diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index 2f7d4ed59a..24b08810ac 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -1330,8 +1330,7 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
XLogReaderState *xlogreader;
char *errormsg;
- xlogreader = XLogReaderAllocate(wal_segment_size, NULL,
- &read_local_xlog_page, NULL);
+ xlogreader = XLogReaderAllocate(wal_segment_size, NULL);
if (!xlogreader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
@@ -1339,7 +1338,13 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
errdetail("Failed while allocating a WAL reading processor.")));
XLogBeginRead(xlogreader, lsn);
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!read_local_xlog_page(xlogreader))
+ break;
+ }
+
if (record == NULL)
ereport(ERROR,
(errcode_for_file_access(),
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index b0ad9376d6..e380eaa186 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -827,13 +827,6 @@ static XLogSource currentSource = XLOG_FROM_ANY;
static bool lastSourceFailed = false;
static bool pendingWalRcvRestart = false;
-typedef struct XLogPageReadPrivate
-{
- int emode;
- bool fetching_ckpt; /* are we fetching a checkpoint record? */
- bool randAccess;
-} XLogPageReadPrivate;
-
/*
* These variables track when we last obtained some WAL data to process,
* and where we got it from. (XLogReceiptSource is initially the same as
@@ -908,8 +901,8 @@ static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath,
static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli,
XLogSource source, bool notfoundOk);
static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, XLogSource source);
-static bool XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
+static bool XLogPageRead(XLogReaderState *xlogreader,
+ bool fetching_ckpt, int emode, bool randAccess);
static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
bool fetching_ckpt, XLogRecPtr tliRecPtr);
static int emode_for_corrupt_record(int emode, XLogRecPtr RecPtr);
@@ -1221,8 +1214,7 @@ XLogInsertRecord(XLogRecData *rdata,
appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len);
if (!debug_reader)
- debug_reader = XLogReaderAllocate(wal_segment_size, NULL,
- NULL, NULL);
+ debug_reader = XLogReaderAllocate(wal_segment_size, NULL);
if (!debug_reader)
{
@@ -4322,15 +4314,10 @@ CleanupBackupHistory(void)
* record is available.
*/
static XLogRecord *
-ReadRecord(XLogReaderState *xlogreader, int emode,
- bool fetching_ckpt)
+ReadRecord(XLogReaderState *xlogreader, int emode, bool fetching_ckpt)
{
XLogRecord *record;
- XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
-
- private->fetching_ckpt = fetching_ckpt;
- private->emode = emode;
- private->randAccess = (xlogreader->ReadRecPtr == InvalidXLogRecPtr);
+ bool randAccess = (xlogreader->ReadRecPtr == InvalidXLogRecPtr);
/* This is the first attempt to read this page. */
lastSourceFailed = false;
@@ -4338,8 +4325,16 @@ ReadRecord(XLogReaderState *xlogreader, int emode,
for (;;)
{
char *errormsg;
+ XLogReadRecordResult result;
+
+ while ((result = XLogReadRecord(xlogreader, &record, &errormsg))
+ == XLREAD_NEED_DATA)
+ {
+ if (!XLogPageRead(xlogreader, fetching_ckpt, emode, randAccess))
+ break;
+
+ }
- record = XLogReadRecord(xlogreader, &errormsg);
ReadRecPtr = xlogreader->ReadRecPtr;
EndRecPtr = xlogreader->EndRecPtr;
if (record == NULL)
@@ -6309,7 +6304,6 @@ StartupXLOG(void)
bool backupFromStandby = false;
DBState dbstate_at_startup;
XLogReaderState *xlogreader;
- XLogPageReadPrivate private;
bool fast_promoted = false;
struct stat st;
@@ -6465,9 +6459,7 @@ StartupXLOG(void)
OwnLatch(&XLogCtl->recoveryWakeupLatch);
/* Set up XLOG reader facility */
- MemSet(&private, 0, sizeof(XLogPageReadPrivate));
- xlogreader = XLogReaderAllocate(wal_segment_size, NULL,
- &XLogPageRead, &private);
+ xlogreader = XLogReaderAllocate(wal_segment_size, NULL);
if (!xlogreader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
@@ -11810,12 +11802,13 @@ CancelBackup(void)
* sleep and retry.
*/
static bool
-XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetRecPtr, char *readBuf)
+XLogPageRead(XLogReaderState *xlogreader,
+ bool fetching_ckpt, int emode, bool randAccess)
{
- XLogPageReadPrivate *private =
- (XLogPageReadPrivate *) xlogreader->private_data;
- int emode = private->emode;
+ char *readBuf = xlogreader->readBuf;
+ XLogRecPtr targetPagePtr = xlogreader->readPagePtr;
+ int reqLen = xlogreader->readLen;
+ XLogRecPtr targetRecPtr = xlogreader->ReadRecPtr;
uint32 targetPageOff;
XLogSegNo targetSegNo PG_USED_FOR_ASSERTS_ONLY;
int r;
@@ -11858,8 +11851,8 @@ retry:
flushedUpto < targetPagePtr + reqLen))
{
if (!WaitForWALToBecomeAvailable(targetPagePtr + reqLen,
- private->randAccess,
- private->fetching_ckpt,
+ randAccess,
+ fetching_ckpt,
targetRecPtr))
{
if (readFile >= 0)
@@ -11964,6 +11957,7 @@ retry:
goto next_record_is_invalid;
}
+ Assert(xlogreader->readPagePtr == targetPagePtr);
xlogreader->readLen = readLen;
return true;
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index 6250093dd9..7c8c6fc314 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -40,7 +40,7 @@ static bool XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr,
int reqLen, bool header_inclusive);
static void XLogReaderInvalReadState(XLogReaderState *state);
static bool ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
- XLogRecPtr PrevRecPtr, XLogRecord *record, bool randAccess);
+ XLogRecPtr PrevRecPtr, XLogRecord *record);
static bool ValidXLogRecord(XLogReaderState *state, XLogRecord *record,
XLogRecPtr recptr);
static void ResetDecoder(XLogReaderState *state);
@@ -70,8 +70,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...)
* Returns NULL if the xlogreader couldn't be allocated.
*/
XLogReaderState *
-XLogReaderAllocate(int wal_segment_size, const char *waldir,
- XLogPageReadCB pagereadfunc, void *private_data)
+XLogReaderAllocate(int wal_segment_size, const char *waldir)
{
XLogReaderState *state;
@@ -102,9 +101,6 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
WALOpenSegmentInit(&state->seg, &state->segcxt, wal_segment_size,
waldir);
- state->read_page = pagereadfunc;
- /* system_identifier initialized to zeroes above */
- state->private_data = private_data;
/* ReadRecPtr, EndRecPtr and readLen initialized to zeroes above */
state->errormsg_buf = palloc_extended(MAX_ERRORMSG_LEN + 1,
MCXT_ALLOC_NO_OOM);
@@ -242,6 +238,7 @@ XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
/* Begin at the passed-in record pointer. */
state->EndRecPtr = RecPtr;
state->ReadRecPtr = InvalidXLogRecPtr;
+ state->readRecordState = XLREAD_NEXT_RECORD;
}
/*
@@ -250,314 +247,452 @@ XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
* XLogBeginRead() or XLogFindNextRecord() must be called before the first call
* to XLogReadRecord().
*
- * If the read_page callback fails to read the requested data, NULL is
- * returned. The callback is expected to have reported the error; errormsg
- * is set to NULL.
+ * This function may return XLREAD_NEED_DATA several times before returning a
+ * result record. The caller shall read in some new data then call this
+ * function again with the same parameters.
*
- * If the reading fails for some other reason, NULL is also returned, and
- * *errormsg is set to a string with details of the failure.
+ * When a record is successfully read, returns XLREAD_SUCCESS with result
+ * record being stored in *record. Otherwise *record is NULL.
*
- * The returned pointer (or *errormsg) points to an internal buffer that's
- * valid until the next call to XLogReadRecord.
+ * Returns XLREAD_NEED_DATA if more data is needed to finish reading the
+ * current record. In that case, state->readPagePtr and state->readLen inform
+ * the desired position and minimum length of data needed. The caller shall
+ * read in the requested data and set state->readBuf to point to a buffer
+ * containing it. The caller must also set state->readPageTLI and
+ * state->readLen to indicate the timeline that it was read from, and the
+ * length of data that is now available (which must be >= given readLen),
+ * respectively.
+ *
+ * If invalid data is encountered, returns XLREAD_FAIL with *record being set to
+ * NULL. *errormsg is set to a string with details of the failure.
+ * The returned pointer (or *errormsg) points to an internal buffer that's valid
+ * until the next call to XLogReadRecord.
+ *
+ *
+ * This function runs a state machine consists of the following states.
+ *
+ * XLREAD_NEXT_RECORD :
+ * The initial state, if called with valid RecPtr, try to read a record at
+ * that position. If invalid RecPtr is given try to read a record just after
+ * the last one previously read.
+ * This state ens after setting ReadRecPtr. Then goes to XLREAD_TOT_LEN.
+ *
+ * XLREAD_TOT_LEN:
+ * Examining record header. Ends after reading record total
+ * length. recordRemainLen and recordGotLen are initialized.
+ *
+ * XLREAD_FIRST_FRAGMENT:
+ * Reading the first fragment. Ends with finishing reading a single
+ * record. Goes to XLREAD_NEXT_RECORD if that's all or
+ * XLREAD_CONTINUATION if we have continuation.
+
+ * XLREAD_CONTINUATION:
+ * Reading continuation of record. Ends with finishing the whole record then
+ * goes to XLREAD_NEXT_RECORD. During this state, recordRemainLen indicates
+ * how much is left and readRecordBuf holds the partially assert
+ * record.recordContRecPtr points to the beginning of the next page where to
+ * continue.
+ *
+ * If wrong data found in any state, the state machine stays at the current
+ * state. This behavior allows to continue reading a reacord switching among
+ * different souces, while streaming replication.
*/
-XLogRecord *
-XLogReadRecord(XLogReaderState *state, char **errormsg)
+XLogReadRecordResult
+XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
{
- XLogRecPtr RecPtr;
- XLogRecord *record;
- XLogRecPtr targetPagePtr;
- bool randAccess;
- uint32 len,
- total_len;
- uint32 targetRecOff;
- uint32 pageHeaderSize;
- bool gotheader;
+ XLogRecord *prec;
- /*
- * randAccess indicates whether to verify the previous-record pointer of
- * the record we're reading. We only do this if we're reading
- * sequentially, which is what we initially assume.
- */
- randAccess = false;
+ *record = NULL;
/* reset error state */
*errormsg = NULL;
state->errormsg_buf[0] = '\0';
- ResetDecoder(state);
-
- RecPtr = state->EndRecPtr;
-
- if (state->ReadRecPtr != InvalidXLogRecPtr)
- {
- /* read the record after the one we just read */
-
- /*
- * EndRecPtr is pointing to end+1 of the previous WAL record. If
- * we're at a page boundary, no more records can fit on the current
- * page. We must skip over the page header, but we can't do that until
- * we've read in the page, since the header size is variable.
- */
- }
- else
- {
- /*
- * Caller supplied a position to start at.
- *
- * In this case, EndRecPtr should already be pointing to a valid
- * record starting position.
- */
- Assert(XRecOffIsValid(RecPtr));
- randAccess = true;
- }
-
- state->currRecPtr = RecPtr;
-
- targetPagePtr = RecPtr - (RecPtr % XLOG_BLCKSZ);
- targetRecOff = RecPtr % XLOG_BLCKSZ;
-
- /*
- * Read the page containing the record into state->readBuf. Request enough
- * byte to cover the whole record header, or at least the part of it that
- * fits on the same page.
- */
- while (XLogNeedData(state, targetPagePtr,
- Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
- targetRecOff != 0))
- {
- if (!state->read_page(state, state->readPagePtr, state->readLen,
- RecPtr, state->readBuf))
- break;
- }
-
- if (!state->page_verified)
- goto err;
-
- /*
- * We have at least the page header, so we can examine it now.
- */
- pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
- if (targetRecOff == 0)
- {
- /*
- * At page start, so skip over page header.
- */
- RecPtr += pageHeaderSize;
- targetRecOff = pageHeaderSize;
- }
- else if (targetRecOff < pageHeaderSize)
- {
- report_invalid_record(state, "invalid record offset at %X/%X",
- (uint32) (RecPtr >> 32), (uint32) RecPtr);
- goto err;
- }
-
- if ((((XLogPageHeader) state->readBuf)->xlp_info & XLP_FIRST_IS_CONTRECORD) &&
- targetRecOff == pageHeaderSize)
- {
- report_invalid_record(state, "contrecord is requested by %X/%X",
- (uint32) (RecPtr >> 32), (uint32) RecPtr);
- goto err;
- }
-
- /* XLogNeedData has verified the page header */
- Assert(pageHeaderSize <= state->readLen);
-
- /*
- * Read the record length.
- *
- * NB: Even though we use an XLogRecord pointer here, the whole record
- * header might not fit on this page. xl_tot_len is the first field of the
- * struct, so it must be on this page (the records are MAXALIGNed), but we
- * cannot access any other fields until we've verified that we got the
- * whole header.
- */
- record = (XLogRecord *) (state->readBuf + RecPtr % XLOG_BLCKSZ);
- total_len = record->xl_tot_len;
-
- /*
- * If the whole record header is on this page, validate it immediately.
- * Otherwise do just a basic sanity check on xl_tot_len, and validate the
- * rest of the header after reading it from the next page. The xl_tot_len
- * check is necessary here to ensure that we enter the "Need to reassemble
- * record" code path below; otherwise we might fail to apply
- * ValidXLogRecordHeader at all.
- */
- if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
- {
- if (!ValidXLogRecordHeader(state, RecPtr, state->ReadRecPtr, record,
- randAccess))
- goto err;
- gotheader = true;
- }
- else
- {
- /* XXX: more validation should be done here */
- if (total_len < SizeOfXLogRecord)
- {
- report_invalid_record(state,
- "invalid record length at %X/%X: wanted %u, got %u",
- (uint32) (RecPtr >> 32), (uint32) RecPtr,
- (uint32) SizeOfXLogRecord, total_len);
- goto err;
- }
- gotheader = false;
- }
-
- len = XLOG_BLCKSZ - RecPtr % XLOG_BLCKSZ;
- if (total_len > len)
+ switch (state->readRecordState)
{
- /* Need to reassemble record */
- char *contdata;
- XLogPageHeader pageHeader;
- char *buffer;
- uint32 gotlen;
-
- /*
- * Enlarge readRecordBuf as needed.
- */
- if (total_len > state->readRecordBufSize &&
- !allocate_recordbuf(state, total_len))
- {
- /* We treat this as a "bogus data" condition */
- report_invalid_record(state, "record length %u at %X/%X too long",
- total_len,
- (uint32) (RecPtr >> 32), (uint32) RecPtr);
- goto err;
- }
+ case XLREAD_NEXT_RECORD:
+ ResetDecoder(state);
- /* Copy the first fragment of the record from the first page. */
- memcpy(state->readRecordBuf,
- state->readBuf + RecPtr % XLOG_BLCKSZ, len);
- buffer = state->readRecordBuf + len;
- gotlen = len;
-
- do
- {
- int rest_len = total_len - gotlen;
-
- /* Calculate pointer to beginning of next page */
- targetPagePtr += XLOG_BLCKSZ;
-
- /* Wait for the next page to become available */
- while (XLogNeedData(state, targetPagePtr,
- Min(rest_len, XLOG_BLCKSZ),
- false))
+ if (state->ReadRecPtr != InvalidXLogRecPtr)
{
- if (!state->read_page(state, state->readPagePtr, state->readLen,
- state->ReadRecPtr, state->readBuf))
- break;
+ /* read the record after the one we just read */
+
+ /*
+ * EndRecPtr is pointing to end+1 of the previous WAL record.
+ * If we're at a page boundary, no more records can fit on the
+ * current page. We must skip over the page header, but we
+ * can't do that until we've read in the page, since the header
+ * size is variable.
+ */
+ state->PrevRecPtr = state->ReadRecPtr;
+ state->ReadRecPtr = state->EndRecPtr;
+ }
+ else
+ {
+ /*
+ * Caller supplied a position to start at.
+ *
+ * In this case, EndRecPtr should already be pointing to a
+ * valid record starting position.
+ */
+ Assert(XRecOffIsValid(state->EndRecPtr));
+ state->ReadRecPtr = state->EndRecPtr;
+
+ /*
+ * We cannot verify the previous-record pointer when we're
+ * seeking to a particular record. Reset PrevRecPtr so that we
+ * won't try doing that.
+ */
+ state->PrevRecPtr = InvalidXLogRecPtr;
+ state->EndRecPtr = InvalidXLogRecPtr; /* to be tidy */
}
+ state->record_verified = false;
+ state->readRecordState = XLREAD_TOT_LEN;
+ /* fall through */
+
+ case XLREAD_TOT_LEN:
+ {
+ uint32 total_len;
+ uint32 pageHeaderSize;
+ XLogRecPtr targetPagePtr;
+ uint32 targetRecOff;
+ XLogPageHeader pageHeader;
+
+ targetPagePtr =
+ state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
+ targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+
+ /*
+ * Check if we have enough data. For the first record in the page,
+ * the requesting length doesn't contain page header.
+ */
+ if (XLogNeedData(state, targetPagePtr,
+ Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+ targetRecOff != 0))
+ return XLREAD_NEED_DATA;
+
+ /* error out if caller supplied bogus page */
if (!state->page_verified)
goto err;
- Assert(SizeOfXLogShortPHD <= state->readLen);
+ /* examine page header now. */
+ pageHeaderSize =
+ XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+ if (targetRecOff == 0)
+ {
+ /* At page start, so skip over page header. */
+ state->ReadRecPtr += pageHeaderSize;
+ targetRecOff = pageHeaderSize;
+ }
+ else if (targetRecOff < pageHeaderSize)
+ {
+ report_invalid_record(state, "invalid record offset at %X/%X",
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr);
+ goto err;
+ }
- /* Check that the continuation on next page looks valid */
pageHeader = (XLogPageHeader) state->readBuf;
- if (!(pageHeader->xlp_info & XLP_FIRST_IS_CONTRECORD))
+ if ((pageHeader->xlp_info & XLP_FIRST_IS_CONTRECORD) &&
+ targetRecOff == pageHeaderSize)
{
- report_invalid_record(state,
- "there is no contrecord flag at %X/%X",
- (uint32) (RecPtr >> 32), (uint32) RecPtr);
+ report_invalid_record(state, "contrecord is requested by %X/%X",
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr);
goto err;
}
- /*
- * Cross-check that xlp_rem_len agrees with how much of the record
- * we expect there to be left.
- */
- if (pageHeader->xlp_rem_len == 0 ||
- total_len != (pageHeader->xlp_rem_len + gotlen))
- {
- report_invalid_record(state,
- "invalid contrecord length %u at %X/%X",
- pageHeader->xlp_rem_len,
- (uint32) (RecPtr >> 32), (uint32) RecPtr);
- goto err;
- }
-
- /* Append the continuation from this page to the buffer */
- pageHeaderSize = XLogPageHeaderSize(pageHeader);
-
+ /* XLogNeedData has verified the page header */
Assert(pageHeaderSize <= state->readLen);
- contdata = (char *) state->readBuf + pageHeaderSize;
- len = XLOG_BLCKSZ - pageHeaderSize;
- if (pageHeader->xlp_rem_len < len)
- len = pageHeader->xlp_rem_len;
+ /*
+ * Read the record length.
+ *
+ * NB: Even though we use an XLogRecord pointer here, the whole
+ * record header might not fit on this page. xl_tot_len is the first
+ * field of the struct, so it must be on this page (the records are
+ * MAXALIGNed), but we cannot access any other fields until we've
+ * verified that we got the whole header.
+ */
+ prec = (XLogRecord *) (state->readBuf +
+ state->ReadRecPtr % XLOG_BLCKSZ);
+ total_len = prec->xl_tot_len;
- Assert (pageHeaderSize + len <= state->readLen);
- memcpy(buffer, (char *) contdata, len);
- buffer += len;
- gotlen += len;
+ /*
+ * If the whole record header is on this page, validate it
+ * immediately. Otherwise do just a basic sanity check on
+ * xl_tot_len, and validate the rest of the header after reading it
+ * from the next page. The xl_tot_len check is necessary here to
+ * ensure that we enter the XLREAD_CONTINUATION state below;
+ * otherwise we might fail to apply ValidXLogRecordHeader at all.
+ */
+ if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
+ {
+ if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
+ state->PrevRecPtr, prec))
+ goto err;
- /* If we just reassembled the record header, validate it. */
- if (!gotheader)
+ state->record_verified = true;
+ }
+ else
{
- record = (XLogRecord *) state->readRecordBuf;
- if (!ValidXLogRecordHeader(state, RecPtr, state->ReadRecPtr,
- record, randAccess))
+ /* XXX: more validation should be done here */
+ if (total_len < SizeOfXLogRecord)
+ {
+ report_invalid_record(state,
+ "invalid record length at %X/%X: wanted %u, got %u",
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr,
+ (uint32) SizeOfXLogRecord, total_len);
goto err;
- gotheader = true;
+ }
}
- } while (gotlen < total_len);
-
- Assert(gotheader);
- record = (XLogRecord *) state->readRecordBuf;
- if (!ValidXLogRecord(state, record, RecPtr))
- goto err;
+ /*
+ * Wait for the rest of the record, or the part of the record that
+ * fit on the first page if crossed a page boundary, to become
+ * available.
+ */
+ state->recordGotLen = 0;
+ state->recordRemainLen = total_len;
+ state->readRecordState = XLREAD_FIRST_FRAGMENT;
+ }
+ /* fall through */
- pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
- state->ReadRecPtr = RecPtr;
- state->EndRecPtr = targetPagePtr + pageHeaderSize
- + MAXALIGN(pageHeader->xlp_rem_len);
- }
- else
- {
- /* Wait for the record data to become available */
- while (XLogNeedData(state, targetPagePtr,
- Min(targetRecOff + total_len, XLOG_BLCKSZ), true))
+ case XLREAD_FIRST_FRAGMENT:
{
- if (!state->read_page(state, state->readPagePtr, state->readLen,
- state->ReadRecPtr, state->readBuf))
+ uint32 total_len = state->recordRemainLen;
+ uint32 request_len;
+ uint32 record_len;
+ XLogRecPtr targetPagePtr;
+ uint32 targetRecOff;
+
+ /*
+ * Wait for the rest of the record on the first page to become
+ * available
+ */
+ targetPagePtr =
+ state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
+ targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+
+ request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ);
+ record_len = request_len - targetRecOff;
+
+ /* ReadRecPtr contains page header */
+ Assert (targetRecOff != 0);
+ if (XLogNeedData(state, targetPagePtr, request_len, true))
+ return XLREAD_NEED_DATA;
+
+ /* error out if caller supplied bogus page */
+ if (!state->page_verified)
+ goto err;
+
+ prec = (XLogRecord *) (state->readBuf + targetRecOff);
+
+ /* validate record header if not yet */
+ if (!state->record_verified && record_len >= SizeOfXLogRecord)
+ {
+ if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
+ state->PrevRecPtr, prec))
+ goto err;
+
+ state->record_verified = true;
+ }
+
+
+ if (total_len == record_len)
+ {
+ /* Record does not cross a page boundary */
+ Assert(state->record_verified);
+
+ if (!ValidXLogRecord(state, prec, state->ReadRecPtr))
+ goto err;
+
+ state->record_verified = true; /* to be tidy */
+
+ /* We already checked the header earlier */
+ state->EndRecPtr = state->ReadRecPtr + MAXALIGN(record_len);
+
+ *record = prec;
+ state->readRecordState = XLREAD_NEXT_RECORD;
break;
+ }
+
+ /*
+ * The record continues on the next page. Need to reassemble
+ * record
+ */
+ Assert(total_len > record_len);
+
+ /* Enlarge readRecordBuf as needed. */
+ if (total_len > state->readRecordBufSize &&
+ !allocate_recordbuf(state, total_len))
+ {
+ /* We treat this as a "bogus data" condition */
+ report_invalid_record(state,
+ "record length %u at %X/%X too long",
+ total_len,
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr);
+ goto err;
+ }
+
+ /* Copy the first fragment of the record from the first page. */
+ memcpy(state->readRecordBuf, state->readBuf + targetRecOff,
+ record_len);
+ state->recordGotLen += record_len;
+ state->recordRemainLen -= record_len;
+
+ /* Calculate pointer to beginning of next page */
+ state->recordContRecPtr = state->ReadRecPtr + record_len;
+ Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0);
+
+ state->readRecordState = XLREAD_CONTINUATION;
}
+ /* fall through */
+
+ case XLREAD_CONTINUATION:
+ {
+ XLogPageHeader pageHeader;
+ uint32 pageHeaderSize;
+ XLogRecPtr targetPagePtr;
+
+ /* we enter this state only if we haven't read the whole record. */
+ Assert (state->recordRemainLen > 0);
+
+ while(state->recordRemainLen > 0)
+ {
+ char *contdata;
+ uint32 request_len;
+ uint32 record_len;
+
+ /* Wait for the next page to become available */
+ targetPagePtr = state->recordContRecPtr;
+
+ /* this request contains page header */
+ Assert (targetPagePtr != 0);
+ if (XLogNeedData(state, targetPagePtr,
+ Min(state->recordRemainLen, XLOG_BLCKSZ),
+ false))
+ return XLREAD_NEED_DATA;
+
+ if (!state->page_verified)
+ goto err;
+
+ Assert(SizeOfXLogShortPHD <= state->readLen);
- if (!state->page_verified)
- goto err;
+ /* Check that the continuation on next page looks valid */
+ pageHeader = (XLogPageHeader) state->readBuf;
+ if (!(pageHeader->xlp_info & XLP_FIRST_IS_CONTRECORD))
+ {
+ report_invalid_record(
+ state,
+ "there is no contrecord flag at %X/%X reading %X/%X",
+ (uint32) (state->recordContRecPtr >> 32),
+ (uint32) state->recordContRecPtr,
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr);
+ goto err;
+ }
- /* Record does not cross a page boundary */
- if (!ValidXLogRecord(state, record, RecPtr))
- goto err;
+ /*
+ * Cross-check that xlp_rem_len agrees with how much of the
+ * record we expect there to be left.
+ */
+ if (pageHeader->xlp_rem_len == 0 ||
+ pageHeader->xlp_rem_len != state->recordRemainLen)
+ {
+ report_invalid_record(
+ state,
+ "invalid contrecord length %u at %X/%X reading %X/%X, expected %u",
+ pageHeader->xlp_rem_len,
+ (uint32) (state->recordContRecPtr >> 32),
+ (uint32) state->recordContRecPtr,
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr,
+ state->recordRemainLen);
+ goto err;
+ }
- state->EndRecPtr = RecPtr + MAXALIGN(total_len);
+ /* Append the continuation from this page to the buffer */
+ pageHeaderSize = XLogPageHeaderSize(pageHeader);
- state->ReadRecPtr = RecPtr;
+ /*
+ * XLogNeedData should have ensured that the whole page header
+ * was read
+ */
+ Assert(state->readLen >= pageHeaderSize);
+
+ contdata = (char *) state->readBuf + pageHeaderSize;
+ record_len = XLOG_BLCKSZ - pageHeaderSize;
+ if (pageHeader->xlp_rem_len < record_len)
+ record_len = pageHeader->xlp_rem_len;
+
+ request_len = record_len + pageHeaderSize;
+
+ /* XLogNeedData should have ensured all needed data was read */
+ Assert (state->readLen >= request_len);
+
+ memcpy(state->readRecordBuf + state->recordGotLen,
+ (char *) contdata, record_len);
+ state->recordGotLen += record_len;
+ state->recordRemainLen -= record_len;
+
+ /* If we just reassembled the record header, validate it. */
+ if (!state->record_verified)
+ {
+ Assert(state->recordGotLen >= SizeOfXLogRecord);
+ if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
+ state->PrevRecPtr,
+ (XLogRecord *) state->readRecordBuf))
+ goto err;
+
+ state->record_verified = true;
+ }
+
+ /* Calculate pointer to beginning of next page, and continue */
+ state->recordContRecPtr += XLOG_BLCKSZ;
+ }
+
+ /* targetPagePtr is pointing the last-read page here */
+ prec = (XLogRecord *) state->readRecordBuf;
+ if (!ValidXLogRecord(state, prec, state->ReadRecPtr))
+ goto err;
+
+ pageHeaderSize =
+ XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+ state->EndRecPtr = targetPagePtr + pageHeaderSize
+ + MAXALIGN(pageHeader->xlp_rem_len);
+
+ *record = prec;
+ state->readRecordState = XLREAD_NEXT_RECORD;
+ break;
+ }
}
/*
* Special processing if it's an XLOG SWITCH record
*/
- if (record->xl_rmid == RM_XLOG_ID &&
- (record->xl_info & ~XLR_INFO_MASK) == XLOG_SWITCH)
+ if ((*record)->xl_rmid == RM_XLOG_ID &&
+ ((*record)->xl_info & ~XLR_INFO_MASK) == XLOG_SWITCH)
{
/* Pretend it extends to end of segment */
state->EndRecPtr += state->segcxt.ws_segsize - 1;
state->EndRecPtr -= XLogSegmentOffset(state->EndRecPtr, state->segcxt.ws_segsize);
}
- if (DecodeXLogRecord(state, record, errormsg))
- return record;
- else
- return NULL;
+ Assert (!*record || state->readLen >= 0);
+ if (DecodeXLogRecord(state, *record, errormsg))
+ return XLREAD_SUCCESS;
+
+ *record = NULL;
+ return XLREAD_FAIL;
err:
/*
- * Invalidate the read state. We might read from a different source after
+ * Invalidate the read page. We might read from a different source after
* failure.
*/
XLogReaderInvalReadState(state);
@@ -565,7 +700,8 @@ err:
if (state->errormsg_buf[0] != '\0')
*errormsg = state->errormsg_buf;
- return NULL;
+ *record = NULL;
+ return XLREAD_FAIL;
}
/*
@@ -718,11 +854,12 @@ XLogReaderInvalReadState(XLogReaderState *state)
*
* This is just a convenience subroutine to avoid duplicated code in
* XLogReadRecord. It's not intended for use from anywhere else.
+ *
+ * If PrevRecPtr is valid, the xl_prev is is cross-checked with it.
*/
static bool
ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
- XLogRecPtr PrevRecPtr, XLogRecord *record,
- bool randAccess)
+ XLogRecPtr PrevRecPtr, XLogRecord *record)
{
if (record->xl_tot_len < SizeOfXLogRecord)
{
@@ -740,7 +877,7 @@ ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
(uint32) RecPtr);
return false;
}
- if (randAccess)
+ if (PrevRecPtr == InvalidXLogRecPtr)
{
/*
* We can't exactly verify the prev-link, but surely it should be less
@@ -972,11 +1109,14 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
* XLogReadRecord() will read the next valid record.
*/
XLogRecPtr
-XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
+XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
+ XLogFindNextRecordCB read_page, void *private)
{
XLogRecPtr tmpRecPtr;
XLogRecPtr found = InvalidXLogRecPtr;
XLogPageHeader header;
+ XLogRecord *record;
+ XLogReadRecordResult result;
char *errormsg;
Assert(!XLogRecPtrIsInvalid(RecPtr));
@@ -1009,8 +1149,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
while(XLogNeedData(state, targetPagePtr, targetRecOff,
targetRecOff != 0))
{
- if (!state->read_page(state, state->readPagePtr, state->readLen,
- state->ReadRecPtr, state->readBuf))
+ if (!read_page(state, private))
break;
}
@@ -1062,8 +1201,16 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
* or we just jumped over the remaining data of a continuation.
*/
XLogBeginRead(state, tmpRecPtr);
- while (XLogReadRecord(state, &errormsg) != NULL)
+ while ((result = XLogReadRecord(state, &record, &errormsg)) !=
+ XLREAD_FAIL)
{
+ if (result == XLREAD_NEED_DATA)
+ {
+ if (!read_page(state, private))
+ goto err;
+ continue;
+ }
+
/* past the record we've found, break out */
if (RecPtr <= state->ReadRecPtr)
{
diff --git a/src/backend/access/transam/xlogutils.c b/src/backend/access/transam/xlogutils.c
index 2464a0d092..82f1ed2d1a 100644
--- a/src/backend/access/transam/xlogutils.c
+++ b/src/backend/access/transam/xlogutils.c
@@ -689,8 +689,7 @@ XLogTruncateRelation(RelFileNode rnode, ForkNumber forkNum,
void
XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wantLength)
{
- const XLogRecPtr lastReadPage = state->seg.ws_segno *
- state->segcxt.ws_segsize + state->readLen;
+ const XLogRecPtr lastReadPage = state->readPagePtr;
Assert(wantPage != InvalidXLogRecPtr && wantPage % XLOG_BLCKSZ == 0);
Assert(wantLength <= XLOG_BLCKSZ);
@@ -705,7 +704,7 @@ XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wa
* current TLI has since become historical.
*/
if (lastReadPage == wantPage &&
- state->readLen != 0 &&
+ state->page_verified &&
lastReadPage + state->readLen >= wantPage + Min(wantLength, XLOG_BLCKSZ - 1))
return;
@@ -823,9 +822,11 @@ wal_segment_open(XLogSegNo nextSegNo, WALSegmentContext * segcxt,
* loop for now.
*/
bool
-read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *cur_page)
+read_local_xlog_page(XLogReaderState *state)
{
+ XLogRecPtr targetPagePtr = state->readPagePtr;
+ int reqLen = state->readLen;
+ char *cur_page = state->readBuf;
XLogRecPtr read_upto,
loc;
TimeLineID tli;
@@ -943,6 +944,7 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
WALReadRaiseError(&errinfo);
/* number of valid bytes in the buffer */
+ state->readPagePtr = targetPagePtr;
state->readLen = count;
return true;
}
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index 5adf253583..7782e3ad2f 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -120,7 +120,7 @@ StartupDecodingContext(List *output_plugin_options,
TransactionId xmin_horizon,
bool need_full_snapshot,
bool fast_forward,
- XLogPageReadCB read_page,
+ LogicalDecodingXLogReadPageCB read_page,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress)
@@ -169,11 +169,12 @@ StartupDecodingContext(List *output_plugin_options,
ctx->slot = slot;
- ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, read_page, ctx);
+ ctx->reader = XLogReaderAllocate(wal_segment_size, NULL);
if (!ctx->reader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
errmsg("out of memory")));
+ ctx->read_page = read_page;
ctx->reorder = ReorderBufferAllocate();
ctx->snapshot_builder =
@@ -230,7 +231,7 @@ CreateInitDecodingContext(char *plugin,
List *output_plugin_options,
bool need_full_snapshot,
XLogRecPtr restart_lsn,
- XLogPageReadCB read_page,
+ LogicalDecodingXLogReadPageCB read_page,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress)
@@ -372,7 +373,7 @@ LogicalDecodingContext *
CreateDecodingContext(XLogRecPtr start_lsn,
List *output_plugin_options,
bool fast_forward,
- XLogPageReadCB read_page,
+ LogicalDecodingXLogReadPageCB read_page,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress)
@@ -479,7 +480,13 @@ DecodingContextFindStartpoint(LogicalDecodingContext *ctx)
char *err = NULL;
/* the read_page callback waits for new WAL */
- record = XLogReadRecord(ctx->reader, &err);
+ while (XLogReadRecord(ctx->reader, &record, &err) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!ctx->read_page(ctx->reader))
+ break;
+ }
+
if (err)
elog(ERROR, "%s", err);
if (!record)
diff --git a/src/backend/replication/logical/logicalfuncs.c b/src/backend/replication/logical/logicalfuncs.c
index f5384f1df8..8f2b2dd1fe 100644
--- a/src/backend/replication/logical/logicalfuncs.c
+++ b/src/backend/replication/logical/logicalfuncs.c
@@ -269,7 +269,13 @@ pg_logical_slot_get_changes_guts(FunctionCallInfo fcinfo, bool confirm, bool bin
XLogRecord *record;
char *errm = NULL;
- record = XLogReadRecord(ctx->reader, &errm);
+ while (XLogReadRecord(ctx->reader, &record, &errm) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!ctx->read_page(ctx->reader))
+ break;
+ }
+
if (errm)
elog(ERROR, "%s", errm);
diff --git a/src/backend/replication/slotfuncs.c b/src/backend/replication/slotfuncs.c
index f776de3df7..bd34042458 100644
--- a/src/backend/replication/slotfuncs.c
+++ b/src/backend/replication/slotfuncs.c
@@ -489,7 +489,13 @@ pg_logical_replication_slot_advance(XLogRecPtr moveto)
* Read records. No changes are generated in fast_forward mode,
* but snapbuilder/slot statuses are updated properly.
*/
- record = XLogReadRecord(ctx->reader, &errm);
+ while (XLogReadRecord(ctx->reader, &record, &errm) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!ctx->read_page(ctx->reader))
+ break;
+ }
+
if (errm)
elog(ERROR, "%s", errm);
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index e1c1de22c5..0dab43545d 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -799,9 +799,11 @@ StartReplication(StartReplicationCmd *cmd)
* set every time WAL is flushed.
*/
static bool
-logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetRecPtr, char *cur_page)
+logical_read_xlog_page(XLogReaderState *state)
{
+ XLogRecPtr targetPagePtr = state->readPagePtr;
+ int reqLen = state->readLen;
+ char *cur_page = state->readBuf;
XLogRecPtr flushptr;
int count;
WALReadError errinfo;
@@ -2834,7 +2836,12 @@ XLogSendLogical(void)
*/
WalSndCaughtUp = false;
- record = XLogReadRecord(logical_decoding_ctx->reader, &errm);
+ while (XLogReadRecord(logical_decoding_ctx->reader, &record, &errm) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!logical_decoding_ctx->read_page(logical_decoding_ctx->reader))
+ break;
+ }
/* xlog record was invalid */
if (errm != NULL)
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index 6cbe108129..86b6540057 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -40,15 +40,9 @@ static int xlogreadfd = -1;
static XLogSegNo xlogreadsegno = -1;
static char xlogfpath[MAXPGPATH];
-typedef struct XLogPageReadPrivate
-{
- const char *restoreCommand;
- int tliIndex;
-} XLogPageReadPrivate;
-
-static bool SimpleXLogPageRead(XLogReaderState *xlogreader,
- XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
+static bool SimpleXLogPageRead(XLogReaderState *xlogreader,
+ const char *datadir, int *tliIndex,
+ const char *restoreCommand);
/*
* Read WAL from the datadir/pg_wal, starting from 'startpoint' on timeline
@@ -62,19 +56,22 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
XLogRecord *record;
XLogReaderState *xlogreader;
char *errormsg;
- XLogPageReadPrivate private;
- private.tliIndex = tliIndex;
- private.restoreCommand = restoreCommand;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir, &SimpleXLogPageRead,
- &private);
+ xlogreader = XLogReaderAllocate(WalSegSz, datadir);
+
if (xlogreader == NULL)
pg_fatal("out of memory");
XLogBeginRead(xlogreader, startpoint);
do
{
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!SimpleXLogPageRead(xlogreader, datadir,
+ &tliIndex, restoreCommand))
+ break;
+ }
if (record == NULL)
{
@@ -112,18 +109,19 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex,
XLogRecord *record;
XLogReaderState *xlogreader;
char *errormsg;
- XLogPageReadPrivate private;
XLogRecPtr endptr;
- private.tliIndex = tliIndex;
- private.restoreCommand = restoreCommand;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir, &SimpleXLogPageRead,
- &private);
+ xlogreader = XLogReaderAllocate(WalSegSz, datadir);
if (xlogreader == NULL)
pg_fatal("out of memory");
XLogBeginRead(xlogreader, ptr);
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!SimpleXLogPageRead(xlogreader, datadir, &tliIndex, restoreCommand))
+ break;
+ }
if (record == NULL)
{
if (errormsg)
@@ -158,7 +156,6 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
XLogRecPtr searchptr;
XLogReaderState *xlogreader;
char *errormsg;
- XLogPageReadPrivate private;
/*
* The given fork pointer points to the end of the last common record,
@@ -174,10 +171,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
forkptr += SizeOfXLogShortPHD;
}
- private.tliIndex = tliIndex;
- private.restoreCommand = restoreCommand;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir, &SimpleXLogPageRead,
- &private);
+ xlogreader = XLogReaderAllocate(WalSegSz, datadir);
if (xlogreader == NULL)
pg_fatal("out of memory");
@@ -187,7 +181,13 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
uint8 info;
XLogBeginRead(xlogreader, searchptr);
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!SimpleXLogPageRead(xlogreader, datadir,
+ &tliIndex, restoreCommand))
+ break;
+ }
if (record == NULL)
{
@@ -234,10 +234,11 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
/* XLogReader callback function, to read a WAL page */
static bool
-SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *readBuf)
+SimpleXLogPageRead(XLogReaderState *xlogreader, const char *datadir,
+ int *tliIndex, const char *restoreCommand)
{
- XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
+ XLogRecPtr targetPagePtr = xlogreader->readPagePtr;
+ char *readBuf = xlogreader->readBuf;
uint32 targetPageOff;
XLogRecPtr targetSegEnd;
XLogSegNo targetSegNo;
@@ -270,14 +271,14 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
* be done both forward and backward, consider also switching timeline
* accordingly.
*/
- while (private->tliIndex < targetNentries - 1 &&
- targetHistory[private->tliIndex].end < targetSegEnd)
- private->tliIndex++;
- while (private->tliIndex > 0 &&
- targetHistory[private->tliIndex].begin >= targetSegEnd)
- private->tliIndex--;
+ while (*tliIndex < targetNentries - 1 &&
+ targetHistory[*tliIndex].end < targetSegEnd)
+ (*tliIndex)++;
+ while (*tliIndex > 0 &&
+ targetHistory[*tliIndex].begin >= targetSegEnd)
+ (*tliIndex)--;
- XLogFileName(xlogfname, targetHistory[private->tliIndex].tli,
+ XLogFileName(xlogfname, targetHistory[*tliIndex].tli,
xlogreadsegno, WalSegSz);
snprintf(xlogfpath, MAXPGPATH, "%s/" XLOGDIR "/%s",
@@ -290,7 +291,7 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
/*
* If we have no restore_command to execute, then exit.
*/
- if (private->restoreCommand == NULL)
+ if (restoreCommand == NULL)
{
pg_log_error("could not open file \"%s\": %m", xlogfpath);
xlogreader->readLen = -1;
@@ -304,7 +305,7 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
xlogreadfd = RestoreArchivedFile(xlogreader->segcxt.ws_dir,
xlogfname,
WalSegSz,
- private->restoreCommand);
+ restoreCommand);
if (xlogreadfd < 0)
{
@@ -346,7 +347,7 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
Assert(targetSegNo == xlogreadsegno);
- xlogreader->seg.ws_tli = targetHistory[private->tliIndex].tli;
+ xlogreader->seg.ws_tli = targetHistory[*tliIndex].tli;
xlogreader->readLen = XLOG_BLCKSZ;
return true;
}
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index 33fde23e7b..1c42050277 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -324,10 +324,12 @@ WALDumpOpenSegment(XLogSegNo nextSegNo, WALSegmentContext *segcxt,
* XLogReader read_page callback
*/
static bool
-WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetPtr, char *readBuff)
+WALDumpReadPage(XLogReaderState *state, void *priv)
{
- XLogDumpPrivate *private = state->private_data;
+ XLogRecPtr targetPagePtr = state->readPagePtr;
+ int reqLen = state->readLen;
+ char *readBuff = state->readBuf;
+ XLogDumpPrivate *private = (XLogDumpPrivate *) priv;
int count = XLOG_BLCKSZ;
WALReadError errinfo;
@@ -366,6 +368,7 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
(Size) errinfo.wre_req);
}
+ Assert(count >= state->readLen);
state->readLen = count;
return true;
}
@@ -1033,13 +1036,14 @@ main(int argc, char **argv)
/* done with argument parsing, do the actual work */
/* we have everything we need, start reading */
- xlogreader_state = XLogReaderAllocate(WalSegSz, waldir, WALDumpReadPage,
- &private);
+ xlogreader_state = XLogReaderAllocate(WalSegSz, waldir);
+
if (!xlogreader_state)
fatal_error("out of memory");
/* first find a valid recptr to start from */
- first_record = XLogFindNextRecord(xlogreader_state, private.startptr);
+ first_record = XLogFindNextRecord(xlogreader_state, private.startptr,
+ &WALDumpReadPage, (void*) &private);
if (first_record == InvalidXLogRecPtr)
fatal_error("could not find a valid record after %X/%X",
@@ -1063,7 +1067,13 @@ main(int argc, char **argv)
for (;;)
{
/* try to read the next record */
- record = XLogReadRecord(xlogreader_state, &errormsg);
+ while (XLogReadRecord(xlogreader_state, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!WALDumpReadPage(xlogreader_state, (void *) &private))
+ break;
+ }
+
if (!record)
{
if (!config.follow || private.endptr_reached)
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index 6ad953eea3..e59e42bee3 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -50,13 +50,6 @@ typedef struct WALSegmentContext
typedef struct XLogReaderState XLogReaderState;
-/* Function type definition for the read_page callback */
-typedef bool (*XLogPageReadCB) (XLogReaderState *xlogreader,
- XLogRecPtr targetPagePtr,
- int reqLen,
- XLogRecPtr targetRecPtr,
- char *readBuf);
-
typedef struct
{
/* Is this block ref in use? */
@@ -86,6 +79,29 @@ typedef struct
uint16 data_bufsz;
} DecodedBkpBlock;
+/* Return code from XLogReadRecord */
+typedef enum XLogReadRecordResult
+{
+ XLREAD_SUCCESS, /* record is successfully read */
+ XLREAD_NEED_DATA, /* need more data. see XLogReadRecord. */
+ XLREAD_FAIL /* failed during reading a record */
+} XLogReadRecordResult;
+
+/*
+ * internal state of XLogReadRecord
+ *
+ * XLogReadState runs a state machine while reading a record. Theses states
+ * are not seen outside the function. Each state may repeat several times
+ * exiting requesting caller for new data. See the comment of XLogReadRecrod
+ * for details.
+ */
+typedef enum XLogReadRecordState {
+ XLREAD_NEXT_RECORD,
+ XLREAD_TOT_LEN,
+ XLREAD_FIRST_FRAGMENT,
+ XLREAD_CONTINUATION
+} XLogReadRecordState;
+
struct XLogReaderState
{
/* ----------------------------------------
@@ -93,46 +109,20 @@ struct XLogReaderState
* ----------------------------------------
*/
- /*
- * Data input callback (mandatory).
- *
- * This callback shall read at least reqLen valid bytes of the xlog page
- * starting at targetPagePtr, and store them in readBuf. The callback
- * shall return the number of bytes read (never more than XLOG_BLCKSZ), or
- * -1 on failure. The callback shall sleep, if necessary, to wait for the
- * requested bytes to become available. The callback will not be invoked
- * again for the same page unless more than the returned number of bytes
- * are needed.
- *
- * targetRecPtr is the position of the WAL record we're reading. Usually
- * it is equal to targetPagePtr + reqLen, but sometimes xlogreader needs
- * to read and verify the page or segment header, before it reads the
- * actual WAL record it's interested in. In that case, targetRecPtr can
- * be used to determine which timeline to read the page from.
- *
- * The callback shall set ->seg.ws_tli to the TLI of the file the page was
- * read from.
- */
- XLogPageReadCB read_page;
-
/*
* System identifier of the xlog files we're about to read. Set to zero
* (the default value) if unknown or unimportant.
*/
uint64 system_identifier;
- /*
- * Opaque data for callbacks to use. Not used by XLogReader.
- */
- void *private_data;
-
/*
* Start and end point of last record read. EndRecPtr is also used as the
* position to read next. Calling XLogBeginRead() sets EndRecPtr to the
* starting position and ReadRecPtr to invalid.
*/
- XLogRecPtr ReadRecPtr; /* start of last record read */
+ XLogRecPtr ReadRecPtr; /* start of last record read or being read */
XLogRecPtr EndRecPtr; /* end+1 of last record read */
+ XLogRecPtr PrevRecPtr; /* start of previous record read */
/* ----------------------------------------
* Communication with page reader
@@ -146,7 +136,9 @@ struct XLogReaderState
* the request, or -1 on error */
TimeLineID readPageTLI; /* TLI for data currently in readBuf */
char *readBuf; /* buffer to store data */
- bool page_verified; /* is the page on the buffer verified? */
+ bool page_verified; /* is the page header on the buffer verified? */
+ bool record_verified;/* is the current record header verified? */
+
/* ----------------------------------------
@@ -186,8 +178,6 @@ struct XLogReaderState
XLogRecPtr latestPagePtr;
TimeLineID latestPageTLI;
- /* beginning of the WAL record being read. */
- XLogRecPtr currRecPtr;
/* timeline to read it from, 0 if a lookup is required */
TimeLineID currTLI;
@@ -214,15 +204,22 @@ struct XLogReaderState
char *readRecordBuf;
uint32 readRecordBufSize;
+ /*
+ * XLogReadRecord() state
+ */
+ XLogReadRecordState readRecordState;/* state machine state */
+ int recordGotLen; /* amount of current record that has
+ * already been read */
+ int recordRemainLen; /* length of current record that remains */
+ XLogRecPtr recordContRecPtr; /* where the current record continues */
+
/* Buffer to hold error message */
char *errormsg_buf;
};
/* Get a new XLogReader */
extern XLogReaderState *XLogReaderAllocate(int wal_segment_size,
- const char *waldir,
- XLogPageReadCB pagereadfunc,
- void *private_data);
+ const char *waldir);
/* Free an XLogReader */
extern void XLogReaderFree(XLogReaderState *state);
@@ -252,12 +249,17 @@ extern void WALOpenSegmentInit(WALOpenSegment *seg, WALSegmentContext *segcxt,
/* Position the XLogReader to given record */
extern void XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr);
#ifdef FRONTEND
-extern XLogRecPtr XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr);
+/* Function type definition for the read_page callback */
+typedef bool (*XLogFindNextRecordCB) (XLogReaderState *xlogreader,
+ void *private);
+extern XLogRecPtr XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
+ XLogFindNextRecordCB read_page, void *private);
#endif /* FRONTEND */
/* Read the next XLog record. Returns NULL on end-of-WAL or failure */
-extern struct XLogRecord *XLogReadRecord(XLogReaderState *state,
- char **errormsg);
+extern XLogReadRecordResult XLogReadRecord(XLogReaderState *state,
+ XLogRecord **record,
+ char **errormsg);
/* Validate a page */
extern bool XLogReaderValidatePageHeader(XLogReaderState *state,
diff --git a/src/include/access/xlogutils.h b/src/include/access/xlogutils.h
index dc7d894e5d..312acb4fa3 100644
--- a/src/include/access/xlogutils.h
+++ b/src/include/access/xlogutils.h
@@ -47,9 +47,7 @@ extern Buffer XLogReadBufferExtended(RelFileNode rnode, ForkNumber forknum,
extern Relation CreateFakeRelcacheEntry(RelFileNode rnode);
extern void FreeFakeRelcacheEntry(Relation fakerel);
-extern bool read_local_xlog_page(XLogReaderState *state,
- XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetRecPtr, char *cur_page);
+extern bool read_local_xlog_page(XLogReaderState *state);
extern void XLogReadDetermineTimeline(XLogReaderState *state,
XLogRecPtr wantPage, uint32 wantLength);
diff --git a/src/include/replication/logical.h b/src/include/replication/logical.h
index 3b7ca7f1da..cbe9ed751c 100644
--- a/src/include/replication/logical.h
+++ b/src/include/replication/logical.h
@@ -29,6 +29,10 @@ typedef void (*LogicalOutputPluginWriterUpdateProgress) (struct LogicalDecodingC
TransactionId xid
);
+typedef struct LogicalDecodingContext LogicalDecodingContext;
+
+typedef bool (*LogicalDecodingXLogReadPageCB)(XLogReaderState *ctx);
+
typedef struct LogicalDecodingContext
{
/* memory context this is all allocated in */
@@ -39,6 +43,7 @@ typedef struct LogicalDecodingContext
/* infrastructure pieces for decoding */
XLogReaderState *reader;
+ LogicalDecodingXLogReadPageCB read_page;
struct ReorderBuffer *reorder;
struct SnapBuild *snapshot_builder;
@@ -95,14 +100,14 @@ extern LogicalDecodingContext *CreateInitDecodingContext(char *plugin,
List *output_plugin_options,
bool need_full_snapshot,
XLogRecPtr restart_lsn,
- XLogPageReadCB read_page,
+ LogicalDecodingXLogReadPageCB read_page,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress);
extern LogicalDecodingContext *CreateDecodingContext(XLogRecPtr start_lsn,
List *output_plugin_options,
bool fast_forward,
- XLogPageReadCB read_page,
+ LogicalDecodingXLogReadPageCB read_page,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress);
--
2.18.2
----Next_Part(Wed_Apr_22_10_12_46_2020_740)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
filename="v9-0003-Remove-globals-readOff-readLen-and-readSegNo.patch"
^ permalink raw reply [nested|flat] 70+ messages in thread
* [PATCH v15 2/4] Move page-reader out of XLogReadRecord
@ 2019-09-10 03:58 Kyotaro Horiguchi <[email protected]>
0 siblings, 0 replies; 70+ messages in thread
From: Kyotaro Horiguchi @ 2019-09-10 03:58 UTC (permalink / raw)
This is the second step of removing callbacks from WAL record reader.
Since it is essential to take in additional data while reading a
record, the function have to ask caller for new data while keeping
working state. Thus the function is turned into a state machine.
---
src/backend/access/transam/twophase.c | 15 +-
src/backend/access/transam/xlog.c | 58 +-
src/backend/access/transam/xlogreader.c | 700 +++++++++++-------
src/backend/access/transam/xlogutils.c | 17 +-
src/backend/replication/logical/logical.c | 26 +-
.../replication/logical/logicalfuncs.c | 13 +-
src/backend/replication/slotfuncs.c | 18 +-
src/backend/replication/walsender.c | 32 +-
src/bin/pg_rewind/parsexlog.c | 84 +--
src/bin/pg_waldump/pg_waldump.c | 36 +-
src/include/access/xlogreader.h | 121 ++-
src/include/access/xlogutils.h | 4 +-
src/include/replication/logical.h | 11 +-
13 files changed, 648 insertions(+), 487 deletions(-)
diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index 9b2e59bf0e..b0d60a0d0f 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -1330,11 +1330,8 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
XLogReaderState *xlogreader;
char *errormsg;
- xlogreader = XLogReaderAllocate(wal_segment_size, NULL,
- XL_ROUTINE(.page_read = &read_local_xlog_page,
- .segment_open = &wal_segment_open,
- .segment_close = &wal_segment_close),
- NULL);
+ xlogreader = XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+
if (!xlogreader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
@@ -1342,7 +1339,13 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
errdetail("Failed while allocating a WAL reading processor.")));
XLogBeginRead(xlogreader, lsn);
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!read_local_xlog_page(xlogreader))
+ break;
+ }
+
if (record == NULL)
ereport(ERROR,
(errcode_for_file_access(),
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index e570e56a24..f9b0108602 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -831,13 +831,6 @@ static XLogSource currentSource = XLOG_FROM_ANY;
static bool lastSourceFailed = false;
static bool pendingWalRcvRestart = false;
-typedef struct XLogPageReadPrivate
-{
- int emode;
- bool fetching_ckpt; /* are we fetching a checkpoint record? */
- bool randAccess;
-} XLogPageReadPrivate;
-
/*
* These variables track when we last obtained some WAL data to process,
* and where we got it from. (XLogReceiptSource is initially the same as
@@ -912,8 +905,8 @@ static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath,
static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli,
XLogSource source, bool notfoundOk);
static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, XLogSource source);
-static bool XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
+static bool XLogPageRead(XLogReaderState *xlogreader,
+ bool fetching_ckpt, int emode, bool randAccess);
static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
bool fetching_ckpt, XLogRecPtr tliRecPtr);
static int emode_for_corrupt_record(int emode, XLogRecPtr RecPtr);
@@ -1225,8 +1218,7 @@ XLogInsertRecord(XLogRecData *rdata,
appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len);
if (!debug_reader)
- debug_reader = XLogReaderAllocate(wal_segment_size, NULL,
- XL_ROUTINE(), NULL);
+ debug_reader = XLogReaderAllocate(wal_segment_size, NULL NULL);
if (!debug_reader)
{
@@ -4326,15 +4318,10 @@ CleanupBackupHistory(void)
* record is available.
*/
static XLogRecord *
-ReadRecord(XLogReaderState *xlogreader, int emode,
- bool fetching_ckpt)
+ReadRecord(XLogReaderState *xlogreader, int emode, bool fetching_ckpt)
{
XLogRecord *record;
- XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
-
- private->fetching_ckpt = fetching_ckpt;
- private->emode = emode;
- private->randAccess = (xlogreader->ReadRecPtr == InvalidXLogRecPtr);
+ bool randAccess = (xlogreader->ReadRecPtr == InvalidXLogRecPtr);
/* This is the first attempt to read this page. */
lastSourceFailed = false;
@@ -4342,8 +4329,16 @@ ReadRecord(XLogReaderState *xlogreader, int emode,
for (;;)
{
char *errormsg;
+ XLogReadRecordResult result;
+
+ while ((result = XLogReadRecord(xlogreader, &record, &errormsg))
+ == XLREAD_NEED_DATA)
+ {
+ if (!XLogPageRead(xlogreader, fetching_ckpt, emode, randAccess))
+ break;
+
+ }
- record = XLogReadRecord(xlogreader, &errormsg);
ReadRecPtr = xlogreader->ReadRecPtr;
EndRecPtr = xlogreader->EndRecPtr;
if (record == NULL)
@@ -6320,7 +6315,6 @@ StartupXLOG(void)
bool backupFromStandby = false;
DBState dbstate_at_startup;
XLogReaderState *xlogreader;
- XLogPageReadPrivate private;
bool fast_promoted = false;
struct stat st;
@@ -6480,13 +6474,9 @@ StartupXLOG(void)
OwnLatch(&XLogCtl->recoveryWakeupLatch);
/* Set up XLOG reader facility */
- MemSet(&private, 0, sizeof(XLogPageReadPrivate));
xlogreader =
- XLogReaderAllocate(wal_segment_size, NULL,
- XL_ROUTINE(.page_read = &XLogPageRead,
- .segment_open = NULL,
- .segment_close = wal_segment_close),
- &private);
+ XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+
if (!xlogreader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
@@ -11860,12 +11850,13 @@ CancelBackup(void)
* sleep and retry.
*/
static bool
-XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetRecPtr, char *readBuf)
+XLogPageRead(XLogReaderState *xlogreader,
+ bool fetching_ckpt, int emode, bool randAccess)
{
- XLogPageReadPrivate *private =
- (XLogPageReadPrivate *) xlogreader->private_data;
- int emode = private->emode;
+ char *readBuf = xlogreader->readBuf;
+ XLogRecPtr targetPagePtr = xlogreader->readPagePtr;
+ int reqLen = xlogreader->readLen;
+ XLogRecPtr targetRecPtr = xlogreader->ReadRecPtr;
uint32 targetPageOff;
XLogSegNo targetSegNo PG_USED_FOR_ASSERTS_ONLY;
int r;
@@ -11908,8 +11899,8 @@ retry:
flushedUpto < targetPagePtr + reqLen))
{
if (!WaitForWALToBecomeAvailable(targetPagePtr + reqLen,
- private->randAccess,
- private->fetching_ckpt,
+ randAccess,
+ fetching_ckpt,
targetRecPtr))
{
if (readFile >= 0)
@@ -12014,6 +12005,7 @@ retry:
goto next_record_is_invalid;
}
+ Assert(xlogreader->readPagePtr == targetPagePtr);
xlogreader->readLen = readLen;
return true;
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index 3d599325ee..9281b57379 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -40,7 +40,7 @@ static bool XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr,
int reqLen, bool header_inclusive);
static void XLogReaderInvalReadState(XLogReaderState *state);
static bool ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
- XLogRecPtr PrevRecPtr, XLogRecord *record, bool randAccess);
+ XLogRecPtr PrevRecPtr, XLogRecord *record);
static bool ValidXLogRecord(XLogReaderState *state, XLogRecord *record,
XLogRecPtr recptr);
static void ResetDecoder(XLogReaderState *state);
@@ -73,7 +73,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...)
*/
XLogReaderState *
XLogReaderAllocate(int wal_segment_size, const char *waldir,
- XLogReaderRoutine *routine, void *private_data)
+ WALSegmentCleanupCB cleanup_cb)
{
XLogReaderState *state;
@@ -84,7 +84,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
return NULL;
/* initialize caller-provided support functions */
- state->routine = *routine;
+ state->cleanup_cb = cleanup_cb;
state->max_block_id = -1;
@@ -107,8 +107,6 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
WALOpenSegmentInit(&state->seg, &state->segcxt, wal_segment_size,
waldir);
- /* system_identifier initialized to zeroes above */
- state->private_data = private_data;
/* ReadRecPtr, EndRecPtr and readLen initialized to zeroes above */
state->errormsg_buf = palloc_extended(MAX_ERRORMSG_LEN + 1,
MCXT_ALLOC_NO_OOM);
@@ -140,8 +138,8 @@ XLogReaderFree(XLogReaderState *state)
{
int block_id;
- if (state->seg.ws_file != -1)
- state->routine.segment_close(state);
+ if (state->seg.ws_file >= 0)
+ state->cleanup_cb(state);
for (block_id = 0; block_id <= XLR_MAX_BLOCK_ID; block_id++)
{
@@ -246,6 +244,7 @@ XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
/* Begin at the passed-in record pointer. */
state->EndRecPtr = RecPtr;
state->ReadRecPtr = InvalidXLogRecPtr;
+ state->readRecordState = XLREAD_NEXT_RECORD;
}
/*
@@ -254,317 +253,452 @@ XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
* XLogBeginRead() or XLogFindNextRecord() must be called before the first call
* to XLogReadRecord().
*
- * If the page_read callback fails to read the requested data, NULL is
- * returned. The callback is expected to have reported the error; errormsg
- * is set to NULL.
+ * This function may return XLREAD_NEED_DATA several times before returning a
+ * result record. The caller shall read in some new data then call this
+ * function again with the same parameters.
*
- * If the reading fails for some other reason, NULL is also returned, and
- * *errormsg is set to a string with details of the failure.
+ * When a record is successfully read, returns XLREAD_SUCCESS with result
+ * record being stored in *record. Otherwise *record is NULL.
*
- * The returned pointer (or *errormsg) points to an internal buffer that's
- * valid until the next call to XLogReadRecord.
+ * Returns XLREAD_NEED_DATA if more data is needed to finish reading the
+ * current record. In that case, state->readPagePtr and state->readLen inform
+ * the desired position and minimum length of data needed. The caller shall
+ * read in the requested data and set state->readBuf to point to a buffer
+ * containing it. The caller must also set state->readPageTLI and
+ * state->readLen to indicate the timeline that it was read from, and the
+ * length of data that is now available (which must be >= given readLen),
+ * respectively.
+ *
+ * If invalid data is encountered, returns XLREAD_FAIL with *record being set to
+ * NULL. *errormsg is set to a string with details of the failure.
+ * The returned pointer (or *errormsg) points to an internal buffer that's valid
+ * until the next call to XLogReadRecord.
+ *
+ *
+ * This function runs a state machine consists of the following states.
+ *
+ * XLREAD_NEXT_RECORD :
+ * The initial state, if called with valid RecPtr, try to read a record at
+ * that position. If invalid RecPtr is given try to read a record just after
+ * the last one previously read.
+ * This state ens after setting ReadRecPtr. Then goes to XLREAD_TOT_LEN.
+ *
+ * XLREAD_TOT_LEN:
+ * Examining record header. Ends after reading record total
+ * length. recordRemainLen and recordGotLen are initialized.
+ *
+ * XLREAD_FIRST_FRAGMENT:
+ * Reading the first fragment. Ends with finishing reading a single
+ * record. Goes to XLREAD_NEXT_RECORD if that's all or
+ * XLREAD_CONTINUATION if we have continuation.
+
+ * XLREAD_CONTINUATION:
+ * Reading continuation of record. Ends with finishing the whole record then
+ * goes to XLREAD_NEXT_RECORD. During this state, recordRemainLen indicates
+ * how much is left and readRecordBuf holds the partially assert
+ * record.recordContRecPtr points to the beginning of the next page where to
+ * continue.
+ *
+ * If wrong data found in any state, the state machine stays at the current
+ * state. This behavior allows to continue reading a reacord switching among
+ * different souces, while streaming replication.
*/
-XLogRecord *
-XLogReadRecord(XLogReaderState *state, char **errormsg)
+XLogReadRecordResult
+XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
{
- XLogRecPtr RecPtr;
- XLogRecord *record;
- XLogRecPtr targetPagePtr;
- bool randAccess;
- uint32 len,
- total_len;
- uint32 targetRecOff;
- uint32 pageHeaderSize;
- bool gotheader;
+ XLogRecord *prec;
- /*
- * randAccess indicates whether to verify the previous-record pointer of
- * the record we're reading. We only do this if we're reading
- * sequentially, which is what we initially assume.
- */
- randAccess = false;
+ *record = NULL;
/* reset error state */
*errormsg = NULL;
state->errormsg_buf[0] = '\0';
- ResetDecoder(state);
-
- RecPtr = state->EndRecPtr;
-
- if (state->ReadRecPtr != InvalidXLogRecPtr)
- {
- /* read the record after the one we just read */
-
- /*
- * EndRecPtr is pointing to end+1 of the previous WAL record. If
- * we're at a page boundary, no more records can fit on the current
- * page. We must skip over the page header, but we can't do that until
- * we've read in the page, since the header size is variable.
- */
- }
- else
- {
- /*
- * Caller supplied a position to start at.
- *
- * In this case, EndRecPtr should already be pointing to a valid
- * record starting position.
- */
- Assert(XRecOffIsValid(RecPtr));
- randAccess = true;
- }
-
- state->currRecPtr = RecPtr;
-
- targetPagePtr = RecPtr - (RecPtr % XLOG_BLCKSZ);
- targetRecOff = RecPtr % XLOG_BLCKSZ;
-
- /*
- * Read the page containing the record into state->readBuf. Request enough
- * byte to cover the whole record header, or at least the part of it that
- * fits on the same page.
- */
- while (XLogNeedData(state, targetPagePtr,
- Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
- targetRecOff != 0))
- {
- if (!state->routine.page_read(state, state->readPagePtr, state->readLen,
- RecPtr, state->readBuf))
- break;
- }
-
- if (!state->page_verified)
- goto err;
-
- /*
- * We have at least the page header, so we can examine it now.
- */
- pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
- if (targetRecOff == 0)
- {
- /*
- * At page start, so skip over page header.
- */
- RecPtr += pageHeaderSize;
- targetRecOff = pageHeaderSize;
- }
- else if (targetRecOff < pageHeaderSize)
- {
- report_invalid_record(state, "invalid record offset at %X/%X",
- (uint32) (RecPtr >> 32), (uint32) RecPtr);
- goto err;
- }
-
- if ((((XLogPageHeader) state->readBuf)->xlp_info & XLP_FIRST_IS_CONTRECORD) &&
- targetRecOff == pageHeaderSize)
- {
- report_invalid_record(state, "contrecord is requested by %X/%X",
- (uint32) (RecPtr >> 32), (uint32) RecPtr);
- goto err;
- }
-
- /* XLogNeedData has verified the page header */
- Assert(pageHeaderSize <= state->readLen);
-
- /*
- * Read the record length.
- *
- * NB: Even though we use an XLogRecord pointer here, the whole record
- * header might not fit on this page. xl_tot_len is the first field of the
- * struct, so it must be on this page (the records are MAXALIGNed), but we
- * cannot access any other fields until we've verified that we got the
- * whole header.
- */
- record = (XLogRecord *) (state->readBuf + RecPtr % XLOG_BLCKSZ);
- total_len = record->xl_tot_len;
-
- /*
- * If the whole record header is on this page, validate it immediately.
- * Otherwise do just a basic sanity check on xl_tot_len, and validate the
- * rest of the header after reading it from the next page. The xl_tot_len
- * check is necessary here to ensure that we enter the "Need to reassemble
- * record" code path below; otherwise we might fail to apply
- * ValidXLogRecordHeader at all.
- */
- if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
- {
- if (!ValidXLogRecordHeader(state, RecPtr, state->ReadRecPtr, record,
- randAccess))
- goto err;
- gotheader = true;
- }
- else
- {
- /* XXX: more validation should be done here */
- if (total_len < SizeOfXLogRecord)
- {
- report_invalid_record(state,
- "invalid record length at %X/%X: wanted %u, got %u",
- (uint32) (RecPtr >> 32), (uint32) RecPtr,
- (uint32) SizeOfXLogRecord, total_len);
- goto err;
- }
- gotheader = false;
- }
-
- len = XLOG_BLCKSZ - RecPtr % XLOG_BLCKSZ;
- if (total_len > len)
+ switch (state->readRecordState)
{
- /* Need to reassemble record */
- char *contdata;
- XLogPageHeader pageHeader;
- char *buffer;
- uint32 gotlen;
-
- /*
- * Enlarge readRecordBuf as needed.
- */
- if (total_len > state->readRecordBufSize &&
- !allocate_recordbuf(state, total_len))
- {
- /* We treat this as a "bogus data" condition */
- report_invalid_record(state, "record length %u at %X/%X too long",
- total_len,
- (uint32) (RecPtr >> 32), (uint32) RecPtr);
- goto err;
- }
+ case XLREAD_NEXT_RECORD:
+ ResetDecoder(state);
- /* Copy the first fragment of the record from the first page. */
- memcpy(state->readRecordBuf,
- state->readBuf + RecPtr % XLOG_BLCKSZ, len);
- buffer = state->readRecordBuf + len;
- gotlen = len;
-
- do
- {
- int rest_len = total_len - gotlen;
-
- /* Calculate pointer to beginning of next page */
- targetPagePtr += XLOG_BLCKSZ;
-
- /* Wait for the next page to become available */
- while (XLogNeedData(state, targetPagePtr,
- Min(rest_len, XLOG_BLCKSZ),
- false))
+ if (state->ReadRecPtr != InvalidXLogRecPtr)
{
- if (!state->routine.page_read(state, state->readPagePtr,
- state->readLen,
- state->ReadRecPtr,
- state->readBuf))
- break;
+ /* read the record after the one we just read */
+
+ /*
+ * EndRecPtr is pointing to end+1 of the previous WAL record.
+ * If we're at a page boundary, no more records can fit on the
+ * current page. We must skip over the page header, but we
+ * can't do that until we've read in the page, since the header
+ * size is variable.
+ */
+ state->PrevRecPtr = state->ReadRecPtr;
+ state->ReadRecPtr = state->EndRecPtr;
+ }
+ else
+ {
+ /*
+ * Caller supplied a position to start at.
+ *
+ * In this case, EndRecPtr should already be pointing to a
+ * valid record starting position.
+ */
+ Assert(XRecOffIsValid(state->EndRecPtr));
+ state->ReadRecPtr = state->EndRecPtr;
+
+ /*
+ * We cannot verify the previous-record pointer when we're
+ * seeking to a particular record. Reset PrevRecPtr so that we
+ * won't try doing that.
+ */
+ state->PrevRecPtr = InvalidXLogRecPtr;
+ state->EndRecPtr = InvalidXLogRecPtr; /* to be tidy */
}
+ state->record_verified = false;
+ state->readRecordState = XLREAD_TOT_LEN;
+ /* fall through */
+
+ case XLREAD_TOT_LEN:
+ {
+ uint32 total_len;
+ uint32 pageHeaderSize;
+ XLogRecPtr targetPagePtr;
+ uint32 targetRecOff;
+ XLogPageHeader pageHeader;
+
+ targetPagePtr =
+ state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
+ targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+
+ /*
+ * Check if we have enough data. For the first record in the page,
+ * the requesting length doesn't contain page header.
+ */
+ if (XLogNeedData(state, targetPagePtr,
+ Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+ targetRecOff != 0))
+ return XLREAD_NEED_DATA;
+
+ /* error out if caller supplied bogus page */
if (!state->page_verified)
goto err;
- Assert(SizeOfXLogShortPHD <= state->readLen);
+ /* examine page header now. */
+ pageHeaderSize =
+ XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+ if (targetRecOff == 0)
+ {
+ /* At page start, so skip over page header. */
+ state->ReadRecPtr += pageHeaderSize;
+ targetRecOff = pageHeaderSize;
+ }
+ else if (targetRecOff < pageHeaderSize)
+ {
+ report_invalid_record(state, "invalid record offset at %X/%X",
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr);
+ goto err;
+ }
- /* Check that the continuation on next page looks valid */
pageHeader = (XLogPageHeader) state->readBuf;
- if (!(pageHeader->xlp_info & XLP_FIRST_IS_CONTRECORD))
+ if ((pageHeader->xlp_info & XLP_FIRST_IS_CONTRECORD) &&
+ targetRecOff == pageHeaderSize)
{
- report_invalid_record(state,
- "there is no contrecord flag at %X/%X",
- (uint32) (RecPtr >> 32), (uint32) RecPtr);
+ report_invalid_record(state, "contrecord is requested by %X/%X",
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr);
goto err;
}
- /*
- * Cross-check that xlp_rem_len agrees with how much of the record
- * we expect there to be left.
- */
- if (pageHeader->xlp_rem_len == 0 ||
- total_len != (pageHeader->xlp_rem_len + gotlen))
- {
- report_invalid_record(state,
- "invalid contrecord length %u at %X/%X",
- pageHeader->xlp_rem_len,
- (uint32) (RecPtr >> 32), (uint32) RecPtr);
- goto err;
- }
-
- /* Append the continuation from this page to the buffer */
- pageHeaderSize = XLogPageHeaderSize(pageHeader);
-
+ /* XLogNeedData has verified the page header */
Assert(pageHeaderSize <= state->readLen);
- contdata = (char *) state->readBuf + pageHeaderSize;
- len = XLOG_BLCKSZ - pageHeaderSize;
- if (pageHeader->xlp_rem_len < len)
- len = pageHeader->xlp_rem_len;
+ /*
+ * Read the record length.
+ *
+ * NB: Even though we use an XLogRecord pointer here, the whole
+ * record header might not fit on this page. xl_tot_len is the first
+ * field of the struct, so it must be on this page (the records are
+ * MAXALIGNed), but we cannot access any other fields until we've
+ * verified that we got the whole header.
+ */
+ prec = (XLogRecord *) (state->readBuf +
+ state->ReadRecPtr % XLOG_BLCKSZ);
+ total_len = prec->xl_tot_len;
- Assert (pageHeaderSize + len <= state->readLen);
- memcpy(buffer, (char *) contdata, len);
- buffer += len;
- gotlen += len;
+ /*
+ * If the whole record header is on this page, validate it
+ * immediately. Otherwise do just a basic sanity check on
+ * xl_tot_len, and validate the rest of the header after reading it
+ * from the next page. The xl_tot_len check is necessary here to
+ * ensure that we enter the XLREAD_CONTINUATION state below;
+ * otherwise we might fail to apply ValidXLogRecordHeader at all.
+ */
+ if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
+ {
+ if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
+ state->PrevRecPtr, prec))
+ goto err;
- /* If we just reassembled the record header, validate it. */
- if (!gotheader)
+ state->record_verified = true;
+ }
+ else
{
- record = (XLogRecord *) state->readRecordBuf;
- if (!ValidXLogRecordHeader(state, RecPtr, state->ReadRecPtr,
- record, randAccess))
+ /* XXX: more validation should be done here */
+ if (total_len < SizeOfXLogRecord)
+ {
+ report_invalid_record(state,
+ "invalid record length at %X/%X: wanted %u, got %u",
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr,
+ (uint32) SizeOfXLogRecord, total_len);
goto err;
- gotheader = true;
+ }
}
- } while (gotlen < total_len);
-
- Assert(gotheader);
- record = (XLogRecord *) state->readRecordBuf;
- if (!ValidXLogRecord(state, record, RecPtr))
- goto err;
+ /*
+ * Wait for the rest of the record, or the part of the record that
+ * fit on the first page if crossed a page boundary, to become
+ * available.
+ */
+ state->recordGotLen = 0;
+ state->recordRemainLen = total_len;
+ state->readRecordState = XLREAD_FIRST_FRAGMENT;
+ }
+ /* fall through */
- pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
- state->ReadRecPtr = RecPtr;
- state->EndRecPtr = targetPagePtr + pageHeaderSize
- + MAXALIGN(pageHeader->xlp_rem_len);
- }
- else
- {
- /* Wait for the record data to become available */
- while (XLogNeedData(state, targetPagePtr,
- Min(targetRecOff + total_len, XLOG_BLCKSZ), true))
+ case XLREAD_FIRST_FRAGMENT:
{
- if (!state->routine.page_read(state, state->readPagePtr,
- state->readLen,
- state->ReadRecPtr, state->readBuf))
+ uint32 total_len = state->recordRemainLen;
+ uint32 request_len;
+ uint32 record_len;
+ XLogRecPtr targetPagePtr;
+ uint32 targetRecOff;
+
+ /*
+ * Wait for the rest of the record on the first page to become
+ * available
+ */
+ targetPagePtr =
+ state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
+ targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+
+ request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ);
+ record_len = request_len - targetRecOff;
+
+ /* ReadRecPtr contains page header */
+ Assert (targetRecOff != 0);
+ if (XLogNeedData(state, targetPagePtr, request_len, true))
+ return XLREAD_NEED_DATA;
+
+ /* error out if caller supplied bogus page */
+ if (!state->page_verified)
+ goto err;
+
+ prec = (XLogRecord *) (state->readBuf + targetRecOff);
+
+ /* validate record header if not yet */
+ if (!state->record_verified && record_len >= SizeOfXLogRecord)
+ {
+ if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
+ state->PrevRecPtr, prec))
+ goto err;
+
+ state->record_verified = true;
+ }
+
+
+ if (total_len == record_len)
+ {
+ /* Record does not cross a page boundary */
+ Assert(state->record_verified);
+
+ if (!ValidXLogRecord(state, prec, state->ReadRecPtr))
+ goto err;
+
+ state->record_verified = true; /* to be tidy */
+
+ /* We already checked the header earlier */
+ state->EndRecPtr = state->ReadRecPtr + MAXALIGN(record_len);
+
+ *record = prec;
+ state->readRecordState = XLREAD_NEXT_RECORD;
break;
+ }
+
+ /*
+ * The record continues on the next page. Need to reassemble
+ * record
+ */
+ Assert(total_len > record_len);
+
+ /* Enlarge readRecordBuf as needed. */
+ if (total_len > state->readRecordBufSize &&
+ !allocate_recordbuf(state, total_len))
+ {
+ /* We treat this as a "bogus data" condition */
+ report_invalid_record(state,
+ "record length %u at %X/%X too long",
+ total_len,
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr);
+ goto err;
+ }
+
+ /* Copy the first fragment of the record from the first page. */
+ memcpy(state->readRecordBuf, state->readBuf + targetRecOff,
+ record_len);
+ state->recordGotLen += record_len;
+ state->recordRemainLen -= record_len;
+
+ /* Calculate pointer to beginning of next page */
+ state->recordContRecPtr = state->ReadRecPtr + record_len;
+ Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0);
+
+ state->readRecordState = XLREAD_CONTINUATION;
}
+ /* fall through */
+
+ case XLREAD_CONTINUATION:
+ {
+ XLogPageHeader pageHeader;
+ uint32 pageHeaderSize;
+ XLogRecPtr targetPagePtr;
+
+ /* we enter this state only if we haven't read the whole record. */
+ Assert (state->recordRemainLen > 0);
+
+ while(state->recordRemainLen > 0)
+ {
+ char *contdata;
+ uint32 request_len;
+ uint32 record_len;
+
+ /* Wait for the next page to become available */
+ targetPagePtr = state->recordContRecPtr;
+
+ /* this request contains page header */
+ Assert (targetPagePtr != 0);
+ if (XLogNeedData(state, targetPagePtr,
+ Min(state->recordRemainLen, XLOG_BLCKSZ),
+ false))
+ return XLREAD_NEED_DATA;
+
+ if (!state->page_verified)
+ goto err;
+
+ Assert(SizeOfXLogShortPHD <= state->readLen);
- if (!state->page_verified)
- goto err;
+ /* Check that the continuation on next page looks valid */
+ pageHeader = (XLogPageHeader) state->readBuf;
+ if (!(pageHeader->xlp_info & XLP_FIRST_IS_CONTRECORD))
+ {
+ report_invalid_record(
+ state,
+ "there is no contrecord flag at %X/%X reading %X/%X",
+ (uint32) (state->recordContRecPtr >> 32),
+ (uint32) state->recordContRecPtr,
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr);
+ goto err;
+ }
- /* Record does not cross a page boundary */
- if (!ValidXLogRecord(state, record, RecPtr))
- goto err;
+ /*
+ * Cross-check that xlp_rem_len agrees with how much of the
+ * record we expect there to be left.
+ */
+ if (pageHeader->xlp_rem_len == 0 ||
+ pageHeader->xlp_rem_len != state->recordRemainLen)
+ {
+ report_invalid_record(
+ state,
+ "invalid contrecord length %u at %X/%X reading %X/%X, expected %u",
+ pageHeader->xlp_rem_len,
+ (uint32) (state->recordContRecPtr >> 32),
+ (uint32) state->recordContRecPtr,
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr,
+ state->recordRemainLen);
+ goto err;
+ }
- state->EndRecPtr = RecPtr + MAXALIGN(total_len);
+ /* Append the continuation from this page to the buffer */
+ pageHeaderSize = XLogPageHeaderSize(pageHeader);
- state->ReadRecPtr = RecPtr;
+ /*
+ * XLogNeedData should have ensured that the whole page header
+ * was read
+ */
+ Assert(state->readLen >= pageHeaderSize);
+
+ contdata = (char *) state->readBuf + pageHeaderSize;
+ record_len = XLOG_BLCKSZ - pageHeaderSize;
+ if (pageHeader->xlp_rem_len < record_len)
+ record_len = pageHeader->xlp_rem_len;
+
+ request_len = record_len + pageHeaderSize;
+
+ /* XLogNeedData should have ensured all needed data was read */
+ Assert (state->readLen >= request_len);
+
+ memcpy(state->readRecordBuf + state->recordGotLen,
+ (char *) contdata, record_len);
+ state->recordGotLen += record_len;
+ state->recordRemainLen -= record_len;
+
+ /* If we just reassembled the record header, validate it. */
+ if (!state->record_verified)
+ {
+ Assert(state->recordGotLen >= SizeOfXLogRecord);
+ if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
+ state->PrevRecPtr,
+ (XLogRecord *) state->readRecordBuf))
+ goto err;
+
+ state->record_verified = true;
+ }
+
+ /* Calculate pointer to beginning of next page, and continue */
+ state->recordContRecPtr += XLOG_BLCKSZ;
+ }
+
+ /* targetPagePtr is pointing the last-read page here */
+ prec = (XLogRecord *) state->readRecordBuf;
+ if (!ValidXLogRecord(state, prec, state->ReadRecPtr))
+ goto err;
+
+ pageHeaderSize =
+ XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+ state->EndRecPtr = targetPagePtr + pageHeaderSize
+ + MAXALIGN(pageHeader->xlp_rem_len);
+
+ *record = prec;
+ state->readRecordState = XLREAD_NEXT_RECORD;
+ break;
+ }
}
/*
* Special processing if it's an XLOG SWITCH record
*/
- if (record->xl_rmid == RM_XLOG_ID &&
- (record->xl_info & ~XLR_INFO_MASK) == XLOG_SWITCH)
+ if ((*record)->xl_rmid == RM_XLOG_ID &&
+ ((*record)->xl_info & ~XLR_INFO_MASK) == XLOG_SWITCH)
{
/* Pretend it extends to end of segment */
state->EndRecPtr += state->segcxt.ws_segsize - 1;
state->EndRecPtr -= XLogSegmentOffset(state->EndRecPtr, state->segcxt.ws_segsize);
}
- if (DecodeXLogRecord(state, record, errormsg))
- return record;
- else
- return NULL;
+ Assert (!*record || state->readLen >= 0);
+ if (DecodeXLogRecord(state, *record, errormsg))
+ return XLREAD_SUCCESS;
+
+ *record = NULL;
+ return XLREAD_FAIL;
err:
/*
- * Invalidate the read state. We might read from a different source after
+ * Invalidate the read page. We might read from a different source after
* failure.
*/
XLogReaderInvalReadState(state);
@@ -572,7 +706,8 @@ err:
if (state->errormsg_buf[0] != '\0')
*errormsg = state->errormsg_buf;
- return NULL;
+ *record = NULL;
+ return XLREAD_FAIL;
}
/*
@@ -724,11 +859,12 @@ XLogReaderInvalReadState(XLogReaderState *state)
*
* This is just a convenience subroutine to avoid duplicated code in
* XLogReadRecord. It's not intended for use from anywhere else.
+ *
+ * If PrevRecPtr is valid, the xl_prev is is cross-checked with it.
*/
static bool
ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
- XLogRecPtr PrevRecPtr, XLogRecord *record,
- bool randAccess)
+ XLogRecPtr PrevRecPtr, XLogRecord *record)
{
if (record->xl_tot_len < SizeOfXLogRecord)
{
@@ -746,7 +882,7 @@ ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
(uint32) RecPtr);
return false;
}
- if (randAccess)
+ if (PrevRecPtr == InvalidXLogRecPtr)
{
/*
* We can't exactly verify the prev-link, but surely it should be less
@@ -978,11 +1114,14 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
* XLogReadRecord() will read the next valid record.
*/
XLogRecPtr
-XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
+XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
+ XLogFindNextRecordCB read_page, void *private)
{
XLogRecPtr tmpRecPtr;
XLogRecPtr found = InvalidXLogRecPtr;
XLogPageHeader header;
+ XLogRecord *record;
+ XLogReadRecordResult result;
char *errormsg;
Assert(!XLogRecPtrIsInvalid(RecPtr));
@@ -1015,9 +1154,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
while(XLogNeedData(state, targetPagePtr, targetRecOff,
targetRecOff != 0))
{
- if (!state->routine.page_read(state, state->readPagePtr,
- state->readLen,
- state->ReadRecPtr, state->readBuf))
+ if (!read_page(state, private))
break;
}
@@ -1069,8 +1206,16 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
* or we just jumped over the remaining data of a continuation.
*/
XLogBeginRead(state, tmpRecPtr);
- while (XLogReadRecord(state, &errormsg) != NULL)
+ while ((result = XLogReadRecord(state, &record, &errormsg)) !=
+ XLREAD_FAIL)
{
+ if (result == XLREAD_NEED_DATA)
+ {
+ if (!read_page(state, private))
+ goto err;
+ continue;
+ }
+
/* past the record we've found, break out */
if (RecPtr <= state->ReadRecPtr)
{
@@ -1090,9 +1235,9 @@ err:
#endif /* FRONTEND */
/*
- * Helper function to ease writing of XLogRoutine->page_read callbacks.
- * If this function is used, caller must supply a segment_open callback in
- * 'state', as that is used here.
+ * Helper function to ease writing of page_read callback.
+ * If this function is used, caller must supply a segment_open callback and
+ * segment_close callback as that is used here.
*
* Read 'count' bytes into 'buf', starting at location 'startptr', from WAL
* fetched from timeline 'tli'.
@@ -1105,6 +1250,7 @@ err:
*/
bool
WALRead(XLogReaderState *state,
+ WALSegmentOpenCB segopenfn, WALSegmentCloseCB segclosefn,
char *buf, XLogRecPtr startptr, Size count, TimeLineID tli,
WALReadError *errinfo)
{
@@ -1136,10 +1282,10 @@ WALRead(XLogReaderState *state,
XLogSegNo nextSegNo;
if (state->seg.ws_file >= 0)
- state->routine.segment_close(state);
+ segclosefn(state);
XLByteToSeg(recptr, nextSegNo, state->segcxt.ws_segsize);
- state->routine.segment_open(state, nextSegNo, &tli);
+ segopenfn(state, nextSegNo, &tli);
/* This shouldn't happen -- indicates a bug in segment_open */
Assert(state->seg.ws_file >= 0);
diff --git a/src/backend/access/transam/xlogutils.c b/src/backend/access/transam/xlogutils.c
index 1b10b6df20..303333571f 100644
--- a/src/backend/access/transam/xlogutils.c
+++ b/src/backend/access/transam/xlogutils.c
@@ -689,8 +689,7 @@ XLogTruncateRelation(RelFileNode rnode, ForkNumber forkNum,
void
XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wantLength)
{
- const XLogRecPtr lastReadPage = state->seg.ws_segno *
- state->segcxt.ws_segsize + state->readLen;
+ const XLogRecPtr lastReadPage = state->readPagePtr;
Assert(wantPage != InvalidXLogRecPtr && wantPage % XLOG_BLCKSZ == 0);
Assert(wantLength <= XLOG_BLCKSZ);
@@ -705,7 +704,7 @@ XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wa
* current TLI has since become historical.
*/
if (lastReadPage == wantPage &&
- state->readLen != 0 &&
+ state->page_verified &&
lastReadPage + state->readLen >= wantPage + Min(wantLength, XLOG_BLCKSZ - 1))
return;
@@ -792,6 +791,7 @@ wal_segment_open(XLogReaderState *state, XLogSegNo nextSegNo,
char path[MAXPGPATH];
XLogFilePath(path, tli, nextSegNo, state->segcxt.ws_segsize);
+ elog(LOG, "HOGE: %lu, %d => %s", nextSegNo, tli, path);
state->seg.ws_file = BasicOpenFile(path, O_RDONLY | PG_BINARY);
if (state->seg.ws_file >= 0)
return;
@@ -829,9 +829,11 @@ wal_segment_close(XLogReaderState *state)
* loop for now.
*/
bool
-read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *cur_page)
+read_local_xlog_page(XLogReaderState *state)
{
+ XLogRecPtr targetPagePtr = state->readPagePtr;
+ int reqLen = state->readLen;
+ char *cur_page = state->readBuf;
XLogRecPtr read_upto,
loc;
TimeLineID tli;
@@ -944,11 +946,12 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
* as 'count', read the whole page anyway. It's guaranteed to be
* zero-padded up to the page boundary if it's incomplete.
*/
- if (!WALRead(state, cur_page, targetPagePtr, XLOG_BLCKSZ, tli,
- &errinfo))
+ if (!WALRead(state, wal_segment_open, wal_segment_close,
+ cur_page, targetPagePtr, XLOG_BLCKSZ, tli, &errinfo))
WALReadRaiseError(&errinfo);
/* number of valid bytes in the buffer */
+ state->readPagePtr = targetPagePtr;
state->readLen = count;
return true;
}
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index 61902be3b0..a15b0b3355 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -120,7 +120,8 @@ StartupDecodingContext(List *output_plugin_options,
TransactionId xmin_horizon,
bool need_full_snapshot,
bool fast_forward,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress)
@@ -169,11 +170,12 @@ StartupDecodingContext(List *output_plugin_options,
ctx->slot = slot;
- ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, xl_routine, ctx);
+ ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, cleanup_cb);
if (!ctx->reader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
errmsg("out of memory")));
+ ctx->page_read = page_read;
ctx->reorder = ReorderBufferAllocate();
ctx->snapshot_builder =
@@ -231,7 +233,8 @@ CreateInitDecodingContext(char *plugin,
List *output_plugin_options,
bool need_full_snapshot,
XLogRecPtr restart_lsn,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress)
@@ -328,7 +331,7 @@ CreateInitDecodingContext(char *plugin,
ctx = StartupDecodingContext(NIL, restart_lsn, xmin_horizon,
need_full_snapshot, false,
- xl_routine, prepare_write, do_write,
+ page_read, cleanup_cb, prepare_write, do_write,
update_progress);
/* call output plugin initialization callback */
@@ -376,7 +379,8 @@ LogicalDecodingContext *
CreateDecodingContext(XLogRecPtr start_lsn,
List *output_plugin_options,
bool fast_forward,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress)
@@ -429,8 +433,8 @@ CreateDecodingContext(XLogRecPtr start_lsn,
ctx = StartupDecodingContext(output_plugin_options,
start_lsn, InvalidTransactionId, false,
- fast_forward, xl_routine, prepare_write,
- do_write, update_progress);
+ fast_forward, page_read, cleanup_cb,
+ prepare_write, do_write, update_progress);
/* call output plugin initialization callback */
old_context = MemoryContextSwitchTo(ctx->context);
@@ -483,7 +487,13 @@ DecodingContextFindStartpoint(LogicalDecodingContext *ctx)
char *err = NULL;
/* the read_page callback waits for new WAL */
- record = XLogReadRecord(ctx->reader, &err);
+ while (XLogReadRecord(ctx->reader, &record, &err) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!ctx->page_read(ctx->reader))
+ break;
+ }
+
if (err)
elog(ERROR, "%s", err);
if (!record)
diff --git a/src/backend/replication/logical/logicalfuncs.c b/src/backend/replication/logical/logicalfuncs.c
index b99c94e848..fcc81b358e 100644
--- a/src/backend/replication/logical/logicalfuncs.c
+++ b/src/backend/replication/logical/logicalfuncs.c
@@ -233,9 +233,8 @@ pg_logical_slot_get_changes_guts(FunctionCallInfo fcinfo, bool confirm, bool bin
ctx = CreateDecodingContext(InvalidXLogRecPtr,
options,
false,
- XL_ROUTINE(.page_read = read_local_xlog_page,
- .segment_open = wal_segment_open,
- .segment_close = wal_segment_close),
+ read_local_xlog_page,
+ wal_segment_close,
LogicalOutputPrepareWrite,
LogicalOutputWrite, NULL);
@@ -284,7 +283,13 @@ pg_logical_slot_get_changes_guts(FunctionCallInfo fcinfo, bool confirm, bool bin
XLogRecord *record;
char *errm = NULL;
- record = XLogReadRecord(ctx->reader, &errm);
+ while (XLogReadRecord(ctx->reader, &record, &errm) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!ctx->page_read(ctx->reader))
+ break;
+ }
+
if (errm)
elog(ERROR, "%s", errm);
diff --git a/src/backend/replication/slotfuncs.c b/src/backend/replication/slotfuncs.c
index 88033a79b2..8a678074d5 100644
--- a/src/backend/replication/slotfuncs.c
+++ b/src/backend/replication/slotfuncs.c
@@ -152,9 +152,8 @@ create_logical_replication_slot(char *name, char *plugin,
ctx = CreateInitDecodingContext(plugin, NIL,
false, /* just catalogs is OK */
restart_lsn,
- XL_ROUTINE(.page_read = read_local_xlog_page,
- .segment_open = wal_segment_open,
- .segment_close = wal_segment_close),
+ read_local_xlog_page,
+ wal_segment_close,
NULL, NULL, NULL);
/*
@@ -486,9 +485,8 @@ pg_logical_replication_slot_advance(XLogRecPtr moveto)
ctx = CreateDecodingContext(InvalidXLogRecPtr,
NIL,
true, /* fast_forward */
- XL_ROUTINE(.page_read = read_local_xlog_page,
- .segment_open = wal_segment_open,
- .segment_close = wal_segment_close),
+ read_local_xlog_page,
+ wal_segment_close,
NULL, NULL, NULL);
/*
@@ -513,7 +511,13 @@ pg_logical_replication_slot_advance(XLogRecPtr moveto)
* Read records. No changes are generated in fast_forward mode,
* but snapbuilder/slot statuses are updated properly.
*/
- record = XLogReadRecord(ctx->reader, &errm);
+ while (XLogReadRecord(ctx->reader, &record, &errm) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!ctx->page_read(ctx->reader))
+ break;
+ }
+
if (errm)
elog(ERROR, "%s", errm);
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index bf2711bddd..2fd24a9b14 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -580,10 +580,7 @@ StartReplication(StartReplicationCmd *cmd)
/* create xlogreader for physical replication */
xlogreader =
- XLogReaderAllocate(wal_segment_size, NULL,
- XL_ROUTINE(.segment_open = WalSndSegmentOpen,
- .segment_close = wal_segment_close),
- NULL);
+ XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
if (!xlogreader)
ereport(ERROR,
@@ -812,9 +809,11 @@ StartReplication(StartReplicationCmd *cmd)
* set every time WAL is flushed.
*/
static bool
-logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetRecPtr, char *cur_page)
+logical_read_xlog_page(XLogReaderState *state)
{
+ XLogRecPtr targetPagePtr = state->readPagePtr;
+ int reqLen = state->readLen;
+ char *cur_page = state->readBuf;
XLogRecPtr flushptr;
int count;
WALReadError errinfo;
@@ -842,7 +841,7 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
count = flushptr - targetPagePtr; /* part of the page available */
/* now actually read the data, we know it's there */
- if (!WALRead(state,
+ if (!WALRead(state, WalSndSegmentOpen, wal_segment_close,
cur_page,
targetPagePtr,
XLOG_BLCKSZ,
@@ -1014,9 +1013,8 @@ CreateReplicationSlot(CreateReplicationSlotCmd *cmd)
ctx = CreateInitDecodingContext(cmd->plugin, NIL, need_full_snapshot,
InvalidXLogRecPtr,
- XL_ROUTINE(.page_read = logical_read_xlog_page,
- .segment_open = WalSndSegmentOpen,
- .segment_close = wal_segment_close),
+ logical_read_xlog_page,
+ wal_segment_close,
WalSndPrepareWrite, WalSndWriteData,
WalSndUpdateProgress);
@@ -1179,9 +1177,8 @@ StartLogicalReplication(StartReplicationCmd *cmd)
*/
logical_decoding_ctx =
CreateDecodingContext(cmd->startpoint, cmd->options, false,
- XL_ROUTINE(.page_read = logical_read_xlog_page,
- .segment_open = WalSndSegmentOpen,
- .segment_close = wal_segment_close),
+ logical_read_xlog_page,
+ wal_segment_close,
WalSndPrepareWrite, WalSndWriteData,
WalSndUpdateProgress);
xlogreader = logical_decoding_ctx->reader;
@@ -2761,7 +2758,7 @@ XLogSendPhysical(void)
enlargeStringInfo(&output_message, nbytes);
retry:
- if (!WALRead(xlogreader,
+ if (!WALRead(xlogreader, WalSndSegmentOpen, wal_segment_close,
&output_message.data[output_message.len],
startptr,
nbytes,
@@ -2859,7 +2856,12 @@ XLogSendLogical(void)
*/
WalSndCaughtUp = false;
- record = XLogReadRecord(logical_decoding_ctx->reader, &errm);
+ while (XLogReadRecord(logical_decoding_ctx->reader, &record, &errm) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!logical_decoding_ctx->page_read(logical_decoding_ctx->reader))
+ break;
+ }
/* xlog record was invalid */
if (errm != NULL)
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index 0b7e73ae79..3dd6df4be6 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -40,15 +40,9 @@ static int xlogreadfd = -1;
static XLogSegNo xlogreadsegno = -1;
static char xlogfpath[MAXPGPATH];
-typedef struct XLogPageReadPrivate
-{
- const char *restoreCommand;
- int tliIndex;
-} XLogPageReadPrivate;
-
-static bool SimpleXLogPageRead(XLogReaderState *xlogreader,
- XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
+static bool SimpleXLogPageRead(XLogReaderState *xlogreader,
+ const char *datadir, int *tliIndex,
+ const char *restoreCommand);
/*
* Read WAL from the datadir/pg_wal, starting from 'startpoint' on timeline
@@ -62,20 +56,22 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
XLogRecord *record;
XLogReaderState *xlogreader;
char *errormsg;
- XLogPageReadPrivate private;
- private.tliIndex = tliIndex;
- private.restoreCommand = restoreCommand;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir,
- XL_ROUTINE(.page_read = &SimpleXLogPageRead),
- &private);
+ xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+
if (xlogreader == NULL)
pg_fatal("out of memory");
XLogBeginRead(xlogreader, startpoint);
do
{
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!SimpleXLogPageRead(xlogreader, datadir,
+ &tliIndex, restoreCommand))
+ break;
+ }
if (record == NULL)
{
@@ -113,19 +109,19 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex,
XLogRecord *record;
XLogReaderState *xlogreader;
char *errormsg;
- XLogPageReadPrivate private;
XLogRecPtr endptr;
- private.tliIndex = tliIndex;
- private.restoreCommand = restoreCommand;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir,
- XL_ROUTINE(.page_read = &SimpleXLogPageRead),
- &private);
+ xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
XLogBeginRead(xlogreader, ptr);
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!SimpleXLogPageRead(xlogreader, datadir, &tliIndex, restoreCommand))
+ break;
+ }
if (record == NULL)
{
if (errormsg)
@@ -160,7 +156,6 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
XLogRecPtr searchptr;
XLogReaderState *xlogreader;
char *errormsg;
- XLogPageReadPrivate private;
/*
* The given fork pointer points to the end of the last common record,
@@ -176,11 +171,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
forkptr += SizeOfXLogShortPHD;
}
- private.tliIndex = tliIndex;
- private.restoreCommand = restoreCommand;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir,
- XL_ROUTINE(.page_read = &SimpleXLogPageRead),
- &private);
+ xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
@@ -190,7 +181,13 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
uint8 info;
XLogBeginRead(xlogreader, searchptr);
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!SimpleXLogPageRead(xlogreader, datadir,
+ &tliIndex, restoreCommand))
+ break;
+ }
if (record == NULL)
{
@@ -237,10 +234,11 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
/* XLogReader callback function, to read a WAL page */
static bool
-SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *readBuf)
+SimpleXLogPageRead(XLogReaderState *xlogreader, const char *datadir,
+ int *tliIndex, const char *restoreCommand)
{
- XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
+ XLogRecPtr targetPagePtr = xlogreader->readPagePtr;
+ char *readBuf = xlogreader->readBuf;
uint32 targetPageOff;
XLogRecPtr targetSegEnd;
XLogSegNo targetSegNo;
@@ -273,14 +271,14 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
* be done both forward and backward, consider also switching timeline
* accordingly.
*/
- while (private->tliIndex < targetNentries - 1 &&
- targetHistory[private->tliIndex].end < targetSegEnd)
- private->tliIndex++;
- while (private->tliIndex > 0 &&
- targetHistory[private->tliIndex].begin >= targetSegEnd)
- private->tliIndex--;
+ while (*tliIndex < targetNentries - 1 &&
+ targetHistory[*tliIndex].end < targetSegEnd)
+ (*tliIndex)++;
+ while (*tliIndex > 0 &&
+ targetHistory[*tliIndex].begin >= targetSegEnd)
+ (*tliIndex)--;
- XLogFileName(xlogfname, targetHistory[private->tliIndex].tli,
+ XLogFileName(xlogfname, targetHistory[*tliIndex].tli,
xlogreadsegno, WalSegSz);
snprintf(xlogfpath, MAXPGPATH, "%s/" XLOGDIR "/%s",
@@ -293,7 +291,7 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
/*
* If we have no restore_command to execute, then exit.
*/
- if (private->restoreCommand == NULL)
+ if (restoreCommand == NULL)
{
pg_log_error("could not open file \"%s\": %m", xlogfpath);
xlogreader->readLen = -1;
@@ -307,7 +305,7 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
xlogreadfd = RestoreArchivedFile(xlogreader->segcxt.ws_dir,
xlogfname,
WalSegSz,
- private->restoreCommand);
+ restoreCommand);
if (xlogreadfd < 0)
{
@@ -349,7 +347,7 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
Assert(targetSegNo == xlogreadsegno);
- xlogreader->seg.ws_tli = targetHistory[private->tliIndex].tli;
+ xlogreader->seg.ws_tli = targetHistory[*tliIndex].tli;
xlogreader->readLen = XLOG_BLCKSZ;
return true;
}
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index ab0c28d9b7..02e71138cc 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -330,12 +330,17 @@ WALDumpCloseSegment(XLogReaderState *state)
state->seg.ws_file = -1;
}
-/* pg_waldump's XLogReaderRoutine->page_read callback */
+/*
+ * pg_waldump's WAL page rader, also used as page_read callback for
+ * XLogFindNextRecord
+ */
static bool
-WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetPtr, char *readBuff)
+WALDumpReadPage(XLogReaderState *state, void *priv)
{
- XLogDumpPrivate *private = state->private_data;
+ XLogRecPtr targetPagePtr = state->readPagePtr;
+ int reqLen = state->readLen;
+ char *readBuff = state->readBuf;
+ XLogDumpPrivate *private = (XLogDumpPrivate *) priv;
int count = XLOG_BLCKSZ;
WALReadError errinfo;
@@ -353,8 +358,8 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
}
}
- if (!WALRead(state, readBuff, targetPagePtr, count, private->timeline,
- &errinfo))
+ if (!WALRead(state, WALDumpOpenSegment, WALDumpCloseSegment,
+ readBuff, targetPagePtr, count, private->timeline, &errinfo))
{
WALOpenSegment *seg = &errinfo.wre_seg;
char fname[MAXPGPATH];
@@ -374,6 +379,7 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
(Size) errinfo.wre_req);
}
+ Assert(count >= state->readLen);
state->readLen = count;
return true;
}
@@ -1042,16 +1048,14 @@ main(int argc, char **argv)
/* we have everything we need, start reading */
xlogreader_state =
- XLogReaderAllocate(WalSegSz, waldir,
- XL_ROUTINE(.page_read = WALDumpReadPage,
- .segment_open = WALDumpOpenSegment,
- .segment_close = WALDumpCloseSegment),
- &private);
+ XLogReaderAllocate(WalSegSz, waldir, WALDumpCloseSegment);
+
if (!xlogreader_state)
fatal_error("out of memory");
/* first find a valid recptr to start from */
- first_record = XLogFindNextRecord(xlogreader_state, private.startptr);
+ first_record = XLogFindNextRecord(xlogreader_state, private.startptr,
+ &WALDumpReadPage, (void*) &private);
if (first_record == InvalidXLogRecPtr)
fatal_error("could not find a valid record after %X/%X",
@@ -1075,7 +1079,13 @@ main(int argc, char **argv)
for (;;)
{
/* try to read the next record */
- record = XLogReadRecord(xlogreader_state, &errormsg);
+ while (XLogReadRecord(xlogreader_state, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!WALDumpReadPage(xlogreader_state, (void *) &private))
+ break;
+ }
+
if (!record)
{
if (!config.follow || private.endptr_reached)
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index 6c8848e14f..6a7fe140cb 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -57,64 +57,15 @@ typedef struct WALSegmentContext
typedef struct XLogReaderState XLogReaderState;
-/* Function type definition for the read_page callback */
-typedef bool (*XLogPageReadCB) (XLogReaderState *xlogreader,
- XLogRecPtr targetPagePtr,
- int reqLen,
- XLogRecPtr targetRecPtr,
- char *readBuf);
+/* Function type definition for the segment cleanup callback */
+typedef void (*WALSegmentCleanupCB) (XLogReaderState *xlogreader);
+
+/* Function type definition for the open/close callbacks for WALRead() */
typedef void (*WALSegmentOpenCB) (XLogReaderState *xlogreader,
XLogSegNo nextSegNo,
TimeLineID *tli_p);
typedef void (*WALSegmentCloseCB) (XLogReaderState *xlogreader);
-typedef struct XLogReaderRoutine
-{
- /*
- * Data input callback
- *
- * This callback shall read at least reqLen valid bytes of the xlog page
- * starting at targetPagePtr, and store them in readBuf. The callback
- * shall return the number of bytes read (never more than XLOG_BLCKSZ), or
- * -1 on failure. The callback shall sleep, if necessary, to wait for the
- * requested bytes to become available. The callback will not be invoked
- * again for the same page unless more than the returned number of bytes
- * are needed.
- *
- * targetRecPtr is the position of the WAL record we're reading. Usually
- * it is equal to targetPagePtr + reqLen, but sometimes xlogreader needs
- * to read and verify the page or segment header, before it reads the
- * actual WAL record it's interested in. In that case, targetRecPtr can
- * be used to determine which timeline to read the page from.
- *
- * The callback shall set ->seg.ws_tli to the TLI of the file the page was
- * read from.
- */
- XLogPageReadCB page_read;
-
- /*
- * Callback to open the specified WAL segment for reading. ->seg.ws_file
- * shall be set to the file descriptor of the opened segment. In case of
- * failure, an error shall be raised by the callback and it shall not
- * return.
- *
- * "nextSegNo" is the number of the segment to be opened.
- *
- * "tli_p" is an input/output argument. WALRead() uses it to pass the
- * timeline in which the new segment should be found, but the callback can
- * use it to return the TLI that it actually opened.
- */
- WALSegmentOpenCB segment_open;
-
- /*
- * WAL segment close callback. ->seg.ws_file shall be set to a negative
- * number.
- */
- WALSegmentCloseCB segment_close;
-} XLogReaderRoutine;
-
-#define XL_ROUTINE(...) &(XLogReaderRoutine){__VA_ARGS__}
-
typedef struct
{
/* Is this block ref in use? */
@@ -144,12 +95,35 @@ typedef struct
uint16 data_bufsz;
} DecodedBkpBlock;
+/* Return code from XLogReadRecord */
+typedef enum XLogReadRecordResult
+{
+ XLREAD_SUCCESS, /* record is successfully read */
+ XLREAD_NEED_DATA, /* need more data. see XLogReadRecord. */
+ XLREAD_FAIL /* failed during reading a record */
+} XLogReadRecordResult;
+
+/*
+ * internal state of XLogReadRecord
+ *
+ * XLogReadState runs a state machine while reading a record. Theses states
+ * are not seen outside the function. Each state may repeat several times
+ * exiting requesting caller for new data. See the comment of XLogReadRecrod
+ * for details.
+ */
+typedef enum XLogReadRecordState {
+ XLREAD_NEXT_RECORD,
+ XLREAD_TOT_LEN,
+ XLREAD_FIRST_FRAGMENT,
+ XLREAD_CONTINUATION
+} XLogReadRecordState;
+
struct XLogReaderState
{
/*
* Operational callbacks
*/
- XLogReaderRoutine routine;
+ WALSegmentCleanupCB cleanup_cb;
/* ----------------------------------------
* Public parameters
@@ -162,18 +136,14 @@ struct XLogReaderState
*/
uint64 system_identifier;
- /*
- * Opaque data for callbacks to use. Not used by XLogReader.
- */
- void *private_data;
-
/*
* Start and end point of last record read. EndRecPtr is also used as the
* position to read next. Calling XLogBeginRead() sets EndRecPtr to the
* starting position and ReadRecPtr to invalid.
*/
- XLogRecPtr ReadRecPtr; /* start of last record read */
+ XLogRecPtr ReadRecPtr; /* start of last record read or being read */
XLogRecPtr EndRecPtr; /* end+1 of last record read */
+ XLogRecPtr PrevRecPtr; /* start of previous record read */
/* ----------------------------------------
* Communication with page reader
@@ -187,7 +157,9 @@ struct XLogReaderState
* the request, or -1 on error */
TimeLineID readPageTLI; /* TLI for data currently in readBuf */
char *readBuf; /* buffer to store data */
- bool page_verified; /* is the page on the buffer verified? */
+ bool page_verified; /* is the page header on the buffer verified? */
+ bool record_verified;/* is the current record header verified? */
+
/* ----------------------------------------
@@ -227,8 +199,6 @@ struct XLogReaderState
XLogRecPtr latestPagePtr;
TimeLineID latestPageTLI;
- /* beginning of the WAL record being read. */
- XLogRecPtr currRecPtr;
/* timeline to read it from, 0 if a lookup is required */
TimeLineID currTLI;
@@ -255,6 +225,15 @@ struct XLogReaderState
char *readRecordBuf;
uint32 readRecordBufSize;
+ /*
+ * XLogReadRecord() state
+ */
+ XLogReadRecordState readRecordState;/* state machine state */
+ int recordGotLen; /* amount of current record that has
+ * already been read */
+ int recordRemainLen; /* length of current record that remains */
+ XLogRecPtr recordContRecPtr; /* where the current record continues */
+
/* Buffer to hold error message */
char *errormsg_buf;
};
@@ -262,9 +241,7 @@ struct XLogReaderState
/* Get a new XLogReader */
extern XLogReaderState *XLogReaderAllocate(int wal_segment_size,
const char *waldir,
- XLogReaderRoutine *routine,
- void *private_data);
-extern XLogReaderRoutine *LocalXLogReaderRoutine(void);
+ WALSegmentCleanupCB cleanup_cb);
/* Free an XLogReader */
extern void XLogReaderFree(XLogReaderState *state);
@@ -272,12 +249,17 @@ extern void XLogReaderFree(XLogReaderState *state);
/* Position the XLogReader to given record */
extern void XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr);
#ifdef FRONTEND
-extern XLogRecPtr XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr);
+/* Function type definition for the read_page callback */
+typedef bool (*XLogFindNextRecordCB) (XLogReaderState *xlogreader,
+ void *private);
+extern XLogRecPtr XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
+ XLogFindNextRecordCB read_page, void *private);
#endif /* FRONTEND */
/* Read the next XLog record. Returns NULL on end-of-WAL or failure */
-extern struct XLogRecord *XLogReadRecord(XLogReaderState *state,
- char **errormsg);
+extern XLogReadRecordResult XLogReadRecord(XLogReaderState *state,
+ XLogRecord **record,
+ char **errormsg);
/* Validate a page */
extern bool XLogReaderValidatePageHeader(XLogReaderState *state,
@@ -297,6 +279,7 @@ typedef struct WALReadError
} WALReadError;
extern bool WALRead(XLogReaderState *state,
+ WALSegmentOpenCB segopenfn, WALSegmentCloseCB sgclosefn,
char *buf, XLogRecPtr startptr, Size count,
TimeLineID tli, WALReadError *errinfo);
diff --git a/src/include/access/xlogutils.h b/src/include/access/xlogutils.h
index c287edf206..482e4ced69 100644
--- a/src/include/access/xlogutils.h
+++ b/src/include/access/xlogutils.h
@@ -47,9 +47,7 @@ extern Buffer XLogReadBufferExtended(RelFileNode rnode, ForkNumber forknum,
extern Relation CreateFakeRelcacheEntry(RelFileNode rnode);
extern void FreeFakeRelcacheEntry(Relation fakerel);
-extern bool read_local_xlog_page(XLogReaderState *state,
- XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetRecPtr, char *cur_page);
+extern bool read_local_xlog_page(XLogReaderState *state);
extern void wal_segment_open(XLogReaderState *state,
XLogSegNo nextSegNo,
TimeLineID *tli_p);
diff --git a/src/include/replication/logical.h b/src/include/replication/logical.h
index c2f2475e5d..a743db49cf 100644
--- a/src/include/replication/logical.h
+++ b/src/include/replication/logical.h
@@ -29,6 +29,10 @@ typedef void (*LogicalOutputPluginWriterUpdateProgress) (struct LogicalDecodingC
TransactionId xid
);
+typedef struct LogicalDecodingContext LogicalDecodingContext;
+
+typedef bool (*LogicalDecodingXLogPageReadCB)(XLogReaderState *ctx);
+
typedef struct LogicalDecodingContext
{
/* memory context this is all allocated in */
@@ -39,6 +43,7 @@ typedef struct LogicalDecodingContext
/* infrastructure pieces for decoding */
XLogReaderState *reader;
+ LogicalDecodingXLogPageReadCB page_read;
struct ReorderBuffer *reorder;
struct SnapBuild *snapshot_builder;
@@ -95,14 +100,16 @@ extern LogicalDecodingContext *CreateInitDecodingContext(char *plugin,
List *output_plugin_options,
bool need_full_snapshot,
XLogRecPtr restart_lsn,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress);
extern LogicalDecodingContext *CreateDecodingContext(XLogRecPtr start_lsn,
List *output_plugin_options,
bool fast_forward,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress);
--
2.18.4
----Next_Part(Thu_Jul__2_13_53_30_2020_072)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
filename="v15-0003-Remove-globals-readOff-readLen-and-readSegNo.patch"
^ permalink raw reply [nested|flat] 70+ messages in thread
* [PATCH v10 2/4] Move page-reader out of XLogReadRecord
@ 2019-09-10 03:58 Kyotaro Horiguchi <[email protected]>
0 siblings, 0 replies; 70+ messages in thread
From: Kyotaro Horiguchi @ 2019-09-10 03:58 UTC (permalink / raw)
This is the second step of removing callbacks from WAL record reader.
Since it is essential to take in additional data while reading a
record, the function have to ask caller for new data while keeping
working state. Thus the function is turned into a state machine.
---
src/backend/access/transam/twophase.c | 15 +-
src/backend/access/transam/xlog.c | 58 +-
src/backend/access/transam/xlogreader.c | 700 +++++++++++-------
src/backend/access/transam/xlogutils.c | 17 +-
src/backend/replication/logical/logical.c | 26 +-
.../replication/logical/logicalfuncs.c | 13 +-
src/backend/replication/slotfuncs.c | 18 +-
src/backend/replication/walsender.c | 47 +-
src/bin/pg_rewind/parsexlog.c | 84 +--
src/bin/pg_waldump/pg_waldump.c | 36 +-
src/include/access/xlogreader.h | 121 ++-
src/include/access/xlogutils.h | 4 +-
src/include/replication/logical.h | 11 +-
13 files changed, 656 insertions(+), 494 deletions(-)
diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index e1904877fa..8630ad2032 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -1330,11 +1330,8 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
XLogReaderState *xlogreader;
char *errormsg;
- xlogreader = XLogReaderAllocate(wal_segment_size, NULL,
- XL_ROUTINE(.page_read = &read_local_xlog_page,
- .segment_open = &wal_segment_open,
- .segment_close = &wal_segment_close),
- NULL);
+ xlogreader = XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+
if (!xlogreader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
@@ -1342,7 +1339,13 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
errdetail("Failed while allocating a WAL reading processor.")));
XLogBeginRead(xlogreader, lsn);
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!read_local_xlog_page(xlogreader))
+ break;
+ }
+
if (record == NULL)
ereport(ERROR,
(errcode_for_file_access(),
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index da468598e4..840218efee 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -828,13 +828,6 @@ static XLogSource currentSource = XLOG_FROM_ANY;
static bool lastSourceFailed = false;
static bool pendingWalRcvRestart = false;
-typedef struct XLogPageReadPrivate
-{
- int emode;
- bool fetching_ckpt; /* are we fetching a checkpoint record? */
- bool randAccess;
-} XLogPageReadPrivate;
-
/*
* These variables track when we last obtained some WAL data to process,
* and where we got it from. (XLogReceiptSource is initially the same as
@@ -909,8 +902,8 @@ static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath,
static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli,
XLogSource source, bool notfoundOk);
static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, XLogSource source);
-static bool XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
+static bool XLogPageRead(XLogReaderState *xlogreader,
+ bool fetching_ckpt, int emode, bool randAccess);
static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
bool fetching_ckpt, XLogRecPtr tliRecPtr);
static int emode_for_corrupt_record(int emode, XLogRecPtr RecPtr);
@@ -1222,8 +1215,7 @@ XLogInsertRecord(XLogRecData *rdata,
appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len);
if (!debug_reader)
- debug_reader = XLogReaderAllocate(wal_segment_size, NULL,
- XL_ROUTINE(), NULL);
+ debug_reader = XLogReaderAllocate(wal_segment_size, NULL NULL);
if (!debug_reader)
{
@@ -4323,15 +4315,10 @@ CleanupBackupHistory(void)
* record is available.
*/
static XLogRecord *
-ReadRecord(XLogReaderState *xlogreader, int emode,
- bool fetching_ckpt)
+ReadRecord(XLogReaderState *xlogreader, int emode, bool fetching_ckpt)
{
XLogRecord *record;
- XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
-
- private->fetching_ckpt = fetching_ckpt;
- private->emode = emode;
- private->randAccess = (xlogreader->ReadRecPtr == InvalidXLogRecPtr);
+ bool randAccess = (xlogreader->ReadRecPtr == InvalidXLogRecPtr);
/* This is the first attempt to read this page. */
lastSourceFailed = false;
@@ -4339,8 +4326,16 @@ ReadRecord(XLogReaderState *xlogreader, int emode,
for (;;)
{
char *errormsg;
+ XLogReadRecordResult result;
+
+ while ((result = XLogReadRecord(xlogreader, &record, &errormsg))
+ == XLREAD_NEED_DATA)
+ {
+ if (!XLogPageRead(xlogreader, fetching_ckpt, emode, randAccess))
+ break;
+
+ }
- record = XLogReadRecord(xlogreader, &errormsg);
ReadRecPtr = xlogreader->ReadRecPtr;
EndRecPtr = xlogreader->EndRecPtr;
if (record == NULL)
@@ -6317,7 +6312,6 @@ StartupXLOG(void)
bool backupFromStandby = false;
DBState dbstate_at_startup;
XLogReaderState *xlogreader;
- XLogPageReadPrivate private;
bool fast_promoted = false;
struct stat st;
@@ -6477,13 +6471,9 @@ StartupXLOG(void)
OwnLatch(&XLogCtl->recoveryWakeupLatch);
/* Set up XLOG reader facility */
- MemSet(&private, 0, sizeof(XLogPageReadPrivate));
xlogreader =
- XLogReaderAllocate(wal_segment_size, NULL,
- XL_ROUTINE(.page_read = &XLogPageRead,
- .segment_open = NULL,
- .segment_close = wal_segment_close),
- &private);
+ XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+
if (!xlogreader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
@@ -11854,12 +11844,13 @@ CancelBackup(void)
* sleep and retry.
*/
static bool
-XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetRecPtr, char *readBuf)
+XLogPageRead(XLogReaderState *xlogreader,
+ bool fetching_ckpt, int emode, bool randAccess)
{
- XLogPageReadPrivate *private =
- (XLogPageReadPrivate *) xlogreader->private_data;
- int emode = private->emode;
+ char *readBuf = xlogreader->readBuf;
+ XLogRecPtr targetPagePtr = xlogreader->readPagePtr;
+ int reqLen = xlogreader->readLen;
+ XLogRecPtr targetRecPtr = xlogreader->ReadRecPtr;
uint32 targetPageOff;
XLogSegNo targetSegNo PG_USED_FOR_ASSERTS_ONLY;
int r;
@@ -11902,8 +11893,8 @@ retry:
flushedUpto < targetPagePtr + reqLen))
{
if (!WaitForWALToBecomeAvailable(targetPagePtr + reqLen,
- private->randAccess,
- private->fetching_ckpt,
+ randAccess,
+ fetching_ckpt,
targetRecPtr))
{
if (readFile >= 0)
@@ -12008,6 +11999,7 @@ retry:
goto next_record_is_invalid;
}
+ Assert(xlogreader->readPagePtr == targetPagePtr);
xlogreader->readLen = readLen;
return true;
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index 399bad0603..bdde75086c 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -40,7 +40,7 @@ static bool XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr,
int reqLen, bool header_inclusive);
static void XLogReaderInvalReadState(XLogReaderState *state);
static bool ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
- XLogRecPtr PrevRecPtr, XLogRecord *record, bool randAccess);
+ XLogRecPtr PrevRecPtr, XLogRecord *record);
static bool ValidXLogRecord(XLogReaderState *state, XLogRecord *record,
XLogRecPtr recptr);
static void ResetDecoder(XLogReaderState *state);
@@ -71,7 +71,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...)
*/
XLogReaderState *
XLogReaderAllocate(int wal_segment_size, const char *waldir,
- XLogReaderRoutine *routine, void *private_data)
+ WALSegmentCleanupCB cleanup_cb)
{
XLogReaderState *state;
@@ -82,7 +82,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
return NULL;
/* initialize caller-provided support functions */
- state->routine = *routine;
+ state->cleanup_cb = cleanup_cb;
state->max_block_id = -1;
@@ -105,8 +105,6 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
WALOpenSegmentInit(&state->seg, &state->segcxt, wal_segment_size,
waldir);
- /* system_identifier initialized to zeroes above */
- state->private_data = private_data;
/* ReadRecPtr, EndRecPtr and readLen initialized to zeroes above */
state->errormsg_buf = palloc_extended(MAX_ERRORMSG_LEN + 1,
MCXT_ALLOC_NO_OOM);
@@ -138,8 +136,8 @@ XLogReaderFree(XLogReaderState *state)
{
int block_id;
- if (state->seg.ws_file != -1)
- state->routine.segment_close(state);
+ if (state->seg.ws_file >= 0)
+ state->cleanup_cb(state);
for (block_id = 0; block_id <= XLR_MAX_BLOCK_ID; block_id++)
{
@@ -244,6 +242,7 @@ XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
/* Begin at the passed-in record pointer. */
state->EndRecPtr = RecPtr;
state->ReadRecPtr = InvalidXLogRecPtr;
+ state->readRecordState = XLREAD_NEXT_RECORD;
}
/*
@@ -252,317 +251,452 @@ XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
* XLogBeginRead() or XLogFindNextRecord() must be called before the first call
* to XLogReadRecord().
*
- * If the page_read callback fails to read the requested data, NULL is
- * returned. The callback is expected to have reported the error; errormsg
- * is set to NULL.
+ * This function may return XLREAD_NEED_DATA several times before returning a
+ * result record. The caller shall read in some new data then call this
+ * function again with the same parameters.
*
- * If the reading fails for some other reason, NULL is also returned, and
- * *errormsg is set to a string with details of the failure.
+ * When a record is successfully read, returns XLREAD_SUCCESS with result
+ * record being stored in *record. Otherwise *record is NULL.
*
- * The returned pointer (or *errormsg) points to an internal buffer that's
- * valid until the next call to XLogReadRecord.
+ * Returns XLREAD_NEED_DATA if more data is needed to finish reading the
+ * current record. In that case, state->readPagePtr and state->readLen inform
+ * the desired position and minimum length of data needed. The caller shall
+ * read in the requested data and set state->readBuf to point to a buffer
+ * containing it. The caller must also set state->readPageTLI and
+ * state->readLen to indicate the timeline that it was read from, and the
+ * length of data that is now available (which must be >= given readLen),
+ * respectively.
+ *
+ * If invalid data is encountered, returns XLREAD_FAIL with *record being set to
+ * NULL. *errormsg is set to a string with details of the failure.
+ * The returned pointer (or *errormsg) points to an internal buffer that's valid
+ * until the next call to XLogReadRecord.
+ *
+ *
+ * This function runs a state machine consists of the following states.
+ *
+ * XLREAD_NEXT_RECORD :
+ * The initial state, if called with valid RecPtr, try to read a record at
+ * that position. If invalid RecPtr is given try to read a record just after
+ * the last one previously read.
+ * This state ens after setting ReadRecPtr. Then goes to XLREAD_TOT_LEN.
+ *
+ * XLREAD_TOT_LEN:
+ * Examining record header. Ends after reading record total
+ * length. recordRemainLen and recordGotLen are initialized.
+ *
+ * XLREAD_FIRST_FRAGMENT:
+ * Reading the first fragment. Ends with finishing reading a single
+ * record. Goes to XLREAD_NEXT_RECORD if that's all or
+ * XLREAD_CONTINUATION if we have continuation.
+
+ * XLREAD_CONTINUATION:
+ * Reading continuation of record. Ends with finishing the whole record then
+ * goes to XLREAD_NEXT_RECORD. During this state, recordRemainLen indicates
+ * how much is left and readRecordBuf holds the partially assert
+ * record.recordContRecPtr points to the beginning of the next page where to
+ * continue.
+ *
+ * If wrong data found in any state, the state machine stays at the current
+ * state. This behavior allows to continue reading a reacord switching among
+ * different souces, while streaming replication.
*/
-XLogRecord *
-XLogReadRecord(XLogReaderState *state, char **errormsg)
+XLogReadRecordResult
+XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
{
- XLogRecPtr RecPtr;
- XLogRecord *record;
- XLogRecPtr targetPagePtr;
- bool randAccess;
- uint32 len,
- total_len;
- uint32 targetRecOff;
- uint32 pageHeaderSize;
- bool gotheader;
+ XLogRecord *prec;
- /*
- * randAccess indicates whether to verify the previous-record pointer of
- * the record we're reading. We only do this if we're reading
- * sequentially, which is what we initially assume.
- */
- randAccess = false;
+ *record = NULL;
/* reset error state */
*errormsg = NULL;
state->errormsg_buf[0] = '\0';
- ResetDecoder(state);
-
- RecPtr = state->EndRecPtr;
-
- if (state->ReadRecPtr != InvalidXLogRecPtr)
- {
- /* read the record after the one we just read */
-
- /*
- * EndRecPtr is pointing to end+1 of the previous WAL record. If
- * we're at a page boundary, no more records can fit on the current
- * page. We must skip over the page header, but we can't do that until
- * we've read in the page, since the header size is variable.
- */
- }
- else
- {
- /*
- * Caller supplied a position to start at.
- *
- * In this case, EndRecPtr should already be pointing to a valid
- * record starting position.
- */
- Assert(XRecOffIsValid(RecPtr));
- randAccess = true;
- }
-
- state->currRecPtr = RecPtr;
-
- targetPagePtr = RecPtr - (RecPtr % XLOG_BLCKSZ);
- targetRecOff = RecPtr % XLOG_BLCKSZ;
-
- /*
- * Read the page containing the record into state->readBuf. Request enough
- * byte to cover the whole record header, or at least the part of it that
- * fits on the same page.
- */
- while (XLogNeedData(state, targetPagePtr,
- Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
- targetRecOff != 0))
- {
- if (!state->routine.page_read(state, state->readPagePtr, state->readLen,
- RecPtr, state->readBuf))
- break;
- }
-
- if (!state->page_verified)
- goto err;
-
- /*
- * We have at least the page header, so we can examine it now.
- */
- pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
- if (targetRecOff == 0)
- {
- /*
- * At page start, so skip over page header.
- */
- RecPtr += pageHeaderSize;
- targetRecOff = pageHeaderSize;
- }
- else if (targetRecOff < pageHeaderSize)
- {
- report_invalid_record(state, "invalid record offset at %X/%X",
- (uint32) (RecPtr >> 32), (uint32) RecPtr);
- goto err;
- }
-
- if ((((XLogPageHeader) state->readBuf)->xlp_info & XLP_FIRST_IS_CONTRECORD) &&
- targetRecOff == pageHeaderSize)
- {
- report_invalid_record(state, "contrecord is requested by %X/%X",
- (uint32) (RecPtr >> 32), (uint32) RecPtr);
- goto err;
- }
-
- /* XLogNeedData has verified the page header */
- Assert(pageHeaderSize <= state->readLen);
-
- /*
- * Read the record length.
- *
- * NB: Even though we use an XLogRecord pointer here, the whole record
- * header might not fit on this page. xl_tot_len is the first field of the
- * struct, so it must be on this page (the records are MAXALIGNed), but we
- * cannot access any other fields until we've verified that we got the
- * whole header.
- */
- record = (XLogRecord *) (state->readBuf + RecPtr % XLOG_BLCKSZ);
- total_len = record->xl_tot_len;
-
- /*
- * If the whole record header is on this page, validate it immediately.
- * Otherwise do just a basic sanity check on xl_tot_len, and validate the
- * rest of the header after reading it from the next page. The xl_tot_len
- * check is necessary here to ensure that we enter the "Need to reassemble
- * record" code path below; otherwise we might fail to apply
- * ValidXLogRecordHeader at all.
- */
- if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
- {
- if (!ValidXLogRecordHeader(state, RecPtr, state->ReadRecPtr, record,
- randAccess))
- goto err;
- gotheader = true;
- }
- else
- {
- /* XXX: more validation should be done here */
- if (total_len < SizeOfXLogRecord)
- {
- report_invalid_record(state,
- "invalid record length at %X/%X: wanted %u, got %u",
- (uint32) (RecPtr >> 32), (uint32) RecPtr,
- (uint32) SizeOfXLogRecord, total_len);
- goto err;
- }
- gotheader = false;
- }
-
- len = XLOG_BLCKSZ - RecPtr % XLOG_BLCKSZ;
- if (total_len > len)
+ switch (state->readRecordState)
{
- /* Need to reassemble record */
- char *contdata;
- XLogPageHeader pageHeader;
- char *buffer;
- uint32 gotlen;
-
- /*
- * Enlarge readRecordBuf as needed.
- */
- if (total_len > state->readRecordBufSize &&
- !allocate_recordbuf(state, total_len))
- {
- /* We treat this as a "bogus data" condition */
- report_invalid_record(state, "record length %u at %X/%X too long",
- total_len,
- (uint32) (RecPtr >> 32), (uint32) RecPtr);
- goto err;
- }
+ case XLREAD_NEXT_RECORD:
+ ResetDecoder(state);
- /* Copy the first fragment of the record from the first page. */
- memcpy(state->readRecordBuf,
- state->readBuf + RecPtr % XLOG_BLCKSZ, len);
- buffer = state->readRecordBuf + len;
- gotlen = len;
-
- do
- {
- int rest_len = total_len - gotlen;
-
- /* Calculate pointer to beginning of next page */
- targetPagePtr += XLOG_BLCKSZ;
-
- /* Wait for the next page to become available */
- while (XLogNeedData(state, targetPagePtr,
- Min(rest_len, XLOG_BLCKSZ),
- false))
+ if (state->ReadRecPtr != InvalidXLogRecPtr)
{
- if (!state->routine.page_read(state, state->readPagePtr,
- state->readLen,
- state->ReadRecPtr,
- state->readBuf))
- break;
+ /* read the record after the one we just read */
+
+ /*
+ * EndRecPtr is pointing to end+1 of the previous WAL record.
+ * If we're at a page boundary, no more records can fit on the
+ * current page. We must skip over the page header, but we
+ * can't do that until we've read in the page, since the header
+ * size is variable.
+ */
+ state->PrevRecPtr = state->ReadRecPtr;
+ state->ReadRecPtr = state->EndRecPtr;
+ }
+ else
+ {
+ /*
+ * Caller supplied a position to start at.
+ *
+ * In this case, EndRecPtr should already be pointing to a
+ * valid record starting position.
+ */
+ Assert(XRecOffIsValid(state->EndRecPtr));
+ state->ReadRecPtr = state->EndRecPtr;
+
+ /*
+ * We cannot verify the previous-record pointer when we're
+ * seeking to a particular record. Reset PrevRecPtr so that we
+ * won't try doing that.
+ */
+ state->PrevRecPtr = InvalidXLogRecPtr;
+ state->EndRecPtr = InvalidXLogRecPtr; /* to be tidy */
}
+ state->record_verified = false;
+ state->readRecordState = XLREAD_TOT_LEN;
+ /* fall through */
+
+ case XLREAD_TOT_LEN:
+ {
+ uint32 total_len;
+ uint32 pageHeaderSize;
+ XLogRecPtr targetPagePtr;
+ uint32 targetRecOff;
+ XLogPageHeader pageHeader;
+
+ targetPagePtr =
+ state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
+ targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+
+ /*
+ * Check if we have enough data. For the first record in the page,
+ * the requesting length doesn't contain page header.
+ */
+ if (XLogNeedData(state, targetPagePtr,
+ Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+ targetRecOff != 0))
+ return XLREAD_NEED_DATA;
+
+ /* error out if caller supplied bogus page */
if (!state->page_verified)
goto err;
- Assert(SizeOfXLogShortPHD <= state->readLen);
+ /* examine page header now. */
+ pageHeaderSize =
+ XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+ if (targetRecOff == 0)
+ {
+ /* At page start, so skip over page header. */
+ state->ReadRecPtr += pageHeaderSize;
+ targetRecOff = pageHeaderSize;
+ }
+ else if (targetRecOff < pageHeaderSize)
+ {
+ report_invalid_record(state, "invalid record offset at %X/%X",
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr);
+ goto err;
+ }
- /* Check that the continuation on next page looks valid */
pageHeader = (XLogPageHeader) state->readBuf;
- if (!(pageHeader->xlp_info & XLP_FIRST_IS_CONTRECORD))
+ if ((pageHeader->xlp_info & XLP_FIRST_IS_CONTRECORD) &&
+ targetRecOff == pageHeaderSize)
{
- report_invalid_record(state,
- "there is no contrecord flag at %X/%X",
- (uint32) (RecPtr >> 32), (uint32) RecPtr);
+ report_invalid_record(state, "contrecord is requested by %X/%X",
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr);
goto err;
}
- /*
- * Cross-check that xlp_rem_len agrees with how much of the record
- * we expect there to be left.
- */
- if (pageHeader->xlp_rem_len == 0 ||
- total_len != (pageHeader->xlp_rem_len + gotlen))
- {
- report_invalid_record(state,
- "invalid contrecord length %u at %X/%X",
- pageHeader->xlp_rem_len,
- (uint32) (RecPtr >> 32), (uint32) RecPtr);
- goto err;
- }
-
- /* Append the continuation from this page to the buffer */
- pageHeaderSize = XLogPageHeaderSize(pageHeader);
-
+ /* XLogNeedData has verified the page header */
Assert(pageHeaderSize <= state->readLen);
- contdata = (char *) state->readBuf + pageHeaderSize;
- len = XLOG_BLCKSZ - pageHeaderSize;
- if (pageHeader->xlp_rem_len < len)
- len = pageHeader->xlp_rem_len;
+ /*
+ * Read the record length.
+ *
+ * NB: Even though we use an XLogRecord pointer here, the whole
+ * record header might not fit on this page. xl_tot_len is the first
+ * field of the struct, so it must be on this page (the records are
+ * MAXALIGNed), but we cannot access any other fields until we've
+ * verified that we got the whole header.
+ */
+ prec = (XLogRecord *) (state->readBuf +
+ state->ReadRecPtr % XLOG_BLCKSZ);
+ total_len = prec->xl_tot_len;
- Assert (pageHeaderSize + len <= state->readLen);
- memcpy(buffer, (char *) contdata, len);
- buffer += len;
- gotlen += len;
+ /*
+ * If the whole record header is on this page, validate it
+ * immediately. Otherwise do just a basic sanity check on
+ * xl_tot_len, and validate the rest of the header after reading it
+ * from the next page. The xl_tot_len check is necessary here to
+ * ensure that we enter the XLREAD_CONTINUATION state below;
+ * otherwise we might fail to apply ValidXLogRecordHeader at all.
+ */
+ if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
+ {
+ if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
+ state->PrevRecPtr, prec))
+ goto err;
- /* If we just reassembled the record header, validate it. */
- if (!gotheader)
+ state->record_verified = true;
+ }
+ else
{
- record = (XLogRecord *) state->readRecordBuf;
- if (!ValidXLogRecordHeader(state, RecPtr, state->ReadRecPtr,
- record, randAccess))
+ /* XXX: more validation should be done here */
+ if (total_len < SizeOfXLogRecord)
+ {
+ report_invalid_record(state,
+ "invalid record length at %X/%X: wanted %u, got %u",
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr,
+ (uint32) SizeOfXLogRecord, total_len);
goto err;
- gotheader = true;
+ }
}
- } while (gotlen < total_len);
-
- Assert(gotheader);
- record = (XLogRecord *) state->readRecordBuf;
- if (!ValidXLogRecord(state, record, RecPtr))
- goto err;
+ /*
+ * Wait for the rest of the record, or the part of the record that
+ * fit on the first page if crossed a page boundary, to become
+ * available.
+ */
+ state->recordGotLen = 0;
+ state->recordRemainLen = total_len;
+ state->readRecordState = XLREAD_FIRST_FRAGMENT;
+ }
+ /* fall through */
- pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
- state->ReadRecPtr = RecPtr;
- state->EndRecPtr = targetPagePtr + pageHeaderSize
- + MAXALIGN(pageHeader->xlp_rem_len);
- }
- else
- {
- /* Wait for the record data to become available */
- while (XLogNeedData(state, targetPagePtr,
- Min(targetRecOff + total_len, XLOG_BLCKSZ), true))
+ case XLREAD_FIRST_FRAGMENT:
{
- if (!state->routine.page_read(state, state->readPagePtr,
- state->readLen,
- state->ReadRecPtr, state->readBuf))
+ uint32 total_len = state->recordRemainLen;
+ uint32 request_len;
+ uint32 record_len;
+ XLogRecPtr targetPagePtr;
+ uint32 targetRecOff;
+
+ /*
+ * Wait for the rest of the record on the first page to become
+ * available
+ */
+ targetPagePtr =
+ state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
+ targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+
+ request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ);
+ record_len = request_len - targetRecOff;
+
+ /* ReadRecPtr contains page header */
+ Assert (targetRecOff != 0);
+ if (XLogNeedData(state, targetPagePtr, request_len, true))
+ return XLREAD_NEED_DATA;
+
+ /* error out if caller supplied bogus page */
+ if (!state->page_verified)
+ goto err;
+
+ prec = (XLogRecord *) (state->readBuf + targetRecOff);
+
+ /* validate record header if not yet */
+ if (!state->record_verified && record_len >= SizeOfXLogRecord)
+ {
+ if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
+ state->PrevRecPtr, prec))
+ goto err;
+
+ state->record_verified = true;
+ }
+
+
+ if (total_len == record_len)
+ {
+ /* Record does not cross a page boundary */
+ Assert(state->record_verified);
+
+ if (!ValidXLogRecord(state, prec, state->ReadRecPtr))
+ goto err;
+
+ state->record_verified = true; /* to be tidy */
+
+ /* We already checked the header earlier */
+ state->EndRecPtr = state->ReadRecPtr + MAXALIGN(record_len);
+
+ *record = prec;
+ state->readRecordState = XLREAD_NEXT_RECORD;
break;
+ }
+
+ /*
+ * The record continues on the next page. Need to reassemble
+ * record
+ */
+ Assert(total_len > record_len);
+
+ /* Enlarge readRecordBuf as needed. */
+ if (total_len > state->readRecordBufSize &&
+ !allocate_recordbuf(state, total_len))
+ {
+ /* We treat this as a "bogus data" condition */
+ report_invalid_record(state,
+ "record length %u at %X/%X too long",
+ total_len,
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr);
+ goto err;
+ }
+
+ /* Copy the first fragment of the record from the first page. */
+ memcpy(state->readRecordBuf, state->readBuf + targetRecOff,
+ record_len);
+ state->recordGotLen += record_len;
+ state->recordRemainLen -= record_len;
+
+ /* Calculate pointer to beginning of next page */
+ state->recordContRecPtr = state->ReadRecPtr + record_len;
+ Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0);
+
+ state->readRecordState = XLREAD_CONTINUATION;
}
+ /* fall through */
+
+ case XLREAD_CONTINUATION:
+ {
+ XLogPageHeader pageHeader;
+ uint32 pageHeaderSize;
+ XLogRecPtr targetPagePtr;
+
+ /* we enter this state only if we haven't read the whole record. */
+ Assert (state->recordRemainLen > 0);
+
+ while(state->recordRemainLen > 0)
+ {
+ char *contdata;
+ uint32 request_len;
+ uint32 record_len;
+
+ /* Wait for the next page to become available */
+ targetPagePtr = state->recordContRecPtr;
+
+ /* this request contains page header */
+ Assert (targetPagePtr != 0);
+ if (XLogNeedData(state, targetPagePtr,
+ Min(state->recordRemainLen, XLOG_BLCKSZ),
+ false))
+ return XLREAD_NEED_DATA;
+
+ if (!state->page_verified)
+ goto err;
+
+ Assert(SizeOfXLogShortPHD <= state->readLen);
- if (!state->page_verified)
- goto err;
+ /* Check that the continuation on next page looks valid */
+ pageHeader = (XLogPageHeader) state->readBuf;
+ if (!(pageHeader->xlp_info & XLP_FIRST_IS_CONTRECORD))
+ {
+ report_invalid_record(
+ state,
+ "there is no contrecord flag at %X/%X reading %X/%X",
+ (uint32) (state->recordContRecPtr >> 32),
+ (uint32) state->recordContRecPtr,
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr);
+ goto err;
+ }
- /* Record does not cross a page boundary */
- if (!ValidXLogRecord(state, record, RecPtr))
- goto err;
+ /*
+ * Cross-check that xlp_rem_len agrees with how much of the
+ * record we expect there to be left.
+ */
+ if (pageHeader->xlp_rem_len == 0 ||
+ pageHeader->xlp_rem_len != state->recordRemainLen)
+ {
+ report_invalid_record(
+ state,
+ "invalid contrecord length %u at %X/%X reading %X/%X, expected %u",
+ pageHeader->xlp_rem_len,
+ (uint32) (state->recordContRecPtr >> 32),
+ (uint32) state->recordContRecPtr,
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr,
+ state->recordRemainLen);
+ goto err;
+ }
- state->EndRecPtr = RecPtr + MAXALIGN(total_len);
+ /* Append the continuation from this page to the buffer */
+ pageHeaderSize = XLogPageHeaderSize(pageHeader);
- state->ReadRecPtr = RecPtr;
+ /*
+ * XLogNeedData should have ensured that the whole page header
+ * was read
+ */
+ Assert(state->readLen >= pageHeaderSize);
+
+ contdata = (char *) state->readBuf + pageHeaderSize;
+ record_len = XLOG_BLCKSZ - pageHeaderSize;
+ if (pageHeader->xlp_rem_len < record_len)
+ record_len = pageHeader->xlp_rem_len;
+
+ request_len = record_len + pageHeaderSize;
+
+ /* XLogNeedData should have ensured all needed data was read */
+ Assert (state->readLen >= request_len);
+
+ memcpy(state->readRecordBuf + state->recordGotLen,
+ (char *) contdata, record_len);
+ state->recordGotLen += record_len;
+ state->recordRemainLen -= record_len;
+
+ /* If we just reassembled the record header, validate it. */
+ if (!state->record_verified)
+ {
+ Assert(state->recordGotLen >= SizeOfXLogRecord);
+ if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
+ state->PrevRecPtr,
+ (XLogRecord *) state->readRecordBuf))
+ goto err;
+
+ state->record_verified = true;
+ }
+
+ /* Calculate pointer to beginning of next page, and continue */
+ state->recordContRecPtr += XLOG_BLCKSZ;
+ }
+
+ /* targetPagePtr is pointing the last-read page here */
+ prec = (XLogRecord *) state->readRecordBuf;
+ if (!ValidXLogRecord(state, prec, state->ReadRecPtr))
+ goto err;
+
+ pageHeaderSize =
+ XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+ state->EndRecPtr = targetPagePtr + pageHeaderSize
+ + MAXALIGN(pageHeader->xlp_rem_len);
+
+ *record = prec;
+ state->readRecordState = XLREAD_NEXT_RECORD;
+ break;
+ }
}
/*
* Special processing if it's an XLOG SWITCH record
*/
- if (record->xl_rmid == RM_XLOG_ID &&
- (record->xl_info & ~XLR_INFO_MASK) == XLOG_SWITCH)
+ if ((*record)->xl_rmid == RM_XLOG_ID &&
+ ((*record)->xl_info & ~XLR_INFO_MASK) == XLOG_SWITCH)
{
/* Pretend it extends to end of segment */
state->EndRecPtr += state->segcxt.ws_segsize - 1;
state->EndRecPtr -= XLogSegmentOffset(state->EndRecPtr, state->segcxt.ws_segsize);
}
- if (DecodeXLogRecord(state, record, errormsg))
- return record;
- else
- return NULL;
+ Assert (!*record || state->readLen >= 0);
+ if (DecodeXLogRecord(state, *record, errormsg))
+ return XLREAD_SUCCESS;
+
+ *record = NULL;
+ return XLREAD_FAIL;
err:
/*
- * Invalidate the read state. We might read from a different source after
+ * Invalidate the read page. We might read from a different source after
* failure.
*/
XLogReaderInvalReadState(state);
@@ -570,7 +704,8 @@ err:
if (state->errormsg_buf[0] != '\0')
*errormsg = state->errormsg_buf;
- return NULL;
+ *record = NULL;
+ return XLREAD_FAIL;
}
/*
@@ -722,11 +857,12 @@ XLogReaderInvalReadState(XLogReaderState *state)
*
* This is just a convenience subroutine to avoid duplicated code in
* XLogReadRecord. It's not intended for use from anywhere else.
+ *
+ * If PrevRecPtr is valid, the xl_prev is is cross-checked with it.
*/
static bool
ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
- XLogRecPtr PrevRecPtr, XLogRecord *record,
- bool randAccess)
+ XLogRecPtr PrevRecPtr, XLogRecord *record)
{
if (record->xl_tot_len < SizeOfXLogRecord)
{
@@ -744,7 +880,7 @@ ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
(uint32) RecPtr);
return false;
}
- if (randAccess)
+ if (PrevRecPtr == InvalidXLogRecPtr)
{
/*
* We can't exactly verify the prev-link, but surely it should be less
@@ -976,11 +1112,14 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
* XLogReadRecord() will read the next valid record.
*/
XLogRecPtr
-XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
+XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
+ XLogFindNextRecordCB read_page, void *private)
{
XLogRecPtr tmpRecPtr;
XLogRecPtr found = InvalidXLogRecPtr;
XLogPageHeader header;
+ XLogRecord *record;
+ XLogReadRecordResult result;
char *errormsg;
Assert(!XLogRecPtrIsInvalid(RecPtr));
@@ -1013,9 +1152,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
while(XLogNeedData(state, targetPagePtr, targetRecOff,
targetRecOff != 0))
{
- if (!state->routine.page_read(state, state->readPagePtr,
- state->readLen,
- state->ReadRecPtr, state->readBuf))
+ if (!read_page(state, private))
break;
}
@@ -1067,8 +1204,16 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
* or we just jumped over the remaining data of a continuation.
*/
XLogBeginRead(state, tmpRecPtr);
- while (XLogReadRecord(state, &errormsg) != NULL)
+ while ((result = XLogReadRecord(state, &record, &errormsg)) !=
+ XLREAD_FAIL)
{
+ if (result == XLREAD_NEED_DATA)
+ {
+ if (!read_page(state, private))
+ goto err;
+ continue;
+ }
+
/* past the record we've found, break out */
if (RecPtr <= state->ReadRecPtr)
{
@@ -1088,9 +1233,9 @@ err:
#endif /* FRONTEND */
/*
- * Helper function to ease writing of XLogRoutine->page_read callbacks.
- * If this function is used, caller must supply a segment_open callback in
- * 'state', as that is used here.
+ * Helper function to ease writing of page_read callback.
+ * If this function is used, caller must supply a segment_open callback and
+ * segment_close callback as that is used here.
*
* Read 'count' bytes into 'buf', starting at location 'startptr', from WAL
* fetched from timeline 'tli'.
@@ -1103,6 +1248,7 @@ err:
*/
bool
WALRead(XLogReaderState *state,
+ WALSegmentOpenCB segopenfn, WALSegmentCloseCB segclosefn,
char *buf, XLogRecPtr startptr, Size count, TimeLineID tli,
WALReadError *errinfo)
{
@@ -1134,10 +1280,10 @@ WALRead(XLogReaderState *state,
XLogSegNo nextSegNo;
if (state->seg.ws_file >= 0)
- state->routine.segment_close(state);
+ segclosefn(state);
XLByteToSeg(recptr, nextSegNo, state->segcxt.ws_segsize);
- state->routine.segment_open(state, nextSegNo, &tli);
+ segopenfn(state, nextSegNo, &tli);
/* This shouldn't happen -- indicates a bug in segment_open */
Assert(state->seg.ws_file >= 0);
diff --git a/src/backend/access/transam/xlogutils.c b/src/backend/access/transam/xlogutils.c
index 1b10b6df20..303333571f 100644
--- a/src/backend/access/transam/xlogutils.c
+++ b/src/backend/access/transam/xlogutils.c
@@ -689,8 +689,7 @@ XLogTruncateRelation(RelFileNode rnode, ForkNumber forkNum,
void
XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wantLength)
{
- const XLogRecPtr lastReadPage = state->seg.ws_segno *
- state->segcxt.ws_segsize + state->readLen;
+ const XLogRecPtr lastReadPage = state->readPagePtr;
Assert(wantPage != InvalidXLogRecPtr && wantPage % XLOG_BLCKSZ == 0);
Assert(wantLength <= XLOG_BLCKSZ);
@@ -705,7 +704,7 @@ XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wa
* current TLI has since become historical.
*/
if (lastReadPage == wantPage &&
- state->readLen != 0 &&
+ state->page_verified &&
lastReadPage + state->readLen >= wantPage + Min(wantLength, XLOG_BLCKSZ - 1))
return;
@@ -792,6 +791,7 @@ wal_segment_open(XLogReaderState *state, XLogSegNo nextSegNo,
char path[MAXPGPATH];
XLogFilePath(path, tli, nextSegNo, state->segcxt.ws_segsize);
+ elog(LOG, "HOGE: %lu, %d => %s", nextSegNo, tli, path);
state->seg.ws_file = BasicOpenFile(path, O_RDONLY | PG_BINARY);
if (state->seg.ws_file >= 0)
return;
@@ -829,9 +829,11 @@ wal_segment_close(XLogReaderState *state)
* loop for now.
*/
bool
-read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *cur_page)
+read_local_xlog_page(XLogReaderState *state)
{
+ XLogRecPtr targetPagePtr = state->readPagePtr;
+ int reqLen = state->readLen;
+ char *cur_page = state->readBuf;
XLogRecPtr read_upto,
loc;
TimeLineID tli;
@@ -944,11 +946,12 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
* as 'count', read the whole page anyway. It's guaranteed to be
* zero-padded up to the page boundary if it's incomplete.
*/
- if (!WALRead(state, cur_page, targetPagePtr, XLOG_BLCKSZ, tli,
- &errinfo))
+ if (!WALRead(state, wal_segment_open, wal_segment_close,
+ cur_page, targetPagePtr, XLOG_BLCKSZ, tli, &errinfo))
WALReadRaiseError(&errinfo);
/* number of valid bytes in the buffer */
+ state->readPagePtr = targetPagePtr;
state->readLen = count;
return true;
}
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index dc69e5ce5f..d99474c173 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -120,7 +120,8 @@ StartupDecodingContext(List *output_plugin_options,
TransactionId xmin_horizon,
bool need_full_snapshot,
bool fast_forward,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress)
@@ -169,11 +170,12 @@ StartupDecodingContext(List *output_plugin_options,
ctx->slot = slot;
- ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, xl_routine, ctx);
+ ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, cleanup_cb);
if (!ctx->reader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
errmsg("out of memory")));
+ ctx->page_read = page_read;
ctx->reorder = ReorderBufferAllocate();
ctx->snapshot_builder =
@@ -231,7 +233,8 @@ CreateInitDecodingContext(char *plugin,
List *output_plugin_options,
bool need_full_snapshot,
XLogRecPtr restart_lsn,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress)
@@ -328,7 +331,7 @@ CreateInitDecodingContext(char *plugin,
ctx = StartupDecodingContext(NIL, restart_lsn, xmin_horizon,
need_full_snapshot, false,
- xl_routine, prepare_write, do_write,
+ page_read, cleanup_cb, prepare_write, do_write,
update_progress);
/* call output plugin initialization callback */
@@ -376,7 +379,8 @@ LogicalDecodingContext *
CreateDecodingContext(XLogRecPtr start_lsn,
List *output_plugin_options,
bool fast_forward,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress)
@@ -429,8 +433,8 @@ CreateDecodingContext(XLogRecPtr start_lsn,
ctx = StartupDecodingContext(output_plugin_options,
start_lsn, InvalidTransactionId, false,
- fast_forward, xl_routine, prepare_write,
- do_write, update_progress);
+ fast_forward, page_read, cleanup_cb,
+ prepare_write, do_write, update_progress);
/* call output plugin initialization callback */
old_context = MemoryContextSwitchTo(ctx->context);
@@ -483,7 +487,13 @@ DecodingContextFindStartpoint(LogicalDecodingContext *ctx)
char *err = NULL;
/* the read_page callback waits for new WAL */
- record = XLogReadRecord(ctx->reader, &err);
+ while (XLogReadRecord(ctx->reader, &record, &err) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!ctx->page_read(ctx->reader))
+ break;
+ }
+
if (err)
elog(ERROR, "%s", err);
if (!record)
diff --git a/src/backend/replication/logical/logicalfuncs.c b/src/backend/replication/logical/logicalfuncs.c
index b99c94e848..fcc81b358e 100644
--- a/src/backend/replication/logical/logicalfuncs.c
+++ b/src/backend/replication/logical/logicalfuncs.c
@@ -233,9 +233,8 @@ pg_logical_slot_get_changes_guts(FunctionCallInfo fcinfo, bool confirm, bool bin
ctx = CreateDecodingContext(InvalidXLogRecPtr,
options,
false,
- XL_ROUTINE(.page_read = read_local_xlog_page,
- .segment_open = wal_segment_open,
- .segment_close = wal_segment_close),
+ read_local_xlog_page,
+ wal_segment_close,
LogicalOutputPrepareWrite,
LogicalOutputWrite, NULL);
@@ -284,7 +283,13 @@ pg_logical_slot_get_changes_guts(FunctionCallInfo fcinfo, bool confirm, bool bin
XLogRecord *record;
char *errm = NULL;
- record = XLogReadRecord(ctx->reader, &errm);
+ while (XLogReadRecord(ctx->reader, &record, &errm) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!ctx->page_read(ctx->reader))
+ break;
+ }
+
if (errm)
elog(ERROR, "%s", errm);
diff --git a/src/backend/replication/slotfuncs.c b/src/backend/replication/slotfuncs.c
index 6fed3cfd23..cac197045d 100644
--- a/src/backend/replication/slotfuncs.c
+++ b/src/backend/replication/slotfuncs.c
@@ -152,9 +152,8 @@ create_logical_replication_slot(char *name, char *plugin,
ctx = CreateInitDecodingContext(plugin, NIL,
false, /* just catalogs is OK */
restart_lsn,
- XL_ROUTINE(.page_read = read_local_xlog_page,
- .segment_open = wal_segment_open,
- .segment_close = wal_segment_close),
+ read_local_xlog_page,
+ wal_segment_close,
NULL, NULL, NULL);
/*
@@ -465,9 +464,8 @@ pg_logical_replication_slot_advance(XLogRecPtr moveto)
ctx = CreateDecodingContext(InvalidXLogRecPtr,
NIL,
true, /* fast_forward */
- XL_ROUTINE(.page_read = read_local_xlog_page,
- .segment_open = wal_segment_open,
- .segment_close = wal_segment_close),
+ read_local_xlog_page,
+ wal_segment_close,
NULL, NULL, NULL);
/*
@@ -492,7 +490,13 @@ pg_logical_replication_slot_advance(XLogRecPtr moveto)
* Read records. No changes are generated in fast_forward mode,
* but snapbuilder/slot statuses are updated properly.
*/
- record = XLogReadRecord(ctx->reader, &errm);
+ while (XLogReadRecord(ctx->reader, &record, &errm) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!ctx->page_read(ctx->reader))
+ break;
+ }
+
if (errm)
elog(ERROR, "%s", errm);
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index d3a94d611f..cc7e7fdac7 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -293,11 +293,9 @@ InitWalSender(void)
if (!am_db_walsender)
{
xlogreader = &fake_xlogreader;
- xlogreader->routine =
- *XL_ROUTINE(.segment_open = WalSndSegmentOpen,
- .segment_close = wal_segment_close);
WALOpenSegmentInit(&xlogreader->seg, &xlogreader->segcxt,
wal_segment_size, NULL);
+ xlogreader->cleanup_cb = wal_segment_close;
}
}
@@ -816,9 +814,11 @@ StartReplication(StartReplicationCmd *cmd)
* set every time WAL is flushed.
*/
static bool
-logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetRecPtr, char *cur_page)
+logical_read_xlog_page(XLogReaderState *state)
{
+ XLogRecPtr targetPagePtr = state->readPagePtr;
+ int reqLen = state->readLen;
+ char *cur_page = state->readBuf;
XLogRecPtr flushptr;
int count;
WALReadError errinfo;
@@ -846,11 +846,11 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
count = flushptr - targetPagePtr; /* part of the page available */
/* now actually read the data, we know it's there */
- if (!WALRead(state,
+ if (!WALRead(state, WalSndSegmentOpen, wal_segment_close,
cur_page,
targetPagePtr,
XLOG_BLCKSZ,
- state->seg.ws_tli, /* Pass the current TLI because only
+ xlogreader->seg.ws_tli, /* Pass the current TLI because only
* WalSndSegmentOpen controls whether new
* TLI is needed. */
&errinfo))
@@ -863,8 +863,8 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
* read() succeeds in that case, but the data we tried to read might
* already have been overwritten with new WAL records.
*/
- XLByteToSeg(targetPagePtr, segno, state->segcxt.ws_segsize);
- CheckXLogRemoved(segno, state->seg.ws_tli);
+ XLByteToSeg(targetPagePtr, segno, xlogreader->segcxt.ws_segsize);
+ CheckXLogRemoved(segno, xlogreader->seg.ws_tli);
state->readLen = count;
return true;
@@ -1018,9 +1018,8 @@ CreateReplicationSlot(CreateReplicationSlotCmd *cmd)
ctx = CreateInitDecodingContext(cmd->plugin, NIL, need_full_snapshot,
InvalidXLogRecPtr,
- XL_ROUTINE(.page_read = logical_read_xlog_page,
- .segment_open = WalSndSegmentOpen,
- .segment_close = wal_segment_close),
+ logical_read_xlog_page,
+ wal_segment_close,
WalSndPrepareWrite, WalSndWriteData,
WalSndUpdateProgress);
@@ -1183,9 +1182,8 @@ StartLogicalReplication(StartReplicationCmd *cmd)
*/
logical_decoding_ctx =
CreateDecodingContext(cmd->startpoint, cmd->options, false,
- XL_ROUTINE(.page_read = logical_read_xlog_page,
- .segment_open = WalSndSegmentOpen,
- .segment_close = wal_segment_close),
+ logical_read_xlog_page,
+ wal_segment_close,
WalSndPrepareWrite, WalSndWriteData,
WalSndUpdateProgress);
xlogreader = logical_decoding_ctx->reader;
@@ -2495,14 +2493,14 @@ WalSndSegmentOpen(XLogReaderState *state, XLogSegNo nextSegNo,
{
XLogSegNo endSegNo;
- XLByteToSeg(sendTimeLineValidUpto, endSegNo, state->segcxt.ws_segsize);
- if (state->seg.ws_segno == endSegNo)
+ XLByteToSeg(sendTimeLineValidUpto, endSegNo, xlogreader->segcxt.ws_segsize);
+ if (xlogreader->seg.ws_segno == endSegNo)
*tli_p = sendTimeLineNextTLI;
}
- XLogFilePath(path, *tli_p, nextSegNo, state->segcxt.ws_segsize);
- state->seg.ws_file = BasicOpenFile(path, O_RDONLY | PG_BINARY);
- if (state->seg.ws_file >= 0)
+ XLogFilePath(path, *tli_p, nextSegNo, xlogreader->segcxt.ws_segsize);
+ xlogreader->seg.ws_file = BasicOpenFile(path, O_RDONLY | PG_BINARY);
+ if (xlogreader->seg.ws_file >= 0)
return;
/*
@@ -2763,7 +2761,7 @@ XLogSendPhysical(void)
enlargeStringInfo(&output_message, nbytes);
retry:
- if (!WALRead(xlogreader,
+ if (!WALRead(xlogreader, WalSndSegmentOpen, wal_segment_close,
&output_message.data[output_message.len],
startptr,
nbytes,
@@ -2861,7 +2859,12 @@ XLogSendLogical(void)
*/
WalSndCaughtUp = false;
- record = XLogReadRecord(logical_decoding_ctx->reader, &errm);
+ while (XLogReadRecord(logical_decoding_ctx->reader, &record, &errm) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!logical_decoding_ctx->page_read(logical_decoding_ctx->reader))
+ break;
+ }
/* xlog record was invalid */
if (errm != NULL)
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index 907db33b9e..cad8e13965 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -40,15 +40,9 @@ static int xlogreadfd = -1;
static XLogSegNo xlogreadsegno = -1;
static char xlogfpath[MAXPGPATH];
-typedef struct XLogPageReadPrivate
-{
- const char *restoreCommand;
- int tliIndex;
-} XLogPageReadPrivate;
-
-static bool SimpleXLogPageRead(XLogReaderState *xlogreader,
- XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
+static bool SimpleXLogPageRead(XLogReaderState *xlogreader,
+ const char *datadir, int *tliIndex,
+ const char *restoreCommand);
/*
* Read WAL from the datadir/pg_wal, starting from 'startpoint' on timeline
@@ -62,20 +56,22 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
XLogRecord *record;
XLogReaderState *xlogreader;
char *errormsg;
- XLogPageReadPrivate private;
- private.tliIndex = tliIndex;
- private.restoreCommand = restoreCommand;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir,
- XL_ROUTINE(.page_read = &SimpleXLogPageRead),
- &private);
+ xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+
if (xlogreader == NULL)
pg_fatal("out of memory");
XLogBeginRead(xlogreader, startpoint);
do
{
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!SimpleXLogPageRead(xlogreader, datadir,
+ &tliIndex, restoreCommand))
+ break;
+ }
if (record == NULL)
{
@@ -113,19 +109,19 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex,
XLogRecord *record;
XLogReaderState *xlogreader;
char *errormsg;
- XLogPageReadPrivate private;
XLogRecPtr endptr;
- private.tliIndex = tliIndex;
- private.restoreCommand = restoreCommand;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir,
- XL_ROUTINE(.page_read = &SimpleXLogPageRead),
- &private);
+ xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
XLogBeginRead(xlogreader, ptr);
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!SimpleXLogPageRead(xlogreader, datadir, &tliIndex, restoreCommand))
+ break;
+ }
if (record == NULL)
{
if (errormsg)
@@ -160,7 +156,6 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
XLogRecPtr searchptr;
XLogReaderState *xlogreader;
char *errormsg;
- XLogPageReadPrivate private;
/*
* The given fork pointer points to the end of the last common record,
@@ -176,11 +171,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
forkptr += SizeOfXLogShortPHD;
}
- private.tliIndex = tliIndex;
- private.restoreCommand = restoreCommand;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir,
- XL_ROUTINE(.page_read = &SimpleXLogPageRead),
- &private);
+ xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
@@ -190,7 +181,13 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
uint8 info;
XLogBeginRead(xlogreader, searchptr);
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!SimpleXLogPageRead(xlogreader, datadir,
+ &tliIndex, restoreCommand))
+ break;
+ }
if (record == NULL)
{
@@ -237,10 +234,11 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
/* XLogReader callback function, to read a WAL page */
static bool
-SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *readBuf)
+SimpleXLogPageRead(XLogReaderState *xlogreader, const char *datadir,
+ int *tliIndex, const char *restoreCommand)
{
- XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
+ XLogRecPtr targetPagePtr = xlogreader->readPagePtr;
+ char *readBuf = xlogreader->readBuf;
uint32 targetPageOff;
XLogRecPtr targetSegEnd;
XLogSegNo targetSegNo;
@@ -273,14 +271,14 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
* be done both forward and backward, consider also switching timeline
* accordingly.
*/
- while (private->tliIndex < targetNentries - 1 &&
- targetHistory[private->tliIndex].end < targetSegEnd)
- private->tliIndex++;
- while (private->tliIndex > 0 &&
- targetHistory[private->tliIndex].begin >= targetSegEnd)
- private->tliIndex--;
+ while (*tliIndex < targetNentries - 1 &&
+ targetHistory[*tliIndex].end < targetSegEnd)
+ (*tliIndex)++;
+ while (*tliIndex > 0 &&
+ targetHistory[*tliIndex].begin >= targetSegEnd)
+ (*tliIndex)--;
- XLogFileName(xlogfname, targetHistory[private->tliIndex].tli,
+ XLogFileName(xlogfname, targetHistory[*tliIndex].tli,
xlogreadsegno, WalSegSz);
snprintf(xlogfpath, MAXPGPATH, "%s/" XLOGDIR "/%s",
@@ -293,7 +291,7 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
/*
* If we have no restore_command to execute, then exit.
*/
- if (private->restoreCommand == NULL)
+ if (restoreCommand == NULL)
{
pg_log_error("could not open file \"%s\": %m", xlogfpath);
xlogreader->readLen = -1;
@@ -307,7 +305,7 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
xlogreadfd = RestoreArchivedFile(xlogreader->segcxt.ws_dir,
xlogfname,
WalSegSz,
- private->restoreCommand);
+ restoreCommand);
if (xlogreadfd < 0)
{
@@ -349,7 +347,7 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
Assert(targetSegNo == xlogreadsegno);
- xlogreader->seg.ws_tli = targetHistory[private->tliIndex].tli;
+ xlogreader->seg.ws_tli = targetHistory[*tliIndex].tli;
xlogreader->readLen = XLOG_BLCKSZ;
return true;
}
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index ab0c28d9b7..02e71138cc 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -330,12 +330,17 @@ WALDumpCloseSegment(XLogReaderState *state)
state->seg.ws_file = -1;
}
-/* pg_waldump's XLogReaderRoutine->page_read callback */
+/*
+ * pg_waldump's WAL page rader, also used as page_read callback for
+ * XLogFindNextRecord
+ */
static bool
-WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetPtr, char *readBuff)
+WALDumpReadPage(XLogReaderState *state, void *priv)
{
- XLogDumpPrivate *private = state->private_data;
+ XLogRecPtr targetPagePtr = state->readPagePtr;
+ int reqLen = state->readLen;
+ char *readBuff = state->readBuf;
+ XLogDumpPrivate *private = (XLogDumpPrivate *) priv;
int count = XLOG_BLCKSZ;
WALReadError errinfo;
@@ -353,8 +358,8 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
}
}
- if (!WALRead(state, readBuff, targetPagePtr, count, private->timeline,
- &errinfo))
+ if (!WALRead(state, WALDumpOpenSegment, WALDumpCloseSegment,
+ readBuff, targetPagePtr, count, private->timeline, &errinfo))
{
WALOpenSegment *seg = &errinfo.wre_seg;
char fname[MAXPGPATH];
@@ -374,6 +379,7 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
(Size) errinfo.wre_req);
}
+ Assert(count >= state->readLen);
state->readLen = count;
return true;
}
@@ -1042,16 +1048,14 @@ main(int argc, char **argv)
/* we have everything we need, start reading */
xlogreader_state =
- XLogReaderAllocate(WalSegSz, waldir,
- XL_ROUTINE(.page_read = WALDumpReadPage,
- .segment_open = WALDumpOpenSegment,
- .segment_close = WALDumpCloseSegment),
- &private);
+ XLogReaderAllocate(WalSegSz, waldir, WALDumpCloseSegment);
+
if (!xlogreader_state)
fatal_error("out of memory");
/* first find a valid recptr to start from */
- first_record = XLogFindNextRecord(xlogreader_state, private.startptr);
+ first_record = XLogFindNextRecord(xlogreader_state, private.startptr,
+ &WALDumpReadPage, (void*) &private);
if (first_record == InvalidXLogRecPtr)
fatal_error("could not find a valid record after %X/%X",
@@ -1075,7 +1079,13 @@ main(int argc, char **argv)
for (;;)
{
/* try to read the next record */
- record = XLogReadRecord(xlogreader_state, &errormsg);
+ while (XLogReadRecord(xlogreader_state, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!WALDumpReadPage(xlogreader_state, (void *) &private))
+ break;
+ }
+
if (!record)
{
if (!config.follow || private.endptr_reached)
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index 53e4212ac4..1d0167e26d 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -57,64 +57,15 @@ typedef struct WALSegmentContext
typedef struct XLogReaderState XLogReaderState;
-/* Function type definition for the read_page callback */
-typedef bool (*XLogPageReadCB) (XLogReaderState *xlogreader,
- XLogRecPtr targetPagePtr,
- int reqLen,
- XLogRecPtr targetRecPtr,
- char *readBuf);
+/* Function type definition for the segment cleanup callback */
+typedef void (*WALSegmentCleanupCB) (XLogReaderState *xlogreader);
+
+/* Function type definition for the open/close callbacks for WALRead() */
typedef void (*WALSegmentOpenCB) (XLogReaderState *xlogreader,
XLogSegNo nextSegNo,
TimeLineID *tli_p);
typedef void (*WALSegmentCloseCB) (XLogReaderState *xlogreader);
-typedef struct XLogReaderRoutine
-{
- /*
- * Data input callback
- *
- * This callback shall read at least reqLen valid bytes of the xlog page
- * starting at targetPagePtr, and store them in readBuf. The callback
- * shall return the number of bytes read (never more than XLOG_BLCKSZ), or
- * -1 on failure. The callback shall sleep, if necessary, to wait for the
- * requested bytes to become available. The callback will not be invoked
- * again for the same page unless more than the returned number of bytes
- * are needed.
- *
- * targetRecPtr is the position of the WAL record we're reading. Usually
- * it is equal to targetPagePtr + reqLen, but sometimes xlogreader needs
- * to read and verify the page or segment header, before it reads the
- * actual WAL record it's interested in. In that case, targetRecPtr can
- * be used to determine which timeline to read the page from.
- *
- * The callback shall set ->seg.ws_tli to the TLI of the file the page was
- * read from.
- */
- XLogPageReadCB page_read;
-
- /*
- * Callback to open the specified WAL segment for reading. ->seg.ws_file
- * shall be set to the file descriptor of the opened segment. In case of
- * failure, an error shall be raised by the callback and it shall not
- * return.
- *
- * "nextSegNo" is the number of the segment to be opened.
- *
- * "tli_p" is an input/output argument. WALRead() uses it to pass the
- * timeline in which the new segment should be found, but the callback can
- * use it to return the TLI that it actually opened.
- */
- WALSegmentOpenCB segment_open;
-
- /*
- * WAL segment close callback. ->seg.ws_file shall be set to a negative
- * number.
- */
- WALSegmentCloseCB segment_close;
-} XLogReaderRoutine;
-
-#define XL_ROUTINE(...) &(XLogReaderRoutine){__VA_ARGS__}
-
typedef struct
{
/* Is this block ref in use? */
@@ -144,12 +95,35 @@ typedef struct
uint16 data_bufsz;
} DecodedBkpBlock;
+/* Return code from XLogReadRecord */
+typedef enum XLogReadRecordResult
+{
+ XLREAD_SUCCESS, /* record is successfully read */
+ XLREAD_NEED_DATA, /* need more data. see XLogReadRecord. */
+ XLREAD_FAIL /* failed during reading a record */
+} XLogReadRecordResult;
+
+/*
+ * internal state of XLogReadRecord
+ *
+ * XLogReadState runs a state machine while reading a record. Theses states
+ * are not seen outside the function. Each state may repeat several times
+ * exiting requesting caller for new data. See the comment of XLogReadRecrod
+ * for details.
+ */
+typedef enum XLogReadRecordState {
+ XLREAD_NEXT_RECORD,
+ XLREAD_TOT_LEN,
+ XLREAD_FIRST_FRAGMENT,
+ XLREAD_CONTINUATION
+} XLogReadRecordState;
+
struct XLogReaderState
{
/*
* Operational callbacks
*/
- XLogReaderRoutine routine;
+ WALSegmentCleanupCB cleanup_cb;
/* ----------------------------------------
* Public parameters
@@ -162,18 +136,14 @@ struct XLogReaderState
*/
uint64 system_identifier;
- /*
- * Opaque data for callbacks to use. Not used by XLogReader.
- */
- void *private_data;
-
/*
* Start and end point of last record read. EndRecPtr is also used as the
* position to read next. Calling XLogBeginRead() sets EndRecPtr to the
* starting position and ReadRecPtr to invalid.
*/
- XLogRecPtr ReadRecPtr; /* start of last record read */
+ XLogRecPtr ReadRecPtr; /* start of last record read or being read */
XLogRecPtr EndRecPtr; /* end+1 of last record read */
+ XLogRecPtr PrevRecPtr; /* start of previous record read */
/* ----------------------------------------
* Communication with page reader
@@ -187,7 +157,9 @@ struct XLogReaderState
* the request, or -1 on error */
TimeLineID readPageTLI; /* TLI for data currently in readBuf */
char *readBuf; /* buffer to store data */
- bool page_verified; /* is the page on the buffer verified? */
+ bool page_verified; /* is the page header on the buffer verified? */
+ bool record_verified;/* is the current record header verified? */
+
/* ----------------------------------------
@@ -227,8 +199,6 @@ struct XLogReaderState
XLogRecPtr latestPagePtr;
TimeLineID latestPageTLI;
- /* beginning of the WAL record being read. */
- XLogRecPtr currRecPtr;
/* timeline to read it from, 0 if a lookup is required */
TimeLineID currTLI;
@@ -255,6 +225,15 @@ struct XLogReaderState
char *readRecordBuf;
uint32 readRecordBufSize;
+ /*
+ * XLogReadRecord() state
+ */
+ XLogReadRecordState readRecordState;/* state machine state */
+ int recordGotLen; /* amount of current record that has
+ * already been read */
+ int recordRemainLen; /* length of current record that remains */
+ XLogRecPtr recordContRecPtr; /* where the current record continues */
+
/* Buffer to hold error message */
char *errormsg_buf;
};
@@ -262,9 +241,7 @@ struct XLogReaderState
/* Get a new XLogReader */
extern XLogReaderState *XLogReaderAllocate(int wal_segment_size,
const char *waldir,
- XLogReaderRoutine *routine,
- void *private_data);
-extern XLogReaderRoutine *LocalXLogReaderRoutine(void);
+ WALSegmentCleanupCB cleanup_cb);
/* Free an XLogReader */
extern void XLogReaderFree(XLogReaderState *state);
@@ -276,12 +253,17 @@ extern void WALOpenSegmentInit(WALOpenSegment *seg, WALSegmentContext *segcxt,
/* Position the XLogReader to given record */
extern void XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr);
#ifdef FRONTEND
-extern XLogRecPtr XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr);
+/* Function type definition for the read_page callback */
+typedef bool (*XLogFindNextRecordCB) (XLogReaderState *xlogreader,
+ void *private);
+extern XLogRecPtr XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
+ XLogFindNextRecordCB read_page, void *private);
#endif /* FRONTEND */
/* Read the next XLog record. Returns NULL on end-of-WAL or failure */
-extern struct XLogRecord *XLogReadRecord(XLogReaderState *state,
- char **errormsg);
+extern XLogReadRecordResult XLogReadRecord(XLogReaderState *state,
+ XLogRecord **record,
+ char **errormsg);
/* Validate a page */
extern bool XLogReaderValidatePageHeader(XLogReaderState *state,
@@ -301,6 +283,7 @@ typedef struct WALReadError
} WALReadError;
extern bool WALRead(XLogReaderState *state,
+ WALSegmentOpenCB segopenfn, WALSegmentCloseCB sgclosefn,
char *buf, XLogRecPtr startptr, Size count,
TimeLineID tli, WALReadError *errinfo);
diff --git a/src/include/access/xlogutils.h b/src/include/access/xlogutils.h
index c287edf206..482e4ced69 100644
--- a/src/include/access/xlogutils.h
+++ b/src/include/access/xlogutils.h
@@ -47,9 +47,7 @@ extern Buffer XLogReadBufferExtended(RelFileNode rnode, ForkNumber forknum,
extern Relation CreateFakeRelcacheEntry(RelFileNode rnode);
extern void FreeFakeRelcacheEntry(Relation fakerel);
-extern bool read_local_xlog_page(XLogReaderState *state,
- XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetRecPtr, char *cur_page);
+extern bool read_local_xlog_page(XLogReaderState *state);
extern void wal_segment_open(XLogReaderState *state,
XLogSegNo nextSegNo,
TimeLineID *tli_p);
diff --git a/src/include/replication/logical.h b/src/include/replication/logical.h
index c2f2475e5d..a743db49cf 100644
--- a/src/include/replication/logical.h
+++ b/src/include/replication/logical.h
@@ -29,6 +29,10 @@ typedef void (*LogicalOutputPluginWriterUpdateProgress) (struct LogicalDecodingC
TransactionId xid
);
+typedef struct LogicalDecodingContext LogicalDecodingContext;
+
+typedef bool (*LogicalDecodingXLogPageReadCB)(XLogReaderState *ctx);
+
typedef struct LogicalDecodingContext
{
/* memory context this is all allocated in */
@@ -39,6 +43,7 @@ typedef struct LogicalDecodingContext
/* infrastructure pieces for decoding */
XLogReaderState *reader;
+ LogicalDecodingXLogPageReadCB page_read;
struct ReorderBuffer *reorder;
struct SnapBuild *snapshot_builder;
@@ -95,14 +100,16 @@ extern LogicalDecodingContext *CreateInitDecodingContext(char *plugin,
List *output_plugin_options,
bool need_full_snapshot,
XLogRecPtr restart_lsn,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress);
extern LogicalDecodingContext *CreateDecodingContext(XLogRecPtr start_lsn,
List *output_plugin_options,
bool fast_forward,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress);
--
2.18.2
----Next_Part(Tue_May_26_16_40_02_2020_373)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
filename="v10-0003-Remove-globals-readOff-readLen-and-readSegNo.patch"
^ permalink raw reply [nested|flat] 70+ messages in thread
* [PATCH v17 2/4] Move page-reader out of XLogReadRecord
@ 2019-09-10 03:58 Kyotaro Horiguchi <[email protected]>
0 siblings, 0 replies; 70+ messages in thread
From: Kyotaro Horiguchi @ 2019-09-10 03:58 UTC (permalink / raw)
This is the second step of removing callbacks from WAL record reader.
Since it is essential to take in additional data while reading a
record, the function have to ask caller for new data while keeping
working state. Thus the function is turned into a state machine.
---
src/backend/access/transam/twophase.c | 15 +-
src/backend/access/transam/xlog.c | 58 +-
src/backend/access/transam/xlogreader.c | 697 +++++++++++-------
src/backend/access/transam/xlogutils.c | 17 +-
src/backend/replication/logical/logical.c | 26 +-
.../replication/logical/logicalfuncs.c | 13 +-
src/backend/replication/slotfuncs.c | 18 +-
src/backend/replication/walsender.c | 32 +-
src/bin/pg_rewind/parsexlog.c | 84 ++-
src/bin/pg_waldump/pg_waldump.c | 36 +-
src/include/access/xlogreader.h | 121 ++-
src/include/access/xlogutils.h | 4 +-
src/include/pg_config_manual.h | 2 +-
src/include/replication/logical.h | 11 +-
14 files changed, 646 insertions(+), 488 deletions(-)
diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index 80d2d20d6c..7d5a96a5f5 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -1325,11 +1325,8 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
XLogReaderState *xlogreader;
char *errormsg;
- xlogreader = XLogReaderAllocate(wal_segment_size, NULL,
- XL_ROUTINE(.page_read = &read_local_xlog_page,
- .segment_open = &wal_segment_open,
- .segment_close = &wal_segment_close),
- NULL);
+ xlogreader = XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+
if (!xlogreader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
@@ -1337,7 +1334,13 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
errdetail("Failed while allocating a WAL reading processor.")));
XLogBeginRead(xlogreader, lsn);
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!read_local_xlog_page(xlogreader))
+ break;
+ }
+
if (record == NULL)
ereport(ERROR,
(errcode_for_file_access(),
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 606328a65a..670996ae67 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -836,13 +836,6 @@ static XLogSource currentSource = XLOG_FROM_ANY;
static bool lastSourceFailed = false;
static bool pendingWalRcvRestart = false;
-typedef struct XLogPageReadPrivate
-{
- int emode;
- bool fetching_ckpt; /* are we fetching a checkpoint record? */
- bool randAccess;
-} XLogPageReadPrivate;
-
/*
* These variables track when we last obtained some WAL data to process,
* and where we got it from. (XLogReceiptSource is initially the same as
@@ -917,8 +910,8 @@ static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath,
static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli,
XLogSource source, bool notfoundOk);
static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, XLogSource source);
-static bool XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
+static bool XLogPageRead(XLogReaderState *xlogreader,
+ bool fetching_ckpt, int emode, bool randAccess);
static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
bool fetching_ckpt, XLogRecPtr tliRecPtr);
static int emode_for_corrupt_record(int emode, XLogRecPtr RecPtr);
@@ -1231,8 +1224,7 @@ XLogInsertRecord(XLogRecData *rdata,
appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len);
if (!debug_reader)
- debug_reader = XLogReaderAllocate(wal_segment_size, NULL,
- XL_ROUTINE(), NULL);
+ debug_reader = XLogReaderAllocate(wal_segment_size, NULL, NULL);
if (!debug_reader)
{
@@ -4344,15 +4336,10 @@ CleanupBackupHistory(void)
* record is available.
*/
static XLogRecord *
-ReadRecord(XLogReaderState *xlogreader, int emode,
- bool fetching_ckpt)
+ReadRecord(XLogReaderState *xlogreader, int emode, bool fetching_ckpt)
{
XLogRecord *record;
- XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
-
- private->fetching_ckpt = fetching_ckpt;
- private->emode = emode;
- private->randAccess = (xlogreader->ReadRecPtr == InvalidXLogRecPtr);
+ bool randAccess = (xlogreader->ReadRecPtr == InvalidXLogRecPtr);
/* This is the first attempt to read this page. */
lastSourceFailed = false;
@@ -4360,8 +4347,16 @@ ReadRecord(XLogReaderState *xlogreader, int emode,
for (;;)
{
char *errormsg;
+ XLogReadRecordResult result;
+
+ while ((result = XLogReadRecord(xlogreader, &record, &errormsg))
+ == XLREAD_NEED_DATA)
+ {
+ if (!XLogPageRead(xlogreader, fetching_ckpt, emode, randAccess))
+ break;
+
+ }
- record = XLogReadRecord(xlogreader, &errormsg);
ReadRecPtr = xlogreader->ReadRecPtr;
EndRecPtr = xlogreader->EndRecPtr;
if (record == NULL)
@@ -6380,7 +6375,6 @@ StartupXLOG(void)
bool backupFromStandby = false;
DBState dbstate_at_startup;
XLogReaderState *xlogreader;
- XLogPageReadPrivate private;
bool promoted = false;
struct stat st;
@@ -6539,13 +6533,9 @@ StartupXLOG(void)
OwnLatch(&XLogCtl->recoveryWakeupLatch);
/* Set up XLOG reader facility */
- MemSet(&private, 0, sizeof(XLogPageReadPrivate));
xlogreader =
- XLogReaderAllocate(wal_segment_size, NULL,
- XL_ROUTINE(.page_read = &XLogPageRead,
- .segment_open = NULL,
- .segment_close = wal_segment_close),
- &private);
+ XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+
if (!xlogreader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
@@ -11987,12 +11977,13 @@ CancelBackup(void)
* sleep and retry.
*/
static bool
-XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetRecPtr, char *readBuf)
+XLogPageRead(XLogReaderState *xlogreader,
+ bool fetching_ckpt, int emode, bool randAccess)
{
- XLogPageReadPrivate *private =
- (XLogPageReadPrivate *) xlogreader->private_data;
- int emode = private->emode;
+ char *readBuf = xlogreader->readBuf;
+ XLogRecPtr targetPagePtr = xlogreader->readPagePtr;
+ int reqLen = xlogreader->readLen;
+ XLogRecPtr targetRecPtr = xlogreader->ReadRecPtr;
uint32 targetPageOff;
XLogSegNo targetSegNo PG_USED_FOR_ASSERTS_ONLY;
int r;
@@ -12035,8 +12026,8 @@ retry:
flushedUpto < targetPagePtr + reqLen))
{
if (!WaitForWALToBecomeAvailable(targetPagePtr + reqLen,
- private->randAccess,
- private->fetching_ckpt,
+ randAccess,
+ fetching_ckpt,
targetRecPtr))
{
if (readFile >= 0)
@@ -12141,6 +12132,7 @@ retry:
goto next_record_is_invalid;
}
+ Assert(xlogreader->readPagePtr == targetPagePtr);
xlogreader->readLen = readLen;
return true;
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index 7d04995b7b..ba3e4868f7 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -40,7 +40,7 @@ static bool XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr,
int reqLen, bool header_inclusive);
static void XLogReaderInvalReadState(XLogReaderState *state);
static bool ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
- XLogRecPtr PrevRecPtr, XLogRecord *record, bool randAccess);
+ XLogRecPtr PrevRecPtr, XLogRecord *record);
static bool ValidXLogRecord(XLogReaderState *state, XLogRecord *record,
XLogRecPtr recptr);
static void ResetDecoder(XLogReaderState *state);
@@ -73,7 +73,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...)
*/
XLogReaderState *
XLogReaderAllocate(int wal_segment_size, const char *waldir,
- XLogReaderRoutine *routine, void *private_data)
+ WALSegmentCleanupCB cleanup_cb)
{
XLogReaderState *state;
@@ -84,7 +84,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
return NULL;
/* initialize caller-provided support functions */
- state->routine = *routine;
+ state->cleanup_cb = cleanup_cb;
state->max_block_id = -1;
@@ -107,8 +107,6 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
WALOpenSegmentInit(&state->seg, &state->segcxt, wal_segment_size,
waldir);
- /* system_identifier initialized to zeroes above */
- state->private_data = private_data;
/* ReadRecPtr, EndRecPtr and readLen initialized to zeroes above */
state->errormsg_buf = palloc_extended(MAX_ERRORMSG_LEN + 1,
MCXT_ALLOC_NO_OOM);
@@ -140,8 +138,8 @@ XLogReaderFree(XLogReaderState *state)
{
int block_id;
- if (state->seg.ws_file != -1)
- state->routine.segment_close(state);
+ if (state->seg.ws_file >= 0)
+ state->cleanup_cb(state);
for (block_id = 0; block_id <= XLR_MAX_BLOCK_ID; block_id++)
{
@@ -246,6 +244,7 @@ XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
/* Begin at the passed-in record pointer. */
state->EndRecPtr = RecPtr;
state->ReadRecPtr = InvalidXLogRecPtr;
+ state->readRecordState = XLREAD_NEXT_RECORD;
}
/*
@@ -254,317 +253,449 @@ XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
* XLogBeginRead() or XLogFindNextRecord() must be called before the first call
* to XLogReadRecord().
*
- * If the page_read callback fails to read the requested data, NULL is
- * returned. The callback is expected to have reported the error; errormsg
- * is set to NULL.
+ * This function may return XLREAD_NEED_DATA several times before returning a
+ * result record. The caller shall read in some new data then call this
+ * function again with the same parameters.
*
- * If the reading fails for some other reason, NULL is also returned, and
- * *errormsg is set to a string with details of the failure.
+ * When a record is successfully read, returns XLREAD_SUCCESS with result
+ * record being stored in *record. Otherwise *record is NULL.
*
- * The returned pointer (or *errormsg) points to an internal buffer that's
- * valid until the next call to XLogReadRecord.
+ * Returns XLREAD_NEED_DATA if more data is needed to finish reading the
+ * current record. In that case, state->readPagePtr and state->readLen inform
+ * the desired position and minimum length of data needed. The caller shall
+ * read in the requested data and set state->readBuf to point to a buffer
+ * containing it. The caller must also set state->readPageTLI and
+ * state->readLen to indicate the timeline that it was read from, and the
+ * length of data that is now available (which must be >= given readLen),
+ * respectively.
+ *
+ * If invalid data is encountered, returns XLREAD_FAIL with *record being set to
+ * NULL. *errormsg is set to a string with details of the failure.
+ * The returned pointer (or *errormsg) points to an internal buffer that's valid
+ * until the next call to XLogReadRecord.
+ *
+ *
+ * This function runs a state machine consists of the following states.
+ *
+ * XLREAD_NEXT_RECORD :
+ * The initial state, if called with valid RecPtr, try to read a record at
+ * that position. If invalid RecPtr is given try to read a record just after
+ * the last one previously read.
+ * This state ens after setting ReadRecPtr. Then goes to XLREAD_TOT_LEN.
+ *
+ * XLREAD_TOT_LEN:
+ * Examining record header. Ends after reading record total
+ * length. recordRemainLen and recordGotLen are initialized.
+ *
+ * XLREAD_FIRST_FRAGMENT:
+ * Reading the first fragment. Ends with finishing reading a single
+ * record. Goes to XLREAD_NEXT_RECORD if that's all or
+ * XLREAD_CONTINUATION if we have continuation.
+
+ * XLREAD_CONTINUATION:
+ * Reading continuation of record. Ends with finishing the whole record then
+ * goes to XLREAD_NEXT_RECORD. During this state, recordRemainLen indicates
+ * how much is left and readRecordBuf holds the partially assert
+ * record.recordContRecPtr points to the beginning of the next page where to
+ * continue.
+ *
+ * If wrong data found in any state, the state machine stays at the current
+ * state. This behavior allows to continue reading a reacord switching among
+ * different souces, while streaming replication.
*/
-XLogRecord *
-XLogReadRecord(XLogReaderState *state, char **errormsg)
+XLogReadRecordResult
+XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
{
- XLogRecPtr RecPtr;
- XLogRecord *record;
- XLogRecPtr targetPagePtr;
- bool randAccess;
- uint32 len,
- total_len;
- uint32 targetRecOff;
- uint32 pageHeaderSize;
- bool gotheader;
+ XLogRecord *prec;
- /*
- * randAccess indicates whether to verify the previous-record pointer of
- * the record we're reading. We only do this if we're reading
- * sequentially, which is what we initially assume.
- */
- randAccess = false;
+ *record = NULL;
/* reset error state */
*errormsg = NULL;
state->errormsg_buf[0] = '\0';
- ResetDecoder(state);
-
- RecPtr = state->EndRecPtr;
-
- if (state->ReadRecPtr != InvalidXLogRecPtr)
- {
- /* read the record after the one we just read */
-
- /*
- * EndRecPtr is pointing to end+1 of the previous WAL record. If
- * we're at a page boundary, no more records can fit on the current
- * page. We must skip over the page header, but we can't do that until
- * we've read in the page, since the header size is variable.
- */
- }
- else
- {
- /*
- * Caller supplied a position to start at.
- *
- * In this case, EndRecPtr should already be pointing to a valid
- * record starting position.
- */
- Assert(XRecOffIsValid(RecPtr));
- randAccess = true;
- }
-
- state->currRecPtr = RecPtr;
-
- targetPagePtr = RecPtr - (RecPtr % XLOG_BLCKSZ);
- targetRecOff = RecPtr % XLOG_BLCKSZ;
-
- /*
- * Read the page containing the record into state->readBuf. Request enough
- * byte to cover the whole record header, or at least the part of it that
- * fits on the same page.
- */
- while (XLogNeedData(state, targetPagePtr,
- Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
- targetRecOff != 0))
- {
- if (!state->routine.page_read(state, state->readPagePtr, state->readLen,
- RecPtr, state->readBuf))
- break;
- }
-
- if (!state->page_verified)
- goto err;
-
- /*
- * We have at least the page header, so we can examine it now.
- */
- pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
- if (targetRecOff == 0)
- {
- /*
- * At page start, so skip over page header.
- */
- RecPtr += pageHeaderSize;
- targetRecOff = pageHeaderSize;
- }
- else if (targetRecOff < pageHeaderSize)
- {
- report_invalid_record(state, "invalid record offset at %X/%X",
- LSN_FORMAT_ARGS(RecPtr));
- goto err;
- }
-
- if ((((XLogPageHeader) state->readBuf)->xlp_info & XLP_FIRST_IS_CONTRECORD) &&
- targetRecOff == pageHeaderSize)
- {
- report_invalid_record(state, "contrecord is requested by %X/%X",
- LSN_FORMAT_ARGS(RecPtr));
- goto err;
- }
-
- /* XLogNeedData has verified the page header */
- Assert(pageHeaderSize <= state->readLen);
-
- /*
- * Read the record length.
- *
- * NB: Even though we use an XLogRecord pointer here, the whole record
- * header might not fit on this page. xl_tot_len is the first field of the
- * struct, so it must be on this page (the records are MAXALIGNed), but we
- * cannot access any other fields until we've verified that we got the
- * whole header.
- */
- record = (XLogRecord *) (state->readBuf + RecPtr % XLOG_BLCKSZ);
- total_len = record->xl_tot_len;
-
- /*
- * If the whole record header is on this page, validate it immediately.
- * Otherwise do just a basic sanity check on xl_tot_len, and validate the
- * rest of the header after reading it from the next page. The xl_tot_len
- * check is necessary here to ensure that we enter the "Need to reassemble
- * record" code path below; otherwise we might fail to apply
- * ValidXLogRecordHeader at all.
- */
- if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
- {
- if (!ValidXLogRecordHeader(state, RecPtr, state->ReadRecPtr, record,
- randAccess))
- goto err;
- gotheader = true;
- }
- else
- {
- /* XXX: more validation should be done here */
- if (total_len < SizeOfXLogRecord)
- {
- report_invalid_record(state,
- "invalid record length at %X/%X: wanted %u, got %u",
- LSN_FORMAT_ARGS(RecPtr),
- (uint32) SizeOfXLogRecord, total_len);
- goto err;
- }
- gotheader = false;
- }
-
- len = XLOG_BLCKSZ - RecPtr % XLOG_BLCKSZ;
- if (total_len > len)
+ switch (state->readRecordState)
{
- /* Need to reassemble record */
- char *contdata;
- XLogPageHeader pageHeader;
- char *buffer;
- uint32 gotlen;
-
- /*
- * Enlarge readRecordBuf as needed.
- */
- if (total_len > state->readRecordBufSize &&
- !allocate_recordbuf(state, total_len))
- {
- /* We treat this as a "bogus data" condition */
- report_invalid_record(state, "record length %u at %X/%X too long",
- total_len, LSN_FORMAT_ARGS(RecPtr));
- goto err;
- }
+ case XLREAD_NEXT_RECORD:
+ ResetDecoder(state);
- /* Copy the first fragment of the record from the first page. */
- memcpy(state->readRecordBuf,
- state->readBuf + RecPtr % XLOG_BLCKSZ, len);
- buffer = state->readRecordBuf + len;
- gotlen = len;
-
- do
- {
- int rest_len = total_len - gotlen;
-
- /* Calculate pointer to beginning of next page */
- targetPagePtr += XLOG_BLCKSZ;
-
- /* Wait for the next page to become available */
- while (XLogNeedData(state, targetPagePtr,
- Min(rest_len, XLOG_BLCKSZ),
- false))
+ if (state->ReadRecPtr != InvalidXLogRecPtr)
{
- if (!state->routine.page_read(state, state->readPagePtr,
- state->readLen,
- state->ReadRecPtr,
- state->readBuf))
- break;
+ /* read the record after the one we just read */
+
+ /*
+ * EndRecPtr is pointing to end+1 of the previous WAL record.
+ * If we're at a page boundary, no more records can fit on the
+ * current page. We must skip over the page header, but we
+ * can't do that until we've read in the page, since the header
+ * size is variable.
+ */
+ state->PrevRecPtr = state->ReadRecPtr;
+ state->ReadRecPtr = state->EndRecPtr;
+ }
+ else
+ {
+ /*
+ * Caller supplied a position to start at.
+ *
+ * In this case, EndRecPtr should already be pointing to a
+ * valid record starting position.
+ */
+ Assert(XRecOffIsValid(state->EndRecPtr));
+ state->ReadRecPtr = state->EndRecPtr;
+
+ /*
+ * We cannot verify the previous-record pointer when we're
+ * seeking to a particular record. Reset PrevRecPtr so that we
+ * won't try doing that.
+ */
+ state->PrevRecPtr = InvalidXLogRecPtr;
+ state->EndRecPtr = InvalidXLogRecPtr; /* to be tidy */
}
+ state->record_verified = false;
+ state->readRecordState = XLREAD_TOT_LEN;
+ /* fall through */
+
+ case XLREAD_TOT_LEN:
+ {
+ uint32 total_len;
+ uint32 pageHeaderSize;
+ XLogRecPtr targetPagePtr;
+ uint32 targetRecOff;
+ XLogPageHeader pageHeader;
+
+ targetPagePtr =
+ state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
+ targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+
+ /*
+ * Check if we have enough data. For the first record in the page,
+ * the requesting length doesn't contain page header.
+ */
+ if (XLogNeedData(state, targetPagePtr,
+ Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+ targetRecOff != 0))
+ return XLREAD_NEED_DATA;
+
+ /* error out if caller supplied bogus page */
if (!state->page_verified)
goto err;
- Assert(SizeOfXLogShortPHD <= state->readLen);
+ /* examine page header now. */
+ pageHeaderSize =
+ XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+ if (targetRecOff == 0)
+ {
+ /* At page start, so skip over page header. */
+ state->ReadRecPtr += pageHeaderSize;
+ targetRecOff = pageHeaderSize;
+ }
+ else if (targetRecOff < pageHeaderSize)
+ {
+ report_invalid_record(state, "invalid record offset at %X/%X",
+ LSN_FORMAT_ARGS(state->ReadRecPtr));
+ goto err;
+ }
- /* Check that the continuation on next page looks valid */
pageHeader = (XLogPageHeader) state->readBuf;
- if (!(pageHeader->xlp_info & XLP_FIRST_IS_CONTRECORD))
+ if ((pageHeader->xlp_info & XLP_FIRST_IS_CONTRECORD) &&
+ targetRecOff == pageHeaderSize)
{
- report_invalid_record(state,
- "there is no contrecord flag at %X/%X",
- LSN_FORMAT_ARGS(RecPtr));
+ report_invalid_record(state, "contrecord is requested by %X/%X",
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr);
goto err;
}
- /*
- * Cross-check that xlp_rem_len agrees with how much of the record
- * we expect there to be left.
- */
- if (pageHeader->xlp_rem_len == 0 ||
- total_len != (pageHeader->xlp_rem_len + gotlen))
- {
- report_invalid_record(state,
- "invalid contrecord length %u (expected %lld) at %X/%X",
- pageHeader->xlp_rem_len,
- ((long long) total_len) - gotlen,
- LSN_FORMAT_ARGS(RecPtr));
- goto err;
- }
-
- /* Append the continuation from this page to the buffer */
- pageHeaderSize = XLogPageHeaderSize(pageHeader);
-
+ /* XLogNeedData has verified the page header */
Assert(pageHeaderSize <= state->readLen);
- contdata = (char *) state->readBuf + pageHeaderSize;
- len = XLOG_BLCKSZ - pageHeaderSize;
- if (pageHeader->xlp_rem_len < len)
- len = pageHeader->xlp_rem_len;
+ /*
+ * Read the record length.
+ *
+ * NB: Even though we use an XLogRecord pointer here, the whole
+ * record header might not fit on this page. xl_tot_len is the first
+ * field of the struct, so it must be on this page (the records are
+ * MAXALIGNed), but we cannot access any other fields until we've
+ * verified that we got the whole header.
+ */
+ prec = (XLogRecord *) (state->readBuf +
+ state->ReadRecPtr % XLOG_BLCKSZ);
+ total_len = prec->xl_tot_len;
- Assert (pageHeaderSize + len <= state->readLen);
- memcpy(buffer, (char *) contdata, len);
- buffer += len;
- gotlen += len;
+ /*
+ * If the whole record header is on this page, validate it
+ * immediately. Otherwise do just a basic sanity check on
+ * xl_tot_len, and validate the rest of the header after reading it
+ * from the next page. The xl_tot_len check is necessary here to
+ * ensure that we enter the XLREAD_CONTINUATION state below;
+ * otherwise we might fail to apply ValidXLogRecordHeader at all.
+ */
+ if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
+ {
+ if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
+ state->PrevRecPtr, prec))
+ goto err;
- /* If we just reassembled the record header, validate it. */
- if (!gotheader)
+ state->record_verified = true;
+ }
+ else
{
- record = (XLogRecord *) state->readRecordBuf;
- if (!ValidXLogRecordHeader(state, RecPtr, state->ReadRecPtr,
- record, randAccess))
+ /* XXX: more validation should be done here */
+ if (total_len < SizeOfXLogRecord)
+ {
+ report_invalid_record(state,
+ "invalid record length at %X/%X: wanted %u, got %u",
+ LSN_FORMAT_ARGS(state->ReadRecPtr),
+ (uint32) SizeOfXLogRecord, total_len);
goto err;
- gotheader = true;
+ }
}
- } while (gotlen < total_len);
-
- Assert(gotheader);
- record = (XLogRecord *) state->readRecordBuf;
- if (!ValidXLogRecord(state, record, RecPtr))
- goto err;
+ /*
+ * Wait for the rest of the record, or the part of the record that
+ * fit on the first page if crossed a page boundary, to become
+ * available.
+ */
+ state->recordGotLen = 0;
+ state->recordRemainLen = total_len;
+ state->readRecordState = XLREAD_FIRST_FRAGMENT;
+ }
+ /* fall through */
- pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
- state->ReadRecPtr = RecPtr;
- state->EndRecPtr = targetPagePtr + pageHeaderSize
- + MAXALIGN(pageHeader->xlp_rem_len);
- }
- else
- {
- /* Wait for the record data to become available */
- while (XLogNeedData(state, targetPagePtr,
- Min(targetRecOff + total_len, XLOG_BLCKSZ), true))
+ case XLREAD_FIRST_FRAGMENT:
{
- if (!state->routine.page_read(state, state->readPagePtr,
- state->readLen,
- state->ReadRecPtr, state->readBuf))
+ uint32 total_len = state->recordRemainLen;
+ uint32 request_len;
+ uint32 record_len;
+ XLogRecPtr targetPagePtr;
+ uint32 targetRecOff;
+
+ /*
+ * Wait for the rest of the record on the first page to become
+ * available
+ */
+ targetPagePtr =
+ state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
+ targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+
+ request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ);
+ record_len = request_len - targetRecOff;
+
+ /* ReadRecPtr contains page header */
+ Assert (targetRecOff != 0);
+ if (XLogNeedData(state, targetPagePtr, request_len, true))
+ return XLREAD_NEED_DATA;
+
+ /* error out if caller supplied bogus page */
+ if (!state->page_verified)
+ goto err;
+
+ prec = (XLogRecord *) (state->readBuf + targetRecOff);
+
+ /* validate record header if not yet */
+ if (!state->record_verified && record_len >= SizeOfXLogRecord)
+ {
+ if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
+ state->PrevRecPtr, prec))
+ goto err;
+
+ state->record_verified = true;
+ }
+
+
+ if (total_len == record_len)
+ {
+ /* Record does not cross a page boundary */
+ Assert(state->record_verified);
+
+ if (!ValidXLogRecord(state, prec, state->ReadRecPtr))
+ goto err;
+
+ state->record_verified = true; /* to be tidy */
+
+ /* We already checked the header earlier */
+ state->EndRecPtr = state->ReadRecPtr + MAXALIGN(record_len);
+
+ *record = prec;
+ state->readRecordState = XLREAD_NEXT_RECORD;
break;
+ }
+
+ /*
+ * The record continues on the next page. Need to reassemble
+ * record
+ */
+ Assert(total_len > record_len);
+
+ /* Enlarge readRecordBuf as needed. */
+ if (total_len > state->readRecordBufSize &&
+ !allocate_recordbuf(state, total_len))
+ {
+ /* We treat this as a "bogus data" condition */
+ report_invalid_record(state,
+ "record length %u at %X/%X too long",
+ total_len,
+ LSN_FORMAT_ARGS(state->ReadRecPtr));
+ goto err;
+ }
+
+ /* Copy the first fragment of the record from the first page. */
+ memcpy(state->readRecordBuf, state->readBuf + targetRecOff,
+ record_len);
+ state->recordGotLen += record_len;
+ state->recordRemainLen -= record_len;
+
+ /* Calculate pointer to beginning of next page */
+ state->recordContRecPtr = state->ReadRecPtr + record_len;
+ Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0);
+
+ state->readRecordState = XLREAD_CONTINUATION;
}
+ /* fall through */
+
+ case XLREAD_CONTINUATION:
+ {
+ XLogPageHeader pageHeader;
+ uint32 pageHeaderSize;
+ XLogRecPtr targetPagePtr;
+
+ /* we enter this state only if we haven't read the whole record. */
+ Assert (state->recordRemainLen > 0);
+
+ while(state->recordRemainLen > 0)
+ {
+ char *contdata;
+ uint32 request_len;
+ uint32 record_len;
+
+ /* Wait for the next page to become available */
+ targetPagePtr = state->recordContRecPtr;
+
+ /* this request contains page header */
+ Assert (targetPagePtr != 0);
+ if (XLogNeedData(state, targetPagePtr,
+ Min(state->recordRemainLen, XLOG_BLCKSZ),
+ false))
+ return XLREAD_NEED_DATA;
+
+ if (!state->page_verified)
+ goto err;
+
+ Assert(SizeOfXLogShortPHD <= state->readLen);
- if (!state->page_verified)
- goto err;
+ /* Check that the continuation on next page looks valid */
+ pageHeader = (XLogPageHeader) state->readBuf;
+ if (!(pageHeader->xlp_info & XLP_FIRST_IS_CONTRECORD))
+ {
+ report_invalid_record(
+ state,
+ "there is no contrecord flag at %X/%X reading %X/%X",
+ (uint32) (state->recordContRecPtr >> 32),
+ (uint32) state->recordContRecPtr,
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr);
+ goto err;
+ }
- /* Record does not cross a page boundary */
- if (!ValidXLogRecord(state, record, RecPtr))
- goto err;
+ /*
+ * Cross-check that xlp_rem_len agrees with how much of the
+ * record we expect there to be left.
+ */
+ if (pageHeader->xlp_rem_len == 0 ||
+ pageHeader->xlp_rem_len != state->recordRemainLen)
+ {
+ report_invalid_record(
+ state,
+ "invalid contrecord length %u at %X/%X reading %X/%X, expected %u",
+ pageHeader->xlp_rem_len,
+ (uint32) (state->recordContRecPtr >> 32),
+ (uint32) state->recordContRecPtr,
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr,
+ state->recordRemainLen);
+ goto err;
+ }
- state->EndRecPtr = RecPtr + MAXALIGN(total_len);
+ /* Append the continuation from this page to the buffer */
+ pageHeaderSize = XLogPageHeaderSize(pageHeader);
- state->ReadRecPtr = RecPtr;
+ /*
+ * XLogNeedData should have ensured that the whole page header
+ * was read
+ */
+ Assert(state->readLen >= pageHeaderSize);
+
+ contdata = (char *) state->readBuf + pageHeaderSize;
+ record_len = XLOG_BLCKSZ - pageHeaderSize;
+ if (pageHeader->xlp_rem_len < record_len)
+ record_len = pageHeader->xlp_rem_len;
+
+ request_len = record_len + pageHeaderSize;
+
+ /* XLogNeedData should have ensured all needed data was read */
+ Assert (state->readLen >= request_len);
+
+ memcpy(state->readRecordBuf + state->recordGotLen,
+ (char *) contdata, record_len);
+ state->recordGotLen += record_len;
+ state->recordRemainLen -= record_len;
+
+ /* If we just reassembled the record header, validate it. */
+ if (!state->record_verified)
+ {
+ Assert(state->recordGotLen >= SizeOfXLogRecord);
+ if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
+ state->PrevRecPtr,
+ (XLogRecord *) state->readRecordBuf))
+ goto err;
+
+ state->record_verified = true;
+ }
+
+ /* Calculate pointer to beginning of next page, and continue */
+ state->recordContRecPtr += XLOG_BLCKSZ;
+ }
+
+ /* targetPagePtr is pointing the last-read page here */
+ prec = (XLogRecord *) state->readRecordBuf;
+ if (!ValidXLogRecord(state, prec, state->ReadRecPtr))
+ goto err;
+
+ pageHeaderSize =
+ XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+ state->EndRecPtr = targetPagePtr + pageHeaderSize
+ + MAXALIGN(pageHeader->xlp_rem_len);
+
+ *record = prec;
+ state->readRecordState = XLREAD_NEXT_RECORD;
+ break;
+ }
}
/*
* Special processing if it's an XLOG SWITCH record
*/
- if (record->xl_rmid == RM_XLOG_ID &&
- (record->xl_info & ~XLR_INFO_MASK) == XLOG_SWITCH)
+ if ((*record)->xl_rmid == RM_XLOG_ID &&
+ ((*record)->xl_info & ~XLR_INFO_MASK) == XLOG_SWITCH)
{
/* Pretend it extends to end of segment */
state->EndRecPtr += state->segcxt.ws_segsize - 1;
state->EndRecPtr -= XLogSegmentOffset(state->EndRecPtr, state->segcxt.ws_segsize);
}
- if (DecodeXLogRecord(state, record, errormsg))
- return record;
- else
- return NULL;
+ Assert (!*record || state->readLen >= 0);
+ if (DecodeXLogRecord(state, *record, errormsg))
+ return XLREAD_SUCCESS;
+
+ *record = NULL;
+ return XLREAD_FAIL;
err:
/*
- * Invalidate the read state. We might read from a different source after
+ * Invalidate the read page. We might read from a different source after
* failure.
*/
XLogReaderInvalReadState(state);
@@ -572,7 +703,8 @@ err:
if (state->errormsg_buf[0] != '\0')
*errormsg = state->errormsg_buf;
- return NULL;
+ *record = NULL;
+ return XLREAD_FAIL;
}
/*
@@ -724,11 +856,12 @@ XLogReaderInvalReadState(XLogReaderState *state)
*
* This is just a convenience subroutine to avoid duplicated code in
* XLogReadRecord. It's not intended for use from anywhere else.
+ *
+ * If PrevRecPtr is valid, the xl_prev is is cross-checked with it.
*/
static bool
ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
- XLogRecPtr PrevRecPtr, XLogRecord *record,
- bool randAccess)
+ XLogRecPtr PrevRecPtr, XLogRecord *record)
{
if (record->xl_tot_len < SizeOfXLogRecord)
{
@@ -745,7 +878,7 @@ ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
record->xl_rmid, LSN_FORMAT_ARGS(RecPtr));
return false;
}
- if (randAccess)
+ if (PrevRecPtr == InvalidXLogRecPtr)
{
/*
* We can't exactly verify the prev-link, but surely it should be less
@@ -975,11 +1108,14 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
* XLogReadRecord() will read the next valid record.
*/
XLogRecPtr
-XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
+XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
+ XLogFindNextRecordCB read_page, void *private)
{
XLogRecPtr tmpRecPtr;
XLogRecPtr found = InvalidXLogRecPtr;
XLogPageHeader header;
+ XLogRecord *record;
+ XLogReadRecordResult result;
char *errormsg;
Assert(!XLogRecPtrIsInvalid(RecPtr));
@@ -1012,9 +1148,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
while(XLogNeedData(state, targetPagePtr, targetRecOff,
targetRecOff != 0))
{
- if (!state->routine.page_read(state, state->readPagePtr,
- state->readLen,
- state->ReadRecPtr, state->readBuf))
+ if (!read_page(state, private))
break;
}
@@ -1066,8 +1200,16 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
* or we just jumped over the remaining data of a continuation.
*/
XLogBeginRead(state, tmpRecPtr);
- while (XLogReadRecord(state, &errormsg) != NULL)
+ while ((result = XLogReadRecord(state, &record, &errormsg)) !=
+ XLREAD_FAIL)
{
+ if (result == XLREAD_NEED_DATA)
+ {
+ if (!read_page(state, private))
+ goto err;
+ continue;
+ }
+
/* past the record we've found, break out */
if (RecPtr <= state->ReadRecPtr)
{
@@ -1087,9 +1229,9 @@ err:
#endif /* FRONTEND */
/*
- * Helper function to ease writing of XLogRoutine->page_read callbacks.
- * If this function is used, caller must supply a segment_open callback in
- * 'state', as that is used here.
+ * Helper function to ease writing of page_read callback.
+ * If this function is used, caller must supply a segment_open callback and
+ * segment_close callback as that is used here.
*
* Read 'count' bytes into 'buf', starting at location 'startptr', from WAL
* fetched from timeline 'tli'.
@@ -1102,6 +1244,7 @@ err:
*/
bool
WALRead(XLogReaderState *state,
+ WALSegmentOpenCB segopenfn, WALSegmentCloseCB segclosefn,
char *buf, XLogRecPtr startptr, Size count, TimeLineID tli,
WALReadError *errinfo)
{
@@ -1133,10 +1276,10 @@ WALRead(XLogReaderState *state,
XLogSegNo nextSegNo;
if (state->seg.ws_file >= 0)
- state->routine.segment_close(state);
+ segclosefn(state);
XLByteToSeg(recptr, nextSegNo, state->segcxt.ws_segsize);
- state->routine.segment_open(state, nextSegNo, &tli);
+ segopenfn(state, nextSegNo, &tli);
/* This shouldn't happen -- indicates a bug in segment_open */
Assert(state->seg.ws_file >= 0);
diff --git a/src/backend/access/transam/xlogutils.c b/src/backend/access/transam/xlogutils.c
index 46eda33f25..b003990745 100644
--- a/src/backend/access/transam/xlogutils.c
+++ b/src/backend/access/transam/xlogutils.c
@@ -686,8 +686,7 @@ XLogTruncateRelation(RelFileNode rnode, ForkNumber forkNum,
void
XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wantLength)
{
- const XLogRecPtr lastReadPage = state->seg.ws_segno *
- state->segcxt.ws_segsize + state->readLen;
+ const XLogRecPtr lastReadPage = state->readPagePtr;
Assert(wantPage != InvalidXLogRecPtr && wantPage % XLOG_BLCKSZ == 0);
Assert(wantLength <= XLOG_BLCKSZ);
@@ -702,7 +701,7 @@ XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wa
* current TLI has since become historical.
*/
if (lastReadPage == wantPage &&
- state->readLen != 0 &&
+ state->page_verified &&
lastReadPage + state->readLen >= wantPage + Min(wantLength, XLOG_BLCKSZ - 1))
return;
@@ -788,6 +787,7 @@ wal_segment_open(XLogReaderState *state, XLogSegNo nextSegNo,
char path[MAXPGPATH];
XLogFilePath(path, tli, nextSegNo, state->segcxt.ws_segsize);
+ elog(LOG, "HOGE: %lu, %d => %s", nextSegNo, tli, path);
state->seg.ws_file = BasicOpenFile(path, O_RDONLY | PG_BINARY);
if (state->seg.ws_file >= 0)
return;
@@ -825,9 +825,11 @@ wal_segment_close(XLogReaderState *state)
* loop for now.
*/
bool
-read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *cur_page)
+read_local_xlog_page(XLogReaderState *state)
{
+ XLogRecPtr targetPagePtr = state->readPagePtr;
+ int reqLen = state->readLen;
+ char *cur_page = state->readBuf;
XLogRecPtr read_upto,
loc;
TimeLineID tli;
@@ -940,11 +942,12 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
* as 'count', read the whole page anyway. It's guaranteed to be
* zero-padded up to the page boundary if it's incomplete.
*/
- if (!WALRead(state, cur_page, targetPagePtr, XLOG_BLCKSZ, tli,
- &errinfo))
+ if (!WALRead(state, wal_segment_open, wal_segment_close,
+ cur_page, targetPagePtr, XLOG_BLCKSZ, tli, &errinfo))
WALReadRaiseError(&errinfo);
/* number of valid bytes in the buffer */
+ state->readPagePtr = targetPagePtr;
state->readLen = count;
return true;
}
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index 37b75deb72..9cbd8afa49 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -148,7 +148,8 @@ StartupDecodingContext(List *output_plugin_options,
TransactionId xmin_horizon,
bool need_full_snapshot,
bool fast_forward,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress)
@@ -198,11 +199,12 @@ StartupDecodingContext(List *output_plugin_options,
ctx->slot = slot;
- ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, xl_routine, ctx);
+ ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, cleanup_cb);
if (!ctx->reader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
errmsg("out of memory")));
+ ctx->page_read = page_read;
ctx->reorder = ReorderBufferAllocate();
ctx->snapshot_builder =
@@ -319,7 +321,8 @@ CreateInitDecodingContext(const char *plugin,
List *output_plugin_options,
bool need_full_snapshot,
XLogRecPtr restart_lsn,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress)
@@ -422,7 +425,7 @@ CreateInitDecodingContext(const char *plugin,
ctx = StartupDecodingContext(NIL, restart_lsn, xmin_horizon,
need_full_snapshot, false,
- xl_routine, prepare_write, do_write,
+ page_read, cleanup_cb, prepare_write, do_write,
update_progress);
/* call output plugin initialization callback */
@@ -476,7 +479,8 @@ LogicalDecodingContext *
CreateDecodingContext(XLogRecPtr start_lsn,
List *output_plugin_options,
bool fast_forward,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress)
@@ -528,8 +532,8 @@ CreateDecodingContext(XLogRecPtr start_lsn,
ctx = StartupDecodingContext(output_plugin_options,
start_lsn, InvalidTransactionId, false,
- fast_forward, xl_routine, prepare_write,
- do_write, update_progress);
+ fast_forward, page_read, cleanup_cb,
+ prepare_write, do_write, update_progress);
/* call output plugin initialization callback */
old_context = MemoryContextSwitchTo(ctx->context);
@@ -585,7 +589,13 @@ DecodingContextFindStartpoint(LogicalDecodingContext *ctx)
char *err = NULL;
/* the read_page callback waits for new WAL */
- record = XLogReadRecord(ctx->reader, &err);
+ while (XLogReadRecord(ctx->reader, &record, &err) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!ctx->page_read(ctx->reader))
+ break;
+ }
+
if (err)
elog(ERROR, "%s", err);
if (!record)
diff --git a/src/backend/replication/logical/logicalfuncs.c b/src/backend/replication/logical/logicalfuncs.c
index f7e055874e..b69f6911c5 100644
--- a/src/backend/replication/logical/logicalfuncs.c
+++ b/src/backend/replication/logical/logicalfuncs.c
@@ -233,9 +233,8 @@ pg_logical_slot_get_changes_guts(FunctionCallInfo fcinfo, bool confirm, bool bin
ctx = CreateDecodingContext(InvalidXLogRecPtr,
options,
false,
- XL_ROUTINE(.page_read = read_local_xlog_page,
- .segment_open = wal_segment_open,
- .segment_close = wal_segment_close),
+ read_local_xlog_page,
+ wal_segment_close,
LogicalOutputPrepareWrite,
LogicalOutputWrite, NULL);
@@ -284,7 +283,13 @@ pg_logical_slot_get_changes_guts(FunctionCallInfo fcinfo, bool confirm, bool bin
XLogRecord *record;
char *errm = NULL;
- record = XLogReadRecord(ctx->reader, &errm);
+ while (XLogReadRecord(ctx->reader, &record, &errm) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!ctx->page_read(ctx->reader))
+ break;
+ }
+
if (errm)
elog(ERROR, "%s", errm);
diff --git a/src/backend/replication/slotfuncs.c b/src/backend/replication/slotfuncs.c
index 9817b44113..deb7cf15eb 100644
--- a/src/backend/replication/slotfuncs.c
+++ b/src/backend/replication/slotfuncs.c
@@ -153,9 +153,8 @@ create_logical_replication_slot(char *name, char *plugin,
ctx = CreateInitDecodingContext(plugin, NIL,
false, /* just catalogs is OK */
restart_lsn,
- XL_ROUTINE(.page_read = read_local_xlog_page,
- .segment_open = wal_segment_open,
- .segment_close = wal_segment_close),
+ read_local_xlog_page,
+ wal_segment_close,
NULL, NULL, NULL);
/*
@@ -512,9 +511,8 @@ pg_logical_replication_slot_advance(XLogRecPtr moveto)
ctx = CreateDecodingContext(InvalidXLogRecPtr,
NIL,
true, /* fast_forward */
- XL_ROUTINE(.page_read = read_local_xlog_page,
- .segment_open = wal_segment_open,
- .segment_close = wal_segment_close),
+ read_local_xlog_page,
+ wal_segment_close,
NULL, NULL, NULL);
/*
@@ -536,7 +534,13 @@ pg_logical_replication_slot_advance(XLogRecPtr moveto)
* Read records. No changes are generated in fast_forward mode,
* but snapbuilder/slot statuses are updated properly.
*/
- record = XLogReadRecord(ctx->reader, &errm);
+ while (XLogReadRecord(ctx->reader, &record, &errm) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!ctx->page_read(ctx->reader))
+ break;
+ }
+
if (errm)
elog(ERROR, "%s", errm);
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index c36b91ae5e..062093727d 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -580,10 +580,7 @@ StartReplication(StartReplicationCmd *cmd)
/* create xlogreader for physical replication */
xlogreader =
- XLogReaderAllocate(wal_segment_size, NULL,
- XL_ROUTINE(.segment_open = WalSndSegmentOpen,
- .segment_close = wal_segment_close),
- NULL);
+ XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
if (!xlogreader)
ereport(ERROR,
@@ -807,9 +804,11 @@ StartReplication(StartReplicationCmd *cmd)
* set every time WAL is flushed.
*/
static bool
-logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetRecPtr, char *cur_page)
+logical_read_xlog_page(XLogReaderState *state)
{
+ XLogRecPtr targetPagePtr = state->readPagePtr;
+ int reqLen = state->readLen;
+ char *cur_page = state->readBuf;
XLogRecPtr flushptr;
int count;
WALReadError errinfo;
@@ -837,7 +836,7 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
count = flushptr - targetPagePtr; /* part of the page available */
/* now actually read the data, we know it's there */
- if (!WALRead(state,
+ if (!WALRead(state, WalSndSegmentOpen, wal_segment_close,
cur_page,
targetPagePtr,
XLOG_BLCKSZ,
@@ -1011,9 +1010,8 @@ CreateReplicationSlot(CreateReplicationSlotCmd *cmd)
ctx = CreateInitDecodingContext(cmd->plugin, NIL, need_full_snapshot,
InvalidXLogRecPtr,
- XL_ROUTINE(.page_read = logical_read_xlog_page,
- .segment_open = WalSndSegmentOpen,
- .segment_close = wal_segment_close),
+ logical_read_xlog_page,
+ wal_segment_close,
WalSndPrepareWrite, WalSndWriteData,
WalSndUpdateProgress);
@@ -1171,9 +1169,8 @@ StartLogicalReplication(StartReplicationCmd *cmd)
*/
logical_decoding_ctx =
CreateDecodingContext(cmd->startpoint, cmd->options, false,
- XL_ROUTINE(.page_read = logical_read_xlog_page,
- .segment_open = WalSndSegmentOpen,
- .segment_close = wal_segment_close),
+ logical_read_xlog_page,
+ wal_segment_close,
WalSndPrepareWrite, WalSndWriteData,
WalSndUpdateProgress);
xlogreader = logical_decoding_ctx->reader;
@@ -2749,7 +2746,7 @@ XLogSendPhysical(void)
enlargeStringInfo(&output_message, nbytes);
retry:
- if (!WALRead(xlogreader,
+ if (!WALRead(xlogreader, WalSndSegmentOpen, wal_segment_close,
&output_message.data[output_message.len],
startptr,
nbytes,
@@ -2847,7 +2844,12 @@ XLogSendLogical(void)
*/
WalSndCaughtUp = false;
- record = XLogReadRecord(logical_decoding_ctx->reader, &errm);
+ while (XLogReadRecord(logical_decoding_ctx->reader, &record, &errm) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!logical_decoding_ctx->page_read(logical_decoding_ctx->reader))
+ break;
+ }
/* xlog record was invalid */
if (errm != NULL)
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index cf119848b0..712c85281c 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -41,15 +41,9 @@ static int xlogreadfd = -1;
static XLogSegNo xlogreadsegno = -1;
static char xlogfpath[MAXPGPATH];
-typedef struct XLogPageReadPrivate
-{
- const char *restoreCommand;
- int tliIndex;
-} XLogPageReadPrivate;
-
-static bool SimpleXLogPageRead(XLogReaderState *xlogreader,
- XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
+static bool SimpleXLogPageRead(XLogReaderState *xlogreader,
+ const char *datadir, int *tliIndex,
+ const char *restoreCommand);
/*
* Read WAL from the datadir/pg_wal, starting from 'startpoint' on timeline
@@ -66,20 +60,22 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
XLogRecord *record;
XLogReaderState *xlogreader;
char *errormsg;
- XLogPageReadPrivate private;
- private.tliIndex = tliIndex;
- private.restoreCommand = restoreCommand;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir,
- XL_ROUTINE(.page_read = &SimpleXLogPageRead),
- &private);
+ xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+
if (xlogreader == NULL)
pg_fatal("out of memory");
XLogBeginRead(xlogreader, startpoint);
do
{
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!SimpleXLogPageRead(xlogreader, datadir,
+ &tliIndex, restoreCommand))
+ break;
+ }
if (record == NULL)
{
@@ -123,19 +119,19 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex,
XLogRecord *record;
XLogReaderState *xlogreader;
char *errormsg;
- XLogPageReadPrivate private;
XLogRecPtr endptr;
- private.tliIndex = tliIndex;
- private.restoreCommand = restoreCommand;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir,
- XL_ROUTINE(.page_read = &SimpleXLogPageRead),
- &private);
+ xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
XLogBeginRead(xlogreader, ptr);
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!SimpleXLogPageRead(xlogreader, datadir, &tliIndex, restoreCommand))
+ break;
+ }
if (record == NULL)
{
if (errormsg)
@@ -170,7 +166,6 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
XLogRecPtr searchptr;
XLogReaderState *xlogreader;
char *errormsg;
- XLogPageReadPrivate private;
/*
* The given fork pointer points to the end of the last common record,
@@ -186,11 +181,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
forkptr += SizeOfXLogShortPHD;
}
- private.tliIndex = tliIndex;
- private.restoreCommand = restoreCommand;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir,
- XL_ROUTINE(.page_read = &SimpleXLogPageRead),
- &private);
+ xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
@@ -200,7 +191,13 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
uint8 info;
XLogBeginRead(xlogreader, searchptr);
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!SimpleXLogPageRead(xlogreader, datadir,
+ &tliIndex, restoreCommand))
+ break;
+ }
if (record == NULL)
{
@@ -247,10 +244,11 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
/* XLogReader callback function, to read a WAL page */
static bool
-SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *readBuf)
+SimpleXLogPageRead(XLogReaderState *xlogreader, const char *datadir,
+ int *tliIndex, const char *restoreCommand)
{
- XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
+ XLogRecPtr targetPagePtr = xlogreader->readPagePtr;
+ char *readBuf = xlogreader->readBuf;
uint32 targetPageOff;
XLogRecPtr targetSegEnd;
XLogSegNo targetSegNo;
@@ -283,14 +281,14 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
* be done both forward and backward, consider also switching timeline
* accordingly.
*/
- while (private->tliIndex < targetNentries - 1 &&
- targetHistory[private->tliIndex].end < targetSegEnd)
- private->tliIndex++;
- while (private->tliIndex > 0 &&
- targetHistory[private->tliIndex].begin >= targetSegEnd)
- private->tliIndex--;
+ while (*tliIndex < targetNentries - 1 &&
+ targetHistory[*tliIndex].end < targetSegEnd)
+ (*tliIndex)++;
+ while (*tliIndex > 0 &&
+ targetHistory[*tliIndex].begin >= targetSegEnd)
+ (*tliIndex)--;
- XLogFileName(xlogfname, targetHistory[private->tliIndex].tli,
+ XLogFileName(xlogfname, targetHistory[*tliIndex].tli,
xlogreadsegno, WalSegSz);
snprintf(xlogfpath, MAXPGPATH, "%s/" XLOGDIR "/%s",
@@ -303,7 +301,7 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
/*
* If we have no restore_command to execute, then exit.
*/
- if (private->restoreCommand == NULL)
+ if (restoreCommand == NULL)
{
pg_log_error("could not open file \"%s\": %m", xlogfpath);
xlogreader->readLen = -1;
@@ -317,7 +315,7 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
xlogreadfd = RestoreArchivedFile(xlogreader->segcxt.ws_dir,
xlogfname,
WalSegSz,
- private->restoreCommand);
+ restoreCommand);
if (xlogreadfd < 0)
{
@@ -359,7 +357,7 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
Assert(targetSegNo == xlogreadsegno);
- xlogreader->seg.ws_tli = targetHistory[private->tliIndex].tli;
+ xlogreader->seg.ws_tli = targetHistory[*tliIndex].tli;
xlogreader->readLen = XLOG_BLCKSZ;
return true;
}
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index 0f0e0723a3..f514de6b13 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -330,12 +330,17 @@ WALDumpCloseSegment(XLogReaderState *state)
state->seg.ws_file = -1;
}
-/* pg_waldump's XLogReaderRoutine->page_read callback */
+/*
+ * pg_waldump's WAL page rader, also used as page_read callback for
+ * XLogFindNextRecord
+ */
static bool
-WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetPtr, char *readBuff)
+WALDumpReadPage(XLogReaderState *state, void *priv)
{
- XLogDumpPrivate *private = state->private_data;
+ XLogRecPtr targetPagePtr = state->readPagePtr;
+ int reqLen = state->readLen;
+ char *readBuff = state->readBuf;
+ XLogDumpPrivate *private = (XLogDumpPrivate *) priv;
int count = XLOG_BLCKSZ;
WALReadError errinfo;
@@ -353,8 +358,8 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
}
}
- if (!WALRead(state, readBuff, targetPagePtr, count, private->timeline,
- &errinfo))
+ if (!WALRead(state, WALDumpOpenSegment, WALDumpCloseSegment,
+ readBuff, targetPagePtr, count, private->timeline, &errinfo))
{
WALOpenSegment *seg = &errinfo.wre_seg;
char fname[MAXPGPATH];
@@ -374,6 +379,7 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
(Size) errinfo.wre_req);
}
+ Assert(count >= state->readLen);
state->readLen = count;
return true;
}
@@ -1035,16 +1041,14 @@ main(int argc, char **argv)
/* we have everything we need, start reading */
xlogreader_state =
- XLogReaderAllocate(WalSegSz, waldir,
- XL_ROUTINE(.page_read = WALDumpReadPage,
- .segment_open = WALDumpOpenSegment,
- .segment_close = WALDumpCloseSegment),
- &private);
+ XLogReaderAllocate(WalSegSz, waldir, WALDumpCloseSegment);
+
if (!xlogreader_state)
fatal_error("out of memory");
/* first find a valid recptr to start from */
- first_record = XLogFindNextRecord(xlogreader_state, private.startptr);
+ first_record = XLogFindNextRecord(xlogreader_state, private.startptr,
+ &WALDumpReadPage, (void*) &private);
if (first_record == InvalidXLogRecPtr)
fatal_error("could not find a valid record after %X/%X",
@@ -1067,7 +1071,13 @@ main(int argc, char **argv)
for (;;)
{
/* try to read the next record */
- record = XLogReadRecord(xlogreader_state, &errormsg);
+ while (XLogReadRecord(xlogreader_state, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!WALDumpReadPage(xlogreader_state, (void *) &private))
+ break;
+ }
+
if (!record)
{
if (!config.follow || private.endptr_reached)
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index 23d9e70a59..4e9268b553 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -57,64 +57,15 @@ typedef struct WALSegmentContext
typedef struct XLogReaderState XLogReaderState;
-/* Function type definition for the read_page callback */
-typedef bool (*XLogPageReadCB) (XLogReaderState *xlogreader,
- XLogRecPtr targetPagePtr,
- int reqLen,
- XLogRecPtr targetRecPtr,
- char *readBuf);
+/* Function type definition for the segment cleanup callback */
+typedef void (*WALSegmentCleanupCB) (XLogReaderState *xlogreader);
+
+/* Function type definition for the open/close callbacks for WALRead() */
typedef void (*WALSegmentOpenCB) (XLogReaderState *xlogreader,
XLogSegNo nextSegNo,
TimeLineID *tli_p);
typedef void (*WALSegmentCloseCB) (XLogReaderState *xlogreader);
-typedef struct XLogReaderRoutine
-{
- /*
- * Data input callback
- *
- * This callback shall read at least reqLen valid bytes of the xlog page
- * starting at targetPagePtr, and store them in readBuf. The callback
- * shall return the number of bytes read (never more than XLOG_BLCKSZ), or
- * -1 on failure. The callback shall sleep, if necessary, to wait for the
- * requested bytes to become available. The callback will not be invoked
- * again for the same page unless more than the returned number of bytes
- * are needed.
- *
- * targetRecPtr is the position of the WAL record we're reading. Usually
- * it is equal to targetPagePtr + reqLen, but sometimes xlogreader needs
- * to read and verify the page or segment header, before it reads the
- * actual WAL record it's interested in. In that case, targetRecPtr can
- * be used to determine which timeline to read the page from.
- *
- * The callback shall set ->seg.ws_tli to the TLI of the file the page was
- * read from.
- */
- XLogPageReadCB page_read;
-
- /*
- * Callback to open the specified WAL segment for reading. ->seg.ws_file
- * shall be set to the file descriptor of the opened segment. In case of
- * failure, an error shall be raised by the callback and it shall not
- * return.
- *
- * "nextSegNo" is the number of the segment to be opened.
- *
- * "tli_p" is an input/output argument. WALRead() uses it to pass the
- * timeline in which the new segment should be found, but the callback can
- * use it to return the TLI that it actually opened.
- */
- WALSegmentOpenCB segment_open;
-
- /*
- * WAL segment close callback. ->seg.ws_file shall be set to a negative
- * number.
- */
- WALSegmentCloseCB segment_close;
-} XLogReaderRoutine;
-
-#define XL_ROUTINE(...) &(XLogReaderRoutine){__VA_ARGS__}
-
typedef struct
{
/* Is this block ref in use? */
@@ -144,12 +95,35 @@ typedef struct
uint16 data_bufsz;
} DecodedBkpBlock;
+/* Return code from XLogReadRecord */
+typedef enum XLogReadRecordResult
+{
+ XLREAD_SUCCESS, /* record is successfully read */
+ XLREAD_NEED_DATA, /* need more data. see XLogReadRecord. */
+ XLREAD_FAIL /* failed during reading a record */
+} XLogReadRecordResult;
+
+/*
+ * internal state of XLogReadRecord
+ *
+ * XLogReadState runs a state machine while reading a record. Theses states
+ * are not seen outside the function. Each state may repeat several times
+ * exiting requesting caller for new data. See the comment of XLogReadRecrod
+ * for details.
+ */
+typedef enum XLogReadRecordState {
+ XLREAD_NEXT_RECORD,
+ XLREAD_TOT_LEN,
+ XLREAD_FIRST_FRAGMENT,
+ XLREAD_CONTINUATION
+} XLogReadRecordState;
+
struct XLogReaderState
{
/*
* Operational callbacks
*/
- XLogReaderRoutine routine;
+ WALSegmentCleanupCB cleanup_cb;
/* ----------------------------------------
* Public parameters
@@ -162,18 +136,14 @@ struct XLogReaderState
*/
uint64 system_identifier;
- /*
- * Opaque data for callbacks to use. Not used by XLogReader.
- */
- void *private_data;
-
/*
* Start and end point of last record read. EndRecPtr is also used as the
* position to read next. Calling XLogBeginRead() sets EndRecPtr to the
* starting position and ReadRecPtr to invalid.
*/
- XLogRecPtr ReadRecPtr; /* start of last record read */
+ XLogRecPtr ReadRecPtr; /* start of last record read or being read */
XLogRecPtr EndRecPtr; /* end+1 of last record read */
+ XLogRecPtr PrevRecPtr; /* start of previous record read */
/* ----------------------------------------
* Communication with page reader
@@ -187,7 +157,9 @@ struct XLogReaderState
* the request, or -1 on error */
TimeLineID readPageTLI; /* TLI for data currently in readBuf */
char *readBuf; /* buffer to store data */
- bool page_verified; /* is the page on the buffer verified? */
+ bool page_verified; /* is the page header on the buffer verified? */
+ bool record_verified;/* is the current record header verified? */
+
/* ----------------------------------------
@@ -229,8 +201,6 @@ struct XLogReaderState
XLogRecPtr latestPagePtr;
TimeLineID latestPageTLI;
- /* beginning of the WAL record being read. */
- XLogRecPtr currRecPtr;
/* timeline to read it from, 0 if a lookup is required */
TimeLineID currTLI;
@@ -257,6 +227,15 @@ struct XLogReaderState
char *readRecordBuf;
uint32 readRecordBufSize;
+ /*
+ * XLogReadRecord() state
+ */
+ XLogReadRecordState readRecordState;/* state machine state */
+ int recordGotLen; /* amount of current record that has
+ * already been read */
+ int recordRemainLen; /* length of current record that remains */
+ XLogRecPtr recordContRecPtr; /* where the current record continues */
+
/* Buffer to hold error message */
char *errormsg_buf;
};
@@ -264,9 +243,7 @@ struct XLogReaderState
/* Get a new XLogReader */
extern XLogReaderState *XLogReaderAllocate(int wal_segment_size,
const char *waldir,
- XLogReaderRoutine *routine,
- void *private_data);
-extern XLogReaderRoutine *LocalXLogReaderRoutine(void);
+ WALSegmentCleanupCB cleanup_cb);
/* Free an XLogReader */
extern void XLogReaderFree(XLogReaderState *state);
@@ -274,12 +251,17 @@ extern void XLogReaderFree(XLogReaderState *state);
/* Position the XLogReader to given record */
extern void XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr);
#ifdef FRONTEND
-extern XLogRecPtr XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr);
+/* Function type definition for the read_page callback */
+typedef bool (*XLogFindNextRecordCB) (XLogReaderState *xlogreader,
+ void *private);
+extern XLogRecPtr XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
+ XLogFindNextRecordCB read_page, void *private);
#endif /* FRONTEND */
/* Read the next XLog record. Returns NULL on end-of-WAL or failure */
-extern struct XLogRecord *XLogReadRecord(XLogReaderState *state,
- char **errormsg);
+extern XLogReadRecordResult XLogReadRecord(XLogReaderState *state,
+ XLogRecord **record,
+ char **errormsg);
/* Validate a page */
extern bool XLogReaderValidatePageHeader(XLogReaderState *state,
@@ -299,6 +281,7 @@ typedef struct WALReadError
} WALReadError;
extern bool WALRead(XLogReaderState *state,
+ WALSegmentOpenCB segopenfn, WALSegmentCloseCB sgclosefn,
char *buf, XLogRecPtr startptr, Size count,
TimeLineID tli, WALReadError *errinfo);
diff --git a/src/include/access/xlogutils.h b/src/include/access/xlogutils.h
index 364a21c4ea..397fb27fc2 100644
--- a/src/include/access/xlogutils.h
+++ b/src/include/access/xlogutils.h
@@ -47,9 +47,7 @@ extern Buffer XLogReadBufferExtended(RelFileNode rnode, ForkNumber forknum,
extern Relation CreateFakeRelcacheEntry(RelFileNode rnode);
extern void FreeFakeRelcacheEntry(Relation fakerel);
-extern bool read_local_xlog_page(XLogReaderState *state,
- XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetRecPtr, char *cur_page);
+extern bool read_local_xlog_page(XLogReaderState *state);
extern void wal_segment_open(XLogReaderState *state,
XLogSegNo nextSegNo,
TimeLineID *tli_p);
diff --git a/src/include/pg_config_manual.h b/src/include/pg_config_manual.h
index f10ad0acd6..94749609b4 100644
--- a/src/include/pg_config_manual.h
+++ b/src/include/pg_config_manual.h
@@ -377,7 +377,7 @@
* Enable debugging print statements for WAL-related operations; see
* also the wal_debug GUC var.
*/
-/* #define WAL_DEBUG */
+#define WAL_DEBUG
/*
* Enable tracing of resource consumption during sort operations;
diff --git a/src/include/replication/logical.h b/src/include/replication/logical.h
index c253403372..b936f31393 100644
--- a/src/include/replication/logical.h
+++ b/src/include/replication/logical.h
@@ -29,6 +29,10 @@ typedef void (*LogicalOutputPluginWriterUpdateProgress) (struct LogicalDecodingC
TransactionId xid
);
+typedef struct LogicalDecodingContext LogicalDecodingContext;
+
+typedef bool (*LogicalDecodingXLogPageReadCB)(XLogReaderState *ctx);
+
typedef struct LogicalDecodingContext
{
/* memory context this is all allocated in */
@@ -39,6 +43,7 @@ typedef struct LogicalDecodingContext
/* infrastructure pieces for decoding */
XLogReaderState *reader;
+ LogicalDecodingXLogPageReadCB page_read;
struct ReorderBuffer *reorder;
struct SnapBuild *snapshot_builder;
@@ -105,14 +110,16 @@ extern LogicalDecodingContext *CreateInitDecodingContext(const char *plugin,
List *output_plugin_options,
bool need_full_snapshot,
XLogRecPtr restart_lsn,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress);
extern LogicalDecodingContext *CreateDecodingContext(XLogRecPtr start_lsn,
List *output_plugin_options,
bool fast_forward,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress);
--
2.27.0
----Next_Part(Thu_Mar__4_11_28_53_2021_014)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
filename="v17-0003-Remove-globals-readOff-readLen-and-readSegNo.patch"
^ permalink raw reply [nested|flat] 70+ messages in thread
* [PATCH v15 2/4] Move page-reader out of XLogReadRecord
@ 2019-09-10 03:58 Kyotaro Horiguchi <[email protected]>
0 siblings, 0 replies; 70+ messages in thread
From: Kyotaro Horiguchi @ 2019-09-10 03:58 UTC (permalink / raw)
This is the second step of removing callbacks from WAL record reader.
Since it is essential to take in additional data while reading a
record, the function have to ask caller for new data while keeping
working state. Thus the function is turned into a state machine.
---
src/backend/access/transam/twophase.c | 15 +-
src/backend/access/transam/xlog.c | 58 +-
src/backend/access/transam/xlogreader.c | 700 +++++++++++-------
src/backend/access/transam/xlogutils.c | 17 +-
src/backend/replication/logical/logical.c | 26 +-
.../replication/logical/logicalfuncs.c | 13 +-
src/backend/replication/slotfuncs.c | 18 +-
src/backend/replication/walsender.c | 32 +-
src/bin/pg_rewind/parsexlog.c | 84 +--
src/bin/pg_waldump/pg_waldump.c | 36 +-
src/include/access/xlogreader.h | 121 ++-
src/include/access/xlogutils.h | 4 +-
src/include/replication/logical.h | 11 +-
13 files changed, 648 insertions(+), 487 deletions(-)
diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index 9b2e59bf0e..b0d60a0d0f 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -1330,11 +1330,8 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
XLogReaderState *xlogreader;
char *errormsg;
- xlogreader = XLogReaderAllocate(wal_segment_size, NULL,
- XL_ROUTINE(.page_read = &read_local_xlog_page,
- .segment_open = &wal_segment_open,
- .segment_close = &wal_segment_close),
- NULL);
+ xlogreader = XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+
if (!xlogreader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
@@ -1342,7 +1339,13 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
errdetail("Failed while allocating a WAL reading processor.")));
XLogBeginRead(xlogreader, lsn);
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!read_local_xlog_page(xlogreader))
+ break;
+ }
+
if (record == NULL)
ereport(ERROR,
(errcode_for_file_access(),
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index e570e56a24..f9b0108602 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -831,13 +831,6 @@ static XLogSource currentSource = XLOG_FROM_ANY;
static bool lastSourceFailed = false;
static bool pendingWalRcvRestart = false;
-typedef struct XLogPageReadPrivate
-{
- int emode;
- bool fetching_ckpt; /* are we fetching a checkpoint record? */
- bool randAccess;
-} XLogPageReadPrivate;
-
/*
* These variables track when we last obtained some WAL data to process,
* and where we got it from. (XLogReceiptSource is initially the same as
@@ -912,8 +905,8 @@ static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath,
static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli,
XLogSource source, bool notfoundOk);
static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, XLogSource source);
-static bool XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
+static bool XLogPageRead(XLogReaderState *xlogreader,
+ bool fetching_ckpt, int emode, bool randAccess);
static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
bool fetching_ckpt, XLogRecPtr tliRecPtr);
static int emode_for_corrupt_record(int emode, XLogRecPtr RecPtr);
@@ -1225,8 +1218,7 @@ XLogInsertRecord(XLogRecData *rdata,
appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len);
if (!debug_reader)
- debug_reader = XLogReaderAllocate(wal_segment_size, NULL,
- XL_ROUTINE(), NULL);
+ debug_reader = XLogReaderAllocate(wal_segment_size, NULL NULL);
if (!debug_reader)
{
@@ -4326,15 +4318,10 @@ CleanupBackupHistory(void)
* record is available.
*/
static XLogRecord *
-ReadRecord(XLogReaderState *xlogreader, int emode,
- bool fetching_ckpt)
+ReadRecord(XLogReaderState *xlogreader, int emode, bool fetching_ckpt)
{
XLogRecord *record;
- XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
-
- private->fetching_ckpt = fetching_ckpt;
- private->emode = emode;
- private->randAccess = (xlogreader->ReadRecPtr == InvalidXLogRecPtr);
+ bool randAccess = (xlogreader->ReadRecPtr == InvalidXLogRecPtr);
/* This is the first attempt to read this page. */
lastSourceFailed = false;
@@ -4342,8 +4329,16 @@ ReadRecord(XLogReaderState *xlogreader, int emode,
for (;;)
{
char *errormsg;
+ XLogReadRecordResult result;
+
+ while ((result = XLogReadRecord(xlogreader, &record, &errormsg))
+ == XLREAD_NEED_DATA)
+ {
+ if (!XLogPageRead(xlogreader, fetching_ckpt, emode, randAccess))
+ break;
+
+ }
- record = XLogReadRecord(xlogreader, &errormsg);
ReadRecPtr = xlogreader->ReadRecPtr;
EndRecPtr = xlogreader->EndRecPtr;
if (record == NULL)
@@ -6320,7 +6315,6 @@ StartupXLOG(void)
bool backupFromStandby = false;
DBState dbstate_at_startup;
XLogReaderState *xlogreader;
- XLogPageReadPrivate private;
bool fast_promoted = false;
struct stat st;
@@ -6480,13 +6474,9 @@ StartupXLOG(void)
OwnLatch(&XLogCtl->recoveryWakeupLatch);
/* Set up XLOG reader facility */
- MemSet(&private, 0, sizeof(XLogPageReadPrivate));
xlogreader =
- XLogReaderAllocate(wal_segment_size, NULL,
- XL_ROUTINE(.page_read = &XLogPageRead,
- .segment_open = NULL,
- .segment_close = wal_segment_close),
- &private);
+ XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+
if (!xlogreader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
@@ -11860,12 +11850,13 @@ CancelBackup(void)
* sleep and retry.
*/
static bool
-XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetRecPtr, char *readBuf)
+XLogPageRead(XLogReaderState *xlogreader,
+ bool fetching_ckpt, int emode, bool randAccess)
{
- XLogPageReadPrivate *private =
- (XLogPageReadPrivate *) xlogreader->private_data;
- int emode = private->emode;
+ char *readBuf = xlogreader->readBuf;
+ XLogRecPtr targetPagePtr = xlogreader->readPagePtr;
+ int reqLen = xlogreader->readLen;
+ XLogRecPtr targetRecPtr = xlogreader->ReadRecPtr;
uint32 targetPageOff;
XLogSegNo targetSegNo PG_USED_FOR_ASSERTS_ONLY;
int r;
@@ -11908,8 +11899,8 @@ retry:
flushedUpto < targetPagePtr + reqLen))
{
if (!WaitForWALToBecomeAvailable(targetPagePtr + reqLen,
- private->randAccess,
- private->fetching_ckpt,
+ randAccess,
+ fetching_ckpt,
targetRecPtr))
{
if (readFile >= 0)
@@ -12014,6 +12005,7 @@ retry:
goto next_record_is_invalid;
}
+ Assert(xlogreader->readPagePtr == targetPagePtr);
xlogreader->readLen = readLen;
return true;
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index 3d599325ee..9281b57379 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -40,7 +40,7 @@ static bool XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr,
int reqLen, bool header_inclusive);
static void XLogReaderInvalReadState(XLogReaderState *state);
static bool ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
- XLogRecPtr PrevRecPtr, XLogRecord *record, bool randAccess);
+ XLogRecPtr PrevRecPtr, XLogRecord *record);
static bool ValidXLogRecord(XLogReaderState *state, XLogRecord *record,
XLogRecPtr recptr);
static void ResetDecoder(XLogReaderState *state);
@@ -73,7 +73,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...)
*/
XLogReaderState *
XLogReaderAllocate(int wal_segment_size, const char *waldir,
- XLogReaderRoutine *routine, void *private_data)
+ WALSegmentCleanupCB cleanup_cb)
{
XLogReaderState *state;
@@ -84,7 +84,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
return NULL;
/* initialize caller-provided support functions */
- state->routine = *routine;
+ state->cleanup_cb = cleanup_cb;
state->max_block_id = -1;
@@ -107,8 +107,6 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
WALOpenSegmentInit(&state->seg, &state->segcxt, wal_segment_size,
waldir);
- /* system_identifier initialized to zeroes above */
- state->private_data = private_data;
/* ReadRecPtr, EndRecPtr and readLen initialized to zeroes above */
state->errormsg_buf = palloc_extended(MAX_ERRORMSG_LEN + 1,
MCXT_ALLOC_NO_OOM);
@@ -140,8 +138,8 @@ XLogReaderFree(XLogReaderState *state)
{
int block_id;
- if (state->seg.ws_file != -1)
- state->routine.segment_close(state);
+ if (state->seg.ws_file >= 0)
+ state->cleanup_cb(state);
for (block_id = 0; block_id <= XLR_MAX_BLOCK_ID; block_id++)
{
@@ -246,6 +244,7 @@ XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
/* Begin at the passed-in record pointer. */
state->EndRecPtr = RecPtr;
state->ReadRecPtr = InvalidXLogRecPtr;
+ state->readRecordState = XLREAD_NEXT_RECORD;
}
/*
@@ -254,317 +253,452 @@ XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
* XLogBeginRead() or XLogFindNextRecord() must be called before the first call
* to XLogReadRecord().
*
- * If the page_read callback fails to read the requested data, NULL is
- * returned. The callback is expected to have reported the error; errormsg
- * is set to NULL.
+ * This function may return XLREAD_NEED_DATA several times before returning a
+ * result record. The caller shall read in some new data then call this
+ * function again with the same parameters.
*
- * If the reading fails for some other reason, NULL is also returned, and
- * *errormsg is set to a string with details of the failure.
+ * When a record is successfully read, returns XLREAD_SUCCESS with result
+ * record being stored in *record. Otherwise *record is NULL.
*
- * The returned pointer (or *errormsg) points to an internal buffer that's
- * valid until the next call to XLogReadRecord.
+ * Returns XLREAD_NEED_DATA if more data is needed to finish reading the
+ * current record. In that case, state->readPagePtr and state->readLen inform
+ * the desired position and minimum length of data needed. The caller shall
+ * read in the requested data and set state->readBuf to point to a buffer
+ * containing it. The caller must also set state->readPageTLI and
+ * state->readLen to indicate the timeline that it was read from, and the
+ * length of data that is now available (which must be >= given readLen),
+ * respectively.
+ *
+ * If invalid data is encountered, returns XLREAD_FAIL with *record being set to
+ * NULL. *errormsg is set to a string with details of the failure.
+ * The returned pointer (or *errormsg) points to an internal buffer that's valid
+ * until the next call to XLogReadRecord.
+ *
+ *
+ * This function runs a state machine consists of the following states.
+ *
+ * XLREAD_NEXT_RECORD :
+ * The initial state, if called with valid RecPtr, try to read a record at
+ * that position. If invalid RecPtr is given try to read a record just after
+ * the last one previously read.
+ * This state ens after setting ReadRecPtr. Then goes to XLREAD_TOT_LEN.
+ *
+ * XLREAD_TOT_LEN:
+ * Examining record header. Ends after reading record total
+ * length. recordRemainLen and recordGotLen are initialized.
+ *
+ * XLREAD_FIRST_FRAGMENT:
+ * Reading the first fragment. Ends with finishing reading a single
+ * record. Goes to XLREAD_NEXT_RECORD if that's all or
+ * XLREAD_CONTINUATION if we have continuation.
+
+ * XLREAD_CONTINUATION:
+ * Reading continuation of record. Ends with finishing the whole record then
+ * goes to XLREAD_NEXT_RECORD. During this state, recordRemainLen indicates
+ * how much is left and readRecordBuf holds the partially assert
+ * record.recordContRecPtr points to the beginning of the next page where to
+ * continue.
+ *
+ * If wrong data found in any state, the state machine stays at the current
+ * state. This behavior allows to continue reading a reacord switching among
+ * different souces, while streaming replication.
*/
-XLogRecord *
-XLogReadRecord(XLogReaderState *state, char **errormsg)
+XLogReadRecordResult
+XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
{
- XLogRecPtr RecPtr;
- XLogRecord *record;
- XLogRecPtr targetPagePtr;
- bool randAccess;
- uint32 len,
- total_len;
- uint32 targetRecOff;
- uint32 pageHeaderSize;
- bool gotheader;
+ XLogRecord *prec;
- /*
- * randAccess indicates whether to verify the previous-record pointer of
- * the record we're reading. We only do this if we're reading
- * sequentially, which is what we initially assume.
- */
- randAccess = false;
+ *record = NULL;
/* reset error state */
*errormsg = NULL;
state->errormsg_buf[0] = '\0';
- ResetDecoder(state);
-
- RecPtr = state->EndRecPtr;
-
- if (state->ReadRecPtr != InvalidXLogRecPtr)
- {
- /* read the record after the one we just read */
-
- /*
- * EndRecPtr is pointing to end+1 of the previous WAL record. If
- * we're at a page boundary, no more records can fit on the current
- * page. We must skip over the page header, but we can't do that until
- * we've read in the page, since the header size is variable.
- */
- }
- else
- {
- /*
- * Caller supplied a position to start at.
- *
- * In this case, EndRecPtr should already be pointing to a valid
- * record starting position.
- */
- Assert(XRecOffIsValid(RecPtr));
- randAccess = true;
- }
-
- state->currRecPtr = RecPtr;
-
- targetPagePtr = RecPtr - (RecPtr % XLOG_BLCKSZ);
- targetRecOff = RecPtr % XLOG_BLCKSZ;
-
- /*
- * Read the page containing the record into state->readBuf. Request enough
- * byte to cover the whole record header, or at least the part of it that
- * fits on the same page.
- */
- while (XLogNeedData(state, targetPagePtr,
- Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
- targetRecOff != 0))
- {
- if (!state->routine.page_read(state, state->readPagePtr, state->readLen,
- RecPtr, state->readBuf))
- break;
- }
-
- if (!state->page_verified)
- goto err;
-
- /*
- * We have at least the page header, so we can examine it now.
- */
- pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
- if (targetRecOff == 0)
- {
- /*
- * At page start, so skip over page header.
- */
- RecPtr += pageHeaderSize;
- targetRecOff = pageHeaderSize;
- }
- else if (targetRecOff < pageHeaderSize)
- {
- report_invalid_record(state, "invalid record offset at %X/%X",
- (uint32) (RecPtr >> 32), (uint32) RecPtr);
- goto err;
- }
-
- if ((((XLogPageHeader) state->readBuf)->xlp_info & XLP_FIRST_IS_CONTRECORD) &&
- targetRecOff == pageHeaderSize)
- {
- report_invalid_record(state, "contrecord is requested by %X/%X",
- (uint32) (RecPtr >> 32), (uint32) RecPtr);
- goto err;
- }
-
- /* XLogNeedData has verified the page header */
- Assert(pageHeaderSize <= state->readLen);
-
- /*
- * Read the record length.
- *
- * NB: Even though we use an XLogRecord pointer here, the whole record
- * header might not fit on this page. xl_tot_len is the first field of the
- * struct, so it must be on this page (the records are MAXALIGNed), but we
- * cannot access any other fields until we've verified that we got the
- * whole header.
- */
- record = (XLogRecord *) (state->readBuf + RecPtr % XLOG_BLCKSZ);
- total_len = record->xl_tot_len;
-
- /*
- * If the whole record header is on this page, validate it immediately.
- * Otherwise do just a basic sanity check on xl_tot_len, and validate the
- * rest of the header after reading it from the next page. The xl_tot_len
- * check is necessary here to ensure that we enter the "Need to reassemble
- * record" code path below; otherwise we might fail to apply
- * ValidXLogRecordHeader at all.
- */
- if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
- {
- if (!ValidXLogRecordHeader(state, RecPtr, state->ReadRecPtr, record,
- randAccess))
- goto err;
- gotheader = true;
- }
- else
- {
- /* XXX: more validation should be done here */
- if (total_len < SizeOfXLogRecord)
- {
- report_invalid_record(state,
- "invalid record length at %X/%X: wanted %u, got %u",
- (uint32) (RecPtr >> 32), (uint32) RecPtr,
- (uint32) SizeOfXLogRecord, total_len);
- goto err;
- }
- gotheader = false;
- }
-
- len = XLOG_BLCKSZ - RecPtr % XLOG_BLCKSZ;
- if (total_len > len)
+ switch (state->readRecordState)
{
- /* Need to reassemble record */
- char *contdata;
- XLogPageHeader pageHeader;
- char *buffer;
- uint32 gotlen;
-
- /*
- * Enlarge readRecordBuf as needed.
- */
- if (total_len > state->readRecordBufSize &&
- !allocate_recordbuf(state, total_len))
- {
- /* We treat this as a "bogus data" condition */
- report_invalid_record(state, "record length %u at %X/%X too long",
- total_len,
- (uint32) (RecPtr >> 32), (uint32) RecPtr);
- goto err;
- }
+ case XLREAD_NEXT_RECORD:
+ ResetDecoder(state);
- /* Copy the first fragment of the record from the first page. */
- memcpy(state->readRecordBuf,
- state->readBuf + RecPtr % XLOG_BLCKSZ, len);
- buffer = state->readRecordBuf + len;
- gotlen = len;
-
- do
- {
- int rest_len = total_len - gotlen;
-
- /* Calculate pointer to beginning of next page */
- targetPagePtr += XLOG_BLCKSZ;
-
- /* Wait for the next page to become available */
- while (XLogNeedData(state, targetPagePtr,
- Min(rest_len, XLOG_BLCKSZ),
- false))
+ if (state->ReadRecPtr != InvalidXLogRecPtr)
{
- if (!state->routine.page_read(state, state->readPagePtr,
- state->readLen,
- state->ReadRecPtr,
- state->readBuf))
- break;
+ /* read the record after the one we just read */
+
+ /*
+ * EndRecPtr is pointing to end+1 of the previous WAL record.
+ * If we're at a page boundary, no more records can fit on the
+ * current page. We must skip over the page header, but we
+ * can't do that until we've read in the page, since the header
+ * size is variable.
+ */
+ state->PrevRecPtr = state->ReadRecPtr;
+ state->ReadRecPtr = state->EndRecPtr;
+ }
+ else
+ {
+ /*
+ * Caller supplied a position to start at.
+ *
+ * In this case, EndRecPtr should already be pointing to a
+ * valid record starting position.
+ */
+ Assert(XRecOffIsValid(state->EndRecPtr));
+ state->ReadRecPtr = state->EndRecPtr;
+
+ /*
+ * We cannot verify the previous-record pointer when we're
+ * seeking to a particular record. Reset PrevRecPtr so that we
+ * won't try doing that.
+ */
+ state->PrevRecPtr = InvalidXLogRecPtr;
+ state->EndRecPtr = InvalidXLogRecPtr; /* to be tidy */
}
+ state->record_verified = false;
+ state->readRecordState = XLREAD_TOT_LEN;
+ /* fall through */
+
+ case XLREAD_TOT_LEN:
+ {
+ uint32 total_len;
+ uint32 pageHeaderSize;
+ XLogRecPtr targetPagePtr;
+ uint32 targetRecOff;
+ XLogPageHeader pageHeader;
+
+ targetPagePtr =
+ state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
+ targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+
+ /*
+ * Check if we have enough data. For the first record in the page,
+ * the requesting length doesn't contain page header.
+ */
+ if (XLogNeedData(state, targetPagePtr,
+ Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+ targetRecOff != 0))
+ return XLREAD_NEED_DATA;
+
+ /* error out if caller supplied bogus page */
if (!state->page_verified)
goto err;
- Assert(SizeOfXLogShortPHD <= state->readLen);
+ /* examine page header now. */
+ pageHeaderSize =
+ XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+ if (targetRecOff == 0)
+ {
+ /* At page start, so skip over page header. */
+ state->ReadRecPtr += pageHeaderSize;
+ targetRecOff = pageHeaderSize;
+ }
+ else if (targetRecOff < pageHeaderSize)
+ {
+ report_invalid_record(state, "invalid record offset at %X/%X",
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr);
+ goto err;
+ }
- /* Check that the continuation on next page looks valid */
pageHeader = (XLogPageHeader) state->readBuf;
- if (!(pageHeader->xlp_info & XLP_FIRST_IS_CONTRECORD))
+ if ((pageHeader->xlp_info & XLP_FIRST_IS_CONTRECORD) &&
+ targetRecOff == pageHeaderSize)
{
- report_invalid_record(state,
- "there is no contrecord flag at %X/%X",
- (uint32) (RecPtr >> 32), (uint32) RecPtr);
+ report_invalid_record(state, "contrecord is requested by %X/%X",
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr);
goto err;
}
- /*
- * Cross-check that xlp_rem_len agrees with how much of the record
- * we expect there to be left.
- */
- if (pageHeader->xlp_rem_len == 0 ||
- total_len != (pageHeader->xlp_rem_len + gotlen))
- {
- report_invalid_record(state,
- "invalid contrecord length %u at %X/%X",
- pageHeader->xlp_rem_len,
- (uint32) (RecPtr >> 32), (uint32) RecPtr);
- goto err;
- }
-
- /* Append the continuation from this page to the buffer */
- pageHeaderSize = XLogPageHeaderSize(pageHeader);
-
+ /* XLogNeedData has verified the page header */
Assert(pageHeaderSize <= state->readLen);
- contdata = (char *) state->readBuf + pageHeaderSize;
- len = XLOG_BLCKSZ - pageHeaderSize;
- if (pageHeader->xlp_rem_len < len)
- len = pageHeader->xlp_rem_len;
+ /*
+ * Read the record length.
+ *
+ * NB: Even though we use an XLogRecord pointer here, the whole
+ * record header might not fit on this page. xl_tot_len is the first
+ * field of the struct, so it must be on this page (the records are
+ * MAXALIGNed), but we cannot access any other fields until we've
+ * verified that we got the whole header.
+ */
+ prec = (XLogRecord *) (state->readBuf +
+ state->ReadRecPtr % XLOG_BLCKSZ);
+ total_len = prec->xl_tot_len;
- Assert (pageHeaderSize + len <= state->readLen);
- memcpy(buffer, (char *) contdata, len);
- buffer += len;
- gotlen += len;
+ /*
+ * If the whole record header is on this page, validate it
+ * immediately. Otherwise do just a basic sanity check on
+ * xl_tot_len, and validate the rest of the header after reading it
+ * from the next page. The xl_tot_len check is necessary here to
+ * ensure that we enter the XLREAD_CONTINUATION state below;
+ * otherwise we might fail to apply ValidXLogRecordHeader at all.
+ */
+ if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
+ {
+ if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
+ state->PrevRecPtr, prec))
+ goto err;
- /* If we just reassembled the record header, validate it. */
- if (!gotheader)
+ state->record_verified = true;
+ }
+ else
{
- record = (XLogRecord *) state->readRecordBuf;
- if (!ValidXLogRecordHeader(state, RecPtr, state->ReadRecPtr,
- record, randAccess))
+ /* XXX: more validation should be done here */
+ if (total_len < SizeOfXLogRecord)
+ {
+ report_invalid_record(state,
+ "invalid record length at %X/%X: wanted %u, got %u",
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr,
+ (uint32) SizeOfXLogRecord, total_len);
goto err;
- gotheader = true;
+ }
}
- } while (gotlen < total_len);
-
- Assert(gotheader);
- record = (XLogRecord *) state->readRecordBuf;
- if (!ValidXLogRecord(state, record, RecPtr))
- goto err;
+ /*
+ * Wait for the rest of the record, or the part of the record that
+ * fit on the first page if crossed a page boundary, to become
+ * available.
+ */
+ state->recordGotLen = 0;
+ state->recordRemainLen = total_len;
+ state->readRecordState = XLREAD_FIRST_FRAGMENT;
+ }
+ /* fall through */
- pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
- state->ReadRecPtr = RecPtr;
- state->EndRecPtr = targetPagePtr + pageHeaderSize
- + MAXALIGN(pageHeader->xlp_rem_len);
- }
- else
- {
- /* Wait for the record data to become available */
- while (XLogNeedData(state, targetPagePtr,
- Min(targetRecOff + total_len, XLOG_BLCKSZ), true))
+ case XLREAD_FIRST_FRAGMENT:
{
- if (!state->routine.page_read(state, state->readPagePtr,
- state->readLen,
- state->ReadRecPtr, state->readBuf))
+ uint32 total_len = state->recordRemainLen;
+ uint32 request_len;
+ uint32 record_len;
+ XLogRecPtr targetPagePtr;
+ uint32 targetRecOff;
+
+ /*
+ * Wait for the rest of the record on the first page to become
+ * available
+ */
+ targetPagePtr =
+ state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
+ targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+
+ request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ);
+ record_len = request_len - targetRecOff;
+
+ /* ReadRecPtr contains page header */
+ Assert (targetRecOff != 0);
+ if (XLogNeedData(state, targetPagePtr, request_len, true))
+ return XLREAD_NEED_DATA;
+
+ /* error out if caller supplied bogus page */
+ if (!state->page_verified)
+ goto err;
+
+ prec = (XLogRecord *) (state->readBuf + targetRecOff);
+
+ /* validate record header if not yet */
+ if (!state->record_verified && record_len >= SizeOfXLogRecord)
+ {
+ if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
+ state->PrevRecPtr, prec))
+ goto err;
+
+ state->record_verified = true;
+ }
+
+
+ if (total_len == record_len)
+ {
+ /* Record does not cross a page boundary */
+ Assert(state->record_verified);
+
+ if (!ValidXLogRecord(state, prec, state->ReadRecPtr))
+ goto err;
+
+ state->record_verified = true; /* to be tidy */
+
+ /* We already checked the header earlier */
+ state->EndRecPtr = state->ReadRecPtr + MAXALIGN(record_len);
+
+ *record = prec;
+ state->readRecordState = XLREAD_NEXT_RECORD;
break;
+ }
+
+ /*
+ * The record continues on the next page. Need to reassemble
+ * record
+ */
+ Assert(total_len > record_len);
+
+ /* Enlarge readRecordBuf as needed. */
+ if (total_len > state->readRecordBufSize &&
+ !allocate_recordbuf(state, total_len))
+ {
+ /* We treat this as a "bogus data" condition */
+ report_invalid_record(state,
+ "record length %u at %X/%X too long",
+ total_len,
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr);
+ goto err;
+ }
+
+ /* Copy the first fragment of the record from the first page. */
+ memcpy(state->readRecordBuf, state->readBuf + targetRecOff,
+ record_len);
+ state->recordGotLen += record_len;
+ state->recordRemainLen -= record_len;
+
+ /* Calculate pointer to beginning of next page */
+ state->recordContRecPtr = state->ReadRecPtr + record_len;
+ Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0);
+
+ state->readRecordState = XLREAD_CONTINUATION;
}
+ /* fall through */
+
+ case XLREAD_CONTINUATION:
+ {
+ XLogPageHeader pageHeader;
+ uint32 pageHeaderSize;
+ XLogRecPtr targetPagePtr;
+
+ /* we enter this state only if we haven't read the whole record. */
+ Assert (state->recordRemainLen > 0);
+
+ while(state->recordRemainLen > 0)
+ {
+ char *contdata;
+ uint32 request_len;
+ uint32 record_len;
+
+ /* Wait for the next page to become available */
+ targetPagePtr = state->recordContRecPtr;
+
+ /* this request contains page header */
+ Assert (targetPagePtr != 0);
+ if (XLogNeedData(state, targetPagePtr,
+ Min(state->recordRemainLen, XLOG_BLCKSZ),
+ false))
+ return XLREAD_NEED_DATA;
+
+ if (!state->page_verified)
+ goto err;
+
+ Assert(SizeOfXLogShortPHD <= state->readLen);
- if (!state->page_verified)
- goto err;
+ /* Check that the continuation on next page looks valid */
+ pageHeader = (XLogPageHeader) state->readBuf;
+ if (!(pageHeader->xlp_info & XLP_FIRST_IS_CONTRECORD))
+ {
+ report_invalid_record(
+ state,
+ "there is no contrecord flag at %X/%X reading %X/%X",
+ (uint32) (state->recordContRecPtr >> 32),
+ (uint32) state->recordContRecPtr,
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr);
+ goto err;
+ }
- /* Record does not cross a page boundary */
- if (!ValidXLogRecord(state, record, RecPtr))
- goto err;
+ /*
+ * Cross-check that xlp_rem_len agrees with how much of the
+ * record we expect there to be left.
+ */
+ if (pageHeader->xlp_rem_len == 0 ||
+ pageHeader->xlp_rem_len != state->recordRemainLen)
+ {
+ report_invalid_record(
+ state,
+ "invalid contrecord length %u at %X/%X reading %X/%X, expected %u",
+ pageHeader->xlp_rem_len,
+ (uint32) (state->recordContRecPtr >> 32),
+ (uint32) state->recordContRecPtr,
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr,
+ state->recordRemainLen);
+ goto err;
+ }
- state->EndRecPtr = RecPtr + MAXALIGN(total_len);
+ /* Append the continuation from this page to the buffer */
+ pageHeaderSize = XLogPageHeaderSize(pageHeader);
- state->ReadRecPtr = RecPtr;
+ /*
+ * XLogNeedData should have ensured that the whole page header
+ * was read
+ */
+ Assert(state->readLen >= pageHeaderSize);
+
+ contdata = (char *) state->readBuf + pageHeaderSize;
+ record_len = XLOG_BLCKSZ - pageHeaderSize;
+ if (pageHeader->xlp_rem_len < record_len)
+ record_len = pageHeader->xlp_rem_len;
+
+ request_len = record_len + pageHeaderSize;
+
+ /* XLogNeedData should have ensured all needed data was read */
+ Assert (state->readLen >= request_len);
+
+ memcpy(state->readRecordBuf + state->recordGotLen,
+ (char *) contdata, record_len);
+ state->recordGotLen += record_len;
+ state->recordRemainLen -= record_len;
+
+ /* If we just reassembled the record header, validate it. */
+ if (!state->record_verified)
+ {
+ Assert(state->recordGotLen >= SizeOfXLogRecord);
+ if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
+ state->PrevRecPtr,
+ (XLogRecord *) state->readRecordBuf))
+ goto err;
+
+ state->record_verified = true;
+ }
+
+ /* Calculate pointer to beginning of next page, and continue */
+ state->recordContRecPtr += XLOG_BLCKSZ;
+ }
+
+ /* targetPagePtr is pointing the last-read page here */
+ prec = (XLogRecord *) state->readRecordBuf;
+ if (!ValidXLogRecord(state, prec, state->ReadRecPtr))
+ goto err;
+
+ pageHeaderSize =
+ XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+ state->EndRecPtr = targetPagePtr + pageHeaderSize
+ + MAXALIGN(pageHeader->xlp_rem_len);
+
+ *record = prec;
+ state->readRecordState = XLREAD_NEXT_RECORD;
+ break;
+ }
}
/*
* Special processing if it's an XLOG SWITCH record
*/
- if (record->xl_rmid == RM_XLOG_ID &&
- (record->xl_info & ~XLR_INFO_MASK) == XLOG_SWITCH)
+ if ((*record)->xl_rmid == RM_XLOG_ID &&
+ ((*record)->xl_info & ~XLR_INFO_MASK) == XLOG_SWITCH)
{
/* Pretend it extends to end of segment */
state->EndRecPtr += state->segcxt.ws_segsize - 1;
state->EndRecPtr -= XLogSegmentOffset(state->EndRecPtr, state->segcxt.ws_segsize);
}
- if (DecodeXLogRecord(state, record, errormsg))
- return record;
- else
- return NULL;
+ Assert (!*record || state->readLen >= 0);
+ if (DecodeXLogRecord(state, *record, errormsg))
+ return XLREAD_SUCCESS;
+
+ *record = NULL;
+ return XLREAD_FAIL;
err:
/*
- * Invalidate the read state. We might read from a different source after
+ * Invalidate the read page. We might read from a different source after
* failure.
*/
XLogReaderInvalReadState(state);
@@ -572,7 +706,8 @@ err:
if (state->errormsg_buf[0] != '\0')
*errormsg = state->errormsg_buf;
- return NULL;
+ *record = NULL;
+ return XLREAD_FAIL;
}
/*
@@ -724,11 +859,12 @@ XLogReaderInvalReadState(XLogReaderState *state)
*
* This is just a convenience subroutine to avoid duplicated code in
* XLogReadRecord. It's not intended for use from anywhere else.
+ *
+ * If PrevRecPtr is valid, the xl_prev is is cross-checked with it.
*/
static bool
ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
- XLogRecPtr PrevRecPtr, XLogRecord *record,
- bool randAccess)
+ XLogRecPtr PrevRecPtr, XLogRecord *record)
{
if (record->xl_tot_len < SizeOfXLogRecord)
{
@@ -746,7 +882,7 @@ ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
(uint32) RecPtr);
return false;
}
- if (randAccess)
+ if (PrevRecPtr == InvalidXLogRecPtr)
{
/*
* We can't exactly verify the prev-link, but surely it should be less
@@ -978,11 +1114,14 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
* XLogReadRecord() will read the next valid record.
*/
XLogRecPtr
-XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
+XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
+ XLogFindNextRecordCB read_page, void *private)
{
XLogRecPtr tmpRecPtr;
XLogRecPtr found = InvalidXLogRecPtr;
XLogPageHeader header;
+ XLogRecord *record;
+ XLogReadRecordResult result;
char *errormsg;
Assert(!XLogRecPtrIsInvalid(RecPtr));
@@ -1015,9 +1154,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
while(XLogNeedData(state, targetPagePtr, targetRecOff,
targetRecOff != 0))
{
- if (!state->routine.page_read(state, state->readPagePtr,
- state->readLen,
- state->ReadRecPtr, state->readBuf))
+ if (!read_page(state, private))
break;
}
@@ -1069,8 +1206,16 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
* or we just jumped over the remaining data of a continuation.
*/
XLogBeginRead(state, tmpRecPtr);
- while (XLogReadRecord(state, &errormsg) != NULL)
+ while ((result = XLogReadRecord(state, &record, &errormsg)) !=
+ XLREAD_FAIL)
{
+ if (result == XLREAD_NEED_DATA)
+ {
+ if (!read_page(state, private))
+ goto err;
+ continue;
+ }
+
/* past the record we've found, break out */
if (RecPtr <= state->ReadRecPtr)
{
@@ -1090,9 +1235,9 @@ err:
#endif /* FRONTEND */
/*
- * Helper function to ease writing of XLogRoutine->page_read callbacks.
- * If this function is used, caller must supply a segment_open callback in
- * 'state', as that is used here.
+ * Helper function to ease writing of page_read callback.
+ * If this function is used, caller must supply a segment_open callback and
+ * segment_close callback as that is used here.
*
* Read 'count' bytes into 'buf', starting at location 'startptr', from WAL
* fetched from timeline 'tli'.
@@ -1105,6 +1250,7 @@ err:
*/
bool
WALRead(XLogReaderState *state,
+ WALSegmentOpenCB segopenfn, WALSegmentCloseCB segclosefn,
char *buf, XLogRecPtr startptr, Size count, TimeLineID tli,
WALReadError *errinfo)
{
@@ -1136,10 +1282,10 @@ WALRead(XLogReaderState *state,
XLogSegNo nextSegNo;
if (state->seg.ws_file >= 0)
- state->routine.segment_close(state);
+ segclosefn(state);
XLByteToSeg(recptr, nextSegNo, state->segcxt.ws_segsize);
- state->routine.segment_open(state, nextSegNo, &tli);
+ segopenfn(state, nextSegNo, &tli);
/* This shouldn't happen -- indicates a bug in segment_open */
Assert(state->seg.ws_file >= 0);
diff --git a/src/backend/access/transam/xlogutils.c b/src/backend/access/transam/xlogutils.c
index 1b10b6df20..303333571f 100644
--- a/src/backend/access/transam/xlogutils.c
+++ b/src/backend/access/transam/xlogutils.c
@@ -689,8 +689,7 @@ XLogTruncateRelation(RelFileNode rnode, ForkNumber forkNum,
void
XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wantLength)
{
- const XLogRecPtr lastReadPage = state->seg.ws_segno *
- state->segcxt.ws_segsize + state->readLen;
+ const XLogRecPtr lastReadPage = state->readPagePtr;
Assert(wantPage != InvalidXLogRecPtr && wantPage % XLOG_BLCKSZ == 0);
Assert(wantLength <= XLOG_BLCKSZ);
@@ -705,7 +704,7 @@ XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wa
* current TLI has since become historical.
*/
if (lastReadPage == wantPage &&
- state->readLen != 0 &&
+ state->page_verified &&
lastReadPage + state->readLen >= wantPage + Min(wantLength, XLOG_BLCKSZ - 1))
return;
@@ -792,6 +791,7 @@ wal_segment_open(XLogReaderState *state, XLogSegNo nextSegNo,
char path[MAXPGPATH];
XLogFilePath(path, tli, nextSegNo, state->segcxt.ws_segsize);
+ elog(LOG, "HOGE: %lu, %d => %s", nextSegNo, tli, path);
state->seg.ws_file = BasicOpenFile(path, O_RDONLY | PG_BINARY);
if (state->seg.ws_file >= 0)
return;
@@ -829,9 +829,11 @@ wal_segment_close(XLogReaderState *state)
* loop for now.
*/
bool
-read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *cur_page)
+read_local_xlog_page(XLogReaderState *state)
{
+ XLogRecPtr targetPagePtr = state->readPagePtr;
+ int reqLen = state->readLen;
+ char *cur_page = state->readBuf;
XLogRecPtr read_upto,
loc;
TimeLineID tli;
@@ -944,11 +946,12 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
* as 'count', read the whole page anyway. It's guaranteed to be
* zero-padded up to the page boundary if it's incomplete.
*/
- if (!WALRead(state, cur_page, targetPagePtr, XLOG_BLCKSZ, tli,
- &errinfo))
+ if (!WALRead(state, wal_segment_open, wal_segment_close,
+ cur_page, targetPagePtr, XLOG_BLCKSZ, tli, &errinfo))
WALReadRaiseError(&errinfo);
/* number of valid bytes in the buffer */
+ state->readPagePtr = targetPagePtr;
state->readLen = count;
return true;
}
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index 61902be3b0..a15b0b3355 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -120,7 +120,8 @@ StartupDecodingContext(List *output_plugin_options,
TransactionId xmin_horizon,
bool need_full_snapshot,
bool fast_forward,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress)
@@ -169,11 +170,12 @@ StartupDecodingContext(List *output_plugin_options,
ctx->slot = slot;
- ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, xl_routine, ctx);
+ ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, cleanup_cb);
if (!ctx->reader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
errmsg("out of memory")));
+ ctx->page_read = page_read;
ctx->reorder = ReorderBufferAllocate();
ctx->snapshot_builder =
@@ -231,7 +233,8 @@ CreateInitDecodingContext(char *plugin,
List *output_plugin_options,
bool need_full_snapshot,
XLogRecPtr restart_lsn,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress)
@@ -328,7 +331,7 @@ CreateInitDecodingContext(char *plugin,
ctx = StartupDecodingContext(NIL, restart_lsn, xmin_horizon,
need_full_snapshot, false,
- xl_routine, prepare_write, do_write,
+ page_read, cleanup_cb, prepare_write, do_write,
update_progress);
/* call output plugin initialization callback */
@@ -376,7 +379,8 @@ LogicalDecodingContext *
CreateDecodingContext(XLogRecPtr start_lsn,
List *output_plugin_options,
bool fast_forward,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress)
@@ -429,8 +433,8 @@ CreateDecodingContext(XLogRecPtr start_lsn,
ctx = StartupDecodingContext(output_plugin_options,
start_lsn, InvalidTransactionId, false,
- fast_forward, xl_routine, prepare_write,
- do_write, update_progress);
+ fast_forward, page_read, cleanup_cb,
+ prepare_write, do_write, update_progress);
/* call output plugin initialization callback */
old_context = MemoryContextSwitchTo(ctx->context);
@@ -483,7 +487,13 @@ DecodingContextFindStartpoint(LogicalDecodingContext *ctx)
char *err = NULL;
/* the read_page callback waits for new WAL */
- record = XLogReadRecord(ctx->reader, &err);
+ while (XLogReadRecord(ctx->reader, &record, &err) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!ctx->page_read(ctx->reader))
+ break;
+ }
+
if (err)
elog(ERROR, "%s", err);
if (!record)
diff --git a/src/backend/replication/logical/logicalfuncs.c b/src/backend/replication/logical/logicalfuncs.c
index b99c94e848..fcc81b358e 100644
--- a/src/backend/replication/logical/logicalfuncs.c
+++ b/src/backend/replication/logical/logicalfuncs.c
@@ -233,9 +233,8 @@ pg_logical_slot_get_changes_guts(FunctionCallInfo fcinfo, bool confirm, bool bin
ctx = CreateDecodingContext(InvalidXLogRecPtr,
options,
false,
- XL_ROUTINE(.page_read = read_local_xlog_page,
- .segment_open = wal_segment_open,
- .segment_close = wal_segment_close),
+ read_local_xlog_page,
+ wal_segment_close,
LogicalOutputPrepareWrite,
LogicalOutputWrite, NULL);
@@ -284,7 +283,13 @@ pg_logical_slot_get_changes_guts(FunctionCallInfo fcinfo, bool confirm, bool bin
XLogRecord *record;
char *errm = NULL;
- record = XLogReadRecord(ctx->reader, &errm);
+ while (XLogReadRecord(ctx->reader, &record, &errm) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!ctx->page_read(ctx->reader))
+ break;
+ }
+
if (errm)
elog(ERROR, "%s", errm);
diff --git a/src/backend/replication/slotfuncs.c b/src/backend/replication/slotfuncs.c
index 88033a79b2..8a678074d5 100644
--- a/src/backend/replication/slotfuncs.c
+++ b/src/backend/replication/slotfuncs.c
@@ -152,9 +152,8 @@ create_logical_replication_slot(char *name, char *plugin,
ctx = CreateInitDecodingContext(plugin, NIL,
false, /* just catalogs is OK */
restart_lsn,
- XL_ROUTINE(.page_read = read_local_xlog_page,
- .segment_open = wal_segment_open,
- .segment_close = wal_segment_close),
+ read_local_xlog_page,
+ wal_segment_close,
NULL, NULL, NULL);
/*
@@ -486,9 +485,8 @@ pg_logical_replication_slot_advance(XLogRecPtr moveto)
ctx = CreateDecodingContext(InvalidXLogRecPtr,
NIL,
true, /* fast_forward */
- XL_ROUTINE(.page_read = read_local_xlog_page,
- .segment_open = wal_segment_open,
- .segment_close = wal_segment_close),
+ read_local_xlog_page,
+ wal_segment_close,
NULL, NULL, NULL);
/*
@@ -513,7 +511,13 @@ pg_logical_replication_slot_advance(XLogRecPtr moveto)
* Read records. No changes are generated in fast_forward mode,
* but snapbuilder/slot statuses are updated properly.
*/
- record = XLogReadRecord(ctx->reader, &errm);
+ while (XLogReadRecord(ctx->reader, &record, &errm) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!ctx->page_read(ctx->reader))
+ break;
+ }
+
if (errm)
elog(ERROR, "%s", errm);
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index bf2711bddd..2fd24a9b14 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -580,10 +580,7 @@ StartReplication(StartReplicationCmd *cmd)
/* create xlogreader for physical replication */
xlogreader =
- XLogReaderAllocate(wal_segment_size, NULL,
- XL_ROUTINE(.segment_open = WalSndSegmentOpen,
- .segment_close = wal_segment_close),
- NULL);
+ XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
if (!xlogreader)
ereport(ERROR,
@@ -812,9 +809,11 @@ StartReplication(StartReplicationCmd *cmd)
* set every time WAL is flushed.
*/
static bool
-logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetRecPtr, char *cur_page)
+logical_read_xlog_page(XLogReaderState *state)
{
+ XLogRecPtr targetPagePtr = state->readPagePtr;
+ int reqLen = state->readLen;
+ char *cur_page = state->readBuf;
XLogRecPtr flushptr;
int count;
WALReadError errinfo;
@@ -842,7 +841,7 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
count = flushptr - targetPagePtr; /* part of the page available */
/* now actually read the data, we know it's there */
- if (!WALRead(state,
+ if (!WALRead(state, WalSndSegmentOpen, wal_segment_close,
cur_page,
targetPagePtr,
XLOG_BLCKSZ,
@@ -1014,9 +1013,8 @@ CreateReplicationSlot(CreateReplicationSlotCmd *cmd)
ctx = CreateInitDecodingContext(cmd->plugin, NIL, need_full_snapshot,
InvalidXLogRecPtr,
- XL_ROUTINE(.page_read = logical_read_xlog_page,
- .segment_open = WalSndSegmentOpen,
- .segment_close = wal_segment_close),
+ logical_read_xlog_page,
+ wal_segment_close,
WalSndPrepareWrite, WalSndWriteData,
WalSndUpdateProgress);
@@ -1179,9 +1177,8 @@ StartLogicalReplication(StartReplicationCmd *cmd)
*/
logical_decoding_ctx =
CreateDecodingContext(cmd->startpoint, cmd->options, false,
- XL_ROUTINE(.page_read = logical_read_xlog_page,
- .segment_open = WalSndSegmentOpen,
- .segment_close = wal_segment_close),
+ logical_read_xlog_page,
+ wal_segment_close,
WalSndPrepareWrite, WalSndWriteData,
WalSndUpdateProgress);
xlogreader = logical_decoding_ctx->reader;
@@ -2761,7 +2758,7 @@ XLogSendPhysical(void)
enlargeStringInfo(&output_message, nbytes);
retry:
- if (!WALRead(xlogreader,
+ if (!WALRead(xlogreader, WalSndSegmentOpen, wal_segment_close,
&output_message.data[output_message.len],
startptr,
nbytes,
@@ -2859,7 +2856,12 @@ XLogSendLogical(void)
*/
WalSndCaughtUp = false;
- record = XLogReadRecord(logical_decoding_ctx->reader, &errm);
+ while (XLogReadRecord(logical_decoding_ctx->reader, &record, &errm) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!logical_decoding_ctx->page_read(logical_decoding_ctx->reader))
+ break;
+ }
/* xlog record was invalid */
if (errm != NULL)
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index 0b7e73ae79..3dd6df4be6 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -40,15 +40,9 @@ static int xlogreadfd = -1;
static XLogSegNo xlogreadsegno = -1;
static char xlogfpath[MAXPGPATH];
-typedef struct XLogPageReadPrivate
-{
- const char *restoreCommand;
- int tliIndex;
-} XLogPageReadPrivate;
-
-static bool SimpleXLogPageRead(XLogReaderState *xlogreader,
- XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
+static bool SimpleXLogPageRead(XLogReaderState *xlogreader,
+ const char *datadir, int *tliIndex,
+ const char *restoreCommand);
/*
* Read WAL from the datadir/pg_wal, starting from 'startpoint' on timeline
@@ -62,20 +56,22 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
XLogRecord *record;
XLogReaderState *xlogreader;
char *errormsg;
- XLogPageReadPrivate private;
- private.tliIndex = tliIndex;
- private.restoreCommand = restoreCommand;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir,
- XL_ROUTINE(.page_read = &SimpleXLogPageRead),
- &private);
+ xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+
if (xlogreader == NULL)
pg_fatal("out of memory");
XLogBeginRead(xlogreader, startpoint);
do
{
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!SimpleXLogPageRead(xlogreader, datadir,
+ &tliIndex, restoreCommand))
+ break;
+ }
if (record == NULL)
{
@@ -113,19 +109,19 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex,
XLogRecord *record;
XLogReaderState *xlogreader;
char *errormsg;
- XLogPageReadPrivate private;
XLogRecPtr endptr;
- private.tliIndex = tliIndex;
- private.restoreCommand = restoreCommand;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir,
- XL_ROUTINE(.page_read = &SimpleXLogPageRead),
- &private);
+ xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
XLogBeginRead(xlogreader, ptr);
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!SimpleXLogPageRead(xlogreader, datadir, &tliIndex, restoreCommand))
+ break;
+ }
if (record == NULL)
{
if (errormsg)
@@ -160,7 +156,6 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
XLogRecPtr searchptr;
XLogReaderState *xlogreader;
char *errormsg;
- XLogPageReadPrivate private;
/*
* The given fork pointer points to the end of the last common record,
@@ -176,11 +171,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
forkptr += SizeOfXLogShortPHD;
}
- private.tliIndex = tliIndex;
- private.restoreCommand = restoreCommand;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir,
- XL_ROUTINE(.page_read = &SimpleXLogPageRead),
- &private);
+ xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
@@ -190,7 +181,13 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
uint8 info;
XLogBeginRead(xlogreader, searchptr);
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!SimpleXLogPageRead(xlogreader, datadir,
+ &tliIndex, restoreCommand))
+ break;
+ }
if (record == NULL)
{
@@ -237,10 +234,11 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
/* XLogReader callback function, to read a WAL page */
static bool
-SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *readBuf)
+SimpleXLogPageRead(XLogReaderState *xlogreader, const char *datadir,
+ int *tliIndex, const char *restoreCommand)
{
- XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
+ XLogRecPtr targetPagePtr = xlogreader->readPagePtr;
+ char *readBuf = xlogreader->readBuf;
uint32 targetPageOff;
XLogRecPtr targetSegEnd;
XLogSegNo targetSegNo;
@@ -273,14 +271,14 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
* be done both forward and backward, consider also switching timeline
* accordingly.
*/
- while (private->tliIndex < targetNentries - 1 &&
- targetHistory[private->tliIndex].end < targetSegEnd)
- private->tliIndex++;
- while (private->tliIndex > 0 &&
- targetHistory[private->tliIndex].begin >= targetSegEnd)
- private->tliIndex--;
+ while (*tliIndex < targetNentries - 1 &&
+ targetHistory[*tliIndex].end < targetSegEnd)
+ (*tliIndex)++;
+ while (*tliIndex > 0 &&
+ targetHistory[*tliIndex].begin >= targetSegEnd)
+ (*tliIndex)--;
- XLogFileName(xlogfname, targetHistory[private->tliIndex].tli,
+ XLogFileName(xlogfname, targetHistory[*tliIndex].tli,
xlogreadsegno, WalSegSz);
snprintf(xlogfpath, MAXPGPATH, "%s/" XLOGDIR "/%s",
@@ -293,7 +291,7 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
/*
* If we have no restore_command to execute, then exit.
*/
- if (private->restoreCommand == NULL)
+ if (restoreCommand == NULL)
{
pg_log_error("could not open file \"%s\": %m", xlogfpath);
xlogreader->readLen = -1;
@@ -307,7 +305,7 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
xlogreadfd = RestoreArchivedFile(xlogreader->segcxt.ws_dir,
xlogfname,
WalSegSz,
- private->restoreCommand);
+ restoreCommand);
if (xlogreadfd < 0)
{
@@ -349,7 +347,7 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
Assert(targetSegNo == xlogreadsegno);
- xlogreader->seg.ws_tli = targetHistory[private->tliIndex].tli;
+ xlogreader->seg.ws_tli = targetHistory[*tliIndex].tli;
xlogreader->readLen = XLOG_BLCKSZ;
return true;
}
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index ab0c28d9b7..02e71138cc 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -330,12 +330,17 @@ WALDumpCloseSegment(XLogReaderState *state)
state->seg.ws_file = -1;
}
-/* pg_waldump's XLogReaderRoutine->page_read callback */
+/*
+ * pg_waldump's WAL page rader, also used as page_read callback for
+ * XLogFindNextRecord
+ */
static bool
-WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetPtr, char *readBuff)
+WALDumpReadPage(XLogReaderState *state, void *priv)
{
- XLogDumpPrivate *private = state->private_data;
+ XLogRecPtr targetPagePtr = state->readPagePtr;
+ int reqLen = state->readLen;
+ char *readBuff = state->readBuf;
+ XLogDumpPrivate *private = (XLogDumpPrivate *) priv;
int count = XLOG_BLCKSZ;
WALReadError errinfo;
@@ -353,8 +358,8 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
}
}
- if (!WALRead(state, readBuff, targetPagePtr, count, private->timeline,
- &errinfo))
+ if (!WALRead(state, WALDumpOpenSegment, WALDumpCloseSegment,
+ readBuff, targetPagePtr, count, private->timeline, &errinfo))
{
WALOpenSegment *seg = &errinfo.wre_seg;
char fname[MAXPGPATH];
@@ -374,6 +379,7 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
(Size) errinfo.wre_req);
}
+ Assert(count >= state->readLen);
state->readLen = count;
return true;
}
@@ -1042,16 +1048,14 @@ main(int argc, char **argv)
/* we have everything we need, start reading */
xlogreader_state =
- XLogReaderAllocate(WalSegSz, waldir,
- XL_ROUTINE(.page_read = WALDumpReadPage,
- .segment_open = WALDumpOpenSegment,
- .segment_close = WALDumpCloseSegment),
- &private);
+ XLogReaderAllocate(WalSegSz, waldir, WALDumpCloseSegment);
+
if (!xlogreader_state)
fatal_error("out of memory");
/* first find a valid recptr to start from */
- first_record = XLogFindNextRecord(xlogreader_state, private.startptr);
+ first_record = XLogFindNextRecord(xlogreader_state, private.startptr,
+ &WALDumpReadPage, (void*) &private);
if (first_record == InvalidXLogRecPtr)
fatal_error("could not find a valid record after %X/%X",
@@ -1075,7 +1079,13 @@ main(int argc, char **argv)
for (;;)
{
/* try to read the next record */
- record = XLogReadRecord(xlogreader_state, &errormsg);
+ while (XLogReadRecord(xlogreader_state, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!WALDumpReadPage(xlogreader_state, (void *) &private))
+ break;
+ }
+
if (!record)
{
if (!config.follow || private.endptr_reached)
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index 6c8848e14f..6a7fe140cb 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -57,64 +57,15 @@ typedef struct WALSegmentContext
typedef struct XLogReaderState XLogReaderState;
-/* Function type definition for the read_page callback */
-typedef bool (*XLogPageReadCB) (XLogReaderState *xlogreader,
- XLogRecPtr targetPagePtr,
- int reqLen,
- XLogRecPtr targetRecPtr,
- char *readBuf);
+/* Function type definition for the segment cleanup callback */
+typedef void (*WALSegmentCleanupCB) (XLogReaderState *xlogreader);
+
+/* Function type definition for the open/close callbacks for WALRead() */
typedef void (*WALSegmentOpenCB) (XLogReaderState *xlogreader,
XLogSegNo nextSegNo,
TimeLineID *tli_p);
typedef void (*WALSegmentCloseCB) (XLogReaderState *xlogreader);
-typedef struct XLogReaderRoutine
-{
- /*
- * Data input callback
- *
- * This callback shall read at least reqLen valid bytes of the xlog page
- * starting at targetPagePtr, and store them in readBuf. The callback
- * shall return the number of bytes read (never more than XLOG_BLCKSZ), or
- * -1 on failure. The callback shall sleep, if necessary, to wait for the
- * requested bytes to become available. The callback will not be invoked
- * again for the same page unless more than the returned number of bytes
- * are needed.
- *
- * targetRecPtr is the position of the WAL record we're reading. Usually
- * it is equal to targetPagePtr + reqLen, but sometimes xlogreader needs
- * to read and verify the page or segment header, before it reads the
- * actual WAL record it's interested in. In that case, targetRecPtr can
- * be used to determine which timeline to read the page from.
- *
- * The callback shall set ->seg.ws_tli to the TLI of the file the page was
- * read from.
- */
- XLogPageReadCB page_read;
-
- /*
- * Callback to open the specified WAL segment for reading. ->seg.ws_file
- * shall be set to the file descriptor of the opened segment. In case of
- * failure, an error shall be raised by the callback and it shall not
- * return.
- *
- * "nextSegNo" is the number of the segment to be opened.
- *
- * "tli_p" is an input/output argument. WALRead() uses it to pass the
- * timeline in which the new segment should be found, but the callback can
- * use it to return the TLI that it actually opened.
- */
- WALSegmentOpenCB segment_open;
-
- /*
- * WAL segment close callback. ->seg.ws_file shall be set to a negative
- * number.
- */
- WALSegmentCloseCB segment_close;
-} XLogReaderRoutine;
-
-#define XL_ROUTINE(...) &(XLogReaderRoutine){__VA_ARGS__}
-
typedef struct
{
/* Is this block ref in use? */
@@ -144,12 +95,35 @@ typedef struct
uint16 data_bufsz;
} DecodedBkpBlock;
+/* Return code from XLogReadRecord */
+typedef enum XLogReadRecordResult
+{
+ XLREAD_SUCCESS, /* record is successfully read */
+ XLREAD_NEED_DATA, /* need more data. see XLogReadRecord. */
+ XLREAD_FAIL /* failed during reading a record */
+} XLogReadRecordResult;
+
+/*
+ * internal state of XLogReadRecord
+ *
+ * XLogReadState runs a state machine while reading a record. Theses states
+ * are not seen outside the function. Each state may repeat several times
+ * exiting requesting caller for new data. See the comment of XLogReadRecrod
+ * for details.
+ */
+typedef enum XLogReadRecordState {
+ XLREAD_NEXT_RECORD,
+ XLREAD_TOT_LEN,
+ XLREAD_FIRST_FRAGMENT,
+ XLREAD_CONTINUATION
+} XLogReadRecordState;
+
struct XLogReaderState
{
/*
* Operational callbacks
*/
- XLogReaderRoutine routine;
+ WALSegmentCleanupCB cleanup_cb;
/* ----------------------------------------
* Public parameters
@@ -162,18 +136,14 @@ struct XLogReaderState
*/
uint64 system_identifier;
- /*
- * Opaque data for callbacks to use. Not used by XLogReader.
- */
- void *private_data;
-
/*
* Start and end point of last record read. EndRecPtr is also used as the
* position to read next. Calling XLogBeginRead() sets EndRecPtr to the
* starting position and ReadRecPtr to invalid.
*/
- XLogRecPtr ReadRecPtr; /* start of last record read */
+ XLogRecPtr ReadRecPtr; /* start of last record read or being read */
XLogRecPtr EndRecPtr; /* end+1 of last record read */
+ XLogRecPtr PrevRecPtr; /* start of previous record read */
/* ----------------------------------------
* Communication with page reader
@@ -187,7 +157,9 @@ struct XLogReaderState
* the request, or -1 on error */
TimeLineID readPageTLI; /* TLI for data currently in readBuf */
char *readBuf; /* buffer to store data */
- bool page_verified; /* is the page on the buffer verified? */
+ bool page_verified; /* is the page header on the buffer verified? */
+ bool record_verified;/* is the current record header verified? */
+
/* ----------------------------------------
@@ -227,8 +199,6 @@ struct XLogReaderState
XLogRecPtr latestPagePtr;
TimeLineID latestPageTLI;
- /* beginning of the WAL record being read. */
- XLogRecPtr currRecPtr;
/* timeline to read it from, 0 if a lookup is required */
TimeLineID currTLI;
@@ -255,6 +225,15 @@ struct XLogReaderState
char *readRecordBuf;
uint32 readRecordBufSize;
+ /*
+ * XLogReadRecord() state
+ */
+ XLogReadRecordState readRecordState;/* state machine state */
+ int recordGotLen; /* amount of current record that has
+ * already been read */
+ int recordRemainLen; /* length of current record that remains */
+ XLogRecPtr recordContRecPtr; /* where the current record continues */
+
/* Buffer to hold error message */
char *errormsg_buf;
};
@@ -262,9 +241,7 @@ struct XLogReaderState
/* Get a new XLogReader */
extern XLogReaderState *XLogReaderAllocate(int wal_segment_size,
const char *waldir,
- XLogReaderRoutine *routine,
- void *private_data);
-extern XLogReaderRoutine *LocalXLogReaderRoutine(void);
+ WALSegmentCleanupCB cleanup_cb);
/* Free an XLogReader */
extern void XLogReaderFree(XLogReaderState *state);
@@ -272,12 +249,17 @@ extern void XLogReaderFree(XLogReaderState *state);
/* Position the XLogReader to given record */
extern void XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr);
#ifdef FRONTEND
-extern XLogRecPtr XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr);
+/* Function type definition for the read_page callback */
+typedef bool (*XLogFindNextRecordCB) (XLogReaderState *xlogreader,
+ void *private);
+extern XLogRecPtr XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
+ XLogFindNextRecordCB read_page, void *private);
#endif /* FRONTEND */
/* Read the next XLog record. Returns NULL on end-of-WAL or failure */
-extern struct XLogRecord *XLogReadRecord(XLogReaderState *state,
- char **errormsg);
+extern XLogReadRecordResult XLogReadRecord(XLogReaderState *state,
+ XLogRecord **record,
+ char **errormsg);
/* Validate a page */
extern bool XLogReaderValidatePageHeader(XLogReaderState *state,
@@ -297,6 +279,7 @@ typedef struct WALReadError
} WALReadError;
extern bool WALRead(XLogReaderState *state,
+ WALSegmentOpenCB segopenfn, WALSegmentCloseCB sgclosefn,
char *buf, XLogRecPtr startptr, Size count,
TimeLineID tli, WALReadError *errinfo);
diff --git a/src/include/access/xlogutils.h b/src/include/access/xlogutils.h
index c287edf206..482e4ced69 100644
--- a/src/include/access/xlogutils.h
+++ b/src/include/access/xlogutils.h
@@ -47,9 +47,7 @@ extern Buffer XLogReadBufferExtended(RelFileNode rnode, ForkNumber forknum,
extern Relation CreateFakeRelcacheEntry(RelFileNode rnode);
extern void FreeFakeRelcacheEntry(Relation fakerel);
-extern bool read_local_xlog_page(XLogReaderState *state,
- XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetRecPtr, char *cur_page);
+extern bool read_local_xlog_page(XLogReaderState *state);
extern void wal_segment_open(XLogReaderState *state,
XLogSegNo nextSegNo,
TimeLineID *tli_p);
diff --git a/src/include/replication/logical.h b/src/include/replication/logical.h
index c2f2475e5d..a743db49cf 100644
--- a/src/include/replication/logical.h
+++ b/src/include/replication/logical.h
@@ -29,6 +29,10 @@ typedef void (*LogicalOutputPluginWriterUpdateProgress) (struct LogicalDecodingC
TransactionId xid
);
+typedef struct LogicalDecodingContext LogicalDecodingContext;
+
+typedef bool (*LogicalDecodingXLogPageReadCB)(XLogReaderState *ctx);
+
typedef struct LogicalDecodingContext
{
/* memory context this is all allocated in */
@@ -39,6 +43,7 @@ typedef struct LogicalDecodingContext
/* infrastructure pieces for decoding */
XLogReaderState *reader;
+ LogicalDecodingXLogPageReadCB page_read;
struct ReorderBuffer *reorder;
struct SnapBuild *snapshot_builder;
@@ -95,14 +100,16 @@ extern LogicalDecodingContext *CreateInitDecodingContext(char *plugin,
List *output_plugin_options,
bool need_full_snapshot,
XLogRecPtr restart_lsn,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress);
extern LogicalDecodingContext *CreateDecodingContext(XLogRecPtr start_lsn,
List *output_plugin_options,
bool fast_forward,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress);
--
2.18.4
----Next_Part(Thu_Jul__2_13_53_30_2020_072)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
filename="v15-0003-Remove-globals-readOff-readLen-and-readSegNo.patch"
^ permalink raw reply [nested|flat] 70+ messages in thread
* [PATCH v15 2/4] Move page-reader out of XLogReadRecord
@ 2019-09-10 03:58 Kyotaro Horiguchi <[email protected]>
0 siblings, 0 replies; 70+ messages in thread
From: Kyotaro Horiguchi @ 2019-09-10 03:58 UTC (permalink / raw)
This is the second step of removing callbacks from WAL record reader.
Since it is essential to take in additional data while reading a
record, the function have to ask caller for new data while keeping
working state. Thus the function is turned into a state machine.
---
src/backend/access/transam/twophase.c | 15 +-
src/backend/access/transam/xlog.c | 58 +-
src/backend/access/transam/xlogreader.c | 700 +++++++++++-------
src/backend/access/transam/xlogutils.c | 17 +-
src/backend/replication/logical/logical.c | 26 +-
.../replication/logical/logicalfuncs.c | 13 +-
src/backend/replication/slotfuncs.c | 18 +-
src/backend/replication/walsender.c | 32 +-
src/bin/pg_rewind/parsexlog.c | 84 +--
src/bin/pg_waldump/pg_waldump.c | 36 +-
src/include/access/xlogreader.h | 121 ++-
src/include/access/xlogutils.h | 4 +-
src/include/replication/logical.h | 11 +-
13 files changed, 648 insertions(+), 487 deletions(-)
diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index 9b2e59bf0e..b0d60a0d0f 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -1330,11 +1330,8 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
XLogReaderState *xlogreader;
char *errormsg;
- xlogreader = XLogReaderAllocate(wal_segment_size, NULL,
- XL_ROUTINE(.page_read = &read_local_xlog_page,
- .segment_open = &wal_segment_open,
- .segment_close = &wal_segment_close),
- NULL);
+ xlogreader = XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+
if (!xlogreader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
@@ -1342,7 +1339,13 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
errdetail("Failed while allocating a WAL reading processor.")));
XLogBeginRead(xlogreader, lsn);
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!read_local_xlog_page(xlogreader))
+ break;
+ }
+
if (record == NULL)
ereport(ERROR,
(errcode_for_file_access(),
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index e570e56a24..f9b0108602 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -831,13 +831,6 @@ static XLogSource currentSource = XLOG_FROM_ANY;
static bool lastSourceFailed = false;
static bool pendingWalRcvRestart = false;
-typedef struct XLogPageReadPrivate
-{
- int emode;
- bool fetching_ckpt; /* are we fetching a checkpoint record? */
- bool randAccess;
-} XLogPageReadPrivate;
-
/*
* These variables track when we last obtained some WAL data to process,
* and where we got it from. (XLogReceiptSource is initially the same as
@@ -912,8 +905,8 @@ static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath,
static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli,
XLogSource source, bool notfoundOk);
static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, XLogSource source);
-static bool XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
+static bool XLogPageRead(XLogReaderState *xlogreader,
+ bool fetching_ckpt, int emode, bool randAccess);
static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
bool fetching_ckpt, XLogRecPtr tliRecPtr);
static int emode_for_corrupt_record(int emode, XLogRecPtr RecPtr);
@@ -1225,8 +1218,7 @@ XLogInsertRecord(XLogRecData *rdata,
appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len);
if (!debug_reader)
- debug_reader = XLogReaderAllocate(wal_segment_size, NULL,
- XL_ROUTINE(), NULL);
+ debug_reader = XLogReaderAllocate(wal_segment_size, NULL NULL);
if (!debug_reader)
{
@@ -4326,15 +4318,10 @@ CleanupBackupHistory(void)
* record is available.
*/
static XLogRecord *
-ReadRecord(XLogReaderState *xlogreader, int emode,
- bool fetching_ckpt)
+ReadRecord(XLogReaderState *xlogreader, int emode, bool fetching_ckpt)
{
XLogRecord *record;
- XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
-
- private->fetching_ckpt = fetching_ckpt;
- private->emode = emode;
- private->randAccess = (xlogreader->ReadRecPtr == InvalidXLogRecPtr);
+ bool randAccess = (xlogreader->ReadRecPtr == InvalidXLogRecPtr);
/* This is the first attempt to read this page. */
lastSourceFailed = false;
@@ -4342,8 +4329,16 @@ ReadRecord(XLogReaderState *xlogreader, int emode,
for (;;)
{
char *errormsg;
+ XLogReadRecordResult result;
+
+ while ((result = XLogReadRecord(xlogreader, &record, &errormsg))
+ == XLREAD_NEED_DATA)
+ {
+ if (!XLogPageRead(xlogreader, fetching_ckpt, emode, randAccess))
+ break;
+
+ }
- record = XLogReadRecord(xlogreader, &errormsg);
ReadRecPtr = xlogreader->ReadRecPtr;
EndRecPtr = xlogreader->EndRecPtr;
if (record == NULL)
@@ -6320,7 +6315,6 @@ StartupXLOG(void)
bool backupFromStandby = false;
DBState dbstate_at_startup;
XLogReaderState *xlogreader;
- XLogPageReadPrivate private;
bool fast_promoted = false;
struct stat st;
@@ -6480,13 +6474,9 @@ StartupXLOG(void)
OwnLatch(&XLogCtl->recoveryWakeupLatch);
/* Set up XLOG reader facility */
- MemSet(&private, 0, sizeof(XLogPageReadPrivate));
xlogreader =
- XLogReaderAllocate(wal_segment_size, NULL,
- XL_ROUTINE(.page_read = &XLogPageRead,
- .segment_open = NULL,
- .segment_close = wal_segment_close),
- &private);
+ XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+
if (!xlogreader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
@@ -11860,12 +11850,13 @@ CancelBackup(void)
* sleep and retry.
*/
static bool
-XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetRecPtr, char *readBuf)
+XLogPageRead(XLogReaderState *xlogreader,
+ bool fetching_ckpt, int emode, bool randAccess)
{
- XLogPageReadPrivate *private =
- (XLogPageReadPrivate *) xlogreader->private_data;
- int emode = private->emode;
+ char *readBuf = xlogreader->readBuf;
+ XLogRecPtr targetPagePtr = xlogreader->readPagePtr;
+ int reqLen = xlogreader->readLen;
+ XLogRecPtr targetRecPtr = xlogreader->ReadRecPtr;
uint32 targetPageOff;
XLogSegNo targetSegNo PG_USED_FOR_ASSERTS_ONLY;
int r;
@@ -11908,8 +11899,8 @@ retry:
flushedUpto < targetPagePtr + reqLen))
{
if (!WaitForWALToBecomeAvailable(targetPagePtr + reqLen,
- private->randAccess,
- private->fetching_ckpt,
+ randAccess,
+ fetching_ckpt,
targetRecPtr))
{
if (readFile >= 0)
@@ -12014,6 +12005,7 @@ retry:
goto next_record_is_invalid;
}
+ Assert(xlogreader->readPagePtr == targetPagePtr);
xlogreader->readLen = readLen;
return true;
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index 3d599325ee..9281b57379 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -40,7 +40,7 @@ static bool XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr,
int reqLen, bool header_inclusive);
static void XLogReaderInvalReadState(XLogReaderState *state);
static bool ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
- XLogRecPtr PrevRecPtr, XLogRecord *record, bool randAccess);
+ XLogRecPtr PrevRecPtr, XLogRecord *record);
static bool ValidXLogRecord(XLogReaderState *state, XLogRecord *record,
XLogRecPtr recptr);
static void ResetDecoder(XLogReaderState *state);
@@ -73,7 +73,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...)
*/
XLogReaderState *
XLogReaderAllocate(int wal_segment_size, const char *waldir,
- XLogReaderRoutine *routine, void *private_data)
+ WALSegmentCleanupCB cleanup_cb)
{
XLogReaderState *state;
@@ -84,7 +84,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
return NULL;
/* initialize caller-provided support functions */
- state->routine = *routine;
+ state->cleanup_cb = cleanup_cb;
state->max_block_id = -1;
@@ -107,8 +107,6 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
WALOpenSegmentInit(&state->seg, &state->segcxt, wal_segment_size,
waldir);
- /* system_identifier initialized to zeroes above */
- state->private_data = private_data;
/* ReadRecPtr, EndRecPtr and readLen initialized to zeroes above */
state->errormsg_buf = palloc_extended(MAX_ERRORMSG_LEN + 1,
MCXT_ALLOC_NO_OOM);
@@ -140,8 +138,8 @@ XLogReaderFree(XLogReaderState *state)
{
int block_id;
- if (state->seg.ws_file != -1)
- state->routine.segment_close(state);
+ if (state->seg.ws_file >= 0)
+ state->cleanup_cb(state);
for (block_id = 0; block_id <= XLR_MAX_BLOCK_ID; block_id++)
{
@@ -246,6 +244,7 @@ XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
/* Begin at the passed-in record pointer. */
state->EndRecPtr = RecPtr;
state->ReadRecPtr = InvalidXLogRecPtr;
+ state->readRecordState = XLREAD_NEXT_RECORD;
}
/*
@@ -254,317 +253,452 @@ XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
* XLogBeginRead() or XLogFindNextRecord() must be called before the first call
* to XLogReadRecord().
*
- * If the page_read callback fails to read the requested data, NULL is
- * returned. The callback is expected to have reported the error; errormsg
- * is set to NULL.
+ * This function may return XLREAD_NEED_DATA several times before returning a
+ * result record. The caller shall read in some new data then call this
+ * function again with the same parameters.
*
- * If the reading fails for some other reason, NULL is also returned, and
- * *errormsg is set to a string with details of the failure.
+ * When a record is successfully read, returns XLREAD_SUCCESS with result
+ * record being stored in *record. Otherwise *record is NULL.
*
- * The returned pointer (or *errormsg) points to an internal buffer that's
- * valid until the next call to XLogReadRecord.
+ * Returns XLREAD_NEED_DATA if more data is needed to finish reading the
+ * current record. In that case, state->readPagePtr and state->readLen inform
+ * the desired position and minimum length of data needed. The caller shall
+ * read in the requested data and set state->readBuf to point to a buffer
+ * containing it. The caller must also set state->readPageTLI and
+ * state->readLen to indicate the timeline that it was read from, and the
+ * length of data that is now available (which must be >= given readLen),
+ * respectively.
+ *
+ * If invalid data is encountered, returns XLREAD_FAIL with *record being set to
+ * NULL. *errormsg is set to a string with details of the failure.
+ * The returned pointer (or *errormsg) points to an internal buffer that's valid
+ * until the next call to XLogReadRecord.
+ *
+ *
+ * This function runs a state machine consists of the following states.
+ *
+ * XLREAD_NEXT_RECORD :
+ * The initial state, if called with valid RecPtr, try to read a record at
+ * that position. If invalid RecPtr is given try to read a record just after
+ * the last one previously read.
+ * This state ens after setting ReadRecPtr. Then goes to XLREAD_TOT_LEN.
+ *
+ * XLREAD_TOT_LEN:
+ * Examining record header. Ends after reading record total
+ * length. recordRemainLen and recordGotLen are initialized.
+ *
+ * XLREAD_FIRST_FRAGMENT:
+ * Reading the first fragment. Ends with finishing reading a single
+ * record. Goes to XLREAD_NEXT_RECORD if that's all or
+ * XLREAD_CONTINUATION if we have continuation.
+
+ * XLREAD_CONTINUATION:
+ * Reading continuation of record. Ends with finishing the whole record then
+ * goes to XLREAD_NEXT_RECORD. During this state, recordRemainLen indicates
+ * how much is left and readRecordBuf holds the partially assert
+ * record.recordContRecPtr points to the beginning of the next page where to
+ * continue.
+ *
+ * If wrong data found in any state, the state machine stays at the current
+ * state. This behavior allows to continue reading a reacord switching among
+ * different souces, while streaming replication.
*/
-XLogRecord *
-XLogReadRecord(XLogReaderState *state, char **errormsg)
+XLogReadRecordResult
+XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
{
- XLogRecPtr RecPtr;
- XLogRecord *record;
- XLogRecPtr targetPagePtr;
- bool randAccess;
- uint32 len,
- total_len;
- uint32 targetRecOff;
- uint32 pageHeaderSize;
- bool gotheader;
+ XLogRecord *prec;
- /*
- * randAccess indicates whether to verify the previous-record pointer of
- * the record we're reading. We only do this if we're reading
- * sequentially, which is what we initially assume.
- */
- randAccess = false;
+ *record = NULL;
/* reset error state */
*errormsg = NULL;
state->errormsg_buf[0] = '\0';
- ResetDecoder(state);
-
- RecPtr = state->EndRecPtr;
-
- if (state->ReadRecPtr != InvalidXLogRecPtr)
- {
- /* read the record after the one we just read */
-
- /*
- * EndRecPtr is pointing to end+1 of the previous WAL record. If
- * we're at a page boundary, no more records can fit on the current
- * page. We must skip over the page header, but we can't do that until
- * we've read in the page, since the header size is variable.
- */
- }
- else
- {
- /*
- * Caller supplied a position to start at.
- *
- * In this case, EndRecPtr should already be pointing to a valid
- * record starting position.
- */
- Assert(XRecOffIsValid(RecPtr));
- randAccess = true;
- }
-
- state->currRecPtr = RecPtr;
-
- targetPagePtr = RecPtr - (RecPtr % XLOG_BLCKSZ);
- targetRecOff = RecPtr % XLOG_BLCKSZ;
-
- /*
- * Read the page containing the record into state->readBuf. Request enough
- * byte to cover the whole record header, or at least the part of it that
- * fits on the same page.
- */
- while (XLogNeedData(state, targetPagePtr,
- Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
- targetRecOff != 0))
- {
- if (!state->routine.page_read(state, state->readPagePtr, state->readLen,
- RecPtr, state->readBuf))
- break;
- }
-
- if (!state->page_verified)
- goto err;
-
- /*
- * We have at least the page header, so we can examine it now.
- */
- pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
- if (targetRecOff == 0)
- {
- /*
- * At page start, so skip over page header.
- */
- RecPtr += pageHeaderSize;
- targetRecOff = pageHeaderSize;
- }
- else if (targetRecOff < pageHeaderSize)
- {
- report_invalid_record(state, "invalid record offset at %X/%X",
- (uint32) (RecPtr >> 32), (uint32) RecPtr);
- goto err;
- }
-
- if ((((XLogPageHeader) state->readBuf)->xlp_info & XLP_FIRST_IS_CONTRECORD) &&
- targetRecOff == pageHeaderSize)
- {
- report_invalid_record(state, "contrecord is requested by %X/%X",
- (uint32) (RecPtr >> 32), (uint32) RecPtr);
- goto err;
- }
-
- /* XLogNeedData has verified the page header */
- Assert(pageHeaderSize <= state->readLen);
-
- /*
- * Read the record length.
- *
- * NB: Even though we use an XLogRecord pointer here, the whole record
- * header might not fit on this page. xl_tot_len is the first field of the
- * struct, so it must be on this page (the records are MAXALIGNed), but we
- * cannot access any other fields until we've verified that we got the
- * whole header.
- */
- record = (XLogRecord *) (state->readBuf + RecPtr % XLOG_BLCKSZ);
- total_len = record->xl_tot_len;
-
- /*
- * If the whole record header is on this page, validate it immediately.
- * Otherwise do just a basic sanity check on xl_tot_len, and validate the
- * rest of the header after reading it from the next page. The xl_tot_len
- * check is necessary here to ensure that we enter the "Need to reassemble
- * record" code path below; otherwise we might fail to apply
- * ValidXLogRecordHeader at all.
- */
- if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
- {
- if (!ValidXLogRecordHeader(state, RecPtr, state->ReadRecPtr, record,
- randAccess))
- goto err;
- gotheader = true;
- }
- else
- {
- /* XXX: more validation should be done here */
- if (total_len < SizeOfXLogRecord)
- {
- report_invalid_record(state,
- "invalid record length at %X/%X: wanted %u, got %u",
- (uint32) (RecPtr >> 32), (uint32) RecPtr,
- (uint32) SizeOfXLogRecord, total_len);
- goto err;
- }
- gotheader = false;
- }
-
- len = XLOG_BLCKSZ - RecPtr % XLOG_BLCKSZ;
- if (total_len > len)
+ switch (state->readRecordState)
{
- /* Need to reassemble record */
- char *contdata;
- XLogPageHeader pageHeader;
- char *buffer;
- uint32 gotlen;
-
- /*
- * Enlarge readRecordBuf as needed.
- */
- if (total_len > state->readRecordBufSize &&
- !allocate_recordbuf(state, total_len))
- {
- /* We treat this as a "bogus data" condition */
- report_invalid_record(state, "record length %u at %X/%X too long",
- total_len,
- (uint32) (RecPtr >> 32), (uint32) RecPtr);
- goto err;
- }
+ case XLREAD_NEXT_RECORD:
+ ResetDecoder(state);
- /* Copy the first fragment of the record from the first page. */
- memcpy(state->readRecordBuf,
- state->readBuf + RecPtr % XLOG_BLCKSZ, len);
- buffer = state->readRecordBuf + len;
- gotlen = len;
-
- do
- {
- int rest_len = total_len - gotlen;
-
- /* Calculate pointer to beginning of next page */
- targetPagePtr += XLOG_BLCKSZ;
-
- /* Wait for the next page to become available */
- while (XLogNeedData(state, targetPagePtr,
- Min(rest_len, XLOG_BLCKSZ),
- false))
+ if (state->ReadRecPtr != InvalidXLogRecPtr)
{
- if (!state->routine.page_read(state, state->readPagePtr,
- state->readLen,
- state->ReadRecPtr,
- state->readBuf))
- break;
+ /* read the record after the one we just read */
+
+ /*
+ * EndRecPtr is pointing to end+1 of the previous WAL record.
+ * If we're at a page boundary, no more records can fit on the
+ * current page. We must skip over the page header, but we
+ * can't do that until we've read in the page, since the header
+ * size is variable.
+ */
+ state->PrevRecPtr = state->ReadRecPtr;
+ state->ReadRecPtr = state->EndRecPtr;
+ }
+ else
+ {
+ /*
+ * Caller supplied a position to start at.
+ *
+ * In this case, EndRecPtr should already be pointing to a
+ * valid record starting position.
+ */
+ Assert(XRecOffIsValid(state->EndRecPtr));
+ state->ReadRecPtr = state->EndRecPtr;
+
+ /*
+ * We cannot verify the previous-record pointer when we're
+ * seeking to a particular record. Reset PrevRecPtr so that we
+ * won't try doing that.
+ */
+ state->PrevRecPtr = InvalidXLogRecPtr;
+ state->EndRecPtr = InvalidXLogRecPtr; /* to be tidy */
}
+ state->record_verified = false;
+ state->readRecordState = XLREAD_TOT_LEN;
+ /* fall through */
+
+ case XLREAD_TOT_LEN:
+ {
+ uint32 total_len;
+ uint32 pageHeaderSize;
+ XLogRecPtr targetPagePtr;
+ uint32 targetRecOff;
+ XLogPageHeader pageHeader;
+
+ targetPagePtr =
+ state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
+ targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+
+ /*
+ * Check if we have enough data. For the first record in the page,
+ * the requesting length doesn't contain page header.
+ */
+ if (XLogNeedData(state, targetPagePtr,
+ Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+ targetRecOff != 0))
+ return XLREAD_NEED_DATA;
+
+ /* error out if caller supplied bogus page */
if (!state->page_verified)
goto err;
- Assert(SizeOfXLogShortPHD <= state->readLen);
+ /* examine page header now. */
+ pageHeaderSize =
+ XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+ if (targetRecOff == 0)
+ {
+ /* At page start, so skip over page header. */
+ state->ReadRecPtr += pageHeaderSize;
+ targetRecOff = pageHeaderSize;
+ }
+ else if (targetRecOff < pageHeaderSize)
+ {
+ report_invalid_record(state, "invalid record offset at %X/%X",
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr);
+ goto err;
+ }
- /* Check that the continuation on next page looks valid */
pageHeader = (XLogPageHeader) state->readBuf;
- if (!(pageHeader->xlp_info & XLP_FIRST_IS_CONTRECORD))
+ if ((pageHeader->xlp_info & XLP_FIRST_IS_CONTRECORD) &&
+ targetRecOff == pageHeaderSize)
{
- report_invalid_record(state,
- "there is no contrecord flag at %X/%X",
- (uint32) (RecPtr >> 32), (uint32) RecPtr);
+ report_invalid_record(state, "contrecord is requested by %X/%X",
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr);
goto err;
}
- /*
- * Cross-check that xlp_rem_len agrees with how much of the record
- * we expect there to be left.
- */
- if (pageHeader->xlp_rem_len == 0 ||
- total_len != (pageHeader->xlp_rem_len + gotlen))
- {
- report_invalid_record(state,
- "invalid contrecord length %u at %X/%X",
- pageHeader->xlp_rem_len,
- (uint32) (RecPtr >> 32), (uint32) RecPtr);
- goto err;
- }
-
- /* Append the continuation from this page to the buffer */
- pageHeaderSize = XLogPageHeaderSize(pageHeader);
-
+ /* XLogNeedData has verified the page header */
Assert(pageHeaderSize <= state->readLen);
- contdata = (char *) state->readBuf + pageHeaderSize;
- len = XLOG_BLCKSZ - pageHeaderSize;
- if (pageHeader->xlp_rem_len < len)
- len = pageHeader->xlp_rem_len;
+ /*
+ * Read the record length.
+ *
+ * NB: Even though we use an XLogRecord pointer here, the whole
+ * record header might not fit on this page. xl_tot_len is the first
+ * field of the struct, so it must be on this page (the records are
+ * MAXALIGNed), but we cannot access any other fields until we've
+ * verified that we got the whole header.
+ */
+ prec = (XLogRecord *) (state->readBuf +
+ state->ReadRecPtr % XLOG_BLCKSZ);
+ total_len = prec->xl_tot_len;
- Assert (pageHeaderSize + len <= state->readLen);
- memcpy(buffer, (char *) contdata, len);
- buffer += len;
- gotlen += len;
+ /*
+ * If the whole record header is on this page, validate it
+ * immediately. Otherwise do just a basic sanity check on
+ * xl_tot_len, and validate the rest of the header after reading it
+ * from the next page. The xl_tot_len check is necessary here to
+ * ensure that we enter the XLREAD_CONTINUATION state below;
+ * otherwise we might fail to apply ValidXLogRecordHeader at all.
+ */
+ if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
+ {
+ if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
+ state->PrevRecPtr, prec))
+ goto err;
- /* If we just reassembled the record header, validate it. */
- if (!gotheader)
+ state->record_verified = true;
+ }
+ else
{
- record = (XLogRecord *) state->readRecordBuf;
- if (!ValidXLogRecordHeader(state, RecPtr, state->ReadRecPtr,
- record, randAccess))
+ /* XXX: more validation should be done here */
+ if (total_len < SizeOfXLogRecord)
+ {
+ report_invalid_record(state,
+ "invalid record length at %X/%X: wanted %u, got %u",
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr,
+ (uint32) SizeOfXLogRecord, total_len);
goto err;
- gotheader = true;
+ }
}
- } while (gotlen < total_len);
-
- Assert(gotheader);
- record = (XLogRecord *) state->readRecordBuf;
- if (!ValidXLogRecord(state, record, RecPtr))
- goto err;
+ /*
+ * Wait for the rest of the record, or the part of the record that
+ * fit on the first page if crossed a page boundary, to become
+ * available.
+ */
+ state->recordGotLen = 0;
+ state->recordRemainLen = total_len;
+ state->readRecordState = XLREAD_FIRST_FRAGMENT;
+ }
+ /* fall through */
- pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
- state->ReadRecPtr = RecPtr;
- state->EndRecPtr = targetPagePtr + pageHeaderSize
- + MAXALIGN(pageHeader->xlp_rem_len);
- }
- else
- {
- /* Wait for the record data to become available */
- while (XLogNeedData(state, targetPagePtr,
- Min(targetRecOff + total_len, XLOG_BLCKSZ), true))
+ case XLREAD_FIRST_FRAGMENT:
{
- if (!state->routine.page_read(state, state->readPagePtr,
- state->readLen,
- state->ReadRecPtr, state->readBuf))
+ uint32 total_len = state->recordRemainLen;
+ uint32 request_len;
+ uint32 record_len;
+ XLogRecPtr targetPagePtr;
+ uint32 targetRecOff;
+
+ /*
+ * Wait for the rest of the record on the first page to become
+ * available
+ */
+ targetPagePtr =
+ state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
+ targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+
+ request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ);
+ record_len = request_len - targetRecOff;
+
+ /* ReadRecPtr contains page header */
+ Assert (targetRecOff != 0);
+ if (XLogNeedData(state, targetPagePtr, request_len, true))
+ return XLREAD_NEED_DATA;
+
+ /* error out if caller supplied bogus page */
+ if (!state->page_verified)
+ goto err;
+
+ prec = (XLogRecord *) (state->readBuf + targetRecOff);
+
+ /* validate record header if not yet */
+ if (!state->record_verified && record_len >= SizeOfXLogRecord)
+ {
+ if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
+ state->PrevRecPtr, prec))
+ goto err;
+
+ state->record_verified = true;
+ }
+
+
+ if (total_len == record_len)
+ {
+ /* Record does not cross a page boundary */
+ Assert(state->record_verified);
+
+ if (!ValidXLogRecord(state, prec, state->ReadRecPtr))
+ goto err;
+
+ state->record_verified = true; /* to be tidy */
+
+ /* We already checked the header earlier */
+ state->EndRecPtr = state->ReadRecPtr + MAXALIGN(record_len);
+
+ *record = prec;
+ state->readRecordState = XLREAD_NEXT_RECORD;
break;
+ }
+
+ /*
+ * The record continues on the next page. Need to reassemble
+ * record
+ */
+ Assert(total_len > record_len);
+
+ /* Enlarge readRecordBuf as needed. */
+ if (total_len > state->readRecordBufSize &&
+ !allocate_recordbuf(state, total_len))
+ {
+ /* We treat this as a "bogus data" condition */
+ report_invalid_record(state,
+ "record length %u at %X/%X too long",
+ total_len,
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr);
+ goto err;
+ }
+
+ /* Copy the first fragment of the record from the first page. */
+ memcpy(state->readRecordBuf, state->readBuf + targetRecOff,
+ record_len);
+ state->recordGotLen += record_len;
+ state->recordRemainLen -= record_len;
+
+ /* Calculate pointer to beginning of next page */
+ state->recordContRecPtr = state->ReadRecPtr + record_len;
+ Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0);
+
+ state->readRecordState = XLREAD_CONTINUATION;
}
+ /* fall through */
+
+ case XLREAD_CONTINUATION:
+ {
+ XLogPageHeader pageHeader;
+ uint32 pageHeaderSize;
+ XLogRecPtr targetPagePtr;
+
+ /* we enter this state only if we haven't read the whole record. */
+ Assert (state->recordRemainLen > 0);
+
+ while(state->recordRemainLen > 0)
+ {
+ char *contdata;
+ uint32 request_len;
+ uint32 record_len;
+
+ /* Wait for the next page to become available */
+ targetPagePtr = state->recordContRecPtr;
+
+ /* this request contains page header */
+ Assert (targetPagePtr != 0);
+ if (XLogNeedData(state, targetPagePtr,
+ Min(state->recordRemainLen, XLOG_BLCKSZ),
+ false))
+ return XLREAD_NEED_DATA;
+
+ if (!state->page_verified)
+ goto err;
+
+ Assert(SizeOfXLogShortPHD <= state->readLen);
- if (!state->page_verified)
- goto err;
+ /* Check that the continuation on next page looks valid */
+ pageHeader = (XLogPageHeader) state->readBuf;
+ if (!(pageHeader->xlp_info & XLP_FIRST_IS_CONTRECORD))
+ {
+ report_invalid_record(
+ state,
+ "there is no contrecord flag at %X/%X reading %X/%X",
+ (uint32) (state->recordContRecPtr >> 32),
+ (uint32) state->recordContRecPtr,
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr);
+ goto err;
+ }
- /* Record does not cross a page boundary */
- if (!ValidXLogRecord(state, record, RecPtr))
- goto err;
+ /*
+ * Cross-check that xlp_rem_len agrees with how much of the
+ * record we expect there to be left.
+ */
+ if (pageHeader->xlp_rem_len == 0 ||
+ pageHeader->xlp_rem_len != state->recordRemainLen)
+ {
+ report_invalid_record(
+ state,
+ "invalid contrecord length %u at %X/%X reading %X/%X, expected %u",
+ pageHeader->xlp_rem_len,
+ (uint32) (state->recordContRecPtr >> 32),
+ (uint32) state->recordContRecPtr,
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr,
+ state->recordRemainLen);
+ goto err;
+ }
- state->EndRecPtr = RecPtr + MAXALIGN(total_len);
+ /* Append the continuation from this page to the buffer */
+ pageHeaderSize = XLogPageHeaderSize(pageHeader);
- state->ReadRecPtr = RecPtr;
+ /*
+ * XLogNeedData should have ensured that the whole page header
+ * was read
+ */
+ Assert(state->readLen >= pageHeaderSize);
+
+ contdata = (char *) state->readBuf + pageHeaderSize;
+ record_len = XLOG_BLCKSZ - pageHeaderSize;
+ if (pageHeader->xlp_rem_len < record_len)
+ record_len = pageHeader->xlp_rem_len;
+
+ request_len = record_len + pageHeaderSize;
+
+ /* XLogNeedData should have ensured all needed data was read */
+ Assert (state->readLen >= request_len);
+
+ memcpy(state->readRecordBuf + state->recordGotLen,
+ (char *) contdata, record_len);
+ state->recordGotLen += record_len;
+ state->recordRemainLen -= record_len;
+
+ /* If we just reassembled the record header, validate it. */
+ if (!state->record_verified)
+ {
+ Assert(state->recordGotLen >= SizeOfXLogRecord);
+ if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
+ state->PrevRecPtr,
+ (XLogRecord *) state->readRecordBuf))
+ goto err;
+
+ state->record_verified = true;
+ }
+
+ /* Calculate pointer to beginning of next page, and continue */
+ state->recordContRecPtr += XLOG_BLCKSZ;
+ }
+
+ /* targetPagePtr is pointing the last-read page here */
+ prec = (XLogRecord *) state->readRecordBuf;
+ if (!ValidXLogRecord(state, prec, state->ReadRecPtr))
+ goto err;
+
+ pageHeaderSize =
+ XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+ state->EndRecPtr = targetPagePtr + pageHeaderSize
+ + MAXALIGN(pageHeader->xlp_rem_len);
+
+ *record = prec;
+ state->readRecordState = XLREAD_NEXT_RECORD;
+ break;
+ }
}
/*
* Special processing if it's an XLOG SWITCH record
*/
- if (record->xl_rmid == RM_XLOG_ID &&
- (record->xl_info & ~XLR_INFO_MASK) == XLOG_SWITCH)
+ if ((*record)->xl_rmid == RM_XLOG_ID &&
+ ((*record)->xl_info & ~XLR_INFO_MASK) == XLOG_SWITCH)
{
/* Pretend it extends to end of segment */
state->EndRecPtr += state->segcxt.ws_segsize - 1;
state->EndRecPtr -= XLogSegmentOffset(state->EndRecPtr, state->segcxt.ws_segsize);
}
- if (DecodeXLogRecord(state, record, errormsg))
- return record;
- else
- return NULL;
+ Assert (!*record || state->readLen >= 0);
+ if (DecodeXLogRecord(state, *record, errormsg))
+ return XLREAD_SUCCESS;
+
+ *record = NULL;
+ return XLREAD_FAIL;
err:
/*
- * Invalidate the read state. We might read from a different source after
+ * Invalidate the read page. We might read from a different source after
* failure.
*/
XLogReaderInvalReadState(state);
@@ -572,7 +706,8 @@ err:
if (state->errormsg_buf[0] != '\0')
*errormsg = state->errormsg_buf;
- return NULL;
+ *record = NULL;
+ return XLREAD_FAIL;
}
/*
@@ -724,11 +859,12 @@ XLogReaderInvalReadState(XLogReaderState *state)
*
* This is just a convenience subroutine to avoid duplicated code in
* XLogReadRecord. It's not intended for use from anywhere else.
+ *
+ * If PrevRecPtr is valid, the xl_prev is is cross-checked with it.
*/
static bool
ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
- XLogRecPtr PrevRecPtr, XLogRecord *record,
- bool randAccess)
+ XLogRecPtr PrevRecPtr, XLogRecord *record)
{
if (record->xl_tot_len < SizeOfXLogRecord)
{
@@ -746,7 +882,7 @@ ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
(uint32) RecPtr);
return false;
}
- if (randAccess)
+ if (PrevRecPtr == InvalidXLogRecPtr)
{
/*
* We can't exactly verify the prev-link, but surely it should be less
@@ -978,11 +1114,14 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
* XLogReadRecord() will read the next valid record.
*/
XLogRecPtr
-XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
+XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
+ XLogFindNextRecordCB read_page, void *private)
{
XLogRecPtr tmpRecPtr;
XLogRecPtr found = InvalidXLogRecPtr;
XLogPageHeader header;
+ XLogRecord *record;
+ XLogReadRecordResult result;
char *errormsg;
Assert(!XLogRecPtrIsInvalid(RecPtr));
@@ -1015,9 +1154,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
while(XLogNeedData(state, targetPagePtr, targetRecOff,
targetRecOff != 0))
{
- if (!state->routine.page_read(state, state->readPagePtr,
- state->readLen,
- state->ReadRecPtr, state->readBuf))
+ if (!read_page(state, private))
break;
}
@@ -1069,8 +1206,16 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
* or we just jumped over the remaining data of a continuation.
*/
XLogBeginRead(state, tmpRecPtr);
- while (XLogReadRecord(state, &errormsg) != NULL)
+ while ((result = XLogReadRecord(state, &record, &errormsg)) !=
+ XLREAD_FAIL)
{
+ if (result == XLREAD_NEED_DATA)
+ {
+ if (!read_page(state, private))
+ goto err;
+ continue;
+ }
+
/* past the record we've found, break out */
if (RecPtr <= state->ReadRecPtr)
{
@@ -1090,9 +1235,9 @@ err:
#endif /* FRONTEND */
/*
- * Helper function to ease writing of XLogRoutine->page_read callbacks.
- * If this function is used, caller must supply a segment_open callback in
- * 'state', as that is used here.
+ * Helper function to ease writing of page_read callback.
+ * If this function is used, caller must supply a segment_open callback and
+ * segment_close callback as that is used here.
*
* Read 'count' bytes into 'buf', starting at location 'startptr', from WAL
* fetched from timeline 'tli'.
@@ -1105,6 +1250,7 @@ err:
*/
bool
WALRead(XLogReaderState *state,
+ WALSegmentOpenCB segopenfn, WALSegmentCloseCB segclosefn,
char *buf, XLogRecPtr startptr, Size count, TimeLineID tli,
WALReadError *errinfo)
{
@@ -1136,10 +1282,10 @@ WALRead(XLogReaderState *state,
XLogSegNo nextSegNo;
if (state->seg.ws_file >= 0)
- state->routine.segment_close(state);
+ segclosefn(state);
XLByteToSeg(recptr, nextSegNo, state->segcxt.ws_segsize);
- state->routine.segment_open(state, nextSegNo, &tli);
+ segopenfn(state, nextSegNo, &tli);
/* This shouldn't happen -- indicates a bug in segment_open */
Assert(state->seg.ws_file >= 0);
diff --git a/src/backend/access/transam/xlogutils.c b/src/backend/access/transam/xlogutils.c
index 1b10b6df20..303333571f 100644
--- a/src/backend/access/transam/xlogutils.c
+++ b/src/backend/access/transam/xlogutils.c
@@ -689,8 +689,7 @@ XLogTruncateRelation(RelFileNode rnode, ForkNumber forkNum,
void
XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wantLength)
{
- const XLogRecPtr lastReadPage = state->seg.ws_segno *
- state->segcxt.ws_segsize + state->readLen;
+ const XLogRecPtr lastReadPage = state->readPagePtr;
Assert(wantPage != InvalidXLogRecPtr && wantPage % XLOG_BLCKSZ == 0);
Assert(wantLength <= XLOG_BLCKSZ);
@@ -705,7 +704,7 @@ XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wa
* current TLI has since become historical.
*/
if (lastReadPage == wantPage &&
- state->readLen != 0 &&
+ state->page_verified &&
lastReadPage + state->readLen >= wantPage + Min(wantLength, XLOG_BLCKSZ - 1))
return;
@@ -792,6 +791,7 @@ wal_segment_open(XLogReaderState *state, XLogSegNo nextSegNo,
char path[MAXPGPATH];
XLogFilePath(path, tli, nextSegNo, state->segcxt.ws_segsize);
+ elog(LOG, "HOGE: %lu, %d => %s", nextSegNo, tli, path);
state->seg.ws_file = BasicOpenFile(path, O_RDONLY | PG_BINARY);
if (state->seg.ws_file >= 0)
return;
@@ -829,9 +829,11 @@ wal_segment_close(XLogReaderState *state)
* loop for now.
*/
bool
-read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *cur_page)
+read_local_xlog_page(XLogReaderState *state)
{
+ XLogRecPtr targetPagePtr = state->readPagePtr;
+ int reqLen = state->readLen;
+ char *cur_page = state->readBuf;
XLogRecPtr read_upto,
loc;
TimeLineID tli;
@@ -944,11 +946,12 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
* as 'count', read the whole page anyway. It's guaranteed to be
* zero-padded up to the page boundary if it's incomplete.
*/
- if (!WALRead(state, cur_page, targetPagePtr, XLOG_BLCKSZ, tli,
- &errinfo))
+ if (!WALRead(state, wal_segment_open, wal_segment_close,
+ cur_page, targetPagePtr, XLOG_BLCKSZ, tli, &errinfo))
WALReadRaiseError(&errinfo);
/* number of valid bytes in the buffer */
+ state->readPagePtr = targetPagePtr;
state->readLen = count;
return true;
}
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index 61902be3b0..a15b0b3355 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -120,7 +120,8 @@ StartupDecodingContext(List *output_plugin_options,
TransactionId xmin_horizon,
bool need_full_snapshot,
bool fast_forward,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress)
@@ -169,11 +170,12 @@ StartupDecodingContext(List *output_plugin_options,
ctx->slot = slot;
- ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, xl_routine, ctx);
+ ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, cleanup_cb);
if (!ctx->reader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
errmsg("out of memory")));
+ ctx->page_read = page_read;
ctx->reorder = ReorderBufferAllocate();
ctx->snapshot_builder =
@@ -231,7 +233,8 @@ CreateInitDecodingContext(char *plugin,
List *output_plugin_options,
bool need_full_snapshot,
XLogRecPtr restart_lsn,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress)
@@ -328,7 +331,7 @@ CreateInitDecodingContext(char *plugin,
ctx = StartupDecodingContext(NIL, restart_lsn, xmin_horizon,
need_full_snapshot, false,
- xl_routine, prepare_write, do_write,
+ page_read, cleanup_cb, prepare_write, do_write,
update_progress);
/* call output plugin initialization callback */
@@ -376,7 +379,8 @@ LogicalDecodingContext *
CreateDecodingContext(XLogRecPtr start_lsn,
List *output_plugin_options,
bool fast_forward,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress)
@@ -429,8 +433,8 @@ CreateDecodingContext(XLogRecPtr start_lsn,
ctx = StartupDecodingContext(output_plugin_options,
start_lsn, InvalidTransactionId, false,
- fast_forward, xl_routine, prepare_write,
- do_write, update_progress);
+ fast_forward, page_read, cleanup_cb,
+ prepare_write, do_write, update_progress);
/* call output plugin initialization callback */
old_context = MemoryContextSwitchTo(ctx->context);
@@ -483,7 +487,13 @@ DecodingContextFindStartpoint(LogicalDecodingContext *ctx)
char *err = NULL;
/* the read_page callback waits for new WAL */
- record = XLogReadRecord(ctx->reader, &err);
+ while (XLogReadRecord(ctx->reader, &record, &err) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!ctx->page_read(ctx->reader))
+ break;
+ }
+
if (err)
elog(ERROR, "%s", err);
if (!record)
diff --git a/src/backend/replication/logical/logicalfuncs.c b/src/backend/replication/logical/logicalfuncs.c
index b99c94e848..fcc81b358e 100644
--- a/src/backend/replication/logical/logicalfuncs.c
+++ b/src/backend/replication/logical/logicalfuncs.c
@@ -233,9 +233,8 @@ pg_logical_slot_get_changes_guts(FunctionCallInfo fcinfo, bool confirm, bool bin
ctx = CreateDecodingContext(InvalidXLogRecPtr,
options,
false,
- XL_ROUTINE(.page_read = read_local_xlog_page,
- .segment_open = wal_segment_open,
- .segment_close = wal_segment_close),
+ read_local_xlog_page,
+ wal_segment_close,
LogicalOutputPrepareWrite,
LogicalOutputWrite, NULL);
@@ -284,7 +283,13 @@ pg_logical_slot_get_changes_guts(FunctionCallInfo fcinfo, bool confirm, bool bin
XLogRecord *record;
char *errm = NULL;
- record = XLogReadRecord(ctx->reader, &errm);
+ while (XLogReadRecord(ctx->reader, &record, &errm) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!ctx->page_read(ctx->reader))
+ break;
+ }
+
if (errm)
elog(ERROR, "%s", errm);
diff --git a/src/backend/replication/slotfuncs.c b/src/backend/replication/slotfuncs.c
index 88033a79b2..8a678074d5 100644
--- a/src/backend/replication/slotfuncs.c
+++ b/src/backend/replication/slotfuncs.c
@@ -152,9 +152,8 @@ create_logical_replication_slot(char *name, char *plugin,
ctx = CreateInitDecodingContext(plugin, NIL,
false, /* just catalogs is OK */
restart_lsn,
- XL_ROUTINE(.page_read = read_local_xlog_page,
- .segment_open = wal_segment_open,
- .segment_close = wal_segment_close),
+ read_local_xlog_page,
+ wal_segment_close,
NULL, NULL, NULL);
/*
@@ -486,9 +485,8 @@ pg_logical_replication_slot_advance(XLogRecPtr moveto)
ctx = CreateDecodingContext(InvalidXLogRecPtr,
NIL,
true, /* fast_forward */
- XL_ROUTINE(.page_read = read_local_xlog_page,
- .segment_open = wal_segment_open,
- .segment_close = wal_segment_close),
+ read_local_xlog_page,
+ wal_segment_close,
NULL, NULL, NULL);
/*
@@ -513,7 +511,13 @@ pg_logical_replication_slot_advance(XLogRecPtr moveto)
* Read records. No changes are generated in fast_forward mode,
* but snapbuilder/slot statuses are updated properly.
*/
- record = XLogReadRecord(ctx->reader, &errm);
+ while (XLogReadRecord(ctx->reader, &record, &errm) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!ctx->page_read(ctx->reader))
+ break;
+ }
+
if (errm)
elog(ERROR, "%s", errm);
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index bf2711bddd..2fd24a9b14 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -580,10 +580,7 @@ StartReplication(StartReplicationCmd *cmd)
/* create xlogreader for physical replication */
xlogreader =
- XLogReaderAllocate(wal_segment_size, NULL,
- XL_ROUTINE(.segment_open = WalSndSegmentOpen,
- .segment_close = wal_segment_close),
- NULL);
+ XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
if (!xlogreader)
ereport(ERROR,
@@ -812,9 +809,11 @@ StartReplication(StartReplicationCmd *cmd)
* set every time WAL is flushed.
*/
static bool
-logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetRecPtr, char *cur_page)
+logical_read_xlog_page(XLogReaderState *state)
{
+ XLogRecPtr targetPagePtr = state->readPagePtr;
+ int reqLen = state->readLen;
+ char *cur_page = state->readBuf;
XLogRecPtr flushptr;
int count;
WALReadError errinfo;
@@ -842,7 +841,7 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
count = flushptr - targetPagePtr; /* part of the page available */
/* now actually read the data, we know it's there */
- if (!WALRead(state,
+ if (!WALRead(state, WalSndSegmentOpen, wal_segment_close,
cur_page,
targetPagePtr,
XLOG_BLCKSZ,
@@ -1014,9 +1013,8 @@ CreateReplicationSlot(CreateReplicationSlotCmd *cmd)
ctx = CreateInitDecodingContext(cmd->plugin, NIL, need_full_snapshot,
InvalidXLogRecPtr,
- XL_ROUTINE(.page_read = logical_read_xlog_page,
- .segment_open = WalSndSegmentOpen,
- .segment_close = wal_segment_close),
+ logical_read_xlog_page,
+ wal_segment_close,
WalSndPrepareWrite, WalSndWriteData,
WalSndUpdateProgress);
@@ -1179,9 +1177,8 @@ StartLogicalReplication(StartReplicationCmd *cmd)
*/
logical_decoding_ctx =
CreateDecodingContext(cmd->startpoint, cmd->options, false,
- XL_ROUTINE(.page_read = logical_read_xlog_page,
- .segment_open = WalSndSegmentOpen,
- .segment_close = wal_segment_close),
+ logical_read_xlog_page,
+ wal_segment_close,
WalSndPrepareWrite, WalSndWriteData,
WalSndUpdateProgress);
xlogreader = logical_decoding_ctx->reader;
@@ -2761,7 +2758,7 @@ XLogSendPhysical(void)
enlargeStringInfo(&output_message, nbytes);
retry:
- if (!WALRead(xlogreader,
+ if (!WALRead(xlogreader, WalSndSegmentOpen, wal_segment_close,
&output_message.data[output_message.len],
startptr,
nbytes,
@@ -2859,7 +2856,12 @@ XLogSendLogical(void)
*/
WalSndCaughtUp = false;
- record = XLogReadRecord(logical_decoding_ctx->reader, &errm);
+ while (XLogReadRecord(logical_decoding_ctx->reader, &record, &errm) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!logical_decoding_ctx->page_read(logical_decoding_ctx->reader))
+ break;
+ }
/* xlog record was invalid */
if (errm != NULL)
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index 0b7e73ae79..3dd6df4be6 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -40,15 +40,9 @@ static int xlogreadfd = -1;
static XLogSegNo xlogreadsegno = -1;
static char xlogfpath[MAXPGPATH];
-typedef struct XLogPageReadPrivate
-{
- const char *restoreCommand;
- int tliIndex;
-} XLogPageReadPrivate;
-
-static bool SimpleXLogPageRead(XLogReaderState *xlogreader,
- XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
+static bool SimpleXLogPageRead(XLogReaderState *xlogreader,
+ const char *datadir, int *tliIndex,
+ const char *restoreCommand);
/*
* Read WAL from the datadir/pg_wal, starting from 'startpoint' on timeline
@@ -62,20 +56,22 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
XLogRecord *record;
XLogReaderState *xlogreader;
char *errormsg;
- XLogPageReadPrivate private;
- private.tliIndex = tliIndex;
- private.restoreCommand = restoreCommand;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir,
- XL_ROUTINE(.page_read = &SimpleXLogPageRead),
- &private);
+ xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+
if (xlogreader == NULL)
pg_fatal("out of memory");
XLogBeginRead(xlogreader, startpoint);
do
{
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!SimpleXLogPageRead(xlogreader, datadir,
+ &tliIndex, restoreCommand))
+ break;
+ }
if (record == NULL)
{
@@ -113,19 +109,19 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex,
XLogRecord *record;
XLogReaderState *xlogreader;
char *errormsg;
- XLogPageReadPrivate private;
XLogRecPtr endptr;
- private.tliIndex = tliIndex;
- private.restoreCommand = restoreCommand;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir,
- XL_ROUTINE(.page_read = &SimpleXLogPageRead),
- &private);
+ xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
XLogBeginRead(xlogreader, ptr);
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!SimpleXLogPageRead(xlogreader, datadir, &tliIndex, restoreCommand))
+ break;
+ }
if (record == NULL)
{
if (errormsg)
@@ -160,7 +156,6 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
XLogRecPtr searchptr;
XLogReaderState *xlogreader;
char *errormsg;
- XLogPageReadPrivate private;
/*
* The given fork pointer points to the end of the last common record,
@@ -176,11 +171,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
forkptr += SizeOfXLogShortPHD;
}
- private.tliIndex = tliIndex;
- private.restoreCommand = restoreCommand;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir,
- XL_ROUTINE(.page_read = &SimpleXLogPageRead),
- &private);
+ xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
@@ -190,7 +181,13 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
uint8 info;
XLogBeginRead(xlogreader, searchptr);
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!SimpleXLogPageRead(xlogreader, datadir,
+ &tliIndex, restoreCommand))
+ break;
+ }
if (record == NULL)
{
@@ -237,10 +234,11 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
/* XLogReader callback function, to read a WAL page */
static bool
-SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *readBuf)
+SimpleXLogPageRead(XLogReaderState *xlogreader, const char *datadir,
+ int *tliIndex, const char *restoreCommand)
{
- XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
+ XLogRecPtr targetPagePtr = xlogreader->readPagePtr;
+ char *readBuf = xlogreader->readBuf;
uint32 targetPageOff;
XLogRecPtr targetSegEnd;
XLogSegNo targetSegNo;
@@ -273,14 +271,14 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
* be done both forward and backward, consider also switching timeline
* accordingly.
*/
- while (private->tliIndex < targetNentries - 1 &&
- targetHistory[private->tliIndex].end < targetSegEnd)
- private->tliIndex++;
- while (private->tliIndex > 0 &&
- targetHistory[private->tliIndex].begin >= targetSegEnd)
- private->tliIndex--;
+ while (*tliIndex < targetNentries - 1 &&
+ targetHistory[*tliIndex].end < targetSegEnd)
+ (*tliIndex)++;
+ while (*tliIndex > 0 &&
+ targetHistory[*tliIndex].begin >= targetSegEnd)
+ (*tliIndex)--;
- XLogFileName(xlogfname, targetHistory[private->tliIndex].tli,
+ XLogFileName(xlogfname, targetHistory[*tliIndex].tli,
xlogreadsegno, WalSegSz);
snprintf(xlogfpath, MAXPGPATH, "%s/" XLOGDIR "/%s",
@@ -293,7 +291,7 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
/*
* If we have no restore_command to execute, then exit.
*/
- if (private->restoreCommand == NULL)
+ if (restoreCommand == NULL)
{
pg_log_error("could not open file \"%s\": %m", xlogfpath);
xlogreader->readLen = -1;
@@ -307,7 +305,7 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
xlogreadfd = RestoreArchivedFile(xlogreader->segcxt.ws_dir,
xlogfname,
WalSegSz,
- private->restoreCommand);
+ restoreCommand);
if (xlogreadfd < 0)
{
@@ -349,7 +347,7 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
Assert(targetSegNo == xlogreadsegno);
- xlogreader->seg.ws_tli = targetHistory[private->tliIndex].tli;
+ xlogreader->seg.ws_tli = targetHistory[*tliIndex].tli;
xlogreader->readLen = XLOG_BLCKSZ;
return true;
}
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index ab0c28d9b7..02e71138cc 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -330,12 +330,17 @@ WALDumpCloseSegment(XLogReaderState *state)
state->seg.ws_file = -1;
}
-/* pg_waldump's XLogReaderRoutine->page_read callback */
+/*
+ * pg_waldump's WAL page rader, also used as page_read callback for
+ * XLogFindNextRecord
+ */
static bool
-WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetPtr, char *readBuff)
+WALDumpReadPage(XLogReaderState *state, void *priv)
{
- XLogDumpPrivate *private = state->private_data;
+ XLogRecPtr targetPagePtr = state->readPagePtr;
+ int reqLen = state->readLen;
+ char *readBuff = state->readBuf;
+ XLogDumpPrivate *private = (XLogDumpPrivate *) priv;
int count = XLOG_BLCKSZ;
WALReadError errinfo;
@@ -353,8 +358,8 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
}
}
- if (!WALRead(state, readBuff, targetPagePtr, count, private->timeline,
- &errinfo))
+ if (!WALRead(state, WALDumpOpenSegment, WALDumpCloseSegment,
+ readBuff, targetPagePtr, count, private->timeline, &errinfo))
{
WALOpenSegment *seg = &errinfo.wre_seg;
char fname[MAXPGPATH];
@@ -374,6 +379,7 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
(Size) errinfo.wre_req);
}
+ Assert(count >= state->readLen);
state->readLen = count;
return true;
}
@@ -1042,16 +1048,14 @@ main(int argc, char **argv)
/* we have everything we need, start reading */
xlogreader_state =
- XLogReaderAllocate(WalSegSz, waldir,
- XL_ROUTINE(.page_read = WALDumpReadPage,
- .segment_open = WALDumpOpenSegment,
- .segment_close = WALDumpCloseSegment),
- &private);
+ XLogReaderAllocate(WalSegSz, waldir, WALDumpCloseSegment);
+
if (!xlogreader_state)
fatal_error("out of memory");
/* first find a valid recptr to start from */
- first_record = XLogFindNextRecord(xlogreader_state, private.startptr);
+ first_record = XLogFindNextRecord(xlogreader_state, private.startptr,
+ &WALDumpReadPage, (void*) &private);
if (first_record == InvalidXLogRecPtr)
fatal_error("could not find a valid record after %X/%X",
@@ -1075,7 +1079,13 @@ main(int argc, char **argv)
for (;;)
{
/* try to read the next record */
- record = XLogReadRecord(xlogreader_state, &errormsg);
+ while (XLogReadRecord(xlogreader_state, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!WALDumpReadPage(xlogreader_state, (void *) &private))
+ break;
+ }
+
if (!record)
{
if (!config.follow || private.endptr_reached)
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index 6c8848e14f..6a7fe140cb 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -57,64 +57,15 @@ typedef struct WALSegmentContext
typedef struct XLogReaderState XLogReaderState;
-/* Function type definition for the read_page callback */
-typedef bool (*XLogPageReadCB) (XLogReaderState *xlogreader,
- XLogRecPtr targetPagePtr,
- int reqLen,
- XLogRecPtr targetRecPtr,
- char *readBuf);
+/* Function type definition for the segment cleanup callback */
+typedef void (*WALSegmentCleanupCB) (XLogReaderState *xlogreader);
+
+/* Function type definition for the open/close callbacks for WALRead() */
typedef void (*WALSegmentOpenCB) (XLogReaderState *xlogreader,
XLogSegNo nextSegNo,
TimeLineID *tli_p);
typedef void (*WALSegmentCloseCB) (XLogReaderState *xlogreader);
-typedef struct XLogReaderRoutine
-{
- /*
- * Data input callback
- *
- * This callback shall read at least reqLen valid bytes of the xlog page
- * starting at targetPagePtr, and store them in readBuf. The callback
- * shall return the number of bytes read (never more than XLOG_BLCKSZ), or
- * -1 on failure. The callback shall sleep, if necessary, to wait for the
- * requested bytes to become available. The callback will not be invoked
- * again for the same page unless more than the returned number of bytes
- * are needed.
- *
- * targetRecPtr is the position of the WAL record we're reading. Usually
- * it is equal to targetPagePtr + reqLen, but sometimes xlogreader needs
- * to read and verify the page or segment header, before it reads the
- * actual WAL record it's interested in. In that case, targetRecPtr can
- * be used to determine which timeline to read the page from.
- *
- * The callback shall set ->seg.ws_tli to the TLI of the file the page was
- * read from.
- */
- XLogPageReadCB page_read;
-
- /*
- * Callback to open the specified WAL segment for reading. ->seg.ws_file
- * shall be set to the file descriptor of the opened segment. In case of
- * failure, an error shall be raised by the callback and it shall not
- * return.
- *
- * "nextSegNo" is the number of the segment to be opened.
- *
- * "tli_p" is an input/output argument. WALRead() uses it to pass the
- * timeline in which the new segment should be found, but the callback can
- * use it to return the TLI that it actually opened.
- */
- WALSegmentOpenCB segment_open;
-
- /*
- * WAL segment close callback. ->seg.ws_file shall be set to a negative
- * number.
- */
- WALSegmentCloseCB segment_close;
-} XLogReaderRoutine;
-
-#define XL_ROUTINE(...) &(XLogReaderRoutine){__VA_ARGS__}
-
typedef struct
{
/* Is this block ref in use? */
@@ -144,12 +95,35 @@ typedef struct
uint16 data_bufsz;
} DecodedBkpBlock;
+/* Return code from XLogReadRecord */
+typedef enum XLogReadRecordResult
+{
+ XLREAD_SUCCESS, /* record is successfully read */
+ XLREAD_NEED_DATA, /* need more data. see XLogReadRecord. */
+ XLREAD_FAIL /* failed during reading a record */
+} XLogReadRecordResult;
+
+/*
+ * internal state of XLogReadRecord
+ *
+ * XLogReadState runs a state machine while reading a record. Theses states
+ * are not seen outside the function. Each state may repeat several times
+ * exiting requesting caller for new data. See the comment of XLogReadRecrod
+ * for details.
+ */
+typedef enum XLogReadRecordState {
+ XLREAD_NEXT_RECORD,
+ XLREAD_TOT_LEN,
+ XLREAD_FIRST_FRAGMENT,
+ XLREAD_CONTINUATION
+} XLogReadRecordState;
+
struct XLogReaderState
{
/*
* Operational callbacks
*/
- XLogReaderRoutine routine;
+ WALSegmentCleanupCB cleanup_cb;
/* ----------------------------------------
* Public parameters
@@ -162,18 +136,14 @@ struct XLogReaderState
*/
uint64 system_identifier;
- /*
- * Opaque data for callbacks to use. Not used by XLogReader.
- */
- void *private_data;
-
/*
* Start and end point of last record read. EndRecPtr is also used as the
* position to read next. Calling XLogBeginRead() sets EndRecPtr to the
* starting position and ReadRecPtr to invalid.
*/
- XLogRecPtr ReadRecPtr; /* start of last record read */
+ XLogRecPtr ReadRecPtr; /* start of last record read or being read */
XLogRecPtr EndRecPtr; /* end+1 of last record read */
+ XLogRecPtr PrevRecPtr; /* start of previous record read */
/* ----------------------------------------
* Communication with page reader
@@ -187,7 +157,9 @@ struct XLogReaderState
* the request, or -1 on error */
TimeLineID readPageTLI; /* TLI for data currently in readBuf */
char *readBuf; /* buffer to store data */
- bool page_verified; /* is the page on the buffer verified? */
+ bool page_verified; /* is the page header on the buffer verified? */
+ bool record_verified;/* is the current record header verified? */
+
/* ----------------------------------------
@@ -227,8 +199,6 @@ struct XLogReaderState
XLogRecPtr latestPagePtr;
TimeLineID latestPageTLI;
- /* beginning of the WAL record being read. */
- XLogRecPtr currRecPtr;
/* timeline to read it from, 0 if a lookup is required */
TimeLineID currTLI;
@@ -255,6 +225,15 @@ struct XLogReaderState
char *readRecordBuf;
uint32 readRecordBufSize;
+ /*
+ * XLogReadRecord() state
+ */
+ XLogReadRecordState readRecordState;/* state machine state */
+ int recordGotLen; /* amount of current record that has
+ * already been read */
+ int recordRemainLen; /* length of current record that remains */
+ XLogRecPtr recordContRecPtr; /* where the current record continues */
+
/* Buffer to hold error message */
char *errormsg_buf;
};
@@ -262,9 +241,7 @@ struct XLogReaderState
/* Get a new XLogReader */
extern XLogReaderState *XLogReaderAllocate(int wal_segment_size,
const char *waldir,
- XLogReaderRoutine *routine,
- void *private_data);
-extern XLogReaderRoutine *LocalXLogReaderRoutine(void);
+ WALSegmentCleanupCB cleanup_cb);
/* Free an XLogReader */
extern void XLogReaderFree(XLogReaderState *state);
@@ -272,12 +249,17 @@ extern void XLogReaderFree(XLogReaderState *state);
/* Position the XLogReader to given record */
extern void XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr);
#ifdef FRONTEND
-extern XLogRecPtr XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr);
+/* Function type definition for the read_page callback */
+typedef bool (*XLogFindNextRecordCB) (XLogReaderState *xlogreader,
+ void *private);
+extern XLogRecPtr XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
+ XLogFindNextRecordCB read_page, void *private);
#endif /* FRONTEND */
/* Read the next XLog record. Returns NULL on end-of-WAL or failure */
-extern struct XLogRecord *XLogReadRecord(XLogReaderState *state,
- char **errormsg);
+extern XLogReadRecordResult XLogReadRecord(XLogReaderState *state,
+ XLogRecord **record,
+ char **errormsg);
/* Validate a page */
extern bool XLogReaderValidatePageHeader(XLogReaderState *state,
@@ -297,6 +279,7 @@ typedef struct WALReadError
} WALReadError;
extern bool WALRead(XLogReaderState *state,
+ WALSegmentOpenCB segopenfn, WALSegmentCloseCB sgclosefn,
char *buf, XLogRecPtr startptr, Size count,
TimeLineID tli, WALReadError *errinfo);
diff --git a/src/include/access/xlogutils.h b/src/include/access/xlogutils.h
index c287edf206..482e4ced69 100644
--- a/src/include/access/xlogutils.h
+++ b/src/include/access/xlogutils.h
@@ -47,9 +47,7 @@ extern Buffer XLogReadBufferExtended(RelFileNode rnode, ForkNumber forknum,
extern Relation CreateFakeRelcacheEntry(RelFileNode rnode);
extern void FreeFakeRelcacheEntry(Relation fakerel);
-extern bool read_local_xlog_page(XLogReaderState *state,
- XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetRecPtr, char *cur_page);
+extern bool read_local_xlog_page(XLogReaderState *state);
extern void wal_segment_open(XLogReaderState *state,
XLogSegNo nextSegNo,
TimeLineID *tli_p);
diff --git a/src/include/replication/logical.h b/src/include/replication/logical.h
index c2f2475e5d..a743db49cf 100644
--- a/src/include/replication/logical.h
+++ b/src/include/replication/logical.h
@@ -29,6 +29,10 @@ typedef void (*LogicalOutputPluginWriterUpdateProgress) (struct LogicalDecodingC
TransactionId xid
);
+typedef struct LogicalDecodingContext LogicalDecodingContext;
+
+typedef bool (*LogicalDecodingXLogPageReadCB)(XLogReaderState *ctx);
+
typedef struct LogicalDecodingContext
{
/* memory context this is all allocated in */
@@ -39,6 +43,7 @@ typedef struct LogicalDecodingContext
/* infrastructure pieces for decoding */
XLogReaderState *reader;
+ LogicalDecodingXLogPageReadCB page_read;
struct ReorderBuffer *reorder;
struct SnapBuild *snapshot_builder;
@@ -95,14 +100,16 @@ extern LogicalDecodingContext *CreateInitDecodingContext(char *plugin,
List *output_plugin_options,
bool need_full_snapshot,
XLogRecPtr restart_lsn,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress);
extern LogicalDecodingContext *CreateDecodingContext(XLogRecPtr start_lsn,
List *output_plugin_options,
bool fast_forward,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress);
--
2.18.4
----Next_Part(Thu_Jul__2_13_53_30_2020_072)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
filename="v15-0003-Remove-globals-readOff-readLen-and-readSegNo.patch"
^ permalink raw reply [nested|flat] 70+ messages in thread
* [PATCH v17 2/4] Move page-reader out of XLogReadRecord
@ 2019-09-10 03:58 Kyotaro Horiguchi <[email protected]>
0 siblings, 0 replies; 70+ messages in thread
From: Kyotaro Horiguchi @ 2019-09-10 03:58 UTC (permalink / raw)
This is the second step of removing callbacks from WAL record reader.
Since it is essential to take in additional data while reading a
record, the function have to ask caller for new data while keeping
working state. Thus the function is turned into a state machine.
---
src/backend/access/transam/twophase.c | 15 +-
src/backend/access/transam/xlog.c | 58 +-
src/backend/access/transam/xlogreader.c | 697 +++++++++++-------
src/backend/access/transam/xlogutils.c | 17 +-
src/backend/replication/logical/logical.c | 26 +-
.../replication/logical/logicalfuncs.c | 13 +-
src/backend/replication/slotfuncs.c | 18 +-
src/backend/replication/walsender.c | 32 +-
src/bin/pg_rewind/parsexlog.c | 84 ++-
src/bin/pg_waldump/pg_waldump.c | 36 +-
src/include/access/xlogreader.h | 121 ++-
src/include/access/xlogutils.h | 4 +-
src/include/pg_config_manual.h | 2 +-
src/include/replication/logical.h | 11 +-
14 files changed, 646 insertions(+), 488 deletions(-)
diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index 80d2d20d6c..7d5a96a5f5 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -1325,11 +1325,8 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
XLogReaderState *xlogreader;
char *errormsg;
- xlogreader = XLogReaderAllocate(wal_segment_size, NULL,
- XL_ROUTINE(.page_read = &read_local_xlog_page,
- .segment_open = &wal_segment_open,
- .segment_close = &wal_segment_close),
- NULL);
+ xlogreader = XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+
if (!xlogreader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
@@ -1337,7 +1334,13 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
errdetail("Failed while allocating a WAL reading processor.")));
XLogBeginRead(xlogreader, lsn);
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!read_local_xlog_page(xlogreader))
+ break;
+ }
+
if (record == NULL)
ereport(ERROR,
(errcode_for_file_access(),
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 606328a65a..670996ae67 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -836,13 +836,6 @@ static XLogSource currentSource = XLOG_FROM_ANY;
static bool lastSourceFailed = false;
static bool pendingWalRcvRestart = false;
-typedef struct XLogPageReadPrivate
-{
- int emode;
- bool fetching_ckpt; /* are we fetching a checkpoint record? */
- bool randAccess;
-} XLogPageReadPrivate;
-
/*
* These variables track when we last obtained some WAL data to process,
* and where we got it from. (XLogReceiptSource is initially the same as
@@ -917,8 +910,8 @@ static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath,
static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli,
XLogSource source, bool notfoundOk);
static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, XLogSource source);
-static bool XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
+static bool XLogPageRead(XLogReaderState *xlogreader,
+ bool fetching_ckpt, int emode, bool randAccess);
static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
bool fetching_ckpt, XLogRecPtr tliRecPtr);
static int emode_for_corrupt_record(int emode, XLogRecPtr RecPtr);
@@ -1231,8 +1224,7 @@ XLogInsertRecord(XLogRecData *rdata,
appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len);
if (!debug_reader)
- debug_reader = XLogReaderAllocate(wal_segment_size, NULL,
- XL_ROUTINE(), NULL);
+ debug_reader = XLogReaderAllocate(wal_segment_size, NULL, NULL);
if (!debug_reader)
{
@@ -4344,15 +4336,10 @@ CleanupBackupHistory(void)
* record is available.
*/
static XLogRecord *
-ReadRecord(XLogReaderState *xlogreader, int emode,
- bool fetching_ckpt)
+ReadRecord(XLogReaderState *xlogreader, int emode, bool fetching_ckpt)
{
XLogRecord *record;
- XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
-
- private->fetching_ckpt = fetching_ckpt;
- private->emode = emode;
- private->randAccess = (xlogreader->ReadRecPtr == InvalidXLogRecPtr);
+ bool randAccess = (xlogreader->ReadRecPtr == InvalidXLogRecPtr);
/* This is the first attempt to read this page. */
lastSourceFailed = false;
@@ -4360,8 +4347,16 @@ ReadRecord(XLogReaderState *xlogreader, int emode,
for (;;)
{
char *errormsg;
+ XLogReadRecordResult result;
+
+ while ((result = XLogReadRecord(xlogreader, &record, &errormsg))
+ == XLREAD_NEED_DATA)
+ {
+ if (!XLogPageRead(xlogreader, fetching_ckpt, emode, randAccess))
+ break;
+
+ }
- record = XLogReadRecord(xlogreader, &errormsg);
ReadRecPtr = xlogreader->ReadRecPtr;
EndRecPtr = xlogreader->EndRecPtr;
if (record == NULL)
@@ -6380,7 +6375,6 @@ StartupXLOG(void)
bool backupFromStandby = false;
DBState dbstate_at_startup;
XLogReaderState *xlogreader;
- XLogPageReadPrivate private;
bool promoted = false;
struct stat st;
@@ -6539,13 +6533,9 @@ StartupXLOG(void)
OwnLatch(&XLogCtl->recoveryWakeupLatch);
/* Set up XLOG reader facility */
- MemSet(&private, 0, sizeof(XLogPageReadPrivate));
xlogreader =
- XLogReaderAllocate(wal_segment_size, NULL,
- XL_ROUTINE(.page_read = &XLogPageRead,
- .segment_open = NULL,
- .segment_close = wal_segment_close),
- &private);
+ XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+
if (!xlogreader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
@@ -11987,12 +11977,13 @@ CancelBackup(void)
* sleep and retry.
*/
static bool
-XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetRecPtr, char *readBuf)
+XLogPageRead(XLogReaderState *xlogreader,
+ bool fetching_ckpt, int emode, bool randAccess)
{
- XLogPageReadPrivate *private =
- (XLogPageReadPrivate *) xlogreader->private_data;
- int emode = private->emode;
+ char *readBuf = xlogreader->readBuf;
+ XLogRecPtr targetPagePtr = xlogreader->readPagePtr;
+ int reqLen = xlogreader->readLen;
+ XLogRecPtr targetRecPtr = xlogreader->ReadRecPtr;
uint32 targetPageOff;
XLogSegNo targetSegNo PG_USED_FOR_ASSERTS_ONLY;
int r;
@@ -12035,8 +12026,8 @@ retry:
flushedUpto < targetPagePtr + reqLen))
{
if (!WaitForWALToBecomeAvailable(targetPagePtr + reqLen,
- private->randAccess,
- private->fetching_ckpt,
+ randAccess,
+ fetching_ckpt,
targetRecPtr))
{
if (readFile >= 0)
@@ -12141,6 +12132,7 @@ retry:
goto next_record_is_invalid;
}
+ Assert(xlogreader->readPagePtr == targetPagePtr);
xlogreader->readLen = readLen;
return true;
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index 7d04995b7b..ba3e4868f7 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -40,7 +40,7 @@ static bool XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr,
int reqLen, bool header_inclusive);
static void XLogReaderInvalReadState(XLogReaderState *state);
static bool ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
- XLogRecPtr PrevRecPtr, XLogRecord *record, bool randAccess);
+ XLogRecPtr PrevRecPtr, XLogRecord *record);
static bool ValidXLogRecord(XLogReaderState *state, XLogRecord *record,
XLogRecPtr recptr);
static void ResetDecoder(XLogReaderState *state);
@@ -73,7 +73,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...)
*/
XLogReaderState *
XLogReaderAllocate(int wal_segment_size, const char *waldir,
- XLogReaderRoutine *routine, void *private_data)
+ WALSegmentCleanupCB cleanup_cb)
{
XLogReaderState *state;
@@ -84,7 +84,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
return NULL;
/* initialize caller-provided support functions */
- state->routine = *routine;
+ state->cleanup_cb = cleanup_cb;
state->max_block_id = -1;
@@ -107,8 +107,6 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
WALOpenSegmentInit(&state->seg, &state->segcxt, wal_segment_size,
waldir);
- /* system_identifier initialized to zeroes above */
- state->private_data = private_data;
/* ReadRecPtr, EndRecPtr and readLen initialized to zeroes above */
state->errormsg_buf = palloc_extended(MAX_ERRORMSG_LEN + 1,
MCXT_ALLOC_NO_OOM);
@@ -140,8 +138,8 @@ XLogReaderFree(XLogReaderState *state)
{
int block_id;
- if (state->seg.ws_file != -1)
- state->routine.segment_close(state);
+ if (state->seg.ws_file >= 0)
+ state->cleanup_cb(state);
for (block_id = 0; block_id <= XLR_MAX_BLOCK_ID; block_id++)
{
@@ -246,6 +244,7 @@ XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
/* Begin at the passed-in record pointer. */
state->EndRecPtr = RecPtr;
state->ReadRecPtr = InvalidXLogRecPtr;
+ state->readRecordState = XLREAD_NEXT_RECORD;
}
/*
@@ -254,317 +253,449 @@ XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
* XLogBeginRead() or XLogFindNextRecord() must be called before the first call
* to XLogReadRecord().
*
- * If the page_read callback fails to read the requested data, NULL is
- * returned. The callback is expected to have reported the error; errormsg
- * is set to NULL.
+ * This function may return XLREAD_NEED_DATA several times before returning a
+ * result record. The caller shall read in some new data then call this
+ * function again with the same parameters.
*
- * If the reading fails for some other reason, NULL is also returned, and
- * *errormsg is set to a string with details of the failure.
+ * When a record is successfully read, returns XLREAD_SUCCESS with result
+ * record being stored in *record. Otherwise *record is NULL.
*
- * The returned pointer (or *errormsg) points to an internal buffer that's
- * valid until the next call to XLogReadRecord.
+ * Returns XLREAD_NEED_DATA if more data is needed to finish reading the
+ * current record. In that case, state->readPagePtr and state->readLen inform
+ * the desired position and minimum length of data needed. The caller shall
+ * read in the requested data and set state->readBuf to point to a buffer
+ * containing it. The caller must also set state->readPageTLI and
+ * state->readLen to indicate the timeline that it was read from, and the
+ * length of data that is now available (which must be >= given readLen),
+ * respectively.
+ *
+ * If invalid data is encountered, returns XLREAD_FAIL with *record being set to
+ * NULL. *errormsg is set to a string with details of the failure.
+ * The returned pointer (or *errormsg) points to an internal buffer that's valid
+ * until the next call to XLogReadRecord.
+ *
+ *
+ * This function runs a state machine consists of the following states.
+ *
+ * XLREAD_NEXT_RECORD :
+ * The initial state, if called with valid RecPtr, try to read a record at
+ * that position. If invalid RecPtr is given try to read a record just after
+ * the last one previously read.
+ * This state ens after setting ReadRecPtr. Then goes to XLREAD_TOT_LEN.
+ *
+ * XLREAD_TOT_LEN:
+ * Examining record header. Ends after reading record total
+ * length. recordRemainLen and recordGotLen are initialized.
+ *
+ * XLREAD_FIRST_FRAGMENT:
+ * Reading the first fragment. Ends with finishing reading a single
+ * record. Goes to XLREAD_NEXT_RECORD if that's all or
+ * XLREAD_CONTINUATION if we have continuation.
+
+ * XLREAD_CONTINUATION:
+ * Reading continuation of record. Ends with finishing the whole record then
+ * goes to XLREAD_NEXT_RECORD. During this state, recordRemainLen indicates
+ * how much is left and readRecordBuf holds the partially assert
+ * record.recordContRecPtr points to the beginning of the next page where to
+ * continue.
+ *
+ * If wrong data found in any state, the state machine stays at the current
+ * state. This behavior allows to continue reading a reacord switching among
+ * different souces, while streaming replication.
*/
-XLogRecord *
-XLogReadRecord(XLogReaderState *state, char **errormsg)
+XLogReadRecordResult
+XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
{
- XLogRecPtr RecPtr;
- XLogRecord *record;
- XLogRecPtr targetPagePtr;
- bool randAccess;
- uint32 len,
- total_len;
- uint32 targetRecOff;
- uint32 pageHeaderSize;
- bool gotheader;
+ XLogRecord *prec;
- /*
- * randAccess indicates whether to verify the previous-record pointer of
- * the record we're reading. We only do this if we're reading
- * sequentially, which is what we initially assume.
- */
- randAccess = false;
+ *record = NULL;
/* reset error state */
*errormsg = NULL;
state->errormsg_buf[0] = '\0';
- ResetDecoder(state);
-
- RecPtr = state->EndRecPtr;
-
- if (state->ReadRecPtr != InvalidXLogRecPtr)
- {
- /* read the record after the one we just read */
-
- /*
- * EndRecPtr is pointing to end+1 of the previous WAL record. If
- * we're at a page boundary, no more records can fit on the current
- * page. We must skip over the page header, but we can't do that until
- * we've read in the page, since the header size is variable.
- */
- }
- else
- {
- /*
- * Caller supplied a position to start at.
- *
- * In this case, EndRecPtr should already be pointing to a valid
- * record starting position.
- */
- Assert(XRecOffIsValid(RecPtr));
- randAccess = true;
- }
-
- state->currRecPtr = RecPtr;
-
- targetPagePtr = RecPtr - (RecPtr % XLOG_BLCKSZ);
- targetRecOff = RecPtr % XLOG_BLCKSZ;
-
- /*
- * Read the page containing the record into state->readBuf. Request enough
- * byte to cover the whole record header, or at least the part of it that
- * fits on the same page.
- */
- while (XLogNeedData(state, targetPagePtr,
- Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
- targetRecOff != 0))
- {
- if (!state->routine.page_read(state, state->readPagePtr, state->readLen,
- RecPtr, state->readBuf))
- break;
- }
-
- if (!state->page_verified)
- goto err;
-
- /*
- * We have at least the page header, so we can examine it now.
- */
- pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
- if (targetRecOff == 0)
- {
- /*
- * At page start, so skip over page header.
- */
- RecPtr += pageHeaderSize;
- targetRecOff = pageHeaderSize;
- }
- else if (targetRecOff < pageHeaderSize)
- {
- report_invalid_record(state, "invalid record offset at %X/%X",
- LSN_FORMAT_ARGS(RecPtr));
- goto err;
- }
-
- if ((((XLogPageHeader) state->readBuf)->xlp_info & XLP_FIRST_IS_CONTRECORD) &&
- targetRecOff == pageHeaderSize)
- {
- report_invalid_record(state, "contrecord is requested by %X/%X",
- LSN_FORMAT_ARGS(RecPtr));
- goto err;
- }
-
- /* XLogNeedData has verified the page header */
- Assert(pageHeaderSize <= state->readLen);
-
- /*
- * Read the record length.
- *
- * NB: Even though we use an XLogRecord pointer here, the whole record
- * header might not fit on this page. xl_tot_len is the first field of the
- * struct, so it must be on this page (the records are MAXALIGNed), but we
- * cannot access any other fields until we've verified that we got the
- * whole header.
- */
- record = (XLogRecord *) (state->readBuf + RecPtr % XLOG_BLCKSZ);
- total_len = record->xl_tot_len;
-
- /*
- * If the whole record header is on this page, validate it immediately.
- * Otherwise do just a basic sanity check on xl_tot_len, and validate the
- * rest of the header after reading it from the next page. The xl_tot_len
- * check is necessary here to ensure that we enter the "Need to reassemble
- * record" code path below; otherwise we might fail to apply
- * ValidXLogRecordHeader at all.
- */
- if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
- {
- if (!ValidXLogRecordHeader(state, RecPtr, state->ReadRecPtr, record,
- randAccess))
- goto err;
- gotheader = true;
- }
- else
- {
- /* XXX: more validation should be done here */
- if (total_len < SizeOfXLogRecord)
- {
- report_invalid_record(state,
- "invalid record length at %X/%X: wanted %u, got %u",
- LSN_FORMAT_ARGS(RecPtr),
- (uint32) SizeOfXLogRecord, total_len);
- goto err;
- }
- gotheader = false;
- }
-
- len = XLOG_BLCKSZ - RecPtr % XLOG_BLCKSZ;
- if (total_len > len)
+ switch (state->readRecordState)
{
- /* Need to reassemble record */
- char *contdata;
- XLogPageHeader pageHeader;
- char *buffer;
- uint32 gotlen;
-
- /*
- * Enlarge readRecordBuf as needed.
- */
- if (total_len > state->readRecordBufSize &&
- !allocate_recordbuf(state, total_len))
- {
- /* We treat this as a "bogus data" condition */
- report_invalid_record(state, "record length %u at %X/%X too long",
- total_len, LSN_FORMAT_ARGS(RecPtr));
- goto err;
- }
+ case XLREAD_NEXT_RECORD:
+ ResetDecoder(state);
- /* Copy the first fragment of the record from the first page. */
- memcpy(state->readRecordBuf,
- state->readBuf + RecPtr % XLOG_BLCKSZ, len);
- buffer = state->readRecordBuf + len;
- gotlen = len;
-
- do
- {
- int rest_len = total_len - gotlen;
-
- /* Calculate pointer to beginning of next page */
- targetPagePtr += XLOG_BLCKSZ;
-
- /* Wait for the next page to become available */
- while (XLogNeedData(state, targetPagePtr,
- Min(rest_len, XLOG_BLCKSZ),
- false))
+ if (state->ReadRecPtr != InvalidXLogRecPtr)
{
- if (!state->routine.page_read(state, state->readPagePtr,
- state->readLen,
- state->ReadRecPtr,
- state->readBuf))
- break;
+ /* read the record after the one we just read */
+
+ /*
+ * EndRecPtr is pointing to end+1 of the previous WAL record.
+ * If we're at a page boundary, no more records can fit on the
+ * current page. We must skip over the page header, but we
+ * can't do that until we've read in the page, since the header
+ * size is variable.
+ */
+ state->PrevRecPtr = state->ReadRecPtr;
+ state->ReadRecPtr = state->EndRecPtr;
+ }
+ else
+ {
+ /*
+ * Caller supplied a position to start at.
+ *
+ * In this case, EndRecPtr should already be pointing to a
+ * valid record starting position.
+ */
+ Assert(XRecOffIsValid(state->EndRecPtr));
+ state->ReadRecPtr = state->EndRecPtr;
+
+ /*
+ * We cannot verify the previous-record pointer when we're
+ * seeking to a particular record. Reset PrevRecPtr so that we
+ * won't try doing that.
+ */
+ state->PrevRecPtr = InvalidXLogRecPtr;
+ state->EndRecPtr = InvalidXLogRecPtr; /* to be tidy */
}
+ state->record_verified = false;
+ state->readRecordState = XLREAD_TOT_LEN;
+ /* fall through */
+
+ case XLREAD_TOT_LEN:
+ {
+ uint32 total_len;
+ uint32 pageHeaderSize;
+ XLogRecPtr targetPagePtr;
+ uint32 targetRecOff;
+ XLogPageHeader pageHeader;
+
+ targetPagePtr =
+ state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
+ targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+
+ /*
+ * Check if we have enough data. For the first record in the page,
+ * the requesting length doesn't contain page header.
+ */
+ if (XLogNeedData(state, targetPagePtr,
+ Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+ targetRecOff != 0))
+ return XLREAD_NEED_DATA;
+
+ /* error out if caller supplied bogus page */
if (!state->page_verified)
goto err;
- Assert(SizeOfXLogShortPHD <= state->readLen);
+ /* examine page header now. */
+ pageHeaderSize =
+ XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+ if (targetRecOff == 0)
+ {
+ /* At page start, so skip over page header. */
+ state->ReadRecPtr += pageHeaderSize;
+ targetRecOff = pageHeaderSize;
+ }
+ else if (targetRecOff < pageHeaderSize)
+ {
+ report_invalid_record(state, "invalid record offset at %X/%X",
+ LSN_FORMAT_ARGS(state->ReadRecPtr));
+ goto err;
+ }
- /* Check that the continuation on next page looks valid */
pageHeader = (XLogPageHeader) state->readBuf;
- if (!(pageHeader->xlp_info & XLP_FIRST_IS_CONTRECORD))
+ if ((pageHeader->xlp_info & XLP_FIRST_IS_CONTRECORD) &&
+ targetRecOff == pageHeaderSize)
{
- report_invalid_record(state,
- "there is no contrecord flag at %X/%X",
- LSN_FORMAT_ARGS(RecPtr));
+ report_invalid_record(state, "contrecord is requested by %X/%X",
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr);
goto err;
}
- /*
- * Cross-check that xlp_rem_len agrees with how much of the record
- * we expect there to be left.
- */
- if (pageHeader->xlp_rem_len == 0 ||
- total_len != (pageHeader->xlp_rem_len + gotlen))
- {
- report_invalid_record(state,
- "invalid contrecord length %u (expected %lld) at %X/%X",
- pageHeader->xlp_rem_len,
- ((long long) total_len) - gotlen,
- LSN_FORMAT_ARGS(RecPtr));
- goto err;
- }
-
- /* Append the continuation from this page to the buffer */
- pageHeaderSize = XLogPageHeaderSize(pageHeader);
-
+ /* XLogNeedData has verified the page header */
Assert(pageHeaderSize <= state->readLen);
- contdata = (char *) state->readBuf + pageHeaderSize;
- len = XLOG_BLCKSZ - pageHeaderSize;
- if (pageHeader->xlp_rem_len < len)
- len = pageHeader->xlp_rem_len;
+ /*
+ * Read the record length.
+ *
+ * NB: Even though we use an XLogRecord pointer here, the whole
+ * record header might not fit on this page. xl_tot_len is the first
+ * field of the struct, so it must be on this page (the records are
+ * MAXALIGNed), but we cannot access any other fields until we've
+ * verified that we got the whole header.
+ */
+ prec = (XLogRecord *) (state->readBuf +
+ state->ReadRecPtr % XLOG_BLCKSZ);
+ total_len = prec->xl_tot_len;
- Assert (pageHeaderSize + len <= state->readLen);
- memcpy(buffer, (char *) contdata, len);
- buffer += len;
- gotlen += len;
+ /*
+ * If the whole record header is on this page, validate it
+ * immediately. Otherwise do just a basic sanity check on
+ * xl_tot_len, and validate the rest of the header after reading it
+ * from the next page. The xl_tot_len check is necessary here to
+ * ensure that we enter the XLREAD_CONTINUATION state below;
+ * otherwise we might fail to apply ValidXLogRecordHeader at all.
+ */
+ if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
+ {
+ if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
+ state->PrevRecPtr, prec))
+ goto err;
- /* If we just reassembled the record header, validate it. */
- if (!gotheader)
+ state->record_verified = true;
+ }
+ else
{
- record = (XLogRecord *) state->readRecordBuf;
- if (!ValidXLogRecordHeader(state, RecPtr, state->ReadRecPtr,
- record, randAccess))
+ /* XXX: more validation should be done here */
+ if (total_len < SizeOfXLogRecord)
+ {
+ report_invalid_record(state,
+ "invalid record length at %X/%X: wanted %u, got %u",
+ LSN_FORMAT_ARGS(state->ReadRecPtr),
+ (uint32) SizeOfXLogRecord, total_len);
goto err;
- gotheader = true;
+ }
}
- } while (gotlen < total_len);
-
- Assert(gotheader);
- record = (XLogRecord *) state->readRecordBuf;
- if (!ValidXLogRecord(state, record, RecPtr))
- goto err;
+ /*
+ * Wait for the rest of the record, or the part of the record that
+ * fit on the first page if crossed a page boundary, to become
+ * available.
+ */
+ state->recordGotLen = 0;
+ state->recordRemainLen = total_len;
+ state->readRecordState = XLREAD_FIRST_FRAGMENT;
+ }
+ /* fall through */
- pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
- state->ReadRecPtr = RecPtr;
- state->EndRecPtr = targetPagePtr + pageHeaderSize
- + MAXALIGN(pageHeader->xlp_rem_len);
- }
- else
- {
- /* Wait for the record data to become available */
- while (XLogNeedData(state, targetPagePtr,
- Min(targetRecOff + total_len, XLOG_BLCKSZ), true))
+ case XLREAD_FIRST_FRAGMENT:
{
- if (!state->routine.page_read(state, state->readPagePtr,
- state->readLen,
- state->ReadRecPtr, state->readBuf))
+ uint32 total_len = state->recordRemainLen;
+ uint32 request_len;
+ uint32 record_len;
+ XLogRecPtr targetPagePtr;
+ uint32 targetRecOff;
+
+ /*
+ * Wait for the rest of the record on the first page to become
+ * available
+ */
+ targetPagePtr =
+ state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
+ targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+
+ request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ);
+ record_len = request_len - targetRecOff;
+
+ /* ReadRecPtr contains page header */
+ Assert (targetRecOff != 0);
+ if (XLogNeedData(state, targetPagePtr, request_len, true))
+ return XLREAD_NEED_DATA;
+
+ /* error out if caller supplied bogus page */
+ if (!state->page_verified)
+ goto err;
+
+ prec = (XLogRecord *) (state->readBuf + targetRecOff);
+
+ /* validate record header if not yet */
+ if (!state->record_verified && record_len >= SizeOfXLogRecord)
+ {
+ if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
+ state->PrevRecPtr, prec))
+ goto err;
+
+ state->record_verified = true;
+ }
+
+
+ if (total_len == record_len)
+ {
+ /* Record does not cross a page boundary */
+ Assert(state->record_verified);
+
+ if (!ValidXLogRecord(state, prec, state->ReadRecPtr))
+ goto err;
+
+ state->record_verified = true; /* to be tidy */
+
+ /* We already checked the header earlier */
+ state->EndRecPtr = state->ReadRecPtr + MAXALIGN(record_len);
+
+ *record = prec;
+ state->readRecordState = XLREAD_NEXT_RECORD;
break;
+ }
+
+ /*
+ * The record continues on the next page. Need to reassemble
+ * record
+ */
+ Assert(total_len > record_len);
+
+ /* Enlarge readRecordBuf as needed. */
+ if (total_len > state->readRecordBufSize &&
+ !allocate_recordbuf(state, total_len))
+ {
+ /* We treat this as a "bogus data" condition */
+ report_invalid_record(state,
+ "record length %u at %X/%X too long",
+ total_len,
+ LSN_FORMAT_ARGS(state->ReadRecPtr));
+ goto err;
+ }
+
+ /* Copy the first fragment of the record from the first page. */
+ memcpy(state->readRecordBuf, state->readBuf + targetRecOff,
+ record_len);
+ state->recordGotLen += record_len;
+ state->recordRemainLen -= record_len;
+
+ /* Calculate pointer to beginning of next page */
+ state->recordContRecPtr = state->ReadRecPtr + record_len;
+ Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0);
+
+ state->readRecordState = XLREAD_CONTINUATION;
}
+ /* fall through */
+
+ case XLREAD_CONTINUATION:
+ {
+ XLogPageHeader pageHeader;
+ uint32 pageHeaderSize;
+ XLogRecPtr targetPagePtr;
+
+ /* we enter this state only if we haven't read the whole record. */
+ Assert (state->recordRemainLen > 0);
+
+ while(state->recordRemainLen > 0)
+ {
+ char *contdata;
+ uint32 request_len;
+ uint32 record_len;
+
+ /* Wait for the next page to become available */
+ targetPagePtr = state->recordContRecPtr;
+
+ /* this request contains page header */
+ Assert (targetPagePtr != 0);
+ if (XLogNeedData(state, targetPagePtr,
+ Min(state->recordRemainLen, XLOG_BLCKSZ),
+ false))
+ return XLREAD_NEED_DATA;
+
+ if (!state->page_verified)
+ goto err;
+
+ Assert(SizeOfXLogShortPHD <= state->readLen);
- if (!state->page_verified)
- goto err;
+ /* Check that the continuation on next page looks valid */
+ pageHeader = (XLogPageHeader) state->readBuf;
+ if (!(pageHeader->xlp_info & XLP_FIRST_IS_CONTRECORD))
+ {
+ report_invalid_record(
+ state,
+ "there is no contrecord flag at %X/%X reading %X/%X",
+ (uint32) (state->recordContRecPtr >> 32),
+ (uint32) state->recordContRecPtr,
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr);
+ goto err;
+ }
- /* Record does not cross a page boundary */
- if (!ValidXLogRecord(state, record, RecPtr))
- goto err;
+ /*
+ * Cross-check that xlp_rem_len agrees with how much of the
+ * record we expect there to be left.
+ */
+ if (pageHeader->xlp_rem_len == 0 ||
+ pageHeader->xlp_rem_len != state->recordRemainLen)
+ {
+ report_invalid_record(
+ state,
+ "invalid contrecord length %u at %X/%X reading %X/%X, expected %u",
+ pageHeader->xlp_rem_len,
+ (uint32) (state->recordContRecPtr >> 32),
+ (uint32) state->recordContRecPtr,
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr,
+ state->recordRemainLen);
+ goto err;
+ }
- state->EndRecPtr = RecPtr + MAXALIGN(total_len);
+ /* Append the continuation from this page to the buffer */
+ pageHeaderSize = XLogPageHeaderSize(pageHeader);
- state->ReadRecPtr = RecPtr;
+ /*
+ * XLogNeedData should have ensured that the whole page header
+ * was read
+ */
+ Assert(state->readLen >= pageHeaderSize);
+
+ contdata = (char *) state->readBuf + pageHeaderSize;
+ record_len = XLOG_BLCKSZ - pageHeaderSize;
+ if (pageHeader->xlp_rem_len < record_len)
+ record_len = pageHeader->xlp_rem_len;
+
+ request_len = record_len + pageHeaderSize;
+
+ /* XLogNeedData should have ensured all needed data was read */
+ Assert (state->readLen >= request_len);
+
+ memcpy(state->readRecordBuf + state->recordGotLen,
+ (char *) contdata, record_len);
+ state->recordGotLen += record_len;
+ state->recordRemainLen -= record_len;
+
+ /* If we just reassembled the record header, validate it. */
+ if (!state->record_verified)
+ {
+ Assert(state->recordGotLen >= SizeOfXLogRecord);
+ if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
+ state->PrevRecPtr,
+ (XLogRecord *) state->readRecordBuf))
+ goto err;
+
+ state->record_verified = true;
+ }
+
+ /* Calculate pointer to beginning of next page, and continue */
+ state->recordContRecPtr += XLOG_BLCKSZ;
+ }
+
+ /* targetPagePtr is pointing the last-read page here */
+ prec = (XLogRecord *) state->readRecordBuf;
+ if (!ValidXLogRecord(state, prec, state->ReadRecPtr))
+ goto err;
+
+ pageHeaderSize =
+ XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+ state->EndRecPtr = targetPagePtr + pageHeaderSize
+ + MAXALIGN(pageHeader->xlp_rem_len);
+
+ *record = prec;
+ state->readRecordState = XLREAD_NEXT_RECORD;
+ break;
+ }
}
/*
* Special processing if it's an XLOG SWITCH record
*/
- if (record->xl_rmid == RM_XLOG_ID &&
- (record->xl_info & ~XLR_INFO_MASK) == XLOG_SWITCH)
+ if ((*record)->xl_rmid == RM_XLOG_ID &&
+ ((*record)->xl_info & ~XLR_INFO_MASK) == XLOG_SWITCH)
{
/* Pretend it extends to end of segment */
state->EndRecPtr += state->segcxt.ws_segsize - 1;
state->EndRecPtr -= XLogSegmentOffset(state->EndRecPtr, state->segcxt.ws_segsize);
}
- if (DecodeXLogRecord(state, record, errormsg))
- return record;
- else
- return NULL;
+ Assert (!*record || state->readLen >= 0);
+ if (DecodeXLogRecord(state, *record, errormsg))
+ return XLREAD_SUCCESS;
+
+ *record = NULL;
+ return XLREAD_FAIL;
err:
/*
- * Invalidate the read state. We might read from a different source after
+ * Invalidate the read page. We might read from a different source after
* failure.
*/
XLogReaderInvalReadState(state);
@@ -572,7 +703,8 @@ err:
if (state->errormsg_buf[0] != '\0')
*errormsg = state->errormsg_buf;
- return NULL;
+ *record = NULL;
+ return XLREAD_FAIL;
}
/*
@@ -724,11 +856,12 @@ XLogReaderInvalReadState(XLogReaderState *state)
*
* This is just a convenience subroutine to avoid duplicated code in
* XLogReadRecord. It's not intended for use from anywhere else.
+ *
+ * If PrevRecPtr is valid, the xl_prev is is cross-checked with it.
*/
static bool
ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
- XLogRecPtr PrevRecPtr, XLogRecord *record,
- bool randAccess)
+ XLogRecPtr PrevRecPtr, XLogRecord *record)
{
if (record->xl_tot_len < SizeOfXLogRecord)
{
@@ -745,7 +878,7 @@ ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
record->xl_rmid, LSN_FORMAT_ARGS(RecPtr));
return false;
}
- if (randAccess)
+ if (PrevRecPtr == InvalidXLogRecPtr)
{
/*
* We can't exactly verify the prev-link, but surely it should be less
@@ -975,11 +1108,14 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
* XLogReadRecord() will read the next valid record.
*/
XLogRecPtr
-XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
+XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
+ XLogFindNextRecordCB read_page, void *private)
{
XLogRecPtr tmpRecPtr;
XLogRecPtr found = InvalidXLogRecPtr;
XLogPageHeader header;
+ XLogRecord *record;
+ XLogReadRecordResult result;
char *errormsg;
Assert(!XLogRecPtrIsInvalid(RecPtr));
@@ -1012,9 +1148,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
while(XLogNeedData(state, targetPagePtr, targetRecOff,
targetRecOff != 0))
{
- if (!state->routine.page_read(state, state->readPagePtr,
- state->readLen,
- state->ReadRecPtr, state->readBuf))
+ if (!read_page(state, private))
break;
}
@@ -1066,8 +1200,16 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
* or we just jumped over the remaining data of a continuation.
*/
XLogBeginRead(state, tmpRecPtr);
- while (XLogReadRecord(state, &errormsg) != NULL)
+ while ((result = XLogReadRecord(state, &record, &errormsg)) !=
+ XLREAD_FAIL)
{
+ if (result == XLREAD_NEED_DATA)
+ {
+ if (!read_page(state, private))
+ goto err;
+ continue;
+ }
+
/* past the record we've found, break out */
if (RecPtr <= state->ReadRecPtr)
{
@@ -1087,9 +1229,9 @@ err:
#endif /* FRONTEND */
/*
- * Helper function to ease writing of XLogRoutine->page_read callbacks.
- * If this function is used, caller must supply a segment_open callback in
- * 'state', as that is used here.
+ * Helper function to ease writing of page_read callback.
+ * If this function is used, caller must supply a segment_open callback and
+ * segment_close callback as that is used here.
*
* Read 'count' bytes into 'buf', starting at location 'startptr', from WAL
* fetched from timeline 'tli'.
@@ -1102,6 +1244,7 @@ err:
*/
bool
WALRead(XLogReaderState *state,
+ WALSegmentOpenCB segopenfn, WALSegmentCloseCB segclosefn,
char *buf, XLogRecPtr startptr, Size count, TimeLineID tli,
WALReadError *errinfo)
{
@@ -1133,10 +1276,10 @@ WALRead(XLogReaderState *state,
XLogSegNo nextSegNo;
if (state->seg.ws_file >= 0)
- state->routine.segment_close(state);
+ segclosefn(state);
XLByteToSeg(recptr, nextSegNo, state->segcxt.ws_segsize);
- state->routine.segment_open(state, nextSegNo, &tli);
+ segopenfn(state, nextSegNo, &tli);
/* This shouldn't happen -- indicates a bug in segment_open */
Assert(state->seg.ws_file >= 0);
diff --git a/src/backend/access/transam/xlogutils.c b/src/backend/access/transam/xlogutils.c
index 46eda33f25..b003990745 100644
--- a/src/backend/access/transam/xlogutils.c
+++ b/src/backend/access/transam/xlogutils.c
@@ -686,8 +686,7 @@ XLogTruncateRelation(RelFileNode rnode, ForkNumber forkNum,
void
XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wantLength)
{
- const XLogRecPtr lastReadPage = state->seg.ws_segno *
- state->segcxt.ws_segsize + state->readLen;
+ const XLogRecPtr lastReadPage = state->readPagePtr;
Assert(wantPage != InvalidXLogRecPtr && wantPage % XLOG_BLCKSZ == 0);
Assert(wantLength <= XLOG_BLCKSZ);
@@ -702,7 +701,7 @@ XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wa
* current TLI has since become historical.
*/
if (lastReadPage == wantPage &&
- state->readLen != 0 &&
+ state->page_verified &&
lastReadPage + state->readLen >= wantPage + Min(wantLength, XLOG_BLCKSZ - 1))
return;
@@ -788,6 +787,7 @@ wal_segment_open(XLogReaderState *state, XLogSegNo nextSegNo,
char path[MAXPGPATH];
XLogFilePath(path, tli, nextSegNo, state->segcxt.ws_segsize);
+ elog(LOG, "HOGE: %lu, %d => %s", nextSegNo, tli, path);
state->seg.ws_file = BasicOpenFile(path, O_RDONLY | PG_BINARY);
if (state->seg.ws_file >= 0)
return;
@@ -825,9 +825,11 @@ wal_segment_close(XLogReaderState *state)
* loop for now.
*/
bool
-read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *cur_page)
+read_local_xlog_page(XLogReaderState *state)
{
+ XLogRecPtr targetPagePtr = state->readPagePtr;
+ int reqLen = state->readLen;
+ char *cur_page = state->readBuf;
XLogRecPtr read_upto,
loc;
TimeLineID tli;
@@ -940,11 +942,12 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
* as 'count', read the whole page anyway. It's guaranteed to be
* zero-padded up to the page boundary if it's incomplete.
*/
- if (!WALRead(state, cur_page, targetPagePtr, XLOG_BLCKSZ, tli,
- &errinfo))
+ if (!WALRead(state, wal_segment_open, wal_segment_close,
+ cur_page, targetPagePtr, XLOG_BLCKSZ, tli, &errinfo))
WALReadRaiseError(&errinfo);
/* number of valid bytes in the buffer */
+ state->readPagePtr = targetPagePtr;
state->readLen = count;
return true;
}
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index 37b75deb72..9cbd8afa49 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -148,7 +148,8 @@ StartupDecodingContext(List *output_plugin_options,
TransactionId xmin_horizon,
bool need_full_snapshot,
bool fast_forward,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress)
@@ -198,11 +199,12 @@ StartupDecodingContext(List *output_plugin_options,
ctx->slot = slot;
- ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, xl_routine, ctx);
+ ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, cleanup_cb);
if (!ctx->reader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
errmsg("out of memory")));
+ ctx->page_read = page_read;
ctx->reorder = ReorderBufferAllocate();
ctx->snapshot_builder =
@@ -319,7 +321,8 @@ CreateInitDecodingContext(const char *plugin,
List *output_plugin_options,
bool need_full_snapshot,
XLogRecPtr restart_lsn,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress)
@@ -422,7 +425,7 @@ CreateInitDecodingContext(const char *plugin,
ctx = StartupDecodingContext(NIL, restart_lsn, xmin_horizon,
need_full_snapshot, false,
- xl_routine, prepare_write, do_write,
+ page_read, cleanup_cb, prepare_write, do_write,
update_progress);
/* call output plugin initialization callback */
@@ -476,7 +479,8 @@ LogicalDecodingContext *
CreateDecodingContext(XLogRecPtr start_lsn,
List *output_plugin_options,
bool fast_forward,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress)
@@ -528,8 +532,8 @@ CreateDecodingContext(XLogRecPtr start_lsn,
ctx = StartupDecodingContext(output_plugin_options,
start_lsn, InvalidTransactionId, false,
- fast_forward, xl_routine, prepare_write,
- do_write, update_progress);
+ fast_forward, page_read, cleanup_cb,
+ prepare_write, do_write, update_progress);
/* call output plugin initialization callback */
old_context = MemoryContextSwitchTo(ctx->context);
@@ -585,7 +589,13 @@ DecodingContextFindStartpoint(LogicalDecodingContext *ctx)
char *err = NULL;
/* the read_page callback waits for new WAL */
- record = XLogReadRecord(ctx->reader, &err);
+ while (XLogReadRecord(ctx->reader, &record, &err) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!ctx->page_read(ctx->reader))
+ break;
+ }
+
if (err)
elog(ERROR, "%s", err);
if (!record)
diff --git a/src/backend/replication/logical/logicalfuncs.c b/src/backend/replication/logical/logicalfuncs.c
index f7e055874e..b69f6911c5 100644
--- a/src/backend/replication/logical/logicalfuncs.c
+++ b/src/backend/replication/logical/logicalfuncs.c
@@ -233,9 +233,8 @@ pg_logical_slot_get_changes_guts(FunctionCallInfo fcinfo, bool confirm, bool bin
ctx = CreateDecodingContext(InvalidXLogRecPtr,
options,
false,
- XL_ROUTINE(.page_read = read_local_xlog_page,
- .segment_open = wal_segment_open,
- .segment_close = wal_segment_close),
+ read_local_xlog_page,
+ wal_segment_close,
LogicalOutputPrepareWrite,
LogicalOutputWrite, NULL);
@@ -284,7 +283,13 @@ pg_logical_slot_get_changes_guts(FunctionCallInfo fcinfo, bool confirm, bool bin
XLogRecord *record;
char *errm = NULL;
- record = XLogReadRecord(ctx->reader, &errm);
+ while (XLogReadRecord(ctx->reader, &record, &errm) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!ctx->page_read(ctx->reader))
+ break;
+ }
+
if (errm)
elog(ERROR, "%s", errm);
diff --git a/src/backend/replication/slotfuncs.c b/src/backend/replication/slotfuncs.c
index 9817b44113..deb7cf15eb 100644
--- a/src/backend/replication/slotfuncs.c
+++ b/src/backend/replication/slotfuncs.c
@@ -153,9 +153,8 @@ create_logical_replication_slot(char *name, char *plugin,
ctx = CreateInitDecodingContext(plugin, NIL,
false, /* just catalogs is OK */
restart_lsn,
- XL_ROUTINE(.page_read = read_local_xlog_page,
- .segment_open = wal_segment_open,
- .segment_close = wal_segment_close),
+ read_local_xlog_page,
+ wal_segment_close,
NULL, NULL, NULL);
/*
@@ -512,9 +511,8 @@ pg_logical_replication_slot_advance(XLogRecPtr moveto)
ctx = CreateDecodingContext(InvalidXLogRecPtr,
NIL,
true, /* fast_forward */
- XL_ROUTINE(.page_read = read_local_xlog_page,
- .segment_open = wal_segment_open,
- .segment_close = wal_segment_close),
+ read_local_xlog_page,
+ wal_segment_close,
NULL, NULL, NULL);
/*
@@ -536,7 +534,13 @@ pg_logical_replication_slot_advance(XLogRecPtr moveto)
* Read records. No changes are generated in fast_forward mode,
* but snapbuilder/slot statuses are updated properly.
*/
- record = XLogReadRecord(ctx->reader, &errm);
+ while (XLogReadRecord(ctx->reader, &record, &errm) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!ctx->page_read(ctx->reader))
+ break;
+ }
+
if (errm)
elog(ERROR, "%s", errm);
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index c36b91ae5e..062093727d 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -580,10 +580,7 @@ StartReplication(StartReplicationCmd *cmd)
/* create xlogreader for physical replication */
xlogreader =
- XLogReaderAllocate(wal_segment_size, NULL,
- XL_ROUTINE(.segment_open = WalSndSegmentOpen,
- .segment_close = wal_segment_close),
- NULL);
+ XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
if (!xlogreader)
ereport(ERROR,
@@ -807,9 +804,11 @@ StartReplication(StartReplicationCmd *cmd)
* set every time WAL is flushed.
*/
static bool
-logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetRecPtr, char *cur_page)
+logical_read_xlog_page(XLogReaderState *state)
{
+ XLogRecPtr targetPagePtr = state->readPagePtr;
+ int reqLen = state->readLen;
+ char *cur_page = state->readBuf;
XLogRecPtr flushptr;
int count;
WALReadError errinfo;
@@ -837,7 +836,7 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
count = flushptr - targetPagePtr; /* part of the page available */
/* now actually read the data, we know it's there */
- if (!WALRead(state,
+ if (!WALRead(state, WalSndSegmentOpen, wal_segment_close,
cur_page,
targetPagePtr,
XLOG_BLCKSZ,
@@ -1011,9 +1010,8 @@ CreateReplicationSlot(CreateReplicationSlotCmd *cmd)
ctx = CreateInitDecodingContext(cmd->plugin, NIL, need_full_snapshot,
InvalidXLogRecPtr,
- XL_ROUTINE(.page_read = logical_read_xlog_page,
- .segment_open = WalSndSegmentOpen,
- .segment_close = wal_segment_close),
+ logical_read_xlog_page,
+ wal_segment_close,
WalSndPrepareWrite, WalSndWriteData,
WalSndUpdateProgress);
@@ -1171,9 +1169,8 @@ StartLogicalReplication(StartReplicationCmd *cmd)
*/
logical_decoding_ctx =
CreateDecodingContext(cmd->startpoint, cmd->options, false,
- XL_ROUTINE(.page_read = logical_read_xlog_page,
- .segment_open = WalSndSegmentOpen,
- .segment_close = wal_segment_close),
+ logical_read_xlog_page,
+ wal_segment_close,
WalSndPrepareWrite, WalSndWriteData,
WalSndUpdateProgress);
xlogreader = logical_decoding_ctx->reader;
@@ -2749,7 +2746,7 @@ XLogSendPhysical(void)
enlargeStringInfo(&output_message, nbytes);
retry:
- if (!WALRead(xlogreader,
+ if (!WALRead(xlogreader, WalSndSegmentOpen, wal_segment_close,
&output_message.data[output_message.len],
startptr,
nbytes,
@@ -2847,7 +2844,12 @@ XLogSendLogical(void)
*/
WalSndCaughtUp = false;
- record = XLogReadRecord(logical_decoding_ctx->reader, &errm);
+ while (XLogReadRecord(logical_decoding_ctx->reader, &record, &errm) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!logical_decoding_ctx->page_read(logical_decoding_ctx->reader))
+ break;
+ }
/* xlog record was invalid */
if (errm != NULL)
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index cf119848b0..712c85281c 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -41,15 +41,9 @@ static int xlogreadfd = -1;
static XLogSegNo xlogreadsegno = -1;
static char xlogfpath[MAXPGPATH];
-typedef struct XLogPageReadPrivate
-{
- const char *restoreCommand;
- int tliIndex;
-} XLogPageReadPrivate;
-
-static bool SimpleXLogPageRead(XLogReaderState *xlogreader,
- XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
+static bool SimpleXLogPageRead(XLogReaderState *xlogreader,
+ const char *datadir, int *tliIndex,
+ const char *restoreCommand);
/*
* Read WAL from the datadir/pg_wal, starting from 'startpoint' on timeline
@@ -66,20 +60,22 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
XLogRecord *record;
XLogReaderState *xlogreader;
char *errormsg;
- XLogPageReadPrivate private;
- private.tliIndex = tliIndex;
- private.restoreCommand = restoreCommand;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir,
- XL_ROUTINE(.page_read = &SimpleXLogPageRead),
- &private);
+ xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+
if (xlogreader == NULL)
pg_fatal("out of memory");
XLogBeginRead(xlogreader, startpoint);
do
{
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!SimpleXLogPageRead(xlogreader, datadir,
+ &tliIndex, restoreCommand))
+ break;
+ }
if (record == NULL)
{
@@ -123,19 +119,19 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex,
XLogRecord *record;
XLogReaderState *xlogreader;
char *errormsg;
- XLogPageReadPrivate private;
XLogRecPtr endptr;
- private.tliIndex = tliIndex;
- private.restoreCommand = restoreCommand;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir,
- XL_ROUTINE(.page_read = &SimpleXLogPageRead),
- &private);
+ xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
XLogBeginRead(xlogreader, ptr);
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!SimpleXLogPageRead(xlogreader, datadir, &tliIndex, restoreCommand))
+ break;
+ }
if (record == NULL)
{
if (errormsg)
@@ -170,7 +166,6 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
XLogRecPtr searchptr;
XLogReaderState *xlogreader;
char *errormsg;
- XLogPageReadPrivate private;
/*
* The given fork pointer points to the end of the last common record,
@@ -186,11 +181,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
forkptr += SizeOfXLogShortPHD;
}
- private.tliIndex = tliIndex;
- private.restoreCommand = restoreCommand;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir,
- XL_ROUTINE(.page_read = &SimpleXLogPageRead),
- &private);
+ xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
@@ -200,7 +191,13 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
uint8 info;
XLogBeginRead(xlogreader, searchptr);
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!SimpleXLogPageRead(xlogreader, datadir,
+ &tliIndex, restoreCommand))
+ break;
+ }
if (record == NULL)
{
@@ -247,10 +244,11 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
/* XLogReader callback function, to read a WAL page */
static bool
-SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *readBuf)
+SimpleXLogPageRead(XLogReaderState *xlogreader, const char *datadir,
+ int *tliIndex, const char *restoreCommand)
{
- XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
+ XLogRecPtr targetPagePtr = xlogreader->readPagePtr;
+ char *readBuf = xlogreader->readBuf;
uint32 targetPageOff;
XLogRecPtr targetSegEnd;
XLogSegNo targetSegNo;
@@ -283,14 +281,14 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
* be done both forward and backward, consider also switching timeline
* accordingly.
*/
- while (private->tliIndex < targetNentries - 1 &&
- targetHistory[private->tliIndex].end < targetSegEnd)
- private->tliIndex++;
- while (private->tliIndex > 0 &&
- targetHistory[private->tliIndex].begin >= targetSegEnd)
- private->tliIndex--;
+ while (*tliIndex < targetNentries - 1 &&
+ targetHistory[*tliIndex].end < targetSegEnd)
+ (*tliIndex)++;
+ while (*tliIndex > 0 &&
+ targetHistory[*tliIndex].begin >= targetSegEnd)
+ (*tliIndex)--;
- XLogFileName(xlogfname, targetHistory[private->tliIndex].tli,
+ XLogFileName(xlogfname, targetHistory[*tliIndex].tli,
xlogreadsegno, WalSegSz);
snprintf(xlogfpath, MAXPGPATH, "%s/" XLOGDIR "/%s",
@@ -303,7 +301,7 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
/*
* If we have no restore_command to execute, then exit.
*/
- if (private->restoreCommand == NULL)
+ if (restoreCommand == NULL)
{
pg_log_error("could not open file \"%s\": %m", xlogfpath);
xlogreader->readLen = -1;
@@ -317,7 +315,7 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
xlogreadfd = RestoreArchivedFile(xlogreader->segcxt.ws_dir,
xlogfname,
WalSegSz,
- private->restoreCommand);
+ restoreCommand);
if (xlogreadfd < 0)
{
@@ -359,7 +357,7 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
Assert(targetSegNo == xlogreadsegno);
- xlogreader->seg.ws_tli = targetHistory[private->tliIndex].tli;
+ xlogreader->seg.ws_tli = targetHistory[*tliIndex].tli;
xlogreader->readLen = XLOG_BLCKSZ;
return true;
}
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index 0f0e0723a3..f514de6b13 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -330,12 +330,17 @@ WALDumpCloseSegment(XLogReaderState *state)
state->seg.ws_file = -1;
}
-/* pg_waldump's XLogReaderRoutine->page_read callback */
+/*
+ * pg_waldump's WAL page rader, also used as page_read callback for
+ * XLogFindNextRecord
+ */
static bool
-WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetPtr, char *readBuff)
+WALDumpReadPage(XLogReaderState *state, void *priv)
{
- XLogDumpPrivate *private = state->private_data;
+ XLogRecPtr targetPagePtr = state->readPagePtr;
+ int reqLen = state->readLen;
+ char *readBuff = state->readBuf;
+ XLogDumpPrivate *private = (XLogDumpPrivate *) priv;
int count = XLOG_BLCKSZ;
WALReadError errinfo;
@@ -353,8 +358,8 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
}
}
- if (!WALRead(state, readBuff, targetPagePtr, count, private->timeline,
- &errinfo))
+ if (!WALRead(state, WALDumpOpenSegment, WALDumpCloseSegment,
+ readBuff, targetPagePtr, count, private->timeline, &errinfo))
{
WALOpenSegment *seg = &errinfo.wre_seg;
char fname[MAXPGPATH];
@@ -374,6 +379,7 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
(Size) errinfo.wre_req);
}
+ Assert(count >= state->readLen);
state->readLen = count;
return true;
}
@@ -1035,16 +1041,14 @@ main(int argc, char **argv)
/* we have everything we need, start reading */
xlogreader_state =
- XLogReaderAllocate(WalSegSz, waldir,
- XL_ROUTINE(.page_read = WALDumpReadPage,
- .segment_open = WALDumpOpenSegment,
- .segment_close = WALDumpCloseSegment),
- &private);
+ XLogReaderAllocate(WalSegSz, waldir, WALDumpCloseSegment);
+
if (!xlogreader_state)
fatal_error("out of memory");
/* first find a valid recptr to start from */
- first_record = XLogFindNextRecord(xlogreader_state, private.startptr);
+ first_record = XLogFindNextRecord(xlogreader_state, private.startptr,
+ &WALDumpReadPage, (void*) &private);
if (first_record == InvalidXLogRecPtr)
fatal_error("could not find a valid record after %X/%X",
@@ -1067,7 +1071,13 @@ main(int argc, char **argv)
for (;;)
{
/* try to read the next record */
- record = XLogReadRecord(xlogreader_state, &errormsg);
+ while (XLogReadRecord(xlogreader_state, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!WALDumpReadPage(xlogreader_state, (void *) &private))
+ break;
+ }
+
if (!record)
{
if (!config.follow || private.endptr_reached)
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index 23d9e70a59..4e9268b553 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -57,64 +57,15 @@ typedef struct WALSegmentContext
typedef struct XLogReaderState XLogReaderState;
-/* Function type definition for the read_page callback */
-typedef bool (*XLogPageReadCB) (XLogReaderState *xlogreader,
- XLogRecPtr targetPagePtr,
- int reqLen,
- XLogRecPtr targetRecPtr,
- char *readBuf);
+/* Function type definition for the segment cleanup callback */
+typedef void (*WALSegmentCleanupCB) (XLogReaderState *xlogreader);
+
+/* Function type definition for the open/close callbacks for WALRead() */
typedef void (*WALSegmentOpenCB) (XLogReaderState *xlogreader,
XLogSegNo nextSegNo,
TimeLineID *tli_p);
typedef void (*WALSegmentCloseCB) (XLogReaderState *xlogreader);
-typedef struct XLogReaderRoutine
-{
- /*
- * Data input callback
- *
- * This callback shall read at least reqLen valid bytes of the xlog page
- * starting at targetPagePtr, and store them in readBuf. The callback
- * shall return the number of bytes read (never more than XLOG_BLCKSZ), or
- * -1 on failure. The callback shall sleep, if necessary, to wait for the
- * requested bytes to become available. The callback will not be invoked
- * again for the same page unless more than the returned number of bytes
- * are needed.
- *
- * targetRecPtr is the position of the WAL record we're reading. Usually
- * it is equal to targetPagePtr + reqLen, but sometimes xlogreader needs
- * to read and verify the page or segment header, before it reads the
- * actual WAL record it's interested in. In that case, targetRecPtr can
- * be used to determine which timeline to read the page from.
- *
- * The callback shall set ->seg.ws_tli to the TLI of the file the page was
- * read from.
- */
- XLogPageReadCB page_read;
-
- /*
- * Callback to open the specified WAL segment for reading. ->seg.ws_file
- * shall be set to the file descriptor of the opened segment. In case of
- * failure, an error shall be raised by the callback and it shall not
- * return.
- *
- * "nextSegNo" is the number of the segment to be opened.
- *
- * "tli_p" is an input/output argument. WALRead() uses it to pass the
- * timeline in which the new segment should be found, but the callback can
- * use it to return the TLI that it actually opened.
- */
- WALSegmentOpenCB segment_open;
-
- /*
- * WAL segment close callback. ->seg.ws_file shall be set to a negative
- * number.
- */
- WALSegmentCloseCB segment_close;
-} XLogReaderRoutine;
-
-#define XL_ROUTINE(...) &(XLogReaderRoutine){__VA_ARGS__}
-
typedef struct
{
/* Is this block ref in use? */
@@ -144,12 +95,35 @@ typedef struct
uint16 data_bufsz;
} DecodedBkpBlock;
+/* Return code from XLogReadRecord */
+typedef enum XLogReadRecordResult
+{
+ XLREAD_SUCCESS, /* record is successfully read */
+ XLREAD_NEED_DATA, /* need more data. see XLogReadRecord. */
+ XLREAD_FAIL /* failed during reading a record */
+} XLogReadRecordResult;
+
+/*
+ * internal state of XLogReadRecord
+ *
+ * XLogReadState runs a state machine while reading a record. Theses states
+ * are not seen outside the function. Each state may repeat several times
+ * exiting requesting caller for new data. See the comment of XLogReadRecrod
+ * for details.
+ */
+typedef enum XLogReadRecordState {
+ XLREAD_NEXT_RECORD,
+ XLREAD_TOT_LEN,
+ XLREAD_FIRST_FRAGMENT,
+ XLREAD_CONTINUATION
+} XLogReadRecordState;
+
struct XLogReaderState
{
/*
* Operational callbacks
*/
- XLogReaderRoutine routine;
+ WALSegmentCleanupCB cleanup_cb;
/* ----------------------------------------
* Public parameters
@@ -162,18 +136,14 @@ struct XLogReaderState
*/
uint64 system_identifier;
- /*
- * Opaque data for callbacks to use. Not used by XLogReader.
- */
- void *private_data;
-
/*
* Start and end point of last record read. EndRecPtr is also used as the
* position to read next. Calling XLogBeginRead() sets EndRecPtr to the
* starting position and ReadRecPtr to invalid.
*/
- XLogRecPtr ReadRecPtr; /* start of last record read */
+ XLogRecPtr ReadRecPtr; /* start of last record read or being read */
XLogRecPtr EndRecPtr; /* end+1 of last record read */
+ XLogRecPtr PrevRecPtr; /* start of previous record read */
/* ----------------------------------------
* Communication with page reader
@@ -187,7 +157,9 @@ struct XLogReaderState
* the request, or -1 on error */
TimeLineID readPageTLI; /* TLI for data currently in readBuf */
char *readBuf; /* buffer to store data */
- bool page_verified; /* is the page on the buffer verified? */
+ bool page_verified; /* is the page header on the buffer verified? */
+ bool record_verified;/* is the current record header verified? */
+
/* ----------------------------------------
@@ -229,8 +201,6 @@ struct XLogReaderState
XLogRecPtr latestPagePtr;
TimeLineID latestPageTLI;
- /* beginning of the WAL record being read. */
- XLogRecPtr currRecPtr;
/* timeline to read it from, 0 if a lookup is required */
TimeLineID currTLI;
@@ -257,6 +227,15 @@ struct XLogReaderState
char *readRecordBuf;
uint32 readRecordBufSize;
+ /*
+ * XLogReadRecord() state
+ */
+ XLogReadRecordState readRecordState;/* state machine state */
+ int recordGotLen; /* amount of current record that has
+ * already been read */
+ int recordRemainLen; /* length of current record that remains */
+ XLogRecPtr recordContRecPtr; /* where the current record continues */
+
/* Buffer to hold error message */
char *errormsg_buf;
};
@@ -264,9 +243,7 @@ struct XLogReaderState
/* Get a new XLogReader */
extern XLogReaderState *XLogReaderAllocate(int wal_segment_size,
const char *waldir,
- XLogReaderRoutine *routine,
- void *private_data);
-extern XLogReaderRoutine *LocalXLogReaderRoutine(void);
+ WALSegmentCleanupCB cleanup_cb);
/* Free an XLogReader */
extern void XLogReaderFree(XLogReaderState *state);
@@ -274,12 +251,17 @@ extern void XLogReaderFree(XLogReaderState *state);
/* Position the XLogReader to given record */
extern void XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr);
#ifdef FRONTEND
-extern XLogRecPtr XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr);
+/* Function type definition for the read_page callback */
+typedef bool (*XLogFindNextRecordCB) (XLogReaderState *xlogreader,
+ void *private);
+extern XLogRecPtr XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
+ XLogFindNextRecordCB read_page, void *private);
#endif /* FRONTEND */
/* Read the next XLog record. Returns NULL on end-of-WAL or failure */
-extern struct XLogRecord *XLogReadRecord(XLogReaderState *state,
- char **errormsg);
+extern XLogReadRecordResult XLogReadRecord(XLogReaderState *state,
+ XLogRecord **record,
+ char **errormsg);
/* Validate a page */
extern bool XLogReaderValidatePageHeader(XLogReaderState *state,
@@ -299,6 +281,7 @@ typedef struct WALReadError
} WALReadError;
extern bool WALRead(XLogReaderState *state,
+ WALSegmentOpenCB segopenfn, WALSegmentCloseCB sgclosefn,
char *buf, XLogRecPtr startptr, Size count,
TimeLineID tli, WALReadError *errinfo);
diff --git a/src/include/access/xlogutils.h b/src/include/access/xlogutils.h
index 364a21c4ea..397fb27fc2 100644
--- a/src/include/access/xlogutils.h
+++ b/src/include/access/xlogutils.h
@@ -47,9 +47,7 @@ extern Buffer XLogReadBufferExtended(RelFileNode rnode, ForkNumber forknum,
extern Relation CreateFakeRelcacheEntry(RelFileNode rnode);
extern void FreeFakeRelcacheEntry(Relation fakerel);
-extern bool read_local_xlog_page(XLogReaderState *state,
- XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetRecPtr, char *cur_page);
+extern bool read_local_xlog_page(XLogReaderState *state);
extern void wal_segment_open(XLogReaderState *state,
XLogSegNo nextSegNo,
TimeLineID *tli_p);
diff --git a/src/include/pg_config_manual.h b/src/include/pg_config_manual.h
index f10ad0acd6..94749609b4 100644
--- a/src/include/pg_config_manual.h
+++ b/src/include/pg_config_manual.h
@@ -377,7 +377,7 @@
* Enable debugging print statements for WAL-related operations; see
* also the wal_debug GUC var.
*/
-/* #define WAL_DEBUG */
+#define WAL_DEBUG
/*
* Enable tracing of resource consumption during sort operations;
diff --git a/src/include/replication/logical.h b/src/include/replication/logical.h
index c253403372..b936f31393 100644
--- a/src/include/replication/logical.h
+++ b/src/include/replication/logical.h
@@ -29,6 +29,10 @@ typedef void (*LogicalOutputPluginWriterUpdateProgress) (struct LogicalDecodingC
TransactionId xid
);
+typedef struct LogicalDecodingContext LogicalDecodingContext;
+
+typedef bool (*LogicalDecodingXLogPageReadCB)(XLogReaderState *ctx);
+
typedef struct LogicalDecodingContext
{
/* memory context this is all allocated in */
@@ -39,6 +43,7 @@ typedef struct LogicalDecodingContext
/* infrastructure pieces for decoding */
XLogReaderState *reader;
+ LogicalDecodingXLogPageReadCB page_read;
struct ReorderBuffer *reorder;
struct SnapBuild *snapshot_builder;
@@ -105,14 +110,16 @@ extern LogicalDecodingContext *CreateInitDecodingContext(const char *plugin,
List *output_plugin_options,
bool need_full_snapshot,
XLogRecPtr restart_lsn,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress);
extern LogicalDecodingContext *CreateDecodingContext(XLogRecPtr start_lsn,
List *output_plugin_options,
bool fast_forward,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress);
--
2.27.0
----Next_Part(Thu_Mar__4_11_28_53_2021_014)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
filename="v17-0003-Remove-globals-readOff-readLen-and-readSegNo.patch"
^ permalink raw reply [nested|flat] 70+ messages in thread
* [PATCH v17 2/4] Move page-reader out of XLogReadRecord
@ 2019-09-10 03:58 Kyotaro Horiguchi <[email protected]>
0 siblings, 0 replies; 70+ messages in thread
From: Kyotaro Horiguchi @ 2019-09-10 03:58 UTC (permalink / raw)
This is the second step of removing callbacks from WAL record reader.
Since it is essential to take in additional data while reading a
record, the function have to ask caller for new data while keeping
working state. Thus the function is turned into a state machine.
---
src/backend/access/transam/twophase.c | 15 +-
src/backend/access/transam/xlog.c | 58 +-
src/backend/access/transam/xlogreader.c | 697 +++++++++++-------
src/backend/access/transam/xlogutils.c | 17 +-
src/backend/replication/logical/logical.c | 26 +-
.../replication/logical/logicalfuncs.c | 13 +-
src/backend/replication/slotfuncs.c | 18 +-
src/backend/replication/walsender.c | 32 +-
src/bin/pg_rewind/parsexlog.c | 84 ++-
src/bin/pg_waldump/pg_waldump.c | 36 +-
src/include/access/xlogreader.h | 121 ++-
src/include/access/xlogutils.h | 4 +-
src/include/pg_config_manual.h | 2 +-
src/include/replication/logical.h | 11 +-
14 files changed, 646 insertions(+), 488 deletions(-)
diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index 80d2d20d6c..7d5a96a5f5 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -1325,11 +1325,8 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
XLogReaderState *xlogreader;
char *errormsg;
- xlogreader = XLogReaderAllocate(wal_segment_size, NULL,
- XL_ROUTINE(.page_read = &read_local_xlog_page,
- .segment_open = &wal_segment_open,
- .segment_close = &wal_segment_close),
- NULL);
+ xlogreader = XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+
if (!xlogreader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
@@ -1337,7 +1334,13 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
errdetail("Failed while allocating a WAL reading processor.")));
XLogBeginRead(xlogreader, lsn);
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!read_local_xlog_page(xlogreader))
+ break;
+ }
+
if (record == NULL)
ereport(ERROR,
(errcode_for_file_access(),
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 606328a65a..670996ae67 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -836,13 +836,6 @@ static XLogSource currentSource = XLOG_FROM_ANY;
static bool lastSourceFailed = false;
static bool pendingWalRcvRestart = false;
-typedef struct XLogPageReadPrivate
-{
- int emode;
- bool fetching_ckpt; /* are we fetching a checkpoint record? */
- bool randAccess;
-} XLogPageReadPrivate;
-
/*
* These variables track when we last obtained some WAL data to process,
* and where we got it from. (XLogReceiptSource is initially the same as
@@ -917,8 +910,8 @@ static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath,
static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli,
XLogSource source, bool notfoundOk);
static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, XLogSource source);
-static bool XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
+static bool XLogPageRead(XLogReaderState *xlogreader,
+ bool fetching_ckpt, int emode, bool randAccess);
static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
bool fetching_ckpt, XLogRecPtr tliRecPtr);
static int emode_for_corrupt_record(int emode, XLogRecPtr RecPtr);
@@ -1231,8 +1224,7 @@ XLogInsertRecord(XLogRecData *rdata,
appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len);
if (!debug_reader)
- debug_reader = XLogReaderAllocate(wal_segment_size, NULL,
- XL_ROUTINE(), NULL);
+ debug_reader = XLogReaderAllocate(wal_segment_size, NULL, NULL);
if (!debug_reader)
{
@@ -4344,15 +4336,10 @@ CleanupBackupHistory(void)
* record is available.
*/
static XLogRecord *
-ReadRecord(XLogReaderState *xlogreader, int emode,
- bool fetching_ckpt)
+ReadRecord(XLogReaderState *xlogreader, int emode, bool fetching_ckpt)
{
XLogRecord *record;
- XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
-
- private->fetching_ckpt = fetching_ckpt;
- private->emode = emode;
- private->randAccess = (xlogreader->ReadRecPtr == InvalidXLogRecPtr);
+ bool randAccess = (xlogreader->ReadRecPtr == InvalidXLogRecPtr);
/* This is the first attempt to read this page. */
lastSourceFailed = false;
@@ -4360,8 +4347,16 @@ ReadRecord(XLogReaderState *xlogreader, int emode,
for (;;)
{
char *errormsg;
+ XLogReadRecordResult result;
+
+ while ((result = XLogReadRecord(xlogreader, &record, &errormsg))
+ == XLREAD_NEED_DATA)
+ {
+ if (!XLogPageRead(xlogreader, fetching_ckpt, emode, randAccess))
+ break;
+
+ }
- record = XLogReadRecord(xlogreader, &errormsg);
ReadRecPtr = xlogreader->ReadRecPtr;
EndRecPtr = xlogreader->EndRecPtr;
if (record == NULL)
@@ -6380,7 +6375,6 @@ StartupXLOG(void)
bool backupFromStandby = false;
DBState dbstate_at_startup;
XLogReaderState *xlogreader;
- XLogPageReadPrivate private;
bool promoted = false;
struct stat st;
@@ -6539,13 +6533,9 @@ StartupXLOG(void)
OwnLatch(&XLogCtl->recoveryWakeupLatch);
/* Set up XLOG reader facility */
- MemSet(&private, 0, sizeof(XLogPageReadPrivate));
xlogreader =
- XLogReaderAllocate(wal_segment_size, NULL,
- XL_ROUTINE(.page_read = &XLogPageRead,
- .segment_open = NULL,
- .segment_close = wal_segment_close),
- &private);
+ XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+
if (!xlogreader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
@@ -11987,12 +11977,13 @@ CancelBackup(void)
* sleep and retry.
*/
static bool
-XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetRecPtr, char *readBuf)
+XLogPageRead(XLogReaderState *xlogreader,
+ bool fetching_ckpt, int emode, bool randAccess)
{
- XLogPageReadPrivate *private =
- (XLogPageReadPrivate *) xlogreader->private_data;
- int emode = private->emode;
+ char *readBuf = xlogreader->readBuf;
+ XLogRecPtr targetPagePtr = xlogreader->readPagePtr;
+ int reqLen = xlogreader->readLen;
+ XLogRecPtr targetRecPtr = xlogreader->ReadRecPtr;
uint32 targetPageOff;
XLogSegNo targetSegNo PG_USED_FOR_ASSERTS_ONLY;
int r;
@@ -12035,8 +12026,8 @@ retry:
flushedUpto < targetPagePtr + reqLen))
{
if (!WaitForWALToBecomeAvailable(targetPagePtr + reqLen,
- private->randAccess,
- private->fetching_ckpt,
+ randAccess,
+ fetching_ckpt,
targetRecPtr))
{
if (readFile >= 0)
@@ -12141,6 +12132,7 @@ retry:
goto next_record_is_invalid;
}
+ Assert(xlogreader->readPagePtr == targetPagePtr);
xlogreader->readLen = readLen;
return true;
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index 7d04995b7b..ba3e4868f7 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -40,7 +40,7 @@ static bool XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr,
int reqLen, bool header_inclusive);
static void XLogReaderInvalReadState(XLogReaderState *state);
static bool ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
- XLogRecPtr PrevRecPtr, XLogRecord *record, bool randAccess);
+ XLogRecPtr PrevRecPtr, XLogRecord *record);
static bool ValidXLogRecord(XLogReaderState *state, XLogRecord *record,
XLogRecPtr recptr);
static void ResetDecoder(XLogReaderState *state);
@@ -73,7 +73,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...)
*/
XLogReaderState *
XLogReaderAllocate(int wal_segment_size, const char *waldir,
- XLogReaderRoutine *routine, void *private_data)
+ WALSegmentCleanupCB cleanup_cb)
{
XLogReaderState *state;
@@ -84,7 +84,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
return NULL;
/* initialize caller-provided support functions */
- state->routine = *routine;
+ state->cleanup_cb = cleanup_cb;
state->max_block_id = -1;
@@ -107,8 +107,6 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
WALOpenSegmentInit(&state->seg, &state->segcxt, wal_segment_size,
waldir);
- /* system_identifier initialized to zeroes above */
- state->private_data = private_data;
/* ReadRecPtr, EndRecPtr and readLen initialized to zeroes above */
state->errormsg_buf = palloc_extended(MAX_ERRORMSG_LEN + 1,
MCXT_ALLOC_NO_OOM);
@@ -140,8 +138,8 @@ XLogReaderFree(XLogReaderState *state)
{
int block_id;
- if (state->seg.ws_file != -1)
- state->routine.segment_close(state);
+ if (state->seg.ws_file >= 0)
+ state->cleanup_cb(state);
for (block_id = 0; block_id <= XLR_MAX_BLOCK_ID; block_id++)
{
@@ -246,6 +244,7 @@ XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
/* Begin at the passed-in record pointer. */
state->EndRecPtr = RecPtr;
state->ReadRecPtr = InvalidXLogRecPtr;
+ state->readRecordState = XLREAD_NEXT_RECORD;
}
/*
@@ -254,317 +253,449 @@ XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
* XLogBeginRead() or XLogFindNextRecord() must be called before the first call
* to XLogReadRecord().
*
- * If the page_read callback fails to read the requested data, NULL is
- * returned. The callback is expected to have reported the error; errormsg
- * is set to NULL.
+ * This function may return XLREAD_NEED_DATA several times before returning a
+ * result record. The caller shall read in some new data then call this
+ * function again with the same parameters.
*
- * If the reading fails for some other reason, NULL is also returned, and
- * *errormsg is set to a string with details of the failure.
+ * When a record is successfully read, returns XLREAD_SUCCESS with result
+ * record being stored in *record. Otherwise *record is NULL.
*
- * The returned pointer (or *errormsg) points to an internal buffer that's
- * valid until the next call to XLogReadRecord.
+ * Returns XLREAD_NEED_DATA if more data is needed to finish reading the
+ * current record. In that case, state->readPagePtr and state->readLen inform
+ * the desired position and minimum length of data needed. The caller shall
+ * read in the requested data and set state->readBuf to point to a buffer
+ * containing it. The caller must also set state->readPageTLI and
+ * state->readLen to indicate the timeline that it was read from, and the
+ * length of data that is now available (which must be >= given readLen),
+ * respectively.
+ *
+ * If invalid data is encountered, returns XLREAD_FAIL with *record being set to
+ * NULL. *errormsg is set to a string with details of the failure.
+ * The returned pointer (or *errormsg) points to an internal buffer that's valid
+ * until the next call to XLogReadRecord.
+ *
+ *
+ * This function runs a state machine consists of the following states.
+ *
+ * XLREAD_NEXT_RECORD :
+ * The initial state, if called with valid RecPtr, try to read a record at
+ * that position. If invalid RecPtr is given try to read a record just after
+ * the last one previously read.
+ * This state ens after setting ReadRecPtr. Then goes to XLREAD_TOT_LEN.
+ *
+ * XLREAD_TOT_LEN:
+ * Examining record header. Ends after reading record total
+ * length. recordRemainLen and recordGotLen are initialized.
+ *
+ * XLREAD_FIRST_FRAGMENT:
+ * Reading the first fragment. Ends with finishing reading a single
+ * record. Goes to XLREAD_NEXT_RECORD if that's all or
+ * XLREAD_CONTINUATION if we have continuation.
+
+ * XLREAD_CONTINUATION:
+ * Reading continuation of record. Ends with finishing the whole record then
+ * goes to XLREAD_NEXT_RECORD. During this state, recordRemainLen indicates
+ * how much is left and readRecordBuf holds the partially assert
+ * record.recordContRecPtr points to the beginning of the next page where to
+ * continue.
+ *
+ * If wrong data found in any state, the state machine stays at the current
+ * state. This behavior allows to continue reading a reacord switching among
+ * different souces, while streaming replication.
*/
-XLogRecord *
-XLogReadRecord(XLogReaderState *state, char **errormsg)
+XLogReadRecordResult
+XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
{
- XLogRecPtr RecPtr;
- XLogRecord *record;
- XLogRecPtr targetPagePtr;
- bool randAccess;
- uint32 len,
- total_len;
- uint32 targetRecOff;
- uint32 pageHeaderSize;
- bool gotheader;
+ XLogRecord *prec;
- /*
- * randAccess indicates whether to verify the previous-record pointer of
- * the record we're reading. We only do this if we're reading
- * sequentially, which is what we initially assume.
- */
- randAccess = false;
+ *record = NULL;
/* reset error state */
*errormsg = NULL;
state->errormsg_buf[0] = '\0';
- ResetDecoder(state);
-
- RecPtr = state->EndRecPtr;
-
- if (state->ReadRecPtr != InvalidXLogRecPtr)
- {
- /* read the record after the one we just read */
-
- /*
- * EndRecPtr is pointing to end+1 of the previous WAL record. If
- * we're at a page boundary, no more records can fit on the current
- * page. We must skip over the page header, but we can't do that until
- * we've read in the page, since the header size is variable.
- */
- }
- else
- {
- /*
- * Caller supplied a position to start at.
- *
- * In this case, EndRecPtr should already be pointing to a valid
- * record starting position.
- */
- Assert(XRecOffIsValid(RecPtr));
- randAccess = true;
- }
-
- state->currRecPtr = RecPtr;
-
- targetPagePtr = RecPtr - (RecPtr % XLOG_BLCKSZ);
- targetRecOff = RecPtr % XLOG_BLCKSZ;
-
- /*
- * Read the page containing the record into state->readBuf. Request enough
- * byte to cover the whole record header, or at least the part of it that
- * fits on the same page.
- */
- while (XLogNeedData(state, targetPagePtr,
- Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
- targetRecOff != 0))
- {
- if (!state->routine.page_read(state, state->readPagePtr, state->readLen,
- RecPtr, state->readBuf))
- break;
- }
-
- if (!state->page_verified)
- goto err;
-
- /*
- * We have at least the page header, so we can examine it now.
- */
- pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
- if (targetRecOff == 0)
- {
- /*
- * At page start, so skip over page header.
- */
- RecPtr += pageHeaderSize;
- targetRecOff = pageHeaderSize;
- }
- else if (targetRecOff < pageHeaderSize)
- {
- report_invalid_record(state, "invalid record offset at %X/%X",
- LSN_FORMAT_ARGS(RecPtr));
- goto err;
- }
-
- if ((((XLogPageHeader) state->readBuf)->xlp_info & XLP_FIRST_IS_CONTRECORD) &&
- targetRecOff == pageHeaderSize)
- {
- report_invalid_record(state, "contrecord is requested by %X/%X",
- LSN_FORMAT_ARGS(RecPtr));
- goto err;
- }
-
- /* XLogNeedData has verified the page header */
- Assert(pageHeaderSize <= state->readLen);
-
- /*
- * Read the record length.
- *
- * NB: Even though we use an XLogRecord pointer here, the whole record
- * header might not fit on this page. xl_tot_len is the first field of the
- * struct, so it must be on this page (the records are MAXALIGNed), but we
- * cannot access any other fields until we've verified that we got the
- * whole header.
- */
- record = (XLogRecord *) (state->readBuf + RecPtr % XLOG_BLCKSZ);
- total_len = record->xl_tot_len;
-
- /*
- * If the whole record header is on this page, validate it immediately.
- * Otherwise do just a basic sanity check on xl_tot_len, and validate the
- * rest of the header after reading it from the next page. The xl_tot_len
- * check is necessary here to ensure that we enter the "Need to reassemble
- * record" code path below; otherwise we might fail to apply
- * ValidXLogRecordHeader at all.
- */
- if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
- {
- if (!ValidXLogRecordHeader(state, RecPtr, state->ReadRecPtr, record,
- randAccess))
- goto err;
- gotheader = true;
- }
- else
- {
- /* XXX: more validation should be done here */
- if (total_len < SizeOfXLogRecord)
- {
- report_invalid_record(state,
- "invalid record length at %X/%X: wanted %u, got %u",
- LSN_FORMAT_ARGS(RecPtr),
- (uint32) SizeOfXLogRecord, total_len);
- goto err;
- }
- gotheader = false;
- }
-
- len = XLOG_BLCKSZ - RecPtr % XLOG_BLCKSZ;
- if (total_len > len)
+ switch (state->readRecordState)
{
- /* Need to reassemble record */
- char *contdata;
- XLogPageHeader pageHeader;
- char *buffer;
- uint32 gotlen;
-
- /*
- * Enlarge readRecordBuf as needed.
- */
- if (total_len > state->readRecordBufSize &&
- !allocate_recordbuf(state, total_len))
- {
- /* We treat this as a "bogus data" condition */
- report_invalid_record(state, "record length %u at %X/%X too long",
- total_len, LSN_FORMAT_ARGS(RecPtr));
- goto err;
- }
+ case XLREAD_NEXT_RECORD:
+ ResetDecoder(state);
- /* Copy the first fragment of the record from the first page. */
- memcpy(state->readRecordBuf,
- state->readBuf + RecPtr % XLOG_BLCKSZ, len);
- buffer = state->readRecordBuf + len;
- gotlen = len;
-
- do
- {
- int rest_len = total_len - gotlen;
-
- /* Calculate pointer to beginning of next page */
- targetPagePtr += XLOG_BLCKSZ;
-
- /* Wait for the next page to become available */
- while (XLogNeedData(state, targetPagePtr,
- Min(rest_len, XLOG_BLCKSZ),
- false))
+ if (state->ReadRecPtr != InvalidXLogRecPtr)
{
- if (!state->routine.page_read(state, state->readPagePtr,
- state->readLen,
- state->ReadRecPtr,
- state->readBuf))
- break;
+ /* read the record after the one we just read */
+
+ /*
+ * EndRecPtr is pointing to end+1 of the previous WAL record.
+ * If we're at a page boundary, no more records can fit on the
+ * current page. We must skip over the page header, but we
+ * can't do that until we've read in the page, since the header
+ * size is variable.
+ */
+ state->PrevRecPtr = state->ReadRecPtr;
+ state->ReadRecPtr = state->EndRecPtr;
+ }
+ else
+ {
+ /*
+ * Caller supplied a position to start at.
+ *
+ * In this case, EndRecPtr should already be pointing to a
+ * valid record starting position.
+ */
+ Assert(XRecOffIsValid(state->EndRecPtr));
+ state->ReadRecPtr = state->EndRecPtr;
+
+ /*
+ * We cannot verify the previous-record pointer when we're
+ * seeking to a particular record. Reset PrevRecPtr so that we
+ * won't try doing that.
+ */
+ state->PrevRecPtr = InvalidXLogRecPtr;
+ state->EndRecPtr = InvalidXLogRecPtr; /* to be tidy */
}
+ state->record_verified = false;
+ state->readRecordState = XLREAD_TOT_LEN;
+ /* fall through */
+
+ case XLREAD_TOT_LEN:
+ {
+ uint32 total_len;
+ uint32 pageHeaderSize;
+ XLogRecPtr targetPagePtr;
+ uint32 targetRecOff;
+ XLogPageHeader pageHeader;
+
+ targetPagePtr =
+ state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
+ targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+
+ /*
+ * Check if we have enough data. For the first record in the page,
+ * the requesting length doesn't contain page header.
+ */
+ if (XLogNeedData(state, targetPagePtr,
+ Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+ targetRecOff != 0))
+ return XLREAD_NEED_DATA;
+
+ /* error out if caller supplied bogus page */
if (!state->page_verified)
goto err;
- Assert(SizeOfXLogShortPHD <= state->readLen);
+ /* examine page header now. */
+ pageHeaderSize =
+ XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+ if (targetRecOff == 0)
+ {
+ /* At page start, so skip over page header. */
+ state->ReadRecPtr += pageHeaderSize;
+ targetRecOff = pageHeaderSize;
+ }
+ else if (targetRecOff < pageHeaderSize)
+ {
+ report_invalid_record(state, "invalid record offset at %X/%X",
+ LSN_FORMAT_ARGS(state->ReadRecPtr));
+ goto err;
+ }
- /* Check that the continuation on next page looks valid */
pageHeader = (XLogPageHeader) state->readBuf;
- if (!(pageHeader->xlp_info & XLP_FIRST_IS_CONTRECORD))
+ if ((pageHeader->xlp_info & XLP_FIRST_IS_CONTRECORD) &&
+ targetRecOff == pageHeaderSize)
{
- report_invalid_record(state,
- "there is no contrecord flag at %X/%X",
- LSN_FORMAT_ARGS(RecPtr));
+ report_invalid_record(state, "contrecord is requested by %X/%X",
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr);
goto err;
}
- /*
- * Cross-check that xlp_rem_len agrees with how much of the record
- * we expect there to be left.
- */
- if (pageHeader->xlp_rem_len == 0 ||
- total_len != (pageHeader->xlp_rem_len + gotlen))
- {
- report_invalid_record(state,
- "invalid contrecord length %u (expected %lld) at %X/%X",
- pageHeader->xlp_rem_len,
- ((long long) total_len) - gotlen,
- LSN_FORMAT_ARGS(RecPtr));
- goto err;
- }
-
- /* Append the continuation from this page to the buffer */
- pageHeaderSize = XLogPageHeaderSize(pageHeader);
-
+ /* XLogNeedData has verified the page header */
Assert(pageHeaderSize <= state->readLen);
- contdata = (char *) state->readBuf + pageHeaderSize;
- len = XLOG_BLCKSZ - pageHeaderSize;
- if (pageHeader->xlp_rem_len < len)
- len = pageHeader->xlp_rem_len;
+ /*
+ * Read the record length.
+ *
+ * NB: Even though we use an XLogRecord pointer here, the whole
+ * record header might not fit on this page. xl_tot_len is the first
+ * field of the struct, so it must be on this page (the records are
+ * MAXALIGNed), but we cannot access any other fields until we've
+ * verified that we got the whole header.
+ */
+ prec = (XLogRecord *) (state->readBuf +
+ state->ReadRecPtr % XLOG_BLCKSZ);
+ total_len = prec->xl_tot_len;
- Assert (pageHeaderSize + len <= state->readLen);
- memcpy(buffer, (char *) contdata, len);
- buffer += len;
- gotlen += len;
+ /*
+ * If the whole record header is on this page, validate it
+ * immediately. Otherwise do just a basic sanity check on
+ * xl_tot_len, and validate the rest of the header after reading it
+ * from the next page. The xl_tot_len check is necessary here to
+ * ensure that we enter the XLREAD_CONTINUATION state below;
+ * otherwise we might fail to apply ValidXLogRecordHeader at all.
+ */
+ if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
+ {
+ if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
+ state->PrevRecPtr, prec))
+ goto err;
- /* If we just reassembled the record header, validate it. */
- if (!gotheader)
+ state->record_verified = true;
+ }
+ else
{
- record = (XLogRecord *) state->readRecordBuf;
- if (!ValidXLogRecordHeader(state, RecPtr, state->ReadRecPtr,
- record, randAccess))
+ /* XXX: more validation should be done here */
+ if (total_len < SizeOfXLogRecord)
+ {
+ report_invalid_record(state,
+ "invalid record length at %X/%X: wanted %u, got %u",
+ LSN_FORMAT_ARGS(state->ReadRecPtr),
+ (uint32) SizeOfXLogRecord, total_len);
goto err;
- gotheader = true;
+ }
}
- } while (gotlen < total_len);
-
- Assert(gotheader);
- record = (XLogRecord *) state->readRecordBuf;
- if (!ValidXLogRecord(state, record, RecPtr))
- goto err;
+ /*
+ * Wait for the rest of the record, or the part of the record that
+ * fit on the first page if crossed a page boundary, to become
+ * available.
+ */
+ state->recordGotLen = 0;
+ state->recordRemainLen = total_len;
+ state->readRecordState = XLREAD_FIRST_FRAGMENT;
+ }
+ /* fall through */
- pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
- state->ReadRecPtr = RecPtr;
- state->EndRecPtr = targetPagePtr + pageHeaderSize
- + MAXALIGN(pageHeader->xlp_rem_len);
- }
- else
- {
- /* Wait for the record data to become available */
- while (XLogNeedData(state, targetPagePtr,
- Min(targetRecOff + total_len, XLOG_BLCKSZ), true))
+ case XLREAD_FIRST_FRAGMENT:
{
- if (!state->routine.page_read(state, state->readPagePtr,
- state->readLen,
- state->ReadRecPtr, state->readBuf))
+ uint32 total_len = state->recordRemainLen;
+ uint32 request_len;
+ uint32 record_len;
+ XLogRecPtr targetPagePtr;
+ uint32 targetRecOff;
+
+ /*
+ * Wait for the rest of the record on the first page to become
+ * available
+ */
+ targetPagePtr =
+ state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
+ targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+
+ request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ);
+ record_len = request_len - targetRecOff;
+
+ /* ReadRecPtr contains page header */
+ Assert (targetRecOff != 0);
+ if (XLogNeedData(state, targetPagePtr, request_len, true))
+ return XLREAD_NEED_DATA;
+
+ /* error out if caller supplied bogus page */
+ if (!state->page_verified)
+ goto err;
+
+ prec = (XLogRecord *) (state->readBuf + targetRecOff);
+
+ /* validate record header if not yet */
+ if (!state->record_verified && record_len >= SizeOfXLogRecord)
+ {
+ if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
+ state->PrevRecPtr, prec))
+ goto err;
+
+ state->record_verified = true;
+ }
+
+
+ if (total_len == record_len)
+ {
+ /* Record does not cross a page boundary */
+ Assert(state->record_verified);
+
+ if (!ValidXLogRecord(state, prec, state->ReadRecPtr))
+ goto err;
+
+ state->record_verified = true; /* to be tidy */
+
+ /* We already checked the header earlier */
+ state->EndRecPtr = state->ReadRecPtr + MAXALIGN(record_len);
+
+ *record = prec;
+ state->readRecordState = XLREAD_NEXT_RECORD;
break;
+ }
+
+ /*
+ * The record continues on the next page. Need to reassemble
+ * record
+ */
+ Assert(total_len > record_len);
+
+ /* Enlarge readRecordBuf as needed. */
+ if (total_len > state->readRecordBufSize &&
+ !allocate_recordbuf(state, total_len))
+ {
+ /* We treat this as a "bogus data" condition */
+ report_invalid_record(state,
+ "record length %u at %X/%X too long",
+ total_len,
+ LSN_FORMAT_ARGS(state->ReadRecPtr));
+ goto err;
+ }
+
+ /* Copy the first fragment of the record from the first page. */
+ memcpy(state->readRecordBuf, state->readBuf + targetRecOff,
+ record_len);
+ state->recordGotLen += record_len;
+ state->recordRemainLen -= record_len;
+
+ /* Calculate pointer to beginning of next page */
+ state->recordContRecPtr = state->ReadRecPtr + record_len;
+ Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0);
+
+ state->readRecordState = XLREAD_CONTINUATION;
}
+ /* fall through */
+
+ case XLREAD_CONTINUATION:
+ {
+ XLogPageHeader pageHeader;
+ uint32 pageHeaderSize;
+ XLogRecPtr targetPagePtr;
+
+ /* we enter this state only if we haven't read the whole record. */
+ Assert (state->recordRemainLen > 0);
+
+ while(state->recordRemainLen > 0)
+ {
+ char *contdata;
+ uint32 request_len;
+ uint32 record_len;
+
+ /* Wait for the next page to become available */
+ targetPagePtr = state->recordContRecPtr;
+
+ /* this request contains page header */
+ Assert (targetPagePtr != 0);
+ if (XLogNeedData(state, targetPagePtr,
+ Min(state->recordRemainLen, XLOG_BLCKSZ),
+ false))
+ return XLREAD_NEED_DATA;
+
+ if (!state->page_verified)
+ goto err;
+
+ Assert(SizeOfXLogShortPHD <= state->readLen);
- if (!state->page_verified)
- goto err;
+ /* Check that the continuation on next page looks valid */
+ pageHeader = (XLogPageHeader) state->readBuf;
+ if (!(pageHeader->xlp_info & XLP_FIRST_IS_CONTRECORD))
+ {
+ report_invalid_record(
+ state,
+ "there is no contrecord flag at %X/%X reading %X/%X",
+ (uint32) (state->recordContRecPtr >> 32),
+ (uint32) state->recordContRecPtr,
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr);
+ goto err;
+ }
- /* Record does not cross a page boundary */
- if (!ValidXLogRecord(state, record, RecPtr))
- goto err;
+ /*
+ * Cross-check that xlp_rem_len agrees with how much of the
+ * record we expect there to be left.
+ */
+ if (pageHeader->xlp_rem_len == 0 ||
+ pageHeader->xlp_rem_len != state->recordRemainLen)
+ {
+ report_invalid_record(
+ state,
+ "invalid contrecord length %u at %X/%X reading %X/%X, expected %u",
+ pageHeader->xlp_rem_len,
+ (uint32) (state->recordContRecPtr >> 32),
+ (uint32) state->recordContRecPtr,
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr,
+ state->recordRemainLen);
+ goto err;
+ }
- state->EndRecPtr = RecPtr + MAXALIGN(total_len);
+ /* Append the continuation from this page to the buffer */
+ pageHeaderSize = XLogPageHeaderSize(pageHeader);
- state->ReadRecPtr = RecPtr;
+ /*
+ * XLogNeedData should have ensured that the whole page header
+ * was read
+ */
+ Assert(state->readLen >= pageHeaderSize);
+
+ contdata = (char *) state->readBuf + pageHeaderSize;
+ record_len = XLOG_BLCKSZ - pageHeaderSize;
+ if (pageHeader->xlp_rem_len < record_len)
+ record_len = pageHeader->xlp_rem_len;
+
+ request_len = record_len + pageHeaderSize;
+
+ /* XLogNeedData should have ensured all needed data was read */
+ Assert (state->readLen >= request_len);
+
+ memcpy(state->readRecordBuf + state->recordGotLen,
+ (char *) contdata, record_len);
+ state->recordGotLen += record_len;
+ state->recordRemainLen -= record_len;
+
+ /* If we just reassembled the record header, validate it. */
+ if (!state->record_verified)
+ {
+ Assert(state->recordGotLen >= SizeOfXLogRecord);
+ if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
+ state->PrevRecPtr,
+ (XLogRecord *) state->readRecordBuf))
+ goto err;
+
+ state->record_verified = true;
+ }
+
+ /* Calculate pointer to beginning of next page, and continue */
+ state->recordContRecPtr += XLOG_BLCKSZ;
+ }
+
+ /* targetPagePtr is pointing the last-read page here */
+ prec = (XLogRecord *) state->readRecordBuf;
+ if (!ValidXLogRecord(state, prec, state->ReadRecPtr))
+ goto err;
+
+ pageHeaderSize =
+ XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+ state->EndRecPtr = targetPagePtr + pageHeaderSize
+ + MAXALIGN(pageHeader->xlp_rem_len);
+
+ *record = prec;
+ state->readRecordState = XLREAD_NEXT_RECORD;
+ break;
+ }
}
/*
* Special processing if it's an XLOG SWITCH record
*/
- if (record->xl_rmid == RM_XLOG_ID &&
- (record->xl_info & ~XLR_INFO_MASK) == XLOG_SWITCH)
+ if ((*record)->xl_rmid == RM_XLOG_ID &&
+ ((*record)->xl_info & ~XLR_INFO_MASK) == XLOG_SWITCH)
{
/* Pretend it extends to end of segment */
state->EndRecPtr += state->segcxt.ws_segsize - 1;
state->EndRecPtr -= XLogSegmentOffset(state->EndRecPtr, state->segcxt.ws_segsize);
}
- if (DecodeXLogRecord(state, record, errormsg))
- return record;
- else
- return NULL;
+ Assert (!*record || state->readLen >= 0);
+ if (DecodeXLogRecord(state, *record, errormsg))
+ return XLREAD_SUCCESS;
+
+ *record = NULL;
+ return XLREAD_FAIL;
err:
/*
- * Invalidate the read state. We might read from a different source after
+ * Invalidate the read page. We might read from a different source after
* failure.
*/
XLogReaderInvalReadState(state);
@@ -572,7 +703,8 @@ err:
if (state->errormsg_buf[0] != '\0')
*errormsg = state->errormsg_buf;
- return NULL;
+ *record = NULL;
+ return XLREAD_FAIL;
}
/*
@@ -724,11 +856,12 @@ XLogReaderInvalReadState(XLogReaderState *state)
*
* This is just a convenience subroutine to avoid duplicated code in
* XLogReadRecord. It's not intended for use from anywhere else.
+ *
+ * If PrevRecPtr is valid, the xl_prev is is cross-checked with it.
*/
static bool
ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
- XLogRecPtr PrevRecPtr, XLogRecord *record,
- bool randAccess)
+ XLogRecPtr PrevRecPtr, XLogRecord *record)
{
if (record->xl_tot_len < SizeOfXLogRecord)
{
@@ -745,7 +878,7 @@ ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
record->xl_rmid, LSN_FORMAT_ARGS(RecPtr));
return false;
}
- if (randAccess)
+ if (PrevRecPtr == InvalidXLogRecPtr)
{
/*
* We can't exactly verify the prev-link, but surely it should be less
@@ -975,11 +1108,14 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
* XLogReadRecord() will read the next valid record.
*/
XLogRecPtr
-XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
+XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
+ XLogFindNextRecordCB read_page, void *private)
{
XLogRecPtr tmpRecPtr;
XLogRecPtr found = InvalidXLogRecPtr;
XLogPageHeader header;
+ XLogRecord *record;
+ XLogReadRecordResult result;
char *errormsg;
Assert(!XLogRecPtrIsInvalid(RecPtr));
@@ -1012,9 +1148,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
while(XLogNeedData(state, targetPagePtr, targetRecOff,
targetRecOff != 0))
{
- if (!state->routine.page_read(state, state->readPagePtr,
- state->readLen,
- state->ReadRecPtr, state->readBuf))
+ if (!read_page(state, private))
break;
}
@@ -1066,8 +1200,16 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
* or we just jumped over the remaining data of a continuation.
*/
XLogBeginRead(state, tmpRecPtr);
- while (XLogReadRecord(state, &errormsg) != NULL)
+ while ((result = XLogReadRecord(state, &record, &errormsg)) !=
+ XLREAD_FAIL)
{
+ if (result == XLREAD_NEED_DATA)
+ {
+ if (!read_page(state, private))
+ goto err;
+ continue;
+ }
+
/* past the record we've found, break out */
if (RecPtr <= state->ReadRecPtr)
{
@@ -1087,9 +1229,9 @@ err:
#endif /* FRONTEND */
/*
- * Helper function to ease writing of XLogRoutine->page_read callbacks.
- * If this function is used, caller must supply a segment_open callback in
- * 'state', as that is used here.
+ * Helper function to ease writing of page_read callback.
+ * If this function is used, caller must supply a segment_open callback and
+ * segment_close callback as that is used here.
*
* Read 'count' bytes into 'buf', starting at location 'startptr', from WAL
* fetched from timeline 'tli'.
@@ -1102,6 +1244,7 @@ err:
*/
bool
WALRead(XLogReaderState *state,
+ WALSegmentOpenCB segopenfn, WALSegmentCloseCB segclosefn,
char *buf, XLogRecPtr startptr, Size count, TimeLineID tli,
WALReadError *errinfo)
{
@@ -1133,10 +1276,10 @@ WALRead(XLogReaderState *state,
XLogSegNo nextSegNo;
if (state->seg.ws_file >= 0)
- state->routine.segment_close(state);
+ segclosefn(state);
XLByteToSeg(recptr, nextSegNo, state->segcxt.ws_segsize);
- state->routine.segment_open(state, nextSegNo, &tli);
+ segopenfn(state, nextSegNo, &tli);
/* This shouldn't happen -- indicates a bug in segment_open */
Assert(state->seg.ws_file >= 0);
diff --git a/src/backend/access/transam/xlogutils.c b/src/backend/access/transam/xlogutils.c
index 46eda33f25..b003990745 100644
--- a/src/backend/access/transam/xlogutils.c
+++ b/src/backend/access/transam/xlogutils.c
@@ -686,8 +686,7 @@ XLogTruncateRelation(RelFileNode rnode, ForkNumber forkNum,
void
XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wantLength)
{
- const XLogRecPtr lastReadPage = state->seg.ws_segno *
- state->segcxt.ws_segsize + state->readLen;
+ const XLogRecPtr lastReadPage = state->readPagePtr;
Assert(wantPage != InvalidXLogRecPtr && wantPage % XLOG_BLCKSZ == 0);
Assert(wantLength <= XLOG_BLCKSZ);
@@ -702,7 +701,7 @@ XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wa
* current TLI has since become historical.
*/
if (lastReadPage == wantPage &&
- state->readLen != 0 &&
+ state->page_verified &&
lastReadPage + state->readLen >= wantPage + Min(wantLength, XLOG_BLCKSZ - 1))
return;
@@ -788,6 +787,7 @@ wal_segment_open(XLogReaderState *state, XLogSegNo nextSegNo,
char path[MAXPGPATH];
XLogFilePath(path, tli, nextSegNo, state->segcxt.ws_segsize);
+ elog(LOG, "HOGE: %lu, %d => %s", nextSegNo, tli, path);
state->seg.ws_file = BasicOpenFile(path, O_RDONLY | PG_BINARY);
if (state->seg.ws_file >= 0)
return;
@@ -825,9 +825,11 @@ wal_segment_close(XLogReaderState *state)
* loop for now.
*/
bool
-read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *cur_page)
+read_local_xlog_page(XLogReaderState *state)
{
+ XLogRecPtr targetPagePtr = state->readPagePtr;
+ int reqLen = state->readLen;
+ char *cur_page = state->readBuf;
XLogRecPtr read_upto,
loc;
TimeLineID tli;
@@ -940,11 +942,12 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
* as 'count', read the whole page anyway. It's guaranteed to be
* zero-padded up to the page boundary if it's incomplete.
*/
- if (!WALRead(state, cur_page, targetPagePtr, XLOG_BLCKSZ, tli,
- &errinfo))
+ if (!WALRead(state, wal_segment_open, wal_segment_close,
+ cur_page, targetPagePtr, XLOG_BLCKSZ, tli, &errinfo))
WALReadRaiseError(&errinfo);
/* number of valid bytes in the buffer */
+ state->readPagePtr = targetPagePtr;
state->readLen = count;
return true;
}
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index 37b75deb72..9cbd8afa49 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -148,7 +148,8 @@ StartupDecodingContext(List *output_plugin_options,
TransactionId xmin_horizon,
bool need_full_snapshot,
bool fast_forward,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress)
@@ -198,11 +199,12 @@ StartupDecodingContext(List *output_plugin_options,
ctx->slot = slot;
- ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, xl_routine, ctx);
+ ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, cleanup_cb);
if (!ctx->reader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
errmsg("out of memory")));
+ ctx->page_read = page_read;
ctx->reorder = ReorderBufferAllocate();
ctx->snapshot_builder =
@@ -319,7 +321,8 @@ CreateInitDecodingContext(const char *plugin,
List *output_plugin_options,
bool need_full_snapshot,
XLogRecPtr restart_lsn,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress)
@@ -422,7 +425,7 @@ CreateInitDecodingContext(const char *plugin,
ctx = StartupDecodingContext(NIL, restart_lsn, xmin_horizon,
need_full_snapshot, false,
- xl_routine, prepare_write, do_write,
+ page_read, cleanup_cb, prepare_write, do_write,
update_progress);
/* call output plugin initialization callback */
@@ -476,7 +479,8 @@ LogicalDecodingContext *
CreateDecodingContext(XLogRecPtr start_lsn,
List *output_plugin_options,
bool fast_forward,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress)
@@ -528,8 +532,8 @@ CreateDecodingContext(XLogRecPtr start_lsn,
ctx = StartupDecodingContext(output_plugin_options,
start_lsn, InvalidTransactionId, false,
- fast_forward, xl_routine, prepare_write,
- do_write, update_progress);
+ fast_forward, page_read, cleanup_cb,
+ prepare_write, do_write, update_progress);
/* call output plugin initialization callback */
old_context = MemoryContextSwitchTo(ctx->context);
@@ -585,7 +589,13 @@ DecodingContextFindStartpoint(LogicalDecodingContext *ctx)
char *err = NULL;
/* the read_page callback waits for new WAL */
- record = XLogReadRecord(ctx->reader, &err);
+ while (XLogReadRecord(ctx->reader, &record, &err) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!ctx->page_read(ctx->reader))
+ break;
+ }
+
if (err)
elog(ERROR, "%s", err);
if (!record)
diff --git a/src/backend/replication/logical/logicalfuncs.c b/src/backend/replication/logical/logicalfuncs.c
index f7e055874e..b69f6911c5 100644
--- a/src/backend/replication/logical/logicalfuncs.c
+++ b/src/backend/replication/logical/logicalfuncs.c
@@ -233,9 +233,8 @@ pg_logical_slot_get_changes_guts(FunctionCallInfo fcinfo, bool confirm, bool bin
ctx = CreateDecodingContext(InvalidXLogRecPtr,
options,
false,
- XL_ROUTINE(.page_read = read_local_xlog_page,
- .segment_open = wal_segment_open,
- .segment_close = wal_segment_close),
+ read_local_xlog_page,
+ wal_segment_close,
LogicalOutputPrepareWrite,
LogicalOutputWrite, NULL);
@@ -284,7 +283,13 @@ pg_logical_slot_get_changes_guts(FunctionCallInfo fcinfo, bool confirm, bool bin
XLogRecord *record;
char *errm = NULL;
- record = XLogReadRecord(ctx->reader, &errm);
+ while (XLogReadRecord(ctx->reader, &record, &errm) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!ctx->page_read(ctx->reader))
+ break;
+ }
+
if (errm)
elog(ERROR, "%s", errm);
diff --git a/src/backend/replication/slotfuncs.c b/src/backend/replication/slotfuncs.c
index 9817b44113..deb7cf15eb 100644
--- a/src/backend/replication/slotfuncs.c
+++ b/src/backend/replication/slotfuncs.c
@@ -153,9 +153,8 @@ create_logical_replication_slot(char *name, char *plugin,
ctx = CreateInitDecodingContext(plugin, NIL,
false, /* just catalogs is OK */
restart_lsn,
- XL_ROUTINE(.page_read = read_local_xlog_page,
- .segment_open = wal_segment_open,
- .segment_close = wal_segment_close),
+ read_local_xlog_page,
+ wal_segment_close,
NULL, NULL, NULL);
/*
@@ -512,9 +511,8 @@ pg_logical_replication_slot_advance(XLogRecPtr moveto)
ctx = CreateDecodingContext(InvalidXLogRecPtr,
NIL,
true, /* fast_forward */
- XL_ROUTINE(.page_read = read_local_xlog_page,
- .segment_open = wal_segment_open,
- .segment_close = wal_segment_close),
+ read_local_xlog_page,
+ wal_segment_close,
NULL, NULL, NULL);
/*
@@ -536,7 +534,13 @@ pg_logical_replication_slot_advance(XLogRecPtr moveto)
* Read records. No changes are generated in fast_forward mode,
* but snapbuilder/slot statuses are updated properly.
*/
- record = XLogReadRecord(ctx->reader, &errm);
+ while (XLogReadRecord(ctx->reader, &record, &errm) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!ctx->page_read(ctx->reader))
+ break;
+ }
+
if (errm)
elog(ERROR, "%s", errm);
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index c36b91ae5e..062093727d 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -580,10 +580,7 @@ StartReplication(StartReplicationCmd *cmd)
/* create xlogreader for physical replication */
xlogreader =
- XLogReaderAllocate(wal_segment_size, NULL,
- XL_ROUTINE(.segment_open = WalSndSegmentOpen,
- .segment_close = wal_segment_close),
- NULL);
+ XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
if (!xlogreader)
ereport(ERROR,
@@ -807,9 +804,11 @@ StartReplication(StartReplicationCmd *cmd)
* set every time WAL is flushed.
*/
static bool
-logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetRecPtr, char *cur_page)
+logical_read_xlog_page(XLogReaderState *state)
{
+ XLogRecPtr targetPagePtr = state->readPagePtr;
+ int reqLen = state->readLen;
+ char *cur_page = state->readBuf;
XLogRecPtr flushptr;
int count;
WALReadError errinfo;
@@ -837,7 +836,7 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
count = flushptr - targetPagePtr; /* part of the page available */
/* now actually read the data, we know it's there */
- if (!WALRead(state,
+ if (!WALRead(state, WalSndSegmentOpen, wal_segment_close,
cur_page,
targetPagePtr,
XLOG_BLCKSZ,
@@ -1011,9 +1010,8 @@ CreateReplicationSlot(CreateReplicationSlotCmd *cmd)
ctx = CreateInitDecodingContext(cmd->plugin, NIL, need_full_snapshot,
InvalidXLogRecPtr,
- XL_ROUTINE(.page_read = logical_read_xlog_page,
- .segment_open = WalSndSegmentOpen,
- .segment_close = wal_segment_close),
+ logical_read_xlog_page,
+ wal_segment_close,
WalSndPrepareWrite, WalSndWriteData,
WalSndUpdateProgress);
@@ -1171,9 +1169,8 @@ StartLogicalReplication(StartReplicationCmd *cmd)
*/
logical_decoding_ctx =
CreateDecodingContext(cmd->startpoint, cmd->options, false,
- XL_ROUTINE(.page_read = logical_read_xlog_page,
- .segment_open = WalSndSegmentOpen,
- .segment_close = wal_segment_close),
+ logical_read_xlog_page,
+ wal_segment_close,
WalSndPrepareWrite, WalSndWriteData,
WalSndUpdateProgress);
xlogreader = logical_decoding_ctx->reader;
@@ -2749,7 +2746,7 @@ XLogSendPhysical(void)
enlargeStringInfo(&output_message, nbytes);
retry:
- if (!WALRead(xlogreader,
+ if (!WALRead(xlogreader, WalSndSegmentOpen, wal_segment_close,
&output_message.data[output_message.len],
startptr,
nbytes,
@@ -2847,7 +2844,12 @@ XLogSendLogical(void)
*/
WalSndCaughtUp = false;
- record = XLogReadRecord(logical_decoding_ctx->reader, &errm);
+ while (XLogReadRecord(logical_decoding_ctx->reader, &record, &errm) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!logical_decoding_ctx->page_read(logical_decoding_ctx->reader))
+ break;
+ }
/* xlog record was invalid */
if (errm != NULL)
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index cf119848b0..712c85281c 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -41,15 +41,9 @@ static int xlogreadfd = -1;
static XLogSegNo xlogreadsegno = -1;
static char xlogfpath[MAXPGPATH];
-typedef struct XLogPageReadPrivate
-{
- const char *restoreCommand;
- int tliIndex;
-} XLogPageReadPrivate;
-
-static bool SimpleXLogPageRead(XLogReaderState *xlogreader,
- XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
+static bool SimpleXLogPageRead(XLogReaderState *xlogreader,
+ const char *datadir, int *tliIndex,
+ const char *restoreCommand);
/*
* Read WAL from the datadir/pg_wal, starting from 'startpoint' on timeline
@@ -66,20 +60,22 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
XLogRecord *record;
XLogReaderState *xlogreader;
char *errormsg;
- XLogPageReadPrivate private;
- private.tliIndex = tliIndex;
- private.restoreCommand = restoreCommand;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir,
- XL_ROUTINE(.page_read = &SimpleXLogPageRead),
- &private);
+ xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+
if (xlogreader == NULL)
pg_fatal("out of memory");
XLogBeginRead(xlogreader, startpoint);
do
{
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!SimpleXLogPageRead(xlogreader, datadir,
+ &tliIndex, restoreCommand))
+ break;
+ }
if (record == NULL)
{
@@ -123,19 +119,19 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex,
XLogRecord *record;
XLogReaderState *xlogreader;
char *errormsg;
- XLogPageReadPrivate private;
XLogRecPtr endptr;
- private.tliIndex = tliIndex;
- private.restoreCommand = restoreCommand;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir,
- XL_ROUTINE(.page_read = &SimpleXLogPageRead),
- &private);
+ xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
XLogBeginRead(xlogreader, ptr);
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!SimpleXLogPageRead(xlogreader, datadir, &tliIndex, restoreCommand))
+ break;
+ }
if (record == NULL)
{
if (errormsg)
@@ -170,7 +166,6 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
XLogRecPtr searchptr;
XLogReaderState *xlogreader;
char *errormsg;
- XLogPageReadPrivate private;
/*
* The given fork pointer points to the end of the last common record,
@@ -186,11 +181,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
forkptr += SizeOfXLogShortPHD;
}
- private.tliIndex = tliIndex;
- private.restoreCommand = restoreCommand;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir,
- XL_ROUTINE(.page_read = &SimpleXLogPageRead),
- &private);
+ xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
@@ -200,7 +191,13 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
uint8 info;
XLogBeginRead(xlogreader, searchptr);
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!SimpleXLogPageRead(xlogreader, datadir,
+ &tliIndex, restoreCommand))
+ break;
+ }
if (record == NULL)
{
@@ -247,10 +244,11 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
/* XLogReader callback function, to read a WAL page */
static bool
-SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *readBuf)
+SimpleXLogPageRead(XLogReaderState *xlogreader, const char *datadir,
+ int *tliIndex, const char *restoreCommand)
{
- XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
+ XLogRecPtr targetPagePtr = xlogreader->readPagePtr;
+ char *readBuf = xlogreader->readBuf;
uint32 targetPageOff;
XLogRecPtr targetSegEnd;
XLogSegNo targetSegNo;
@@ -283,14 +281,14 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
* be done both forward and backward, consider also switching timeline
* accordingly.
*/
- while (private->tliIndex < targetNentries - 1 &&
- targetHistory[private->tliIndex].end < targetSegEnd)
- private->tliIndex++;
- while (private->tliIndex > 0 &&
- targetHistory[private->tliIndex].begin >= targetSegEnd)
- private->tliIndex--;
+ while (*tliIndex < targetNentries - 1 &&
+ targetHistory[*tliIndex].end < targetSegEnd)
+ (*tliIndex)++;
+ while (*tliIndex > 0 &&
+ targetHistory[*tliIndex].begin >= targetSegEnd)
+ (*tliIndex)--;
- XLogFileName(xlogfname, targetHistory[private->tliIndex].tli,
+ XLogFileName(xlogfname, targetHistory[*tliIndex].tli,
xlogreadsegno, WalSegSz);
snprintf(xlogfpath, MAXPGPATH, "%s/" XLOGDIR "/%s",
@@ -303,7 +301,7 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
/*
* If we have no restore_command to execute, then exit.
*/
- if (private->restoreCommand == NULL)
+ if (restoreCommand == NULL)
{
pg_log_error("could not open file \"%s\": %m", xlogfpath);
xlogreader->readLen = -1;
@@ -317,7 +315,7 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
xlogreadfd = RestoreArchivedFile(xlogreader->segcxt.ws_dir,
xlogfname,
WalSegSz,
- private->restoreCommand);
+ restoreCommand);
if (xlogreadfd < 0)
{
@@ -359,7 +357,7 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
Assert(targetSegNo == xlogreadsegno);
- xlogreader->seg.ws_tli = targetHistory[private->tliIndex].tli;
+ xlogreader->seg.ws_tli = targetHistory[*tliIndex].tli;
xlogreader->readLen = XLOG_BLCKSZ;
return true;
}
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index 0f0e0723a3..f514de6b13 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -330,12 +330,17 @@ WALDumpCloseSegment(XLogReaderState *state)
state->seg.ws_file = -1;
}
-/* pg_waldump's XLogReaderRoutine->page_read callback */
+/*
+ * pg_waldump's WAL page rader, also used as page_read callback for
+ * XLogFindNextRecord
+ */
static bool
-WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetPtr, char *readBuff)
+WALDumpReadPage(XLogReaderState *state, void *priv)
{
- XLogDumpPrivate *private = state->private_data;
+ XLogRecPtr targetPagePtr = state->readPagePtr;
+ int reqLen = state->readLen;
+ char *readBuff = state->readBuf;
+ XLogDumpPrivate *private = (XLogDumpPrivate *) priv;
int count = XLOG_BLCKSZ;
WALReadError errinfo;
@@ -353,8 +358,8 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
}
}
- if (!WALRead(state, readBuff, targetPagePtr, count, private->timeline,
- &errinfo))
+ if (!WALRead(state, WALDumpOpenSegment, WALDumpCloseSegment,
+ readBuff, targetPagePtr, count, private->timeline, &errinfo))
{
WALOpenSegment *seg = &errinfo.wre_seg;
char fname[MAXPGPATH];
@@ -374,6 +379,7 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
(Size) errinfo.wre_req);
}
+ Assert(count >= state->readLen);
state->readLen = count;
return true;
}
@@ -1035,16 +1041,14 @@ main(int argc, char **argv)
/* we have everything we need, start reading */
xlogreader_state =
- XLogReaderAllocate(WalSegSz, waldir,
- XL_ROUTINE(.page_read = WALDumpReadPage,
- .segment_open = WALDumpOpenSegment,
- .segment_close = WALDumpCloseSegment),
- &private);
+ XLogReaderAllocate(WalSegSz, waldir, WALDumpCloseSegment);
+
if (!xlogreader_state)
fatal_error("out of memory");
/* first find a valid recptr to start from */
- first_record = XLogFindNextRecord(xlogreader_state, private.startptr);
+ first_record = XLogFindNextRecord(xlogreader_state, private.startptr,
+ &WALDumpReadPage, (void*) &private);
if (first_record == InvalidXLogRecPtr)
fatal_error("could not find a valid record after %X/%X",
@@ -1067,7 +1071,13 @@ main(int argc, char **argv)
for (;;)
{
/* try to read the next record */
- record = XLogReadRecord(xlogreader_state, &errormsg);
+ while (XLogReadRecord(xlogreader_state, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!WALDumpReadPage(xlogreader_state, (void *) &private))
+ break;
+ }
+
if (!record)
{
if (!config.follow || private.endptr_reached)
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index 23d9e70a59..4e9268b553 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -57,64 +57,15 @@ typedef struct WALSegmentContext
typedef struct XLogReaderState XLogReaderState;
-/* Function type definition for the read_page callback */
-typedef bool (*XLogPageReadCB) (XLogReaderState *xlogreader,
- XLogRecPtr targetPagePtr,
- int reqLen,
- XLogRecPtr targetRecPtr,
- char *readBuf);
+/* Function type definition for the segment cleanup callback */
+typedef void (*WALSegmentCleanupCB) (XLogReaderState *xlogreader);
+
+/* Function type definition for the open/close callbacks for WALRead() */
typedef void (*WALSegmentOpenCB) (XLogReaderState *xlogreader,
XLogSegNo nextSegNo,
TimeLineID *tli_p);
typedef void (*WALSegmentCloseCB) (XLogReaderState *xlogreader);
-typedef struct XLogReaderRoutine
-{
- /*
- * Data input callback
- *
- * This callback shall read at least reqLen valid bytes of the xlog page
- * starting at targetPagePtr, and store them in readBuf. The callback
- * shall return the number of bytes read (never more than XLOG_BLCKSZ), or
- * -1 on failure. The callback shall sleep, if necessary, to wait for the
- * requested bytes to become available. The callback will not be invoked
- * again for the same page unless more than the returned number of bytes
- * are needed.
- *
- * targetRecPtr is the position of the WAL record we're reading. Usually
- * it is equal to targetPagePtr + reqLen, but sometimes xlogreader needs
- * to read and verify the page or segment header, before it reads the
- * actual WAL record it's interested in. In that case, targetRecPtr can
- * be used to determine which timeline to read the page from.
- *
- * The callback shall set ->seg.ws_tli to the TLI of the file the page was
- * read from.
- */
- XLogPageReadCB page_read;
-
- /*
- * Callback to open the specified WAL segment for reading. ->seg.ws_file
- * shall be set to the file descriptor of the opened segment. In case of
- * failure, an error shall be raised by the callback and it shall not
- * return.
- *
- * "nextSegNo" is the number of the segment to be opened.
- *
- * "tli_p" is an input/output argument. WALRead() uses it to pass the
- * timeline in which the new segment should be found, but the callback can
- * use it to return the TLI that it actually opened.
- */
- WALSegmentOpenCB segment_open;
-
- /*
- * WAL segment close callback. ->seg.ws_file shall be set to a negative
- * number.
- */
- WALSegmentCloseCB segment_close;
-} XLogReaderRoutine;
-
-#define XL_ROUTINE(...) &(XLogReaderRoutine){__VA_ARGS__}
-
typedef struct
{
/* Is this block ref in use? */
@@ -144,12 +95,35 @@ typedef struct
uint16 data_bufsz;
} DecodedBkpBlock;
+/* Return code from XLogReadRecord */
+typedef enum XLogReadRecordResult
+{
+ XLREAD_SUCCESS, /* record is successfully read */
+ XLREAD_NEED_DATA, /* need more data. see XLogReadRecord. */
+ XLREAD_FAIL /* failed during reading a record */
+} XLogReadRecordResult;
+
+/*
+ * internal state of XLogReadRecord
+ *
+ * XLogReadState runs a state machine while reading a record. Theses states
+ * are not seen outside the function. Each state may repeat several times
+ * exiting requesting caller for new data. See the comment of XLogReadRecrod
+ * for details.
+ */
+typedef enum XLogReadRecordState {
+ XLREAD_NEXT_RECORD,
+ XLREAD_TOT_LEN,
+ XLREAD_FIRST_FRAGMENT,
+ XLREAD_CONTINUATION
+} XLogReadRecordState;
+
struct XLogReaderState
{
/*
* Operational callbacks
*/
- XLogReaderRoutine routine;
+ WALSegmentCleanupCB cleanup_cb;
/* ----------------------------------------
* Public parameters
@@ -162,18 +136,14 @@ struct XLogReaderState
*/
uint64 system_identifier;
- /*
- * Opaque data for callbacks to use. Not used by XLogReader.
- */
- void *private_data;
-
/*
* Start and end point of last record read. EndRecPtr is also used as the
* position to read next. Calling XLogBeginRead() sets EndRecPtr to the
* starting position and ReadRecPtr to invalid.
*/
- XLogRecPtr ReadRecPtr; /* start of last record read */
+ XLogRecPtr ReadRecPtr; /* start of last record read or being read */
XLogRecPtr EndRecPtr; /* end+1 of last record read */
+ XLogRecPtr PrevRecPtr; /* start of previous record read */
/* ----------------------------------------
* Communication with page reader
@@ -187,7 +157,9 @@ struct XLogReaderState
* the request, or -1 on error */
TimeLineID readPageTLI; /* TLI for data currently in readBuf */
char *readBuf; /* buffer to store data */
- bool page_verified; /* is the page on the buffer verified? */
+ bool page_verified; /* is the page header on the buffer verified? */
+ bool record_verified;/* is the current record header verified? */
+
/* ----------------------------------------
@@ -229,8 +201,6 @@ struct XLogReaderState
XLogRecPtr latestPagePtr;
TimeLineID latestPageTLI;
- /* beginning of the WAL record being read. */
- XLogRecPtr currRecPtr;
/* timeline to read it from, 0 if a lookup is required */
TimeLineID currTLI;
@@ -257,6 +227,15 @@ struct XLogReaderState
char *readRecordBuf;
uint32 readRecordBufSize;
+ /*
+ * XLogReadRecord() state
+ */
+ XLogReadRecordState readRecordState;/* state machine state */
+ int recordGotLen; /* amount of current record that has
+ * already been read */
+ int recordRemainLen; /* length of current record that remains */
+ XLogRecPtr recordContRecPtr; /* where the current record continues */
+
/* Buffer to hold error message */
char *errormsg_buf;
};
@@ -264,9 +243,7 @@ struct XLogReaderState
/* Get a new XLogReader */
extern XLogReaderState *XLogReaderAllocate(int wal_segment_size,
const char *waldir,
- XLogReaderRoutine *routine,
- void *private_data);
-extern XLogReaderRoutine *LocalXLogReaderRoutine(void);
+ WALSegmentCleanupCB cleanup_cb);
/* Free an XLogReader */
extern void XLogReaderFree(XLogReaderState *state);
@@ -274,12 +251,17 @@ extern void XLogReaderFree(XLogReaderState *state);
/* Position the XLogReader to given record */
extern void XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr);
#ifdef FRONTEND
-extern XLogRecPtr XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr);
+/* Function type definition for the read_page callback */
+typedef bool (*XLogFindNextRecordCB) (XLogReaderState *xlogreader,
+ void *private);
+extern XLogRecPtr XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
+ XLogFindNextRecordCB read_page, void *private);
#endif /* FRONTEND */
/* Read the next XLog record. Returns NULL on end-of-WAL or failure */
-extern struct XLogRecord *XLogReadRecord(XLogReaderState *state,
- char **errormsg);
+extern XLogReadRecordResult XLogReadRecord(XLogReaderState *state,
+ XLogRecord **record,
+ char **errormsg);
/* Validate a page */
extern bool XLogReaderValidatePageHeader(XLogReaderState *state,
@@ -299,6 +281,7 @@ typedef struct WALReadError
} WALReadError;
extern bool WALRead(XLogReaderState *state,
+ WALSegmentOpenCB segopenfn, WALSegmentCloseCB sgclosefn,
char *buf, XLogRecPtr startptr, Size count,
TimeLineID tli, WALReadError *errinfo);
diff --git a/src/include/access/xlogutils.h b/src/include/access/xlogutils.h
index 364a21c4ea..397fb27fc2 100644
--- a/src/include/access/xlogutils.h
+++ b/src/include/access/xlogutils.h
@@ -47,9 +47,7 @@ extern Buffer XLogReadBufferExtended(RelFileNode rnode, ForkNumber forknum,
extern Relation CreateFakeRelcacheEntry(RelFileNode rnode);
extern void FreeFakeRelcacheEntry(Relation fakerel);
-extern bool read_local_xlog_page(XLogReaderState *state,
- XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetRecPtr, char *cur_page);
+extern bool read_local_xlog_page(XLogReaderState *state);
extern void wal_segment_open(XLogReaderState *state,
XLogSegNo nextSegNo,
TimeLineID *tli_p);
diff --git a/src/include/pg_config_manual.h b/src/include/pg_config_manual.h
index f10ad0acd6..94749609b4 100644
--- a/src/include/pg_config_manual.h
+++ b/src/include/pg_config_manual.h
@@ -377,7 +377,7 @@
* Enable debugging print statements for WAL-related operations; see
* also the wal_debug GUC var.
*/
-/* #define WAL_DEBUG */
+#define WAL_DEBUG
/*
* Enable tracing of resource consumption during sort operations;
diff --git a/src/include/replication/logical.h b/src/include/replication/logical.h
index c253403372..b936f31393 100644
--- a/src/include/replication/logical.h
+++ b/src/include/replication/logical.h
@@ -29,6 +29,10 @@ typedef void (*LogicalOutputPluginWriterUpdateProgress) (struct LogicalDecodingC
TransactionId xid
);
+typedef struct LogicalDecodingContext LogicalDecodingContext;
+
+typedef bool (*LogicalDecodingXLogPageReadCB)(XLogReaderState *ctx);
+
typedef struct LogicalDecodingContext
{
/* memory context this is all allocated in */
@@ -39,6 +43,7 @@ typedef struct LogicalDecodingContext
/* infrastructure pieces for decoding */
XLogReaderState *reader;
+ LogicalDecodingXLogPageReadCB page_read;
struct ReorderBuffer *reorder;
struct SnapBuild *snapshot_builder;
@@ -105,14 +110,16 @@ extern LogicalDecodingContext *CreateInitDecodingContext(const char *plugin,
List *output_plugin_options,
bool need_full_snapshot,
XLogRecPtr restart_lsn,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress);
extern LogicalDecodingContext *CreateDecodingContext(XLogRecPtr start_lsn,
List *output_plugin_options,
bool fast_forward,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress);
--
2.27.0
----Next_Part(Thu_Mar__4_11_28_53_2021_014)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
filename="v17-0003-Remove-globals-readOff-readLen-and-readSegNo.patch"
^ permalink raw reply [nested|flat] 70+ messages in thread
* [PATCH v15 2/4] Move page-reader out of XLogReadRecord
@ 2019-09-10 03:58 Kyotaro Horiguchi <[email protected]>
0 siblings, 0 replies; 70+ messages in thread
From: Kyotaro Horiguchi @ 2019-09-10 03:58 UTC (permalink / raw)
This is the second step of removing callbacks from WAL record reader.
Since it is essential to take in additional data while reading a
record, the function have to ask caller for new data while keeping
working state. Thus the function is turned into a state machine.
---
src/backend/access/transam/twophase.c | 15 +-
src/backend/access/transam/xlog.c | 58 +-
src/backend/access/transam/xlogreader.c | 700 +++++++++++-------
src/backend/access/transam/xlogutils.c | 17 +-
src/backend/replication/logical/logical.c | 26 +-
.../replication/logical/logicalfuncs.c | 13 +-
src/backend/replication/slotfuncs.c | 18 +-
src/backend/replication/walsender.c | 32 +-
src/bin/pg_rewind/parsexlog.c | 84 +--
src/bin/pg_waldump/pg_waldump.c | 36 +-
src/include/access/xlogreader.h | 121 ++-
src/include/access/xlogutils.h | 4 +-
src/include/replication/logical.h | 11 +-
13 files changed, 648 insertions(+), 487 deletions(-)
diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index 9b2e59bf0e..b0d60a0d0f 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -1330,11 +1330,8 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
XLogReaderState *xlogreader;
char *errormsg;
- xlogreader = XLogReaderAllocate(wal_segment_size, NULL,
- XL_ROUTINE(.page_read = &read_local_xlog_page,
- .segment_open = &wal_segment_open,
- .segment_close = &wal_segment_close),
- NULL);
+ xlogreader = XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+
if (!xlogreader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
@@ -1342,7 +1339,13 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
errdetail("Failed while allocating a WAL reading processor.")));
XLogBeginRead(xlogreader, lsn);
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!read_local_xlog_page(xlogreader))
+ break;
+ }
+
if (record == NULL)
ereport(ERROR,
(errcode_for_file_access(),
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index e570e56a24..f9b0108602 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -831,13 +831,6 @@ static XLogSource currentSource = XLOG_FROM_ANY;
static bool lastSourceFailed = false;
static bool pendingWalRcvRestart = false;
-typedef struct XLogPageReadPrivate
-{
- int emode;
- bool fetching_ckpt; /* are we fetching a checkpoint record? */
- bool randAccess;
-} XLogPageReadPrivate;
-
/*
* These variables track when we last obtained some WAL data to process,
* and where we got it from. (XLogReceiptSource is initially the same as
@@ -912,8 +905,8 @@ static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath,
static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli,
XLogSource source, bool notfoundOk);
static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, XLogSource source);
-static bool XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
+static bool XLogPageRead(XLogReaderState *xlogreader,
+ bool fetching_ckpt, int emode, bool randAccess);
static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
bool fetching_ckpt, XLogRecPtr tliRecPtr);
static int emode_for_corrupt_record(int emode, XLogRecPtr RecPtr);
@@ -1225,8 +1218,7 @@ XLogInsertRecord(XLogRecData *rdata,
appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len);
if (!debug_reader)
- debug_reader = XLogReaderAllocate(wal_segment_size, NULL,
- XL_ROUTINE(), NULL);
+ debug_reader = XLogReaderAllocate(wal_segment_size, NULL NULL);
if (!debug_reader)
{
@@ -4326,15 +4318,10 @@ CleanupBackupHistory(void)
* record is available.
*/
static XLogRecord *
-ReadRecord(XLogReaderState *xlogreader, int emode,
- bool fetching_ckpt)
+ReadRecord(XLogReaderState *xlogreader, int emode, bool fetching_ckpt)
{
XLogRecord *record;
- XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
-
- private->fetching_ckpt = fetching_ckpt;
- private->emode = emode;
- private->randAccess = (xlogreader->ReadRecPtr == InvalidXLogRecPtr);
+ bool randAccess = (xlogreader->ReadRecPtr == InvalidXLogRecPtr);
/* This is the first attempt to read this page. */
lastSourceFailed = false;
@@ -4342,8 +4329,16 @@ ReadRecord(XLogReaderState *xlogreader, int emode,
for (;;)
{
char *errormsg;
+ XLogReadRecordResult result;
+
+ while ((result = XLogReadRecord(xlogreader, &record, &errormsg))
+ == XLREAD_NEED_DATA)
+ {
+ if (!XLogPageRead(xlogreader, fetching_ckpt, emode, randAccess))
+ break;
+
+ }
- record = XLogReadRecord(xlogreader, &errormsg);
ReadRecPtr = xlogreader->ReadRecPtr;
EndRecPtr = xlogreader->EndRecPtr;
if (record == NULL)
@@ -6320,7 +6315,6 @@ StartupXLOG(void)
bool backupFromStandby = false;
DBState dbstate_at_startup;
XLogReaderState *xlogreader;
- XLogPageReadPrivate private;
bool fast_promoted = false;
struct stat st;
@@ -6480,13 +6474,9 @@ StartupXLOG(void)
OwnLatch(&XLogCtl->recoveryWakeupLatch);
/* Set up XLOG reader facility */
- MemSet(&private, 0, sizeof(XLogPageReadPrivate));
xlogreader =
- XLogReaderAllocate(wal_segment_size, NULL,
- XL_ROUTINE(.page_read = &XLogPageRead,
- .segment_open = NULL,
- .segment_close = wal_segment_close),
- &private);
+ XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+
if (!xlogreader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
@@ -11860,12 +11850,13 @@ CancelBackup(void)
* sleep and retry.
*/
static bool
-XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetRecPtr, char *readBuf)
+XLogPageRead(XLogReaderState *xlogreader,
+ bool fetching_ckpt, int emode, bool randAccess)
{
- XLogPageReadPrivate *private =
- (XLogPageReadPrivate *) xlogreader->private_data;
- int emode = private->emode;
+ char *readBuf = xlogreader->readBuf;
+ XLogRecPtr targetPagePtr = xlogreader->readPagePtr;
+ int reqLen = xlogreader->readLen;
+ XLogRecPtr targetRecPtr = xlogreader->ReadRecPtr;
uint32 targetPageOff;
XLogSegNo targetSegNo PG_USED_FOR_ASSERTS_ONLY;
int r;
@@ -11908,8 +11899,8 @@ retry:
flushedUpto < targetPagePtr + reqLen))
{
if (!WaitForWALToBecomeAvailable(targetPagePtr + reqLen,
- private->randAccess,
- private->fetching_ckpt,
+ randAccess,
+ fetching_ckpt,
targetRecPtr))
{
if (readFile >= 0)
@@ -12014,6 +12005,7 @@ retry:
goto next_record_is_invalid;
}
+ Assert(xlogreader->readPagePtr == targetPagePtr);
xlogreader->readLen = readLen;
return true;
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index 3d599325ee..9281b57379 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -40,7 +40,7 @@ static bool XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr,
int reqLen, bool header_inclusive);
static void XLogReaderInvalReadState(XLogReaderState *state);
static bool ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
- XLogRecPtr PrevRecPtr, XLogRecord *record, bool randAccess);
+ XLogRecPtr PrevRecPtr, XLogRecord *record);
static bool ValidXLogRecord(XLogReaderState *state, XLogRecord *record,
XLogRecPtr recptr);
static void ResetDecoder(XLogReaderState *state);
@@ -73,7 +73,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...)
*/
XLogReaderState *
XLogReaderAllocate(int wal_segment_size, const char *waldir,
- XLogReaderRoutine *routine, void *private_data)
+ WALSegmentCleanupCB cleanup_cb)
{
XLogReaderState *state;
@@ -84,7 +84,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
return NULL;
/* initialize caller-provided support functions */
- state->routine = *routine;
+ state->cleanup_cb = cleanup_cb;
state->max_block_id = -1;
@@ -107,8 +107,6 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
WALOpenSegmentInit(&state->seg, &state->segcxt, wal_segment_size,
waldir);
- /* system_identifier initialized to zeroes above */
- state->private_data = private_data;
/* ReadRecPtr, EndRecPtr and readLen initialized to zeroes above */
state->errormsg_buf = palloc_extended(MAX_ERRORMSG_LEN + 1,
MCXT_ALLOC_NO_OOM);
@@ -140,8 +138,8 @@ XLogReaderFree(XLogReaderState *state)
{
int block_id;
- if (state->seg.ws_file != -1)
- state->routine.segment_close(state);
+ if (state->seg.ws_file >= 0)
+ state->cleanup_cb(state);
for (block_id = 0; block_id <= XLR_MAX_BLOCK_ID; block_id++)
{
@@ -246,6 +244,7 @@ XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
/* Begin at the passed-in record pointer. */
state->EndRecPtr = RecPtr;
state->ReadRecPtr = InvalidXLogRecPtr;
+ state->readRecordState = XLREAD_NEXT_RECORD;
}
/*
@@ -254,317 +253,452 @@ XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
* XLogBeginRead() or XLogFindNextRecord() must be called before the first call
* to XLogReadRecord().
*
- * If the page_read callback fails to read the requested data, NULL is
- * returned. The callback is expected to have reported the error; errormsg
- * is set to NULL.
+ * This function may return XLREAD_NEED_DATA several times before returning a
+ * result record. The caller shall read in some new data then call this
+ * function again with the same parameters.
*
- * If the reading fails for some other reason, NULL is also returned, and
- * *errormsg is set to a string with details of the failure.
+ * When a record is successfully read, returns XLREAD_SUCCESS with result
+ * record being stored in *record. Otherwise *record is NULL.
*
- * The returned pointer (or *errormsg) points to an internal buffer that's
- * valid until the next call to XLogReadRecord.
+ * Returns XLREAD_NEED_DATA if more data is needed to finish reading the
+ * current record. In that case, state->readPagePtr and state->readLen inform
+ * the desired position and minimum length of data needed. The caller shall
+ * read in the requested data and set state->readBuf to point to a buffer
+ * containing it. The caller must also set state->readPageTLI and
+ * state->readLen to indicate the timeline that it was read from, and the
+ * length of data that is now available (which must be >= given readLen),
+ * respectively.
+ *
+ * If invalid data is encountered, returns XLREAD_FAIL with *record being set to
+ * NULL. *errormsg is set to a string with details of the failure.
+ * The returned pointer (or *errormsg) points to an internal buffer that's valid
+ * until the next call to XLogReadRecord.
+ *
+ *
+ * This function runs a state machine consists of the following states.
+ *
+ * XLREAD_NEXT_RECORD :
+ * The initial state, if called with valid RecPtr, try to read a record at
+ * that position. If invalid RecPtr is given try to read a record just after
+ * the last one previously read.
+ * This state ens after setting ReadRecPtr. Then goes to XLREAD_TOT_LEN.
+ *
+ * XLREAD_TOT_LEN:
+ * Examining record header. Ends after reading record total
+ * length. recordRemainLen and recordGotLen are initialized.
+ *
+ * XLREAD_FIRST_FRAGMENT:
+ * Reading the first fragment. Ends with finishing reading a single
+ * record. Goes to XLREAD_NEXT_RECORD if that's all or
+ * XLREAD_CONTINUATION if we have continuation.
+
+ * XLREAD_CONTINUATION:
+ * Reading continuation of record. Ends with finishing the whole record then
+ * goes to XLREAD_NEXT_RECORD. During this state, recordRemainLen indicates
+ * how much is left and readRecordBuf holds the partially assert
+ * record.recordContRecPtr points to the beginning of the next page where to
+ * continue.
+ *
+ * If wrong data found in any state, the state machine stays at the current
+ * state. This behavior allows to continue reading a reacord switching among
+ * different souces, while streaming replication.
*/
-XLogRecord *
-XLogReadRecord(XLogReaderState *state, char **errormsg)
+XLogReadRecordResult
+XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
{
- XLogRecPtr RecPtr;
- XLogRecord *record;
- XLogRecPtr targetPagePtr;
- bool randAccess;
- uint32 len,
- total_len;
- uint32 targetRecOff;
- uint32 pageHeaderSize;
- bool gotheader;
+ XLogRecord *prec;
- /*
- * randAccess indicates whether to verify the previous-record pointer of
- * the record we're reading. We only do this if we're reading
- * sequentially, which is what we initially assume.
- */
- randAccess = false;
+ *record = NULL;
/* reset error state */
*errormsg = NULL;
state->errormsg_buf[0] = '\0';
- ResetDecoder(state);
-
- RecPtr = state->EndRecPtr;
-
- if (state->ReadRecPtr != InvalidXLogRecPtr)
- {
- /* read the record after the one we just read */
-
- /*
- * EndRecPtr is pointing to end+1 of the previous WAL record. If
- * we're at a page boundary, no more records can fit on the current
- * page. We must skip over the page header, but we can't do that until
- * we've read in the page, since the header size is variable.
- */
- }
- else
- {
- /*
- * Caller supplied a position to start at.
- *
- * In this case, EndRecPtr should already be pointing to a valid
- * record starting position.
- */
- Assert(XRecOffIsValid(RecPtr));
- randAccess = true;
- }
-
- state->currRecPtr = RecPtr;
-
- targetPagePtr = RecPtr - (RecPtr % XLOG_BLCKSZ);
- targetRecOff = RecPtr % XLOG_BLCKSZ;
-
- /*
- * Read the page containing the record into state->readBuf. Request enough
- * byte to cover the whole record header, or at least the part of it that
- * fits on the same page.
- */
- while (XLogNeedData(state, targetPagePtr,
- Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
- targetRecOff != 0))
- {
- if (!state->routine.page_read(state, state->readPagePtr, state->readLen,
- RecPtr, state->readBuf))
- break;
- }
-
- if (!state->page_verified)
- goto err;
-
- /*
- * We have at least the page header, so we can examine it now.
- */
- pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
- if (targetRecOff == 0)
- {
- /*
- * At page start, so skip over page header.
- */
- RecPtr += pageHeaderSize;
- targetRecOff = pageHeaderSize;
- }
- else if (targetRecOff < pageHeaderSize)
- {
- report_invalid_record(state, "invalid record offset at %X/%X",
- (uint32) (RecPtr >> 32), (uint32) RecPtr);
- goto err;
- }
-
- if ((((XLogPageHeader) state->readBuf)->xlp_info & XLP_FIRST_IS_CONTRECORD) &&
- targetRecOff == pageHeaderSize)
- {
- report_invalid_record(state, "contrecord is requested by %X/%X",
- (uint32) (RecPtr >> 32), (uint32) RecPtr);
- goto err;
- }
-
- /* XLogNeedData has verified the page header */
- Assert(pageHeaderSize <= state->readLen);
-
- /*
- * Read the record length.
- *
- * NB: Even though we use an XLogRecord pointer here, the whole record
- * header might not fit on this page. xl_tot_len is the first field of the
- * struct, so it must be on this page (the records are MAXALIGNed), but we
- * cannot access any other fields until we've verified that we got the
- * whole header.
- */
- record = (XLogRecord *) (state->readBuf + RecPtr % XLOG_BLCKSZ);
- total_len = record->xl_tot_len;
-
- /*
- * If the whole record header is on this page, validate it immediately.
- * Otherwise do just a basic sanity check on xl_tot_len, and validate the
- * rest of the header after reading it from the next page. The xl_tot_len
- * check is necessary here to ensure that we enter the "Need to reassemble
- * record" code path below; otherwise we might fail to apply
- * ValidXLogRecordHeader at all.
- */
- if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
- {
- if (!ValidXLogRecordHeader(state, RecPtr, state->ReadRecPtr, record,
- randAccess))
- goto err;
- gotheader = true;
- }
- else
- {
- /* XXX: more validation should be done here */
- if (total_len < SizeOfXLogRecord)
- {
- report_invalid_record(state,
- "invalid record length at %X/%X: wanted %u, got %u",
- (uint32) (RecPtr >> 32), (uint32) RecPtr,
- (uint32) SizeOfXLogRecord, total_len);
- goto err;
- }
- gotheader = false;
- }
-
- len = XLOG_BLCKSZ - RecPtr % XLOG_BLCKSZ;
- if (total_len > len)
+ switch (state->readRecordState)
{
- /* Need to reassemble record */
- char *contdata;
- XLogPageHeader pageHeader;
- char *buffer;
- uint32 gotlen;
-
- /*
- * Enlarge readRecordBuf as needed.
- */
- if (total_len > state->readRecordBufSize &&
- !allocate_recordbuf(state, total_len))
- {
- /* We treat this as a "bogus data" condition */
- report_invalid_record(state, "record length %u at %X/%X too long",
- total_len,
- (uint32) (RecPtr >> 32), (uint32) RecPtr);
- goto err;
- }
+ case XLREAD_NEXT_RECORD:
+ ResetDecoder(state);
- /* Copy the first fragment of the record from the first page. */
- memcpy(state->readRecordBuf,
- state->readBuf + RecPtr % XLOG_BLCKSZ, len);
- buffer = state->readRecordBuf + len;
- gotlen = len;
-
- do
- {
- int rest_len = total_len - gotlen;
-
- /* Calculate pointer to beginning of next page */
- targetPagePtr += XLOG_BLCKSZ;
-
- /* Wait for the next page to become available */
- while (XLogNeedData(state, targetPagePtr,
- Min(rest_len, XLOG_BLCKSZ),
- false))
+ if (state->ReadRecPtr != InvalidXLogRecPtr)
{
- if (!state->routine.page_read(state, state->readPagePtr,
- state->readLen,
- state->ReadRecPtr,
- state->readBuf))
- break;
+ /* read the record after the one we just read */
+
+ /*
+ * EndRecPtr is pointing to end+1 of the previous WAL record.
+ * If we're at a page boundary, no more records can fit on the
+ * current page. We must skip over the page header, but we
+ * can't do that until we've read in the page, since the header
+ * size is variable.
+ */
+ state->PrevRecPtr = state->ReadRecPtr;
+ state->ReadRecPtr = state->EndRecPtr;
+ }
+ else
+ {
+ /*
+ * Caller supplied a position to start at.
+ *
+ * In this case, EndRecPtr should already be pointing to a
+ * valid record starting position.
+ */
+ Assert(XRecOffIsValid(state->EndRecPtr));
+ state->ReadRecPtr = state->EndRecPtr;
+
+ /*
+ * We cannot verify the previous-record pointer when we're
+ * seeking to a particular record. Reset PrevRecPtr so that we
+ * won't try doing that.
+ */
+ state->PrevRecPtr = InvalidXLogRecPtr;
+ state->EndRecPtr = InvalidXLogRecPtr; /* to be tidy */
}
+ state->record_verified = false;
+ state->readRecordState = XLREAD_TOT_LEN;
+ /* fall through */
+
+ case XLREAD_TOT_LEN:
+ {
+ uint32 total_len;
+ uint32 pageHeaderSize;
+ XLogRecPtr targetPagePtr;
+ uint32 targetRecOff;
+ XLogPageHeader pageHeader;
+
+ targetPagePtr =
+ state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
+ targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+
+ /*
+ * Check if we have enough data. For the first record in the page,
+ * the requesting length doesn't contain page header.
+ */
+ if (XLogNeedData(state, targetPagePtr,
+ Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+ targetRecOff != 0))
+ return XLREAD_NEED_DATA;
+
+ /* error out if caller supplied bogus page */
if (!state->page_verified)
goto err;
- Assert(SizeOfXLogShortPHD <= state->readLen);
+ /* examine page header now. */
+ pageHeaderSize =
+ XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+ if (targetRecOff == 0)
+ {
+ /* At page start, so skip over page header. */
+ state->ReadRecPtr += pageHeaderSize;
+ targetRecOff = pageHeaderSize;
+ }
+ else if (targetRecOff < pageHeaderSize)
+ {
+ report_invalid_record(state, "invalid record offset at %X/%X",
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr);
+ goto err;
+ }
- /* Check that the continuation on next page looks valid */
pageHeader = (XLogPageHeader) state->readBuf;
- if (!(pageHeader->xlp_info & XLP_FIRST_IS_CONTRECORD))
+ if ((pageHeader->xlp_info & XLP_FIRST_IS_CONTRECORD) &&
+ targetRecOff == pageHeaderSize)
{
- report_invalid_record(state,
- "there is no contrecord flag at %X/%X",
- (uint32) (RecPtr >> 32), (uint32) RecPtr);
+ report_invalid_record(state, "contrecord is requested by %X/%X",
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr);
goto err;
}
- /*
- * Cross-check that xlp_rem_len agrees with how much of the record
- * we expect there to be left.
- */
- if (pageHeader->xlp_rem_len == 0 ||
- total_len != (pageHeader->xlp_rem_len + gotlen))
- {
- report_invalid_record(state,
- "invalid contrecord length %u at %X/%X",
- pageHeader->xlp_rem_len,
- (uint32) (RecPtr >> 32), (uint32) RecPtr);
- goto err;
- }
-
- /* Append the continuation from this page to the buffer */
- pageHeaderSize = XLogPageHeaderSize(pageHeader);
-
+ /* XLogNeedData has verified the page header */
Assert(pageHeaderSize <= state->readLen);
- contdata = (char *) state->readBuf + pageHeaderSize;
- len = XLOG_BLCKSZ - pageHeaderSize;
- if (pageHeader->xlp_rem_len < len)
- len = pageHeader->xlp_rem_len;
+ /*
+ * Read the record length.
+ *
+ * NB: Even though we use an XLogRecord pointer here, the whole
+ * record header might not fit on this page. xl_tot_len is the first
+ * field of the struct, so it must be on this page (the records are
+ * MAXALIGNed), but we cannot access any other fields until we've
+ * verified that we got the whole header.
+ */
+ prec = (XLogRecord *) (state->readBuf +
+ state->ReadRecPtr % XLOG_BLCKSZ);
+ total_len = prec->xl_tot_len;
- Assert (pageHeaderSize + len <= state->readLen);
- memcpy(buffer, (char *) contdata, len);
- buffer += len;
- gotlen += len;
+ /*
+ * If the whole record header is on this page, validate it
+ * immediately. Otherwise do just a basic sanity check on
+ * xl_tot_len, and validate the rest of the header after reading it
+ * from the next page. The xl_tot_len check is necessary here to
+ * ensure that we enter the XLREAD_CONTINUATION state below;
+ * otherwise we might fail to apply ValidXLogRecordHeader at all.
+ */
+ if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
+ {
+ if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
+ state->PrevRecPtr, prec))
+ goto err;
- /* If we just reassembled the record header, validate it. */
- if (!gotheader)
+ state->record_verified = true;
+ }
+ else
{
- record = (XLogRecord *) state->readRecordBuf;
- if (!ValidXLogRecordHeader(state, RecPtr, state->ReadRecPtr,
- record, randAccess))
+ /* XXX: more validation should be done here */
+ if (total_len < SizeOfXLogRecord)
+ {
+ report_invalid_record(state,
+ "invalid record length at %X/%X: wanted %u, got %u",
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr,
+ (uint32) SizeOfXLogRecord, total_len);
goto err;
- gotheader = true;
+ }
}
- } while (gotlen < total_len);
-
- Assert(gotheader);
- record = (XLogRecord *) state->readRecordBuf;
- if (!ValidXLogRecord(state, record, RecPtr))
- goto err;
+ /*
+ * Wait for the rest of the record, or the part of the record that
+ * fit on the first page if crossed a page boundary, to become
+ * available.
+ */
+ state->recordGotLen = 0;
+ state->recordRemainLen = total_len;
+ state->readRecordState = XLREAD_FIRST_FRAGMENT;
+ }
+ /* fall through */
- pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
- state->ReadRecPtr = RecPtr;
- state->EndRecPtr = targetPagePtr + pageHeaderSize
- + MAXALIGN(pageHeader->xlp_rem_len);
- }
- else
- {
- /* Wait for the record data to become available */
- while (XLogNeedData(state, targetPagePtr,
- Min(targetRecOff + total_len, XLOG_BLCKSZ), true))
+ case XLREAD_FIRST_FRAGMENT:
{
- if (!state->routine.page_read(state, state->readPagePtr,
- state->readLen,
- state->ReadRecPtr, state->readBuf))
+ uint32 total_len = state->recordRemainLen;
+ uint32 request_len;
+ uint32 record_len;
+ XLogRecPtr targetPagePtr;
+ uint32 targetRecOff;
+
+ /*
+ * Wait for the rest of the record on the first page to become
+ * available
+ */
+ targetPagePtr =
+ state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
+ targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+
+ request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ);
+ record_len = request_len - targetRecOff;
+
+ /* ReadRecPtr contains page header */
+ Assert (targetRecOff != 0);
+ if (XLogNeedData(state, targetPagePtr, request_len, true))
+ return XLREAD_NEED_DATA;
+
+ /* error out if caller supplied bogus page */
+ if (!state->page_verified)
+ goto err;
+
+ prec = (XLogRecord *) (state->readBuf + targetRecOff);
+
+ /* validate record header if not yet */
+ if (!state->record_verified && record_len >= SizeOfXLogRecord)
+ {
+ if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
+ state->PrevRecPtr, prec))
+ goto err;
+
+ state->record_verified = true;
+ }
+
+
+ if (total_len == record_len)
+ {
+ /* Record does not cross a page boundary */
+ Assert(state->record_verified);
+
+ if (!ValidXLogRecord(state, prec, state->ReadRecPtr))
+ goto err;
+
+ state->record_verified = true; /* to be tidy */
+
+ /* We already checked the header earlier */
+ state->EndRecPtr = state->ReadRecPtr + MAXALIGN(record_len);
+
+ *record = prec;
+ state->readRecordState = XLREAD_NEXT_RECORD;
break;
+ }
+
+ /*
+ * The record continues on the next page. Need to reassemble
+ * record
+ */
+ Assert(total_len > record_len);
+
+ /* Enlarge readRecordBuf as needed. */
+ if (total_len > state->readRecordBufSize &&
+ !allocate_recordbuf(state, total_len))
+ {
+ /* We treat this as a "bogus data" condition */
+ report_invalid_record(state,
+ "record length %u at %X/%X too long",
+ total_len,
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr);
+ goto err;
+ }
+
+ /* Copy the first fragment of the record from the first page. */
+ memcpy(state->readRecordBuf, state->readBuf + targetRecOff,
+ record_len);
+ state->recordGotLen += record_len;
+ state->recordRemainLen -= record_len;
+
+ /* Calculate pointer to beginning of next page */
+ state->recordContRecPtr = state->ReadRecPtr + record_len;
+ Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0);
+
+ state->readRecordState = XLREAD_CONTINUATION;
}
+ /* fall through */
+
+ case XLREAD_CONTINUATION:
+ {
+ XLogPageHeader pageHeader;
+ uint32 pageHeaderSize;
+ XLogRecPtr targetPagePtr;
+
+ /* we enter this state only if we haven't read the whole record. */
+ Assert (state->recordRemainLen > 0);
+
+ while(state->recordRemainLen > 0)
+ {
+ char *contdata;
+ uint32 request_len;
+ uint32 record_len;
+
+ /* Wait for the next page to become available */
+ targetPagePtr = state->recordContRecPtr;
+
+ /* this request contains page header */
+ Assert (targetPagePtr != 0);
+ if (XLogNeedData(state, targetPagePtr,
+ Min(state->recordRemainLen, XLOG_BLCKSZ),
+ false))
+ return XLREAD_NEED_DATA;
+
+ if (!state->page_verified)
+ goto err;
+
+ Assert(SizeOfXLogShortPHD <= state->readLen);
- if (!state->page_verified)
- goto err;
+ /* Check that the continuation on next page looks valid */
+ pageHeader = (XLogPageHeader) state->readBuf;
+ if (!(pageHeader->xlp_info & XLP_FIRST_IS_CONTRECORD))
+ {
+ report_invalid_record(
+ state,
+ "there is no contrecord flag at %X/%X reading %X/%X",
+ (uint32) (state->recordContRecPtr >> 32),
+ (uint32) state->recordContRecPtr,
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr);
+ goto err;
+ }
- /* Record does not cross a page boundary */
- if (!ValidXLogRecord(state, record, RecPtr))
- goto err;
+ /*
+ * Cross-check that xlp_rem_len agrees with how much of the
+ * record we expect there to be left.
+ */
+ if (pageHeader->xlp_rem_len == 0 ||
+ pageHeader->xlp_rem_len != state->recordRemainLen)
+ {
+ report_invalid_record(
+ state,
+ "invalid contrecord length %u at %X/%X reading %X/%X, expected %u",
+ pageHeader->xlp_rem_len,
+ (uint32) (state->recordContRecPtr >> 32),
+ (uint32) state->recordContRecPtr,
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr,
+ state->recordRemainLen);
+ goto err;
+ }
- state->EndRecPtr = RecPtr + MAXALIGN(total_len);
+ /* Append the continuation from this page to the buffer */
+ pageHeaderSize = XLogPageHeaderSize(pageHeader);
- state->ReadRecPtr = RecPtr;
+ /*
+ * XLogNeedData should have ensured that the whole page header
+ * was read
+ */
+ Assert(state->readLen >= pageHeaderSize);
+
+ contdata = (char *) state->readBuf + pageHeaderSize;
+ record_len = XLOG_BLCKSZ - pageHeaderSize;
+ if (pageHeader->xlp_rem_len < record_len)
+ record_len = pageHeader->xlp_rem_len;
+
+ request_len = record_len + pageHeaderSize;
+
+ /* XLogNeedData should have ensured all needed data was read */
+ Assert (state->readLen >= request_len);
+
+ memcpy(state->readRecordBuf + state->recordGotLen,
+ (char *) contdata, record_len);
+ state->recordGotLen += record_len;
+ state->recordRemainLen -= record_len;
+
+ /* If we just reassembled the record header, validate it. */
+ if (!state->record_verified)
+ {
+ Assert(state->recordGotLen >= SizeOfXLogRecord);
+ if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
+ state->PrevRecPtr,
+ (XLogRecord *) state->readRecordBuf))
+ goto err;
+
+ state->record_verified = true;
+ }
+
+ /* Calculate pointer to beginning of next page, and continue */
+ state->recordContRecPtr += XLOG_BLCKSZ;
+ }
+
+ /* targetPagePtr is pointing the last-read page here */
+ prec = (XLogRecord *) state->readRecordBuf;
+ if (!ValidXLogRecord(state, prec, state->ReadRecPtr))
+ goto err;
+
+ pageHeaderSize =
+ XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+ state->EndRecPtr = targetPagePtr + pageHeaderSize
+ + MAXALIGN(pageHeader->xlp_rem_len);
+
+ *record = prec;
+ state->readRecordState = XLREAD_NEXT_RECORD;
+ break;
+ }
}
/*
* Special processing if it's an XLOG SWITCH record
*/
- if (record->xl_rmid == RM_XLOG_ID &&
- (record->xl_info & ~XLR_INFO_MASK) == XLOG_SWITCH)
+ if ((*record)->xl_rmid == RM_XLOG_ID &&
+ ((*record)->xl_info & ~XLR_INFO_MASK) == XLOG_SWITCH)
{
/* Pretend it extends to end of segment */
state->EndRecPtr += state->segcxt.ws_segsize - 1;
state->EndRecPtr -= XLogSegmentOffset(state->EndRecPtr, state->segcxt.ws_segsize);
}
- if (DecodeXLogRecord(state, record, errormsg))
- return record;
- else
- return NULL;
+ Assert (!*record || state->readLen >= 0);
+ if (DecodeXLogRecord(state, *record, errormsg))
+ return XLREAD_SUCCESS;
+
+ *record = NULL;
+ return XLREAD_FAIL;
err:
/*
- * Invalidate the read state. We might read from a different source after
+ * Invalidate the read page. We might read from a different source after
* failure.
*/
XLogReaderInvalReadState(state);
@@ -572,7 +706,8 @@ err:
if (state->errormsg_buf[0] != '\0')
*errormsg = state->errormsg_buf;
- return NULL;
+ *record = NULL;
+ return XLREAD_FAIL;
}
/*
@@ -724,11 +859,12 @@ XLogReaderInvalReadState(XLogReaderState *state)
*
* This is just a convenience subroutine to avoid duplicated code in
* XLogReadRecord. It's not intended for use from anywhere else.
+ *
+ * If PrevRecPtr is valid, the xl_prev is is cross-checked with it.
*/
static bool
ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
- XLogRecPtr PrevRecPtr, XLogRecord *record,
- bool randAccess)
+ XLogRecPtr PrevRecPtr, XLogRecord *record)
{
if (record->xl_tot_len < SizeOfXLogRecord)
{
@@ -746,7 +882,7 @@ ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
(uint32) RecPtr);
return false;
}
- if (randAccess)
+ if (PrevRecPtr == InvalidXLogRecPtr)
{
/*
* We can't exactly verify the prev-link, but surely it should be less
@@ -978,11 +1114,14 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
* XLogReadRecord() will read the next valid record.
*/
XLogRecPtr
-XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
+XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
+ XLogFindNextRecordCB read_page, void *private)
{
XLogRecPtr tmpRecPtr;
XLogRecPtr found = InvalidXLogRecPtr;
XLogPageHeader header;
+ XLogRecord *record;
+ XLogReadRecordResult result;
char *errormsg;
Assert(!XLogRecPtrIsInvalid(RecPtr));
@@ -1015,9 +1154,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
while(XLogNeedData(state, targetPagePtr, targetRecOff,
targetRecOff != 0))
{
- if (!state->routine.page_read(state, state->readPagePtr,
- state->readLen,
- state->ReadRecPtr, state->readBuf))
+ if (!read_page(state, private))
break;
}
@@ -1069,8 +1206,16 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
* or we just jumped over the remaining data of a continuation.
*/
XLogBeginRead(state, tmpRecPtr);
- while (XLogReadRecord(state, &errormsg) != NULL)
+ while ((result = XLogReadRecord(state, &record, &errormsg)) !=
+ XLREAD_FAIL)
{
+ if (result == XLREAD_NEED_DATA)
+ {
+ if (!read_page(state, private))
+ goto err;
+ continue;
+ }
+
/* past the record we've found, break out */
if (RecPtr <= state->ReadRecPtr)
{
@@ -1090,9 +1235,9 @@ err:
#endif /* FRONTEND */
/*
- * Helper function to ease writing of XLogRoutine->page_read callbacks.
- * If this function is used, caller must supply a segment_open callback in
- * 'state', as that is used here.
+ * Helper function to ease writing of page_read callback.
+ * If this function is used, caller must supply a segment_open callback and
+ * segment_close callback as that is used here.
*
* Read 'count' bytes into 'buf', starting at location 'startptr', from WAL
* fetched from timeline 'tli'.
@@ -1105,6 +1250,7 @@ err:
*/
bool
WALRead(XLogReaderState *state,
+ WALSegmentOpenCB segopenfn, WALSegmentCloseCB segclosefn,
char *buf, XLogRecPtr startptr, Size count, TimeLineID tli,
WALReadError *errinfo)
{
@@ -1136,10 +1282,10 @@ WALRead(XLogReaderState *state,
XLogSegNo nextSegNo;
if (state->seg.ws_file >= 0)
- state->routine.segment_close(state);
+ segclosefn(state);
XLByteToSeg(recptr, nextSegNo, state->segcxt.ws_segsize);
- state->routine.segment_open(state, nextSegNo, &tli);
+ segopenfn(state, nextSegNo, &tli);
/* This shouldn't happen -- indicates a bug in segment_open */
Assert(state->seg.ws_file >= 0);
diff --git a/src/backend/access/transam/xlogutils.c b/src/backend/access/transam/xlogutils.c
index 1b10b6df20..303333571f 100644
--- a/src/backend/access/transam/xlogutils.c
+++ b/src/backend/access/transam/xlogutils.c
@@ -689,8 +689,7 @@ XLogTruncateRelation(RelFileNode rnode, ForkNumber forkNum,
void
XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wantLength)
{
- const XLogRecPtr lastReadPage = state->seg.ws_segno *
- state->segcxt.ws_segsize + state->readLen;
+ const XLogRecPtr lastReadPage = state->readPagePtr;
Assert(wantPage != InvalidXLogRecPtr && wantPage % XLOG_BLCKSZ == 0);
Assert(wantLength <= XLOG_BLCKSZ);
@@ -705,7 +704,7 @@ XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wa
* current TLI has since become historical.
*/
if (lastReadPage == wantPage &&
- state->readLen != 0 &&
+ state->page_verified &&
lastReadPage + state->readLen >= wantPage + Min(wantLength, XLOG_BLCKSZ - 1))
return;
@@ -792,6 +791,7 @@ wal_segment_open(XLogReaderState *state, XLogSegNo nextSegNo,
char path[MAXPGPATH];
XLogFilePath(path, tli, nextSegNo, state->segcxt.ws_segsize);
+ elog(LOG, "HOGE: %lu, %d => %s", nextSegNo, tli, path);
state->seg.ws_file = BasicOpenFile(path, O_RDONLY | PG_BINARY);
if (state->seg.ws_file >= 0)
return;
@@ -829,9 +829,11 @@ wal_segment_close(XLogReaderState *state)
* loop for now.
*/
bool
-read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *cur_page)
+read_local_xlog_page(XLogReaderState *state)
{
+ XLogRecPtr targetPagePtr = state->readPagePtr;
+ int reqLen = state->readLen;
+ char *cur_page = state->readBuf;
XLogRecPtr read_upto,
loc;
TimeLineID tli;
@@ -944,11 +946,12 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
* as 'count', read the whole page anyway. It's guaranteed to be
* zero-padded up to the page boundary if it's incomplete.
*/
- if (!WALRead(state, cur_page, targetPagePtr, XLOG_BLCKSZ, tli,
- &errinfo))
+ if (!WALRead(state, wal_segment_open, wal_segment_close,
+ cur_page, targetPagePtr, XLOG_BLCKSZ, tli, &errinfo))
WALReadRaiseError(&errinfo);
/* number of valid bytes in the buffer */
+ state->readPagePtr = targetPagePtr;
state->readLen = count;
return true;
}
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index 61902be3b0..a15b0b3355 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -120,7 +120,8 @@ StartupDecodingContext(List *output_plugin_options,
TransactionId xmin_horizon,
bool need_full_snapshot,
bool fast_forward,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress)
@@ -169,11 +170,12 @@ StartupDecodingContext(List *output_plugin_options,
ctx->slot = slot;
- ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, xl_routine, ctx);
+ ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, cleanup_cb);
if (!ctx->reader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
errmsg("out of memory")));
+ ctx->page_read = page_read;
ctx->reorder = ReorderBufferAllocate();
ctx->snapshot_builder =
@@ -231,7 +233,8 @@ CreateInitDecodingContext(char *plugin,
List *output_plugin_options,
bool need_full_snapshot,
XLogRecPtr restart_lsn,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress)
@@ -328,7 +331,7 @@ CreateInitDecodingContext(char *plugin,
ctx = StartupDecodingContext(NIL, restart_lsn, xmin_horizon,
need_full_snapshot, false,
- xl_routine, prepare_write, do_write,
+ page_read, cleanup_cb, prepare_write, do_write,
update_progress);
/* call output plugin initialization callback */
@@ -376,7 +379,8 @@ LogicalDecodingContext *
CreateDecodingContext(XLogRecPtr start_lsn,
List *output_plugin_options,
bool fast_forward,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress)
@@ -429,8 +433,8 @@ CreateDecodingContext(XLogRecPtr start_lsn,
ctx = StartupDecodingContext(output_plugin_options,
start_lsn, InvalidTransactionId, false,
- fast_forward, xl_routine, prepare_write,
- do_write, update_progress);
+ fast_forward, page_read, cleanup_cb,
+ prepare_write, do_write, update_progress);
/* call output plugin initialization callback */
old_context = MemoryContextSwitchTo(ctx->context);
@@ -483,7 +487,13 @@ DecodingContextFindStartpoint(LogicalDecodingContext *ctx)
char *err = NULL;
/* the read_page callback waits for new WAL */
- record = XLogReadRecord(ctx->reader, &err);
+ while (XLogReadRecord(ctx->reader, &record, &err) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!ctx->page_read(ctx->reader))
+ break;
+ }
+
if (err)
elog(ERROR, "%s", err);
if (!record)
diff --git a/src/backend/replication/logical/logicalfuncs.c b/src/backend/replication/logical/logicalfuncs.c
index b99c94e848..fcc81b358e 100644
--- a/src/backend/replication/logical/logicalfuncs.c
+++ b/src/backend/replication/logical/logicalfuncs.c
@@ -233,9 +233,8 @@ pg_logical_slot_get_changes_guts(FunctionCallInfo fcinfo, bool confirm, bool bin
ctx = CreateDecodingContext(InvalidXLogRecPtr,
options,
false,
- XL_ROUTINE(.page_read = read_local_xlog_page,
- .segment_open = wal_segment_open,
- .segment_close = wal_segment_close),
+ read_local_xlog_page,
+ wal_segment_close,
LogicalOutputPrepareWrite,
LogicalOutputWrite, NULL);
@@ -284,7 +283,13 @@ pg_logical_slot_get_changes_guts(FunctionCallInfo fcinfo, bool confirm, bool bin
XLogRecord *record;
char *errm = NULL;
- record = XLogReadRecord(ctx->reader, &errm);
+ while (XLogReadRecord(ctx->reader, &record, &errm) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!ctx->page_read(ctx->reader))
+ break;
+ }
+
if (errm)
elog(ERROR, "%s", errm);
diff --git a/src/backend/replication/slotfuncs.c b/src/backend/replication/slotfuncs.c
index 88033a79b2..8a678074d5 100644
--- a/src/backend/replication/slotfuncs.c
+++ b/src/backend/replication/slotfuncs.c
@@ -152,9 +152,8 @@ create_logical_replication_slot(char *name, char *plugin,
ctx = CreateInitDecodingContext(plugin, NIL,
false, /* just catalogs is OK */
restart_lsn,
- XL_ROUTINE(.page_read = read_local_xlog_page,
- .segment_open = wal_segment_open,
- .segment_close = wal_segment_close),
+ read_local_xlog_page,
+ wal_segment_close,
NULL, NULL, NULL);
/*
@@ -486,9 +485,8 @@ pg_logical_replication_slot_advance(XLogRecPtr moveto)
ctx = CreateDecodingContext(InvalidXLogRecPtr,
NIL,
true, /* fast_forward */
- XL_ROUTINE(.page_read = read_local_xlog_page,
- .segment_open = wal_segment_open,
- .segment_close = wal_segment_close),
+ read_local_xlog_page,
+ wal_segment_close,
NULL, NULL, NULL);
/*
@@ -513,7 +511,13 @@ pg_logical_replication_slot_advance(XLogRecPtr moveto)
* Read records. No changes are generated in fast_forward mode,
* but snapbuilder/slot statuses are updated properly.
*/
- record = XLogReadRecord(ctx->reader, &errm);
+ while (XLogReadRecord(ctx->reader, &record, &errm) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!ctx->page_read(ctx->reader))
+ break;
+ }
+
if (errm)
elog(ERROR, "%s", errm);
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index bf2711bddd..2fd24a9b14 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -580,10 +580,7 @@ StartReplication(StartReplicationCmd *cmd)
/* create xlogreader for physical replication */
xlogreader =
- XLogReaderAllocate(wal_segment_size, NULL,
- XL_ROUTINE(.segment_open = WalSndSegmentOpen,
- .segment_close = wal_segment_close),
- NULL);
+ XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
if (!xlogreader)
ereport(ERROR,
@@ -812,9 +809,11 @@ StartReplication(StartReplicationCmd *cmd)
* set every time WAL is flushed.
*/
static bool
-logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetRecPtr, char *cur_page)
+logical_read_xlog_page(XLogReaderState *state)
{
+ XLogRecPtr targetPagePtr = state->readPagePtr;
+ int reqLen = state->readLen;
+ char *cur_page = state->readBuf;
XLogRecPtr flushptr;
int count;
WALReadError errinfo;
@@ -842,7 +841,7 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
count = flushptr - targetPagePtr; /* part of the page available */
/* now actually read the data, we know it's there */
- if (!WALRead(state,
+ if (!WALRead(state, WalSndSegmentOpen, wal_segment_close,
cur_page,
targetPagePtr,
XLOG_BLCKSZ,
@@ -1014,9 +1013,8 @@ CreateReplicationSlot(CreateReplicationSlotCmd *cmd)
ctx = CreateInitDecodingContext(cmd->plugin, NIL, need_full_snapshot,
InvalidXLogRecPtr,
- XL_ROUTINE(.page_read = logical_read_xlog_page,
- .segment_open = WalSndSegmentOpen,
- .segment_close = wal_segment_close),
+ logical_read_xlog_page,
+ wal_segment_close,
WalSndPrepareWrite, WalSndWriteData,
WalSndUpdateProgress);
@@ -1179,9 +1177,8 @@ StartLogicalReplication(StartReplicationCmd *cmd)
*/
logical_decoding_ctx =
CreateDecodingContext(cmd->startpoint, cmd->options, false,
- XL_ROUTINE(.page_read = logical_read_xlog_page,
- .segment_open = WalSndSegmentOpen,
- .segment_close = wal_segment_close),
+ logical_read_xlog_page,
+ wal_segment_close,
WalSndPrepareWrite, WalSndWriteData,
WalSndUpdateProgress);
xlogreader = logical_decoding_ctx->reader;
@@ -2761,7 +2758,7 @@ XLogSendPhysical(void)
enlargeStringInfo(&output_message, nbytes);
retry:
- if (!WALRead(xlogreader,
+ if (!WALRead(xlogreader, WalSndSegmentOpen, wal_segment_close,
&output_message.data[output_message.len],
startptr,
nbytes,
@@ -2859,7 +2856,12 @@ XLogSendLogical(void)
*/
WalSndCaughtUp = false;
- record = XLogReadRecord(logical_decoding_ctx->reader, &errm);
+ while (XLogReadRecord(logical_decoding_ctx->reader, &record, &errm) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!logical_decoding_ctx->page_read(logical_decoding_ctx->reader))
+ break;
+ }
/* xlog record was invalid */
if (errm != NULL)
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index 0b7e73ae79..3dd6df4be6 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -40,15 +40,9 @@ static int xlogreadfd = -1;
static XLogSegNo xlogreadsegno = -1;
static char xlogfpath[MAXPGPATH];
-typedef struct XLogPageReadPrivate
-{
- const char *restoreCommand;
- int tliIndex;
-} XLogPageReadPrivate;
-
-static bool SimpleXLogPageRead(XLogReaderState *xlogreader,
- XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
+static bool SimpleXLogPageRead(XLogReaderState *xlogreader,
+ const char *datadir, int *tliIndex,
+ const char *restoreCommand);
/*
* Read WAL from the datadir/pg_wal, starting from 'startpoint' on timeline
@@ -62,20 +56,22 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
XLogRecord *record;
XLogReaderState *xlogreader;
char *errormsg;
- XLogPageReadPrivate private;
- private.tliIndex = tliIndex;
- private.restoreCommand = restoreCommand;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir,
- XL_ROUTINE(.page_read = &SimpleXLogPageRead),
- &private);
+ xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+
if (xlogreader == NULL)
pg_fatal("out of memory");
XLogBeginRead(xlogreader, startpoint);
do
{
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!SimpleXLogPageRead(xlogreader, datadir,
+ &tliIndex, restoreCommand))
+ break;
+ }
if (record == NULL)
{
@@ -113,19 +109,19 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex,
XLogRecord *record;
XLogReaderState *xlogreader;
char *errormsg;
- XLogPageReadPrivate private;
XLogRecPtr endptr;
- private.tliIndex = tliIndex;
- private.restoreCommand = restoreCommand;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir,
- XL_ROUTINE(.page_read = &SimpleXLogPageRead),
- &private);
+ xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
XLogBeginRead(xlogreader, ptr);
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!SimpleXLogPageRead(xlogreader, datadir, &tliIndex, restoreCommand))
+ break;
+ }
if (record == NULL)
{
if (errormsg)
@@ -160,7 +156,6 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
XLogRecPtr searchptr;
XLogReaderState *xlogreader;
char *errormsg;
- XLogPageReadPrivate private;
/*
* The given fork pointer points to the end of the last common record,
@@ -176,11 +171,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
forkptr += SizeOfXLogShortPHD;
}
- private.tliIndex = tliIndex;
- private.restoreCommand = restoreCommand;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir,
- XL_ROUTINE(.page_read = &SimpleXLogPageRead),
- &private);
+ xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
@@ -190,7 +181,13 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
uint8 info;
XLogBeginRead(xlogreader, searchptr);
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!SimpleXLogPageRead(xlogreader, datadir,
+ &tliIndex, restoreCommand))
+ break;
+ }
if (record == NULL)
{
@@ -237,10 +234,11 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
/* XLogReader callback function, to read a WAL page */
static bool
-SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *readBuf)
+SimpleXLogPageRead(XLogReaderState *xlogreader, const char *datadir,
+ int *tliIndex, const char *restoreCommand)
{
- XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
+ XLogRecPtr targetPagePtr = xlogreader->readPagePtr;
+ char *readBuf = xlogreader->readBuf;
uint32 targetPageOff;
XLogRecPtr targetSegEnd;
XLogSegNo targetSegNo;
@@ -273,14 +271,14 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
* be done both forward and backward, consider also switching timeline
* accordingly.
*/
- while (private->tliIndex < targetNentries - 1 &&
- targetHistory[private->tliIndex].end < targetSegEnd)
- private->tliIndex++;
- while (private->tliIndex > 0 &&
- targetHistory[private->tliIndex].begin >= targetSegEnd)
- private->tliIndex--;
+ while (*tliIndex < targetNentries - 1 &&
+ targetHistory[*tliIndex].end < targetSegEnd)
+ (*tliIndex)++;
+ while (*tliIndex > 0 &&
+ targetHistory[*tliIndex].begin >= targetSegEnd)
+ (*tliIndex)--;
- XLogFileName(xlogfname, targetHistory[private->tliIndex].tli,
+ XLogFileName(xlogfname, targetHistory[*tliIndex].tli,
xlogreadsegno, WalSegSz);
snprintf(xlogfpath, MAXPGPATH, "%s/" XLOGDIR "/%s",
@@ -293,7 +291,7 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
/*
* If we have no restore_command to execute, then exit.
*/
- if (private->restoreCommand == NULL)
+ if (restoreCommand == NULL)
{
pg_log_error("could not open file \"%s\": %m", xlogfpath);
xlogreader->readLen = -1;
@@ -307,7 +305,7 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
xlogreadfd = RestoreArchivedFile(xlogreader->segcxt.ws_dir,
xlogfname,
WalSegSz,
- private->restoreCommand);
+ restoreCommand);
if (xlogreadfd < 0)
{
@@ -349,7 +347,7 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
Assert(targetSegNo == xlogreadsegno);
- xlogreader->seg.ws_tli = targetHistory[private->tliIndex].tli;
+ xlogreader->seg.ws_tli = targetHistory[*tliIndex].tli;
xlogreader->readLen = XLOG_BLCKSZ;
return true;
}
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index ab0c28d9b7..02e71138cc 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -330,12 +330,17 @@ WALDumpCloseSegment(XLogReaderState *state)
state->seg.ws_file = -1;
}
-/* pg_waldump's XLogReaderRoutine->page_read callback */
+/*
+ * pg_waldump's WAL page rader, also used as page_read callback for
+ * XLogFindNextRecord
+ */
static bool
-WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetPtr, char *readBuff)
+WALDumpReadPage(XLogReaderState *state, void *priv)
{
- XLogDumpPrivate *private = state->private_data;
+ XLogRecPtr targetPagePtr = state->readPagePtr;
+ int reqLen = state->readLen;
+ char *readBuff = state->readBuf;
+ XLogDumpPrivate *private = (XLogDumpPrivate *) priv;
int count = XLOG_BLCKSZ;
WALReadError errinfo;
@@ -353,8 +358,8 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
}
}
- if (!WALRead(state, readBuff, targetPagePtr, count, private->timeline,
- &errinfo))
+ if (!WALRead(state, WALDumpOpenSegment, WALDumpCloseSegment,
+ readBuff, targetPagePtr, count, private->timeline, &errinfo))
{
WALOpenSegment *seg = &errinfo.wre_seg;
char fname[MAXPGPATH];
@@ -374,6 +379,7 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
(Size) errinfo.wre_req);
}
+ Assert(count >= state->readLen);
state->readLen = count;
return true;
}
@@ -1042,16 +1048,14 @@ main(int argc, char **argv)
/* we have everything we need, start reading */
xlogreader_state =
- XLogReaderAllocate(WalSegSz, waldir,
- XL_ROUTINE(.page_read = WALDumpReadPage,
- .segment_open = WALDumpOpenSegment,
- .segment_close = WALDumpCloseSegment),
- &private);
+ XLogReaderAllocate(WalSegSz, waldir, WALDumpCloseSegment);
+
if (!xlogreader_state)
fatal_error("out of memory");
/* first find a valid recptr to start from */
- first_record = XLogFindNextRecord(xlogreader_state, private.startptr);
+ first_record = XLogFindNextRecord(xlogreader_state, private.startptr,
+ &WALDumpReadPage, (void*) &private);
if (first_record == InvalidXLogRecPtr)
fatal_error("could not find a valid record after %X/%X",
@@ -1075,7 +1079,13 @@ main(int argc, char **argv)
for (;;)
{
/* try to read the next record */
- record = XLogReadRecord(xlogreader_state, &errormsg);
+ while (XLogReadRecord(xlogreader_state, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!WALDumpReadPage(xlogreader_state, (void *) &private))
+ break;
+ }
+
if (!record)
{
if (!config.follow || private.endptr_reached)
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index 6c8848e14f..6a7fe140cb 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -57,64 +57,15 @@ typedef struct WALSegmentContext
typedef struct XLogReaderState XLogReaderState;
-/* Function type definition for the read_page callback */
-typedef bool (*XLogPageReadCB) (XLogReaderState *xlogreader,
- XLogRecPtr targetPagePtr,
- int reqLen,
- XLogRecPtr targetRecPtr,
- char *readBuf);
+/* Function type definition for the segment cleanup callback */
+typedef void (*WALSegmentCleanupCB) (XLogReaderState *xlogreader);
+
+/* Function type definition for the open/close callbacks for WALRead() */
typedef void (*WALSegmentOpenCB) (XLogReaderState *xlogreader,
XLogSegNo nextSegNo,
TimeLineID *tli_p);
typedef void (*WALSegmentCloseCB) (XLogReaderState *xlogreader);
-typedef struct XLogReaderRoutine
-{
- /*
- * Data input callback
- *
- * This callback shall read at least reqLen valid bytes of the xlog page
- * starting at targetPagePtr, and store them in readBuf. The callback
- * shall return the number of bytes read (never more than XLOG_BLCKSZ), or
- * -1 on failure. The callback shall sleep, if necessary, to wait for the
- * requested bytes to become available. The callback will not be invoked
- * again for the same page unless more than the returned number of bytes
- * are needed.
- *
- * targetRecPtr is the position of the WAL record we're reading. Usually
- * it is equal to targetPagePtr + reqLen, but sometimes xlogreader needs
- * to read and verify the page or segment header, before it reads the
- * actual WAL record it's interested in. In that case, targetRecPtr can
- * be used to determine which timeline to read the page from.
- *
- * The callback shall set ->seg.ws_tli to the TLI of the file the page was
- * read from.
- */
- XLogPageReadCB page_read;
-
- /*
- * Callback to open the specified WAL segment for reading. ->seg.ws_file
- * shall be set to the file descriptor of the opened segment. In case of
- * failure, an error shall be raised by the callback and it shall not
- * return.
- *
- * "nextSegNo" is the number of the segment to be opened.
- *
- * "tli_p" is an input/output argument. WALRead() uses it to pass the
- * timeline in which the new segment should be found, but the callback can
- * use it to return the TLI that it actually opened.
- */
- WALSegmentOpenCB segment_open;
-
- /*
- * WAL segment close callback. ->seg.ws_file shall be set to a negative
- * number.
- */
- WALSegmentCloseCB segment_close;
-} XLogReaderRoutine;
-
-#define XL_ROUTINE(...) &(XLogReaderRoutine){__VA_ARGS__}
-
typedef struct
{
/* Is this block ref in use? */
@@ -144,12 +95,35 @@ typedef struct
uint16 data_bufsz;
} DecodedBkpBlock;
+/* Return code from XLogReadRecord */
+typedef enum XLogReadRecordResult
+{
+ XLREAD_SUCCESS, /* record is successfully read */
+ XLREAD_NEED_DATA, /* need more data. see XLogReadRecord. */
+ XLREAD_FAIL /* failed during reading a record */
+} XLogReadRecordResult;
+
+/*
+ * internal state of XLogReadRecord
+ *
+ * XLogReadState runs a state machine while reading a record. Theses states
+ * are not seen outside the function. Each state may repeat several times
+ * exiting requesting caller for new data. See the comment of XLogReadRecrod
+ * for details.
+ */
+typedef enum XLogReadRecordState {
+ XLREAD_NEXT_RECORD,
+ XLREAD_TOT_LEN,
+ XLREAD_FIRST_FRAGMENT,
+ XLREAD_CONTINUATION
+} XLogReadRecordState;
+
struct XLogReaderState
{
/*
* Operational callbacks
*/
- XLogReaderRoutine routine;
+ WALSegmentCleanupCB cleanup_cb;
/* ----------------------------------------
* Public parameters
@@ -162,18 +136,14 @@ struct XLogReaderState
*/
uint64 system_identifier;
- /*
- * Opaque data for callbacks to use. Not used by XLogReader.
- */
- void *private_data;
-
/*
* Start and end point of last record read. EndRecPtr is also used as the
* position to read next. Calling XLogBeginRead() sets EndRecPtr to the
* starting position and ReadRecPtr to invalid.
*/
- XLogRecPtr ReadRecPtr; /* start of last record read */
+ XLogRecPtr ReadRecPtr; /* start of last record read or being read */
XLogRecPtr EndRecPtr; /* end+1 of last record read */
+ XLogRecPtr PrevRecPtr; /* start of previous record read */
/* ----------------------------------------
* Communication with page reader
@@ -187,7 +157,9 @@ struct XLogReaderState
* the request, or -1 on error */
TimeLineID readPageTLI; /* TLI for data currently in readBuf */
char *readBuf; /* buffer to store data */
- bool page_verified; /* is the page on the buffer verified? */
+ bool page_verified; /* is the page header on the buffer verified? */
+ bool record_verified;/* is the current record header verified? */
+
/* ----------------------------------------
@@ -227,8 +199,6 @@ struct XLogReaderState
XLogRecPtr latestPagePtr;
TimeLineID latestPageTLI;
- /* beginning of the WAL record being read. */
- XLogRecPtr currRecPtr;
/* timeline to read it from, 0 if a lookup is required */
TimeLineID currTLI;
@@ -255,6 +225,15 @@ struct XLogReaderState
char *readRecordBuf;
uint32 readRecordBufSize;
+ /*
+ * XLogReadRecord() state
+ */
+ XLogReadRecordState readRecordState;/* state machine state */
+ int recordGotLen; /* amount of current record that has
+ * already been read */
+ int recordRemainLen; /* length of current record that remains */
+ XLogRecPtr recordContRecPtr; /* where the current record continues */
+
/* Buffer to hold error message */
char *errormsg_buf;
};
@@ -262,9 +241,7 @@ struct XLogReaderState
/* Get a new XLogReader */
extern XLogReaderState *XLogReaderAllocate(int wal_segment_size,
const char *waldir,
- XLogReaderRoutine *routine,
- void *private_data);
-extern XLogReaderRoutine *LocalXLogReaderRoutine(void);
+ WALSegmentCleanupCB cleanup_cb);
/* Free an XLogReader */
extern void XLogReaderFree(XLogReaderState *state);
@@ -272,12 +249,17 @@ extern void XLogReaderFree(XLogReaderState *state);
/* Position the XLogReader to given record */
extern void XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr);
#ifdef FRONTEND
-extern XLogRecPtr XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr);
+/* Function type definition for the read_page callback */
+typedef bool (*XLogFindNextRecordCB) (XLogReaderState *xlogreader,
+ void *private);
+extern XLogRecPtr XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
+ XLogFindNextRecordCB read_page, void *private);
#endif /* FRONTEND */
/* Read the next XLog record. Returns NULL on end-of-WAL or failure */
-extern struct XLogRecord *XLogReadRecord(XLogReaderState *state,
- char **errormsg);
+extern XLogReadRecordResult XLogReadRecord(XLogReaderState *state,
+ XLogRecord **record,
+ char **errormsg);
/* Validate a page */
extern bool XLogReaderValidatePageHeader(XLogReaderState *state,
@@ -297,6 +279,7 @@ typedef struct WALReadError
} WALReadError;
extern bool WALRead(XLogReaderState *state,
+ WALSegmentOpenCB segopenfn, WALSegmentCloseCB sgclosefn,
char *buf, XLogRecPtr startptr, Size count,
TimeLineID tli, WALReadError *errinfo);
diff --git a/src/include/access/xlogutils.h b/src/include/access/xlogutils.h
index c287edf206..482e4ced69 100644
--- a/src/include/access/xlogutils.h
+++ b/src/include/access/xlogutils.h
@@ -47,9 +47,7 @@ extern Buffer XLogReadBufferExtended(RelFileNode rnode, ForkNumber forknum,
extern Relation CreateFakeRelcacheEntry(RelFileNode rnode);
extern void FreeFakeRelcacheEntry(Relation fakerel);
-extern bool read_local_xlog_page(XLogReaderState *state,
- XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetRecPtr, char *cur_page);
+extern bool read_local_xlog_page(XLogReaderState *state);
extern void wal_segment_open(XLogReaderState *state,
XLogSegNo nextSegNo,
TimeLineID *tli_p);
diff --git a/src/include/replication/logical.h b/src/include/replication/logical.h
index c2f2475e5d..a743db49cf 100644
--- a/src/include/replication/logical.h
+++ b/src/include/replication/logical.h
@@ -29,6 +29,10 @@ typedef void (*LogicalOutputPluginWriterUpdateProgress) (struct LogicalDecodingC
TransactionId xid
);
+typedef struct LogicalDecodingContext LogicalDecodingContext;
+
+typedef bool (*LogicalDecodingXLogPageReadCB)(XLogReaderState *ctx);
+
typedef struct LogicalDecodingContext
{
/* memory context this is all allocated in */
@@ -39,6 +43,7 @@ typedef struct LogicalDecodingContext
/* infrastructure pieces for decoding */
XLogReaderState *reader;
+ LogicalDecodingXLogPageReadCB page_read;
struct ReorderBuffer *reorder;
struct SnapBuild *snapshot_builder;
@@ -95,14 +100,16 @@ extern LogicalDecodingContext *CreateInitDecodingContext(char *plugin,
List *output_plugin_options,
bool need_full_snapshot,
XLogRecPtr restart_lsn,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress);
extern LogicalDecodingContext *CreateDecodingContext(XLogRecPtr start_lsn,
List *output_plugin_options,
bool fast_forward,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress);
--
2.18.4
----Next_Part(Thu_Jul__2_13_53_30_2020_072)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
filename="v15-0003-Remove-globals-readOff-readLen-and-readSegNo.patch"
^ permalink raw reply [nested|flat] 70+ messages in thread
* [PATCH v15 2/4] Move page-reader out of XLogReadRecord
@ 2019-09-10 03:58 Kyotaro Horiguchi <[email protected]>
0 siblings, 0 replies; 70+ messages in thread
From: Kyotaro Horiguchi @ 2019-09-10 03:58 UTC (permalink / raw)
This is the second step of removing callbacks from WAL record reader.
Since it is essential to take in additional data while reading a
record, the function have to ask caller for new data while keeping
working state. Thus the function is turned into a state machine.
---
src/backend/access/transam/twophase.c | 15 +-
src/backend/access/transam/xlog.c | 58 +-
src/backend/access/transam/xlogreader.c | 700 +++++++++++-------
src/backend/access/transam/xlogutils.c | 17 +-
src/backend/replication/logical/logical.c | 26 +-
.../replication/logical/logicalfuncs.c | 13 +-
src/backend/replication/slotfuncs.c | 18 +-
src/backend/replication/walsender.c | 32 +-
src/bin/pg_rewind/parsexlog.c | 84 +--
src/bin/pg_waldump/pg_waldump.c | 36 +-
src/include/access/xlogreader.h | 121 ++-
src/include/access/xlogutils.h | 4 +-
src/include/replication/logical.h | 11 +-
13 files changed, 648 insertions(+), 487 deletions(-)
diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index 9b2e59bf0e..b0d60a0d0f 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -1330,11 +1330,8 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
XLogReaderState *xlogreader;
char *errormsg;
- xlogreader = XLogReaderAllocate(wal_segment_size, NULL,
- XL_ROUTINE(.page_read = &read_local_xlog_page,
- .segment_open = &wal_segment_open,
- .segment_close = &wal_segment_close),
- NULL);
+ xlogreader = XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+
if (!xlogreader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
@@ -1342,7 +1339,13 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
errdetail("Failed while allocating a WAL reading processor.")));
XLogBeginRead(xlogreader, lsn);
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!read_local_xlog_page(xlogreader))
+ break;
+ }
+
if (record == NULL)
ereport(ERROR,
(errcode_for_file_access(),
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index e570e56a24..f9b0108602 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -831,13 +831,6 @@ static XLogSource currentSource = XLOG_FROM_ANY;
static bool lastSourceFailed = false;
static bool pendingWalRcvRestart = false;
-typedef struct XLogPageReadPrivate
-{
- int emode;
- bool fetching_ckpt; /* are we fetching a checkpoint record? */
- bool randAccess;
-} XLogPageReadPrivate;
-
/*
* These variables track when we last obtained some WAL data to process,
* and where we got it from. (XLogReceiptSource is initially the same as
@@ -912,8 +905,8 @@ static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath,
static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli,
XLogSource source, bool notfoundOk);
static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, XLogSource source);
-static bool XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
+static bool XLogPageRead(XLogReaderState *xlogreader,
+ bool fetching_ckpt, int emode, bool randAccess);
static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
bool fetching_ckpt, XLogRecPtr tliRecPtr);
static int emode_for_corrupt_record(int emode, XLogRecPtr RecPtr);
@@ -1225,8 +1218,7 @@ XLogInsertRecord(XLogRecData *rdata,
appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len);
if (!debug_reader)
- debug_reader = XLogReaderAllocate(wal_segment_size, NULL,
- XL_ROUTINE(), NULL);
+ debug_reader = XLogReaderAllocate(wal_segment_size, NULL NULL);
if (!debug_reader)
{
@@ -4326,15 +4318,10 @@ CleanupBackupHistory(void)
* record is available.
*/
static XLogRecord *
-ReadRecord(XLogReaderState *xlogreader, int emode,
- bool fetching_ckpt)
+ReadRecord(XLogReaderState *xlogreader, int emode, bool fetching_ckpt)
{
XLogRecord *record;
- XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
-
- private->fetching_ckpt = fetching_ckpt;
- private->emode = emode;
- private->randAccess = (xlogreader->ReadRecPtr == InvalidXLogRecPtr);
+ bool randAccess = (xlogreader->ReadRecPtr == InvalidXLogRecPtr);
/* This is the first attempt to read this page. */
lastSourceFailed = false;
@@ -4342,8 +4329,16 @@ ReadRecord(XLogReaderState *xlogreader, int emode,
for (;;)
{
char *errormsg;
+ XLogReadRecordResult result;
+
+ while ((result = XLogReadRecord(xlogreader, &record, &errormsg))
+ == XLREAD_NEED_DATA)
+ {
+ if (!XLogPageRead(xlogreader, fetching_ckpt, emode, randAccess))
+ break;
+
+ }
- record = XLogReadRecord(xlogreader, &errormsg);
ReadRecPtr = xlogreader->ReadRecPtr;
EndRecPtr = xlogreader->EndRecPtr;
if (record == NULL)
@@ -6320,7 +6315,6 @@ StartupXLOG(void)
bool backupFromStandby = false;
DBState dbstate_at_startup;
XLogReaderState *xlogreader;
- XLogPageReadPrivate private;
bool fast_promoted = false;
struct stat st;
@@ -6480,13 +6474,9 @@ StartupXLOG(void)
OwnLatch(&XLogCtl->recoveryWakeupLatch);
/* Set up XLOG reader facility */
- MemSet(&private, 0, sizeof(XLogPageReadPrivate));
xlogreader =
- XLogReaderAllocate(wal_segment_size, NULL,
- XL_ROUTINE(.page_read = &XLogPageRead,
- .segment_open = NULL,
- .segment_close = wal_segment_close),
- &private);
+ XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+
if (!xlogreader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
@@ -11860,12 +11850,13 @@ CancelBackup(void)
* sleep and retry.
*/
static bool
-XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetRecPtr, char *readBuf)
+XLogPageRead(XLogReaderState *xlogreader,
+ bool fetching_ckpt, int emode, bool randAccess)
{
- XLogPageReadPrivate *private =
- (XLogPageReadPrivate *) xlogreader->private_data;
- int emode = private->emode;
+ char *readBuf = xlogreader->readBuf;
+ XLogRecPtr targetPagePtr = xlogreader->readPagePtr;
+ int reqLen = xlogreader->readLen;
+ XLogRecPtr targetRecPtr = xlogreader->ReadRecPtr;
uint32 targetPageOff;
XLogSegNo targetSegNo PG_USED_FOR_ASSERTS_ONLY;
int r;
@@ -11908,8 +11899,8 @@ retry:
flushedUpto < targetPagePtr + reqLen))
{
if (!WaitForWALToBecomeAvailable(targetPagePtr + reqLen,
- private->randAccess,
- private->fetching_ckpt,
+ randAccess,
+ fetching_ckpt,
targetRecPtr))
{
if (readFile >= 0)
@@ -12014,6 +12005,7 @@ retry:
goto next_record_is_invalid;
}
+ Assert(xlogreader->readPagePtr == targetPagePtr);
xlogreader->readLen = readLen;
return true;
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index 3d599325ee..9281b57379 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -40,7 +40,7 @@ static bool XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr,
int reqLen, bool header_inclusive);
static void XLogReaderInvalReadState(XLogReaderState *state);
static bool ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
- XLogRecPtr PrevRecPtr, XLogRecord *record, bool randAccess);
+ XLogRecPtr PrevRecPtr, XLogRecord *record);
static bool ValidXLogRecord(XLogReaderState *state, XLogRecord *record,
XLogRecPtr recptr);
static void ResetDecoder(XLogReaderState *state);
@@ -73,7 +73,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...)
*/
XLogReaderState *
XLogReaderAllocate(int wal_segment_size, const char *waldir,
- XLogReaderRoutine *routine, void *private_data)
+ WALSegmentCleanupCB cleanup_cb)
{
XLogReaderState *state;
@@ -84,7 +84,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
return NULL;
/* initialize caller-provided support functions */
- state->routine = *routine;
+ state->cleanup_cb = cleanup_cb;
state->max_block_id = -1;
@@ -107,8 +107,6 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
WALOpenSegmentInit(&state->seg, &state->segcxt, wal_segment_size,
waldir);
- /* system_identifier initialized to zeroes above */
- state->private_data = private_data;
/* ReadRecPtr, EndRecPtr and readLen initialized to zeroes above */
state->errormsg_buf = palloc_extended(MAX_ERRORMSG_LEN + 1,
MCXT_ALLOC_NO_OOM);
@@ -140,8 +138,8 @@ XLogReaderFree(XLogReaderState *state)
{
int block_id;
- if (state->seg.ws_file != -1)
- state->routine.segment_close(state);
+ if (state->seg.ws_file >= 0)
+ state->cleanup_cb(state);
for (block_id = 0; block_id <= XLR_MAX_BLOCK_ID; block_id++)
{
@@ -246,6 +244,7 @@ XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
/* Begin at the passed-in record pointer. */
state->EndRecPtr = RecPtr;
state->ReadRecPtr = InvalidXLogRecPtr;
+ state->readRecordState = XLREAD_NEXT_RECORD;
}
/*
@@ -254,317 +253,452 @@ XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
* XLogBeginRead() or XLogFindNextRecord() must be called before the first call
* to XLogReadRecord().
*
- * If the page_read callback fails to read the requested data, NULL is
- * returned. The callback is expected to have reported the error; errormsg
- * is set to NULL.
+ * This function may return XLREAD_NEED_DATA several times before returning a
+ * result record. The caller shall read in some new data then call this
+ * function again with the same parameters.
*
- * If the reading fails for some other reason, NULL is also returned, and
- * *errormsg is set to a string with details of the failure.
+ * When a record is successfully read, returns XLREAD_SUCCESS with result
+ * record being stored in *record. Otherwise *record is NULL.
*
- * The returned pointer (or *errormsg) points to an internal buffer that's
- * valid until the next call to XLogReadRecord.
+ * Returns XLREAD_NEED_DATA if more data is needed to finish reading the
+ * current record. In that case, state->readPagePtr and state->readLen inform
+ * the desired position and minimum length of data needed. The caller shall
+ * read in the requested data and set state->readBuf to point to a buffer
+ * containing it. The caller must also set state->readPageTLI and
+ * state->readLen to indicate the timeline that it was read from, and the
+ * length of data that is now available (which must be >= given readLen),
+ * respectively.
+ *
+ * If invalid data is encountered, returns XLREAD_FAIL with *record being set to
+ * NULL. *errormsg is set to a string with details of the failure.
+ * The returned pointer (or *errormsg) points to an internal buffer that's valid
+ * until the next call to XLogReadRecord.
+ *
+ *
+ * This function runs a state machine consists of the following states.
+ *
+ * XLREAD_NEXT_RECORD :
+ * The initial state, if called with valid RecPtr, try to read a record at
+ * that position. If invalid RecPtr is given try to read a record just after
+ * the last one previously read.
+ * This state ens after setting ReadRecPtr. Then goes to XLREAD_TOT_LEN.
+ *
+ * XLREAD_TOT_LEN:
+ * Examining record header. Ends after reading record total
+ * length. recordRemainLen and recordGotLen are initialized.
+ *
+ * XLREAD_FIRST_FRAGMENT:
+ * Reading the first fragment. Ends with finishing reading a single
+ * record. Goes to XLREAD_NEXT_RECORD if that's all or
+ * XLREAD_CONTINUATION if we have continuation.
+
+ * XLREAD_CONTINUATION:
+ * Reading continuation of record. Ends with finishing the whole record then
+ * goes to XLREAD_NEXT_RECORD. During this state, recordRemainLen indicates
+ * how much is left and readRecordBuf holds the partially assert
+ * record.recordContRecPtr points to the beginning of the next page where to
+ * continue.
+ *
+ * If wrong data found in any state, the state machine stays at the current
+ * state. This behavior allows to continue reading a reacord switching among
+ * different souces, while streaming replication.
*/
-XLogRecord *
-XLogReadRecord(XLogReaderState *state, char **errormsg)
+XLogReadRecordResult
+XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
{
- XLogRecPtr RecPtr;
- XLogRecord *record;
- XLogRecPtr targetPagePtr;
- bool randAccess;
- uint32 len,
- total_len;
- uint32 targetRecOff;
- uint32 pageHeaderSize;
- bool gotheader;
+ XLogRecord *prec;
- /*
- * randAccess indicates whether to verify the previous-record pointer of
- * the record we're reading. We only do this if we're reading
- * sequentially, which is what we initially assume.
- */
- randAccess = false;
+ *record = NULL;
/* reset error state */
*errormsg = NULL;
state->errormsg_buf[0] = '\0';
- ResetDecoder(state);
-
- RecPtr = state->EndRecPtr;
-
- if (state->ReadRecPtr != InvalidXLogRecPtr)
- {
- /* read the record after the one we just read */
-
- /*
- * EndRecPtr is pointing to end+1 of the previous WAL record. If
- * we're at a page boundary, no more records can fit on the current
- * page. We must skip over the page header, but we can't do that until
- * we've read in the page, since the header size is variable.
- */
- }
- else
- {
- /*
- * Caller supplied a position to start at.
- *
- * In this case, EndRecPtr should already be pointing to a valid
- * record starting position.
- */
- Assert(XRecOffIsValid(RecPtr));
- randAccess = true;
- }
-
- state->currRecPtr = RecPtr;
-
- targetPagePtr = RecPtr - (RecPtr % XLOG_BLCKSZ);
- targetRecOff = RecPtr % XLOG_BLCKSZ;
-
- /*
- * Read the page containing the record into state->readBuf. Request enough
- * byte to cover the whole record header, or at least the part of it that
- * fits on the same page.
- */
- while (XLogNeedData(state, targetPagePtr,
- Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
- targetRecOff != 0))
- {
- if (!state->routine.page_read(state, state->readPagePtr, state->readLen,
- RecPtr, state->readBuf))
- break;
- }
-
- if (!state->page_verified)
- goto err;
-
- /*
- * We have at least the page header, so we can examine it now.
- */
- pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
- if (targetRecOff == 0)
- {
- /*
- * At page start, so skip over page header.
- */
- RecPtr += pageHeaderSize;
- targetRecOff = pageHeaderSize;
- }
- else if (targetRecOff < pageHeaderSize)
- {
- report_invalid_record(state, "invalid record offset at %X/%X",
- (uint32) (RecPtr >> 32), (uint32) RecPtr);
- goto err;
- }
-
- if ((((XLogPageHeader) state->readBuf)->xlp_info & XLP_FIRST_IS_CONTRECORD) &&
- targetRecOff == pageHeaderSize)
- {
- report_invalid_record(state, "contrecord is requested by %X/%X",
- (uint32) (RecPtr >> 32), (uint32) RecPtr);
- goto err;
- }
-
- /* XLogNeedData has verified the page header */
- Assert(pageHeaderSize <= state->readLen);
-
- /*
- * Read the record length.
- *
- * NB: Even though we use an XLogRecord pointer here, the whole record
- * header might not fit on this page. xl_tot_len is the first field of the
- * struct, so it must be on this page (the records are MAXALIGNed), but we
- * cannot access any other fields until we've verified that we got the
- * whole header.
- */
- record = (XLogRecord *) (state->readBuf + RecPtr % XLOG_BLCKSZ);
- total_len = record->xl_tot_len;
-
- /*
- * If the whole record header is on this page, validate it immediately.
- * Otherwise do just a basic sanity check on xl_tot_len, and validate the
- * rest of the header after reading it from the next page. The xl_tot_len
- * check is necessary here to ensure that we enter the "Need to reassemble
- * record" code path below; otherwise we might fail to apply
- * ValidXLogRecordHeader at all.
- */
- if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
- {
- if (!ValidXLogRecordHeader(state, RecPtr, state->ReadRecPtr, record,
- randAccess))
- goto err;
- gotheader = true;
- }
- else
- {
- /* XXX: more validation should be done here */
- if (total_len < SizeOfXLogRecord)
- {
- report_invalid_record(state,
- "invalid record length at %X/%X: wanted %u, got %u",
- (uint32) (RecPtr >> 32), (uint32) RecPtr,
- (uint32) SizeOfXLogRecord, total_len);
- goto err;
- }
- gotheader = false;
- }
-
- len = XLOG_BLCKSZ - RecPtr % XLOG_BLCKSZ;
- if (total_len > len)
+ switch (state->readRecordState)
{
- /* Need to reassemble record */
- char *contdata;
- XLogPageHeader pageHeader;
- char *buffer;
- uint32 gotlen;
-
- /*
- * Enlarge readRecordBuf as needed.
- */
- if (total_len > state->readRecordBufSize &&
- !allocate_recordbuf(state, total_len))
- {
- /* We treat this as a "bogus data" condition */
- report_invalid_record(state, "record length %u at %X/%X too long",
- total_len,
- (uint32) (RecPtr >> 32), (uint32) RecPtr);
- goto err;
- }
+ case XLREAD_NEXT_RECORD:
+ ResetDecoder(state);
- /* Copy the first fragment of the record from the first page. */
- memcpy(state->readRecordBuf,
- state->readBuf + RecPtr % XLOG_BLCKSZ, len);
- buffer = state->readRecordBuf + len;
- gotlen = len;
-
- do
- {
- int rest_len = total_len - gotlen;
-
- /* Calculate pointer to beginning of next page */
- targetPagePtr += XLOG_BLCKSZ;
-
- /* Wait for the next page to become available */
- while (XLogNeedData(state, targetPagePtr,
- Min(rest_len, XLOG_BLCKSZ),
- false))
+ if (state->ReadRecPtr != InvalidXLogRecPtr)
{
- if (!state->routine.page_read(state, state->readPagePtr,
- state->readLen,
- state->ReadRecPtr,
- state->readBuf))
- break;
+ /* read the record after the one we just read */
+
+ /*
+ * EndRecPtr is pointing to end+1 of the previous WAL record.
+ * If we're at a page boundary, no more records can fit on the
+ * current page. We must skip over the page header, but we
+ * can't do that until we've read in the page, since the header
+ * size is variable.
+ */
+ state->PrevRecPtr = state->ReadRecPtr;
+ state->ReadRecPtr = state->EndRecPtr;
+ }
+ else
+ {
+ /*
+ * Caller supplied a position to start at.
+ *
+ * In this case, EndRecPtr should already be pointing to a
+ * valid record starting position.
+ */
+ Assert(XRecOffIsValid(state->EndRecPtr));
+ state->ReadRecPtr = state->EndRecPtr;
+
+ /*
+ * We cannot verify the previous-record pointer when we're
+ * seeking to a particular record. Reset PrevRecPtr so that we
+ * won't try doing that.
+ */
+ state->PrevRecPtr = InvalidXLogRecPtr;
+ state->EndRecPtr = InvalidXLogRecPtr; /* to be tidy */
}
+ state->record_verified = false;
+ state->readRecordState = XLREAD_TOT_LEN;
+ /* fall through */
+
+ case XLREAD_TOT_LEN:
+ {
+ uint32 total_len;
+ uint32 pageHeaderSize;
+ XLogRecPtr targetPagePtr;
+ uint32 targetRecOff;
+ XLogPageHeader pageHeader;
+
+ targetPagePtr =
+ state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
+ targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+
+ /*
+ * Check if we have enough data. For the first record in the page,
+ * the requesting length doesn't contain page header.
+ */
+ if (XLogNeedData(state, targetPagePtr,
+ Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+ targetRecOff != 0))
+ return XLREAD_NEED_DATA;
+
+ /* error out if caller supplied bogus page */
if (!state->page_verified)
goto err;
- Assert(SizeOfXLogShortPHD <= state->readLen);
+ /* examine page header now. */
+ pageHeaderSize =
+ XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+ if (targetRecOff == 0)
+ {
+ /* At page start, so skip over page header. */
+ state->ReadRecPtr += pageHeaderSize;
+ targetRecOff = pageHeaderSize;
+ }
+ else if (targetRecOff < pageHeaderSize)
+ {
+ report_invalid_record(state, "invalid record offset at %X/%X",
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr);
+ goto err;
+ }
- /* Check that the continuation on next page looks valid */
pageHeader = (XLogPageHeader) state->readBuf;
- if (!(pageHeader->xlp_info & XLP_FIRST_IS_CONTRECORD))
+ if ((pageHeader->xlp_info & XLP_FIRST_IS_CONTRECORD) &&
+ targetRecOff == pageHeaderSize)
{
- report_invalid_record(state,
- "there is no contrecord flag at %X/%X",
- (uint32) (RecPtr >> 32), (uint32) RecPtr);
+ report_invalid_record(state, "contrecord is requested by %X/%X",
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr);
goto err;
}
- /*
- * Cross-check that xlp_rem_len agrees with how much of the record
- * we expect there to be left.
- */
- if (pageHeader->xlp_rem_len == 0 ||
- total_len != (pageHeader->xlp_rem_len + gotlen))
- {
- report_invalid_record(state,
- "invalid contrecord length %u at %X/%X",
- pageHeader->xlp_rem_len,
- (uint32) (RecPtr >> 32), (uint32) RecPtr);
- goto err;
- }
-
- /* Append the continuation from this page to the buffer */
- pageHeaderSize = XLogPageHeaderSize(pageHeader);
-
+ /* XLogNeedData has verified the page header */
Assert(pageHeaderSize <= state->readLen);
- contdata = (char *) state->readBuf + pageHeaderSize;
- len = XLOG_BLCKSZ - pageHeaderSize;
- if (pageHeader->xlp_rem_len < len)
- len = pageHeader->xlp_rem_len;
+ /*
+ * Read the record length.
+ *
+ * NB: Even though we use an XLogRecord pointer here, the whole
+ * record header might not fit on this page. xl_tot_len is the first
+ * field of the struct, so it must be on this page (the records are
+ * MAXALIGNed), but we cannot access any other fields until we've
+ * verified that we got the whole header.
+ */
+ prec = (XLogRecord *) (state->readBuf +
+ state->ReadRecPtr % XLOG_BLCKSZ);
+ total_len = prec->xl_tot_len;
- Assert (pageHeaderSize + len <= state->readLen);
- memcpy(buffer, (char *) contdata, len);
- buffer += len;
- gotlen += len;
+ /*
+ * If the whole record header is on this page, validate it
+ * immediately. Otherwise do just a basic sanity check on
+ * xl_tot_len, and validate the rest of the header after reading it
+ * from the next page. The xl_tot_len check is necessary here to
+ * ensure that we enter the XLREAD_CONTINUATION state below;
+ * otherwise we might fail to apply ValidXLogRecordHeader at all.
+ */
+ if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
+ {
+ if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
+ state->PrevRecPtr, prec))
+ goto err;
- /* If we just reassembled the record header, validate it. */
- if (!gotheader)
+ state->record_verified = true;
+ }
+ else
{
- record = (XLogRecord *) state->readRecordBuf;
- if (!ValidXLogRecordHeader(state, RecPtr, state->ReadRecPtr,
- record, randAccess))
+ /* XXX: more validation should be done here */
+ if (total_len < SizeOfXLogRecord)
+ {
+ report_invalid_record(state,
+ "invalid record length at %X/%X: wanted %u, got %u",
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr,
+ (uint32) SizeOfXLogRecord, total_len);
goto err;
- gotheader = true;
+ }
}
- } while (gotlen < total_len);
-
- Assert(gotheader);
- record = (XLogRecord *) state->readRecordBuf;
- if (!ValidXLogRecord(state, record, RecPtr))
- goto err;
+ /*
+ * Wait for the rest of the record, or the part of the record that
+ * fit on the first page if crossed a page boundary, to become
+ * available.
+ */
+ state->recordGotLen = 0;
+ state->recordRemainLen = total_len;
+ state->readRecordState = XLREAD_FIRST_FRAGMENT;
+ }
+ /* fall through */
- pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
- state->ReadRecPtr = RecPtr;
- state->EndRecPtr = targetPagePtr + pageHeaderSize
- + MAXALIGN(pageHeader->xlp_rem_len);
- }
- else
- {
- /* Wait for the record data to become available */
- while (XLogNeedData(state, targetPagePtr,
- Min(targetRecOff + total_len, XLOG_BLCKSZ), true))
+ case XLREAD_FIRST_FRAGMENT:
{
- if (!state->routine.page_read(state, state->readPagePtr,
- state->readLen,
- state->ReadRecPtr, state->readBuf))
+ uint32 total_len = state->recordRemainLen;
+ uint32 request_len;
+ uint32 record_len;
+ XLogRecPtr targetPagePtr;
+ uint32 targetRecOff;
+
+ /*
+ * Wait for the rest of the record on the first page to become
+ * available
+ */
+ targetPagePtr =
+ state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
+ targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+
+ request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ);
+ record_len = request_len - targetRecOff;
+
+ /* ReadRecPtr contains page header */
+ Assert (targetRecOff != 0);
+ if (XLogNeedData(state, targetPagePtr, request_len, true))
+ return XLREAD_NEED_DATA;
+
+ /* error out if caller supplied bogus page */
+ if (!state->page_verified)
+ goto err;
+
+ prec = (XLogRecord *) (state->readBuf + targetRecOff);
+
+ /* validate record header if not yet */
+ if (!state->record_verified && record_len >= SizeOfXLogRecord)
+ {
+ if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
+ state->PrevRecPtr, prec))
+ goto err;
+
+ state->record_verified = true;
+ }
+
+
+ if (total_len == record_len)
+ {
+ /* Record does not cross a page boundary */
+ Assert(state->record_verified);
+
+ if (!ValidXLogRecord(state, prec, state->ReadRecPtr))
+ goto err;
+
+ state->record_verified = true; /* to be tidy */
+
+ /* We already checked the header earlier */
+ state->EndRecPtr = state->ReadRecPtr + MAXALIGN(record_len);
+
+ *record = prec;
+ state->readRecordState = XLREAD_NEXT_RECORD;
break;
+ }
+
+ /*
+ * The record continues on the next page. Need to reassemble
+ * record
+ */
+ Assert(total_len > record_len);
+
+ /* Enlarge readRecordBuf as needed. */
+ if (total_len > state->readRecordBufSize &&
+ !allocate_recordbuf(state, total_len))
+ {
+ /* We treat this as a "bogus data" condition */
+ report_invalid_record(state,
+ "record length %u at %X/%X too long",
+ total_len,
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr);
+ goto err;
+ }
+
+ /* Copy the first fragment of the record from the first page. */
+ memcpy(state->readRecordBuf, state->readBuf + targetRecOff,
+ record_len);
+ state->recordGotLen += record_len;
+ state->recordRemainLen -= record_len;
+
+ /* Calculate pointer to beginning of next page */
+ state->recordContRecPtr = state->ReadRecPtr + record_len;
+ Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0);
+
+ state->readRecordState = XLREAD_CONTINUATION;
}
+ /* fall through */
+
+ case XLREAD_CONTINUATION:
+ {
+ XLogPageHeader pageHeader;
+ uint32 pageHeaderSize;
+ XLogRecPtr targetPagePtr;
+
+ /* we enter this state only if we haven't read the whole record. */
+ Assert (state->recordRemainLen > 0);
+
+ while(state->recordRemainLen > 0)
+ {
+ char *contdata;
+ uint32 request_len;
+ uint32 record_len;
+
+ /* Wait for the next page to become available */
+ targetPagePtr = state->recordContRecPtr;
+
+ /* this request contains page header */
+ Assert (targetPagePtr != 0);
+ if (XLogNeedData(state, targetPagePtr,
+ Min(state->recordRemainLen, XLOG_BLCKSZ),
+ false))
+ return XLREAD_NEED_DATA;
+
+ if (!state->page_verified)
+ goto err;
+
+ Assert(SizeOfXLogShortPHD <= state->readLen);
- if (!state->page_verified)
- goto err;
+ /* Check that the continuation on next page looks valid */
+ pageHeader = (XLogPageHeader) state->readBuf;
+ if (!(pageHeader->xlp_info & XLP_FIRST_IS_CONTRECORD))
+ {
+ report_invalid_record(
+ state,
+ "there is no contrecord flag at %X/%X reading %X/%X",
+ (uint32) (state->recordContRecPtr >> 32),
+ (uint32) state->recordContRecPtr,
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr);
+ goto err;
+ }
- /* Record does not cross a page boundary */
- if (!ValidXLogRecord(state, record, RecPtr))
- goto err;
+ /*
+ * Cross-check that xlp_rem_len agrees with how much of the
+ * record we expect there to be left.
+ */
+ if (pageHeader->xlp_rem_len == 0 ||
+ pageHeader->xlp_rem_len != state->recordRemainLen)
+ {
+ report_invalid_record(
+ state,
+ "invalid contrecord length %u at %X/%X reading %X/%X, expected %u",
+ pageHeader->xlp_rem_len,
+ (uint32) (state->recordContRecPtr >> 32),
+ (uint32) state->recordContRecPtr,
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr,
+ state->recordRemainLen);
+ goto err;
+ }
- state->EndRecPtr = RecPtr + MAXALIGN(total_len);
+ /* Append the continuation from this page to the buffer */
+ pageHeaderSize = XLogPageHeaderSize(pageHeader);
- state->ReadRecPtr = RecPtr;
+ /*
+ * XLogNeedData should have ensured that the whole page header
+ * was read
+ */
+ Assert(state->readLen >= pageHeaderSize);
+
+ contdata = (char *) state->readBuf + pageHeaderSize;
+ record_len = XLOG_BLCKSZ - pageHeaderSize;
+ if (pageHeader->xlp_rem_len < record_len)
+ record_len = pageHeader->xlp_rem_len;
+
+ request_len = record_len + pageHeaderSize;
+
+ /* XLogNeedData should have ensured all needed data was read */
+ Assert (state->readLen >= request_len);
+
+ memcpy(state->readRecordBuf + state->recordGotLen,
+ (char *) contdata, record_len);
+ state->recordGotLen += record_len;
+ state->recordRemainLen -= record_len;
+
+ /* If we just reassembled the record header, validate it. */
+ if (!state->record_verified)
+ {
+ Assert(state->recordGotLen >= SizeOfXLogRecord);
+ if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
+ state->PrevRecPtr,
+ (XLogRecord *) state->readRecordBuf))
+ goto err;
+
+ state->record_verified = true;
+ }
+
+ /* Calculate pointer to beginning of next page, and continue */
+ state->recordContRecPtr += XLOG_BLCKSZ;
+ }
+
+ /* targetPagePtr is pointing the last-read page here */
+ prec = (XLogRecord *) state->readRecordBuf;
+ if (!ValidXLogRecord(state, prec, state->ReadRecPtr))
+ goto err;
+
+ pageHeaderSize =
+ XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+ state->EndRecPtr = targetPagePtr + pageHeaderSize
+ + MAXALIGN(pageHeader->xlp_rem_len);
+
+ *record = prec;
+ state->readRecordState = XLREAD_NEXT_RECORD;
+ break;
+ }
}
/*
* Special processing if it's an XLOG SWITCH record
*/
- if (record->xl_rmid == RM_XLOG_ID &&
- (record->xl_info & ~XLR_INFO_MASK) == XLOG_SWITCH)
+ if ((*record)->xl_rmid == RM_XLOG_ID &&
+ ((*record)->xl_info & ~XLR_INFO_MASK) == XLOG_SWITCH)
{
/* Pretend it extends to end of segment */
state->EndRecPtr += state->segcxt.ws_segsize - 1;
state->EndRecPtr -= XLogSegmentOffset(state->EndRecPtr, state->segcxt.ws_segsize);
}
- if (DecodeXLogRecord(state, record, errormsg))
- return record;
- else
- return NULL;
+ Assert (!*record || state->readLen >= 0);
+ if (DecodeXLogRecord(state, *record, errormsg))
+ return XLREAD_SUCCESS;
+
+ *record = NULL;
+ return XLREAD_FAIL;
err:
/*
- * Invalidate the read state. We might read from a different source after
+ * Invalidate the read page. We might read from a different source after
* failure.
*/
XLogReaderInvalReadState(state);
@@ -572,7 +706,8 @@ err:
if (state->errormsg_buf[0] != '\0')
*errormsg = state->errormsg_buf;
- return NULL;
+ *record = NULL;
+ return XLREAD_FAIL;
}
/*
@@ -724,11 +859,12 @@ XLogReaderInvalReadState(XLogReaderState *state)
*
* This is just a convenience subroutine to avoid duplicated code in
* XLogReadRecord. It's not intended for use from anywhere else.
+ *
+ * If PrevRecPtr is valid, the xl_prev is is cross-checked with it.
*/
static bool
ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
- XLogRecPtr PrevRecPtr, XLogRecord *record,
- bool randAccess)
+ XLogRecPtr PrevRecPtr, XLogRecord *record)
{
if (record->xl_tot_len < SizeOfXLogRecord)
{
@@ -746,7 +882,7 @@ ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
(uint32) RecPtr);
return false;
}
- if (randAccess)
+ if (PrevRecPtr == InvalidXLogRecPtr)
{
/*
* We can't exactly verify the prev-link, but surely it should be less
@@ -978,11 +1114,14 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
* XLogReadRecord() will read the next valid record.
*/
XLogRecPtr
-XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
+XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
+ XLogFindNextRecordCB read_page, void *private)
{
XLogRecPtr tmpRecPtr;
XLogRecPtr found = InvalidXLogRecPtr;
XLogPageHeader header;
+ XLogRecord *record;
+ XLogReadRecordResult result;
char *errormsg;
Assert(!XLogRecPtrIsInvalid(RecPtr));
@@ -1015,9 +1154,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
while(XLogNeedData(state, targetPagePtr, targetRecOff,
targetRecOff != 0))
{
- if (!state->routine.page_read(state, state->readPagePtr,
- state->readLen,
- state->ReadRecPtr, state->readBuf))
+ if (!read_page(state, private))
break;
}
@@ -1069,8 +1206,16 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
* or we just jumped over the remaining data of a continuation.
*/
XLogBeginRead(state, tmpRecPtr);
- while (XLogReadRecord(state, &errormsg) != NULL)
+ while ((result = XLogReadRecord(state, &record, &errormsg)) !=
+ XLREAD_FAIL)
{
+ if (result == XLREAD_NEED_DATA)
+ {
+ if (!read_page(state, private))
+ goto err;
+ continue;
+ }
+
/* past the record we've found, break out */
if (RecPtr <= state->ReadRecPtr)
{
@@ -1090,9 +1235,9 @@ err:
#endif /* FRONTEND */
/*
- * Helper function to ease writing of XLogRoutine->page_read callbacks.
- * If this function is used, caller must supply a segment_open callback in
- * 'state', as that is used here.
+ * Helper function to ease writing of page_read callback.
+ * If this function is used, caller must supply a segment_open callback and
+ * segment_close callback as that is used here.
*
* Read 'count' bytes into 'buf', starting at location 'startptr', from WAL
* fetched from timeline 'tli'.
@@ -1105,6 +1250,7 @@ err:
*/
bool
WALRead(XLogReaderState *state,
+ WALSegmentOpenCB segopenfn, WALSegmentCloseCB segclosefn,
char *buf, XLogRecPtr startptr, Size count, TimeLineID tli,
WALReadError *errinfo)
{
@@ -1136,10 +1282,10 @@ WALRead(XLogReaderState *state,
XLogSegNo nextSegNo;
if (state->seg.ws_file >= 0)
- state->routine.segment_close(state);
+ segclosefn(state);
XLByteToSeg(recptr, nextSegNo, state->segcxt.ws_segsize);
- state->routine.segment_open(state, nextSegNo, &tli);
+ segopenfn(state, nextSegNo, &tli);
/* This shouldn't happen -- indicates a bug in segment_open */
Assert(state->seg.ws_file >= 0);
diff --git a/src/backend/access/transam/xlogutils.c b/src/backend/access/transam/xlogutils.c
index 1b10b6df20..303333571f 100644
--- a/src/backend/access/transam/xlogutils.c
+++ b/src/backend/access/transam/xlogutils.c
@@ -689,8 +689,7 @@ XLogTruncateRelation(RelFileNode rnode, ForkNumber forkNum,
void
XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wantLength)
{
- const XLogRecPtr lastReadPage = state->seg.ws_segno *
- state->segcxt.ws_segsize + state->readLen;
+ const XLogRecPtr lastReadPage = state->readPagePtr;
Assert(wantPage != InvalidXLogRecPtr && wantPage % XLOG_BLCKSZ == 0);
Assert(wantLength <= XLOG_BLCKSZ);
@@ -705,7 +704,7 @@ XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wa
* current TLI has since become historical.
*/
if (lastReadPage == wantPage &&
- state->readLen != 0 &&
+ state->page_verified &&
lastReadPage + state->readLen >= wantPage + Min(wantLength, XLOG_BLCKSZ - 1))
return;
@@ -792,6 +791,7 @@ wal_segment_open(XLogReaderState *state, XLogSegNo nextSegNo,
char path[MAXPGPATH];
XLogFilePath(path, tli, nextSegNo, state->segcxt.ws_segsize);
+ elog(LOG, "HOGE: %lu, %d => %s", nextSegNo, tli, path);
state->seg.ws_file = BasicOpenFile(path, O_RDONLY | PG_BINARY);
if (state->seg.ws_file >= 0)
return;
@@ -829,9 +829,11 @@ wal_segment_close(XLogReaderState *state)
* loop for now.
*/
bool
-read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *cur_page)
+read_local_xlog_page(XLogReaderState *state)
{
+ XLogRecPtr targetPagePtr = state->readPagePtr;
+ int reqLen = state->readLen;
+ char *cur_page = state->readBuf;
XLogRecPtr read_upto,
loc;
TimeLineID tli;
@@ -944,11 +946,12 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
* as 'count', read the whole page anyway. It's guaranteed to be
* zero-padded up to the page boundary if it's incomplete.
*/
- if (!WALRead(state, cur_page, targetPagePtr, XLOG_BLCKSZ, tli,
- &errinfo))
+ if (!WALRead(state, wal_segment_open, wal_segment_close,
+ cur_page, targetPagePtr, XLOG_BLCKSZ, tli, &errinfo))
WALReadRaiseError(&errinfo);
/* number of valid bytes in the buffer */
+ state->readPagePtr = targetPagePtr;
state->readLen = count;
return true;
}
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index 61902be3b0..a15b0b3355 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -120,7 +120,8 @@ StartupDecodingContext(List *output_plugin_options,
TransactionId xmin_horizon,
bool need_full_snapshot,
bool fast_forward,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress)
@@ -169,11 +170,12 @@ StartupDecodingContext(List *output_plugin_options,
ctx->slot = slot;
- ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, xl_routine, ctx);
+ ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, cleanup_cb);
if (!ctx->reader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
errmsg("out of memory")));
+ ctx->page_read = page_read;
ctx->reorder = ReorderBufferAllocate();
ctx->snapshot_builder =
@@ -231,7 +233,8 @@ CreateInitDecodingContext(char *plugin,
List *output_plugin_options,
bool need_full_snapshot,
XLogRecPtr restart_lsn,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress)
@@ -328,7 +331,7 @@ CreateInitDecodingContext(char *plugin,
ctx = StartupDecodingContext(NIL, restart_lsn, xmin_horizon,
need_full_snapshot, false,
- xl_routine, prepare_write, do_write,
+ page_read, cleanup_cb, prepare_write, do_write,
update_progress);
/* call output plugin initialization callback */
@@ -376,7 +379,8 @@ LogicalDecodingContext *
CreateDecodingContext(XLogRecPtr start_lsn,
List *output_plugin_options,
bool fast_forward,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress)
@@ -429,8 +433,8 @@ CreateDecodingContext(XLogRecPtr start_lsn,
ctx = StartupDecodingContext(output_plugin_options,
start_lsn, InvalidTransactionId, false,
- fast_forward, xl_routine, prepare_write,
- do_write, update_progress);
+ fast_forward, page_read, cleanup_cb,
+ prepare_write, do_write, update_progress);
/* call output plugin initialization callback */
old_context = MemoryContextSwitchTo(ctx->context);
@@ -483,7 +487,13 @@ DecodingContextFindStartpoint(LogicalDecodingContext *ctx)
char *err = NULL;
/* the read_page callback waits for new WAL */
- record = XLogReadRecord(ctx->reader, &err);
+ while (XLogReadRecord(ctx->reader, &record, &err) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!ctx->page_read(ctx->reader))
+ break;
+ }
+
if (err)
elog(ERROR, "%s", err);
if (!record)
diff --git a/src/backend/replication/logical/logicalfuncs.c b/src/backend/replication/logical/logicalfuncs.c
index b99c94e848..fcc81b358e 100644
--- a/src/backend/replication/logical/logicalfuncs.c
+++ b/src/backend/replication/logical/logicalfuncs.c
@@ -233,9 +233,8 @@ pg_logical_slot_get_changes_guts(FunctionCallInfo fcinfo, bool confirm, bool bin
ctx = CreateDecodingContext(InvalidXLogRecPtr,
options,
false,
- XL_ROUTINE(.page_read = read_local_xlog_page,
- .segment_open = wal_segment_open,
- .segment_close = wal_segment_close),
+ read_local_xlog_page,
+ wal_segment_close,
LogicalOutputPrepareWrite,
LogicalOutputWrite, NULL);
@@ -284,7 +283,13 @@ pg_logical_slot_get_changes_guts(FunctionCallInfo fcinfo, bool confirm, bool bin
XLogRecord *record;
char *errm = NULL;
- record = XLogReadRecord(ctx->reader, &errm);
+ while (XLogReadRecord(ctx->reader, &record, &errm) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!ctx->page_read(ctx->reader))
+ break;
+ }
+
if (errm)
elog(ERROR, "%s", errm);
diff --git a/src/backend/replication/slotfuncs.c b/src/backend/replication/slotfuncs.c
index 88033a79b2..8a678074d5 100644
--- a/src/backend/replication/slotfuncs.c
+++ b/src/backend/replication/slotfuncs.c
@@ -152,9 +152,8 @@ create_logical_replication_slot(char *name, char *plugin,
ctx = CreateInitDecodingContext(plugin, NIL,
false, /* just catalogs is OK */
restart_lsn,
- XL_ROUTINE(.page_read = read_local_xlog_page,
- .segment_open = wal_segment_open,
- .segment_close = wal_segment_close),
+ read_local_xlog_page,
+ wal_segment_close,
NULL, NULL, NULL);
/*
@@ -486,9 +485,8 @@ pg_logical_replication_slot_advance(XLogRecPtr moveto)
ctx = CreateDecodingContext(InvalidXLogRecPtr,
NIL,
true, /* fast_forward */
- XL_ROUTINE(.page_read = read_local_xlog_page,
- .segment_open = wal_segment_open,
- .segment_close = wal_segment_close),
+ read_local_xlog_page,
+ wal_segment_close,
NULL, NULL, NULL);
/*
@@ -513,7 +511,13 @@ pg_logical_replication_slot_advance(XLogRecPtr moveto)
* Read records. No changes are generated in fast_forward mode,
* but snapbuilder/slot statuses are updated properly.
*/
- record = XLogReadRecord(ctx->reader, &errm);
+ while (XLogReadRecord(ctx->reader, &record, &errm) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!ctx->page_read(ctx->reader))
+ break;
+ }
+
if (errm)
elog(ERROR, "%s", errm);
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index bf2711bddd..2fd24a9b14 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -580,10 +580,7 @@ StartReplication(StartReplicationCmd *cmd)
/* create xlogreader for physical replication */
xlogreader =
- XLogReaderAllocate(wal_segment_size, NULL,
- XL_ROUTINE(.segment_open = WalSndSegmentOpen,
- .segment_close = wal_segment_close),
- NULL);
+ XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
if (!xlogreader)
ereport(ERROR,
@@ -812,9 +809,11 @@ StartReplication(StartReplicationCmd *cmd)
* set every time WAL is flushed.
*/
static bool
-logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetRecPtr, char *cur_page)
+logical_read_xlog_page(XLogReaderState *state)
{
+ XLogRecPtr targetPagePtr = state->readPagePtr;
+ int reqLen = state->readLen;
+ char *cur_page = state->readBuf;
XLogRecPtr flushptr;
int count;
WALReadError errinfo;
@@ -842,7 +841,7 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
count = flushptr - targetPagePtr; /* part of the page available */
/* now actually read the data, we know it's there */
- if (!WALRead(state,
+ if (!WALRead(state, WalSndSegmentOpen, wal_segment_close,
cur_page,
targetPagePtr,
XLOG_BLCKSZ,
@@ -1014,9 +1013,8 @@ CreateReplicationSlot(CreateReplicationSlotCmd *cmd)
ctx = CreateInitDecodingContext(cmd->plugin, NIL, need_full_snapshot,
InvalidXLogRecPtr,
- XL_ROUTINE(.page_read = logical_read_xlog_page,
- .segment_open = WalSndSegmentOpen,
- .segment_close = wal_segment_close),
+ logical_read_xlog_page,
+ wal_segment_close,
WalSndPrepareWrite, WalSndWriteData,
WalSndUpdateProgress);
@@ -1179,9 +1177,8 @@ StartLogicalReplication(StartReplicationCmd *cmd)
*/
logical_decoding_ctx =
CreateDecodingContext(cmd->startpoint, cmd->options, false,
- XL_ROUTINE(.page_read = logical_read_xlog_page,
- .segment_open = WalSndSegmentOpen,
- .segment_close = wal_segment_close),
+ logical_read_xlog_page,
+ wal_segment_close,
WalSndPrepareWrite, WalSndWriteData,
WalSndUpdateProgress);
xlogreader = logical_decoding_ctx->reader;
@@ -2761,7 +2758,7 @@ XLogSendPhysical(void)
enlargeStringInfo(&output_message, nbytes);
retry:
- if (!WALRead(xlogreader,
+ if (!WALRead(xlogreader, WalSndSegmentOpen, wal_segment_close,
&output_message.data[output_message.len],
startptr,
nbytes,
@@ -2859,7 +2856,12 @@ XLogSendLogical(void)
*/
WalSndCaughtUp = false;
- record = XLogReadRecord(logical_decoding_ctx->reader, &errm);
+ while (XLogReadRecord(logical_decoding_ctx->reader, &record, &errm) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!logical_decoding_ctx->page_read(logical_decoding_ctx->reader))
+ break;
+ }
/* xlog record was invalid */
if (errm != NULL)
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index 0b7e73ae79..3dd6df4be6 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -40,15 +40,9 @@ static int xlogreadfd = -1;
static XLogSegNo xlogreadsegno = -1;
static char xlogfpath[MAXPGPATH];
-typedef struct XLogPageReadPrivate
-{
- const char *restoreCommand;
- int tliIndex;
-} XLogPageReadPrivate;
-
-static bool SimpleXLogPageRead(XLogReaderState *xlogreader,
- XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
+static bool SimpleXLogPageRead(XLogReaderState *xlogreader,
+ const char *datadir, int *tliIndex,
+ const char *restoreCommand);
/*
* Read WAL from the datadir/pg_wal, starting from 'startpoint' on timeline
@@ -62,20 +56,22 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
XLogRecord *record;
XLogReaderState *xlogreader;
char *errormsg;
- XLogPageReadPrivate private;
- private.tliIndex = tliIndex;
- private.restoreCommand = restoreCommand;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir,
- XL_ROUTINE(.page_read = &SimpleXLogPageRead),
- &private);
+ xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+
if (xlogreader == NULL)
pg_fatal("out of memory");
XLogBeginRead(xlogreader, startpoint);
do
{
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!SimpleXLogPageRead(xlogreader, datadir,
+ &tliIndex, restoreCommand))
+ break;
+ }
if (record == NULL)
{
@@ -113,19 +109,19 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex,
XLogRecord *record;
XLogReaderState *xlogreader;
char *errormsg;
- XLogPageReadPrivate private;
XLogRecPtr endptr;
- private.tliIndex = tliIndex;
- private.restoreCommand = restoreCommand;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir,
- XL_ROUTINE(.page_read = &SimpleXLogPageRead),
- &private);
+ xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
XLogBeginRead(xlogreader, ptr);
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!SimpleXLogPageRead(xlogreader, datadir, &tliIndex, restoreCommand))
+ break;
+ }
if (record == NULL)
{
if (errormsg)
@@ -160,7 +156,6 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
XLogRecPtr searchptr;
XLogReaderState *xlogreader;
char *errormsg;
- XLogPageReadPrivate private;
/*
* The given fork pointer points to the end of the last common record,
@@ -176,11 +171,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
forkptr += SizeOfXLogShortPHD;
}
- private.tliIndex = tliIndex;
- private.restoreCommand = restoreCommand;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir,
- XL_ROUTINE(.page_read = &SimpleXLogPageRead),
- &private);
+ xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
@@ -190,7 +181,13 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
uint8 info;
XLogBeginRead(xlogreader, searchptr);
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!SimpleXLogPageRead(xlogreader, datadir,
+ &tliIndex, restoreCommand))
+ break;
+ }
if (record == NULL)
{
@@ -237,10 +234,11 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
/* XLogReader callback function, to read a WAL page */
static bool
-SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *readBuf)
+SimpleXLogPageRead(XLogReaderState *xlogreader, const char *datadir,
+ int *tliIndex, const char *restoreCommand)
{
- XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
+ XLogRecPtr targetPagePtr = xlogreader->readPagePtr;
+ char *readBuf = xlogreader->readBuf;
uint32 targetPageOff;
XLogRecPtr targetSegEnd;
XLogSegNo targetSegNo;
@@ -273,14 +271,14 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
* be done both forward and backward, consider also switching timeline
* accordingly.
*/
- while (private->tliIndex < targetNentries - 1 &&
- targetHistory[private->tliIndex].end < targetSegEnd)
- private->tliIndex++;
- while (private->tliIndex > 0 &&
- targetHistory[private->tliIndex].begin >= targetSegEnd)
- private->tliIndex--;
+ while (*tliIndex < targetNentries - 1 &&
+ targetHistory[*tliIndex].end < targetSegEnd)
+ (*tliIndex)++;
+ while (*tliIndex > 0 &&
+ targetHistory[*tliIndex].begin >= targetSegEnd)
+ (*tliIndex)--;
- XLogFileName(xlogfname, targetHistory[private->tliIndex].tli,
+ XLogFileName(xlogfname, targetHistory[*tliIndex].tli,
xlogreadsegno, WalSegSz);
snprintf(xlogfpath, MAXPGPATH, "%s/" XLOGDIR "/%s",
@@ -293,7 +291,7 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
/*
* If we have no restore_command to execute, then exit.
*/
- if (private->restoreCommand == NULL)
+ if (restoreCommand == NULL)
{
pg_log_error("could not open file \"%s\": %m", xlogfpath);
xlogreader->readLen = -1;
@@ -307,7 +305,7 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
xlogreadfd = RestoreArchivedFile(xlogreader->segcxt.ws_dir,
xlogfname,
WalSegSz,
- private->restoreCommand);
+ restoreCommand);
if (xlogreadfd < 0)
{
@@ -349,7 +347,7 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
Assert(targetSegNo == xlogreadsegno);
- xlogreader->seg.ws_tli = targetHistory[private->tliIndex].tli;
+ xlogreader->seg.ws_tli = targetHistory[*tliIndex].tli;
xlogreader->readLen = XLOG_BLCKSZ;
return true;
}
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index ab0c28d9b7..02e71138cc 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -330,12 +330,17 @@ WALDumpCloseSegment(XLogReaderState *state)
state->seg.ws_file = -1;
}
-/* pg_waldump's XLogReaderRoutine->page_read callback */
+/*
+ * pg_waldump's WAL page rader, also used as page_read callback for
+ * XLogFindNextRecord
+ */
static bool
-WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetPtr, char *readBuff)
+WALDumpReadPage(XLogReaderState *state, void *priv)
{
- XLogDumpPrivate *private = state->private_data;
+ XLogRecPtr targetPagePtr = state->readPagePtr;
+ int reqLen = state->readLen;
+ char *readBuff = state->readBuf;
+ XLogDumpPrivate *private = (XLogDumpPrivate *) priv;
int count = XLOG_BLCKSZ;
WALReadError errinfo;
@@ -353,8 +358,8 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
}
}
- if (!WALRead(state, readBuff, targetPagePtr, count, private->timeline,
- &errinfo))
+ if (!WALRead(state, WALDumpOpenSegment, WALDumpCloseSegment,
+ readBuff, targetPagePtr, count, private->timeline, &errinfo))
{
WALOpenSegment *seg = &errinfo.wre_seg;
char fname[MAXPGPATH];
@@ -374,6 +379,7 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
(Size) errinfo.wre_req);
}
+ Assert(count >= state->readLen);
state->readLen = count;
return true;
}
@@ -1042,16 +1048,14 @@ main(int argc, char **argv)
/* we have everything we need, start reading */
xlogreader_state =
- XLogReaderAllocate(WalSegSz, waldir,
- XL_ROUTINE(.page_read = WALDumpReadPage,
- .segment_open = WALDumpOpenSegment,
- .segment_close = WALDumpCloseSegment),
- &private);
+ XLogReaderAllocate(WalSegSz, waldir, WALDumpCloseSegment);
+
if (!xlogreader_state)
fatal_error("out of memory");
/* first find a valid recptr to start from */
- first_record = XLogFindNextRecord(xlogreader_state, private.startptr);
+ first_record = XLogFindNextRecord(xlogreader_state, private.startptr,
+ &WALDumpReadPage, (void*) &private);
if (first_record == InvalidXLogRecPtr)
fatal_error("could not find a valid record after %X/%X",
@@ -1075,7 +1079,13 @@ main(int argc, char **argv)
for (;;)
{
/* try to read the next record */
- record = XLogReadRecord(xlogreader_state, &errormsg);
+ while (XLogReadRecord(xlogreader_state, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!WALDumpReadPage(xlogreader_state, (void *) &private))
+ break;
+ }
+
if (!record)
{
if (!config.follow || private.endptr_reached)
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index 6c8848e14f..6a7fe140cb 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -57,64 +57,15 @@ typedef struct WALSegmentContext
typedef struct XLogReaderState XLogReaderState;
-/* Function type definition for the read_page callback */
-typedef bool (*XLogPageReadCB) (XLogReaderState *xlogreader,
- XLogRecPtr targetPagePtr,
- int reqLen,
- XLogRecPtr targetRecPtr,
- char *readBuf);
+/* Function type definition for the segment cleanup callback */
+typedef void (*WALSegmentCleanupCB) (XLogReaderState *xlogreader);
+
+/* Function type definition for the open/close callbacks for WALRead() */
typedef void (*WALSegmentOpenCB) (XLogReaderState *xlogreader,
XLogSegNo nextSegNo,
TimeLineID *tli_p);
typedef void (*WALSegmentCloseCB) (XLogReaderState *xlogreader);
-typedef struct XLogReaderRoutine
-{
- /*
- * Data input callback
- *
- * This callback shall read at least reqLen valid bytes of the xlog page
- * starting at targetPagePtr, and store them in readBuf. The callback
- * shall return the number of bytes read (never more than XLOG_BLCKSZ), or
- * -1 on failure. The callback shall sleep, if necessary, to wait for the
- * requested bytes to become available. The callback will not be invoked
- * again for the same page unless more than the returned number of bytes
- * are needed.
- *
- * targetRecPtr is the position of the WAL record we're reading. Usually
- * it is equal to targetPagePtr + reqLen, but sometimes xlogreader needs
- * to read and verify the page or segment header, before it reads the
- * actual WAL record it's interested in. In that case, targetRecPtr can
- * be used to determine which timeline to read the page from.
- *
- * The callback shall set ->seg.ws_tli to the TLI of the file the page was
- * read from.
- */
- XLogPageReadCB page_read;
-
- /*
- * Callback to open the specified WAL segment for reading. ->seg.ws_file
- * shall be set to the file descriptor of the opened segment. In case of
- * failure, an error shall be raised by the callback and it shall not
- * return.
- *
- * "nextSegNo" is the number of the segment to be opened.
- *
- * "tli_p" is an input/output argument. WALRead() uses it to pass the
- * timeline in which the new segment should be found, but the callback can
- * use it to return the TLI that it actually opened.
- */
- WALSegmentOpenCB segment_open;
-
- /*
- * WAL segment close callback. ->seg.ws_file shall be set to a negative
- * number.
- */
- WALSegmentCloseCB segment_close;
-} XLogReaderRoutine;
-
-#define XL_ROUTINE(...) &(XLogReaderRoutine){__VA_ARGS__}
-
typedef struct
{
/* Is this block ref in use? */
@@ -144,12 +95,35 @@ typedef struct
uint16 data_bufsz;
} DecodedBkpBlock;
+/* Return code from XLogReadRecord */
+typedef enum XLogReadRecordResult
+{
+ XLREAD_SUCCESS, /* record is successfully read */
+ XLREAD_NEED_DATA, /* need more data. see XLogReadRecord. */
+ XLREAD_FAIL /* failed during reading a record */
+} XLogReadRecordResult;
+
+/*
+ * internal state of XLogReadRecord
+ *
+ * XLogReadState runs a state machine while reading a record. Theses states
+ * are not seen outside the function. Each state may repeat several times
+ * exiting requesting caller for new data. See the comment of XLogReadRecrod
+ * for details.
+ */
+typedef enum XLogReadRecordState {
+ XLREAD_NEXT_RECORD,
+ XLREAD_TOT_LEN,
+ XLREAD_FIRST_FRAGMENT,
+ XLREAD_CONTINUATION
+} XLogReadRecordState;
+
struct XLogReaderState
{
/*
* Operational callbacks
*/
- XLogReaderRoutine routine;
+ WALSegmentCleanupCB cleanup_cb;
/* ----------------------------------------
* Public parameters
@@ -162,18 +136,14 @@ struct XLogReaderState
*/
uint64 system_identifier;
- /*
- * Opaque data for callbacks to use. Not used by XLogReader.
- */
- void *private_data;
-
/*
* Start and end point of last record read. EndRecPtr is also used as the
* position to read next. Calling XLogBeginRead() sets EndRecPtr to the
* starting position and ReadRecPtr to invalid.
*/
- XLogRecPtr ReadRecPtr; /* start of last record read */
+ XLogRecPtr ReadRecPtr; /* start of last record read or being read */
XLogRecPtr EndRecPtr; /* end+1 of last record read */
+ XLogRecPtr PrevRecPtr; /* start of previous record read */
/* ----------------------------------------
* Communication with page reader
@@ -187,7 +157,9 @@ struct XLogReaderState
* the request, or -1 on error */
TimeLineID readPageTLI; /* TLI for data currently in readBuf */
char *readBuf; /* buffer to store data */
- bool page_verified; /* is the page on the buffer verified? */
+ bool page_verified; /* is the page header on the buffer verified? */
+ bool record_verified;/* is the current record header verified? */
+
/* ----------------------------------------
@@ -227,8 +199,6 @@ struct XLogReaderState
XLogRecPtr latestPagePtr;
TimeLineID latestPageTLI;
- /* beginning of the WAL record being read. */
- XLogRecPtr currRecPtr;
/* timeline to read it from, 0 if a lookup is required */
TimeLineID currTLI;
@@ -255,6 +225,15 @@ struct XLogReaderState
char *readRecordBuf;
uint32 readRecordBufSize;
+ /*
+ * XLogReadRecord() state
+ */
+ XLogReadRecordState readRecordState;/* state machine state */
+ int recordGotLen; /* amount of current record that has
+ * already been read */
+ int recordRemainLen; /* length of current record that remains */
+ XLogRecPtr recordContRecPtr; /* where the current record continues */
+
/* Buffer to hold error message */
char *errormsg_buf;
};
@@ -262,9 +241,7 @@ struct XLogReaderState
/* Get a new XLogReader */
extern XLogReaderState *XLogReaderAllocate(int wal_segment_size,
const char *waldir,
- XLogReaderRoutine *routine,
- void *private_data);
-extern XLogReaderRoutine *LocalXLogReaderRoutine(void);
+ WALSegmentCleanupCB cleanup_cb);
/* Free an XLogReader */
extern void XLogReaderFree(XLogReaderState *state);
@@ -272,12 +249,17 @@ extern void XLogReaderFree(XLogReaderState *state);
/* Position the XLogReader to given record */
extern void XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr);
#ifdef FRONTEND
-extern XLogRecPtr XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr);
+/* Function type definition for the read_page callback */
+typedef bool (*XLogFindNextRecordCB) (XLogReaderState *xlogreader,
+ void *private);
+extern XLogRecPtr XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
+ XLogFindNextRecordCB read_page, void *private);
#endif /* FRONTEND */
/* Read the next XLog record. Returns NULL on end-of-WAL or failure */
-extern struct XLogRecord *XLogReadRecord(XLogReaderState *state,
- char **errormsg);
+extern XLogReadRecordResult XLogReadRecord(XLogReaderState *state,
+ XLogRecord **record,
+ char **errormsg);
/* Validate a page */
extern bool XLogReaderValidatePageHeader(XLogReaderState *state,
@@ -297,6 +279,7 @@ typedef struct WALReadError
} WALReadError;
extern bool WALRead(XLogReaderState *state,
+ WALSegmentOpenCB segopenfn, WALSegmentCloseCB sgclosefn,
char *buf, XLogRecPtr startptr, Size count,
TimeLineID tli, WALReadError *errinfo);
diff --git a/src/include/access/xlogutils.h b/src/include/access/xlogutils.h
index c287edf206..482e4ced69 100644
--- a/src/include/access/xlogutils.h
+++ b/src/include/access/xlogutils.h
@@ -47,9 +47,7 @@ extern Buffer XLogReadBufferExtended(RelFileNode rnode, ForkNumber forknum,
extern Relation CreateFakeRelcacheEntry(RelFileNode rnode);
extern void FreeFakeRelcacheEntry(Relation fakerel);
-extern bool read_local_xlog_page(XLogReaderState *state,
- XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetRecPtr, char *cur_page);
+extern bool read_local_xlog_page(XLogReaderState *state);
extern void wal_segment_open(XLogReaderState *state,
XLogSegNo nextSegNo,
TimeLineID *tli_p);
diff --git a/src/include/replication/logical.h b/src/include/replication/logical.h
index c2f2475e5d..a743db49cf 100644
--- a/src/include/replication/logical.h
+++ b/src/include/replication/logical.h
@@ -29,6 +29,10 @@ typedef void (*LogicalOutputPluginWriterUpdateProgress) (struct LogicalDecodingC
TransactionId xid
);
+typedef struct LogicalDecodingContext LogicalDecodingContext;
+
+typedef bool (*LogicalDecodingXLogPageReadCB)(XLogReaderState *ctx);
+
typedef struct LogicalDecodingContext
{
/* memory context this is all allocated in */
@@ -39,6 +43,7 @@ typedef struct LogicalDecodingContext
/* infrastructure pieces for decoding */
XLogReaderState *reader;
+ LogicalDecodingXLogPageReadCB page_read;
struct ReorderBuffer *reorder;
struct SnapBuild *snapshot_builder;
@@ -95,14 +100,16 @@ extern LogicalDecodingContext *CreateInitDecodingContext(char *plugin,
List *output_plugin_options,
bool need_full_snapshot,
XLogRecPtr restart_lsn,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress);
extern LogicalDecodingContext *CreateDecodingContext(XLogRecPtr start_lsn,
List *output_plugin_options,
bool fast_forward,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress);
--
2.18.4
----Next_Part(Thu_Jul__2_13_53_30_2020_072)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
filename="v15-0003-Remove-globals-readOff-readLen-and-readSegNo.patch"
^ permalink raw reply [nested|flat] 70+ messages in thread
* [PATCH v15 2/4] Move page-reader out of XLogReadRecord
@ 2019-09-10 03:58 Kyotaro Horiguchi <[email protected]>
0 siblings, 0 replies; 70+ messages in thread
From: Kyotaro Horiguchi @ 2019-09-10 03:58 UTC (permalink / raw)
This is the second step of removing callbacks from WAL record reader.
Since it is essential to take in additional data while reading a
record, the function have to ask caller for new data while keeping
working state. Thus the function is turned into a state machine.
---
src/backend/access/transam/twophase.c | 15 +-
src/backend/access/transam/xlog.c | 58 +-
src/backend/access/transam/xlogreader.c | 700 +++++++++++-------
src/backend/access/transam/xlogutils.c | 17 +-
src/backend/replication/logical/logical.c | 26 +-
.../replication/logical/logicalfuncs.c | 13 +-
src/backend/replication/slotfuncs.c | 18 +-
src/backend/replication/walsender.c | 32 +-
src/bin/pg_rewind/parsexlog.c | 84 +--
src/bin/pg_waldump/pg_waldump.c | 36 +-
src/include/access/xlogreader.h | 121 ++-
src/include/access/xlogutils.h | 4 +-
src/include/replication/logical.h | 11 +-
13 files changed, 648 insertions(+), 487 deletions(-)
diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index 9b2e59bf0e..b0d60a0d0f 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -1330,11 +1330,8 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
XLogReaderState *xlogreader;
char *errormsg;
- xlogreader = XLogReaderAllocate(wal_segment_size, NULL,
- XL_ROUTINE(.page_read = &read_local_xlog_page,
- .segment_open = &wal_segment_open,
- .segment_close = &wal_segment_close),
- NULL);
+ xlogreader = XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+
if (!xlogreader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
@@ -1342,7 +1339,13 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
errdetail("Failed while allocating a WAL reading processor.")));
XLogBeginRead(xlogreader, lsn);
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!read_local_xlog_page(xlogreader))
+ break;
+ }
+
if (record == NULL)
ereport(ERROR,
(errcode_for_file_access(),
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index e570e56a24..f9b0108602 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -831,13 +831,6 @@ static XLogSource currentSource = XLOG_FROM_ANY;
static bool lastSourceFailed = false;
static bool pendingWalRcvRestart = false;
-typedef struct XLogPageReadPrivate
-{
- int emode;
- bool fetching_ckpt; /* are we fetching a checkpoint record? */
- bool randAccess;
-} XLogPageReadPrivate;
-
/*
* These variables track when we last obtained some WAL data to process,
* and where we got it from. (XLogReceiptSource is initially the same as
@@ -912,8 +905,8 @@ static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath,
static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli,
XLogSource source, bool notfoundOk);
static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, XLogSource source);
-static bool XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
+static bool XLogPageRead(XLogReaderState *xlogreader,
+ bool fetching_ckpt, int emode, bool randAccess);
static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
bool fetching_ckpt, XLogRecPtr tliRecPtr);
static int emode_for_corrupt_record(int emode, XLogRecPtr RecPtr);
@@ -1225,8 +1218,7 @@ XLogInsertRecord(XLogRecData *rdata,
appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len);
if (!debug_reader)
- debug_reader = XLogReaderAllocate(wal_segment_size, NULL,
- XL_ROUTINE(), NULL);
+ debug_reader = XLogReaderAllocate(wal_segment_size, NULL NULL);
if (!debug_reader)
{
@@ -4326,15 +4318,10 @@ CleanupBackupHistory(void)
* record is available.
*/
static XLogRecord *
-ReadRecord(XLogReaderState *xlogreader, int emode,
- bool fetching_ckpt)
+ReadRecord(XLogReaderState *xlogreader, int emode, bool fetching_ckpt)
{
XLogRecord *record;
- XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
-
- private->fetching_ckpt = fetching_ckpt;
- private->emode = emode;
- private->randAccess = (xlogreader->ReadRecPtr == InvalidXLogRecPtr);
+ bool randAccess = (xlogreader->ReadRecPtr == InvalidXLogRecPtr);
/* This is the first attempt to read this page. */
lastSourceFailed = false;
@@ -4342,8 +4329,16 @@ ReadRecord(XLogReaderState *xlogreader, int emode,
for (;;)
{
char *errormsg;
+ XLogReadRecordResult result;
+
+ while ((result = XLogReadRecord(xlogreader, &record, &errormsg))
+ == XLREAD_NEED_DATA)
+ {
+ if (!XLogPageRead(xlogreader, fetching_ckpt, emode, randAccess))
+ break;
+
+ }
- record = XLogReadRecord(xlogreader, &errormsg);
ReadRecPtr = xlogreader->ReadRecPtr;
EndRecPtr = xlogreader->EndRecPtr;
if (record == NULL)
@@ -6320,7 +6315,6 @@ StartupXLOG(void)
bool backupFromStandby = false;
DBState dbstate_at_startup;
XLogReaderState *xlogreader;
- XLogPageReadPrivate private;
bool fast_promoted = false;
struct stat st;
@@ -6480,13 +6474,9 @@ StartupXLOG(void)
OwnLatch(&XLogCtl->recoveryWakeupLatch);
/* Set up XLOG reader facility */
- MemSet(&private, 0, sizeof(XLogPageReadPrivate));
xlogreader =
- XLogReaderAllocate(wal_segment_size, NULL,
- XL_ROUTINE(.page_read = &XLogPageRead,
- .segment_open = NULL,
- .segment_close = wal_segment_close),
- &private);
+ XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+
if (!xlogreader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
@@ -11860,12 +11850,13 @@ CancelBackup(void)
* sleep and retry.
*/
static bool
-XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetRecPtr, char *readBuf)
+XLogPageRead(XLogReaderState *xlogreader,
+ bool fetching_ckpt, int emode, bool randAccess)
{
- XLogPageReadPrivate *private =
- (XLogPageReadPrivate *) xlogreader->private_data;
- int emode = private->emode;
+ char *readBuf = xlogreader->readBuf;
+ XLogRecPtr targetPagePtr = xlogreader->readPagePtr;
+ int reqLen = xlogreader->readLen;
+ XLogRecPtr targetRecPtr = xlogreader->ReadRecPtr;
uint32 targetPageOff;
XLogSegNo targetSegNo PG_USED_FOR_ASSERTS_ONLY;
int r;
@@ -11908,8 +11899,8 @@ retry:
flushedUpto < targetPagePtr + reqLen))
{
if (!WaitForWALToBecomeAvailable(targetPagePtr + reqLen,
- private->randAccess,
- private->fetching_ckpt,
+ randAccess,
+ fetching_ckpt,
targetRecPtr))
{
if (readFile >= 0)
@@ -12014,6 +12005,7 @@ retry:
goto next_record_is_invalid;
}
+ Assert(xlogreader->readPagePtr == targetPagePtr);
xlogreader->readLen = readLen;
return true;
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index 3d599325ee..9281b57379 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -40,7 +40,7 @@ static bool XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr,
int reqLen, bool header_inclusive);
static void XLogReaderInvalReadState(XLogReaderState *state);
static bool ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
- XLogRecPtr PrevRecPtr, XLogRecord *record, bool randAccess);
+ XLogRecPtr PrevRecPtr, XLogRecord *record);
static bool ValidXLogRecord(XLogReaderState *state, XLogRecord *record,
XLogRecPtr recptr);
static void ResetDecoder(XLogReaderState *state);
@@ -73,7 +73,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...)
*/
XLogReaderState *
XLogReaderAllocate(int wal_segment_size, const char *waldir,
- XLogReaderRoutine *routine, void *private_data)
+ WALSegmentCleanupCB cleanup_cb)
{
XLogReaderState *state;
@@ -84,7 +84,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
return NULL;
/* initialize caller-provided support functions */
- state->routine = *routine;
+ state->cleanup_cb = cleanup_cb;
state->max_block_id = -1;
@@ -107,8 +107,6 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
WALOpenSegmentInit(&state->seg, &state->segcxt, wal_segment_size,
waldir);
- /* system_identifier initialized to zeroes above */
- state->private_data = private_data;
/* ReadRecPtr, EndRecPtr and readLen initialized to zeroes above */
state->errormsg_buf = palloc_extended(MAX_ERRORMSG_LEN + 1,
MCXT_ALLOC_NO_OOM);
@@ -140,8 +138,8 @@ XLogReaderFree(XLogReaderState *state)
{
int block_id;
- if (state->seg.ws_file != -1)
- state->routine.segment_close(state);
+ if (state->seg.ws_file >= 0)
+ state->cleanup_cb(state);
for (block_id = 0; block_id <= XLR_MAX_BLOCK_ID; block_id++)
{
@@ -246,6 +244,7 @@ XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
/* Begin at the passed-in record pointer. */
state->EndRecPtr = RecPtr;
state->ReadRecPtr = InvalidXLogRecPtr;
+ state->readRecordState = XLREAD_NEXT_RECORD;
}
/*
@@ -254,317 +253,452 @@ XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
* XLogBeginRead() or XLogFindNextRecord() must be called before the first call
* to XLogReadRecord().
*
- * If the page_read callback fails to read the requested data, NULL is
- * returned. The callback is expected to have reported the error; errormsg
- * is set to NULL.
+ * This function may return XLREAD_NEED_DATA several times before returning a
+ * result record. The caller shall read in some new data then call this
+ * function again with the same parameters.
*
- * If the reading fails for some other reason, NULL is also returned, and
- * *errormsg is set to a string with details of the failure.
+ * When a record is successfully read, returns XLREAD_SUCCESS with result
+ * record being stored in *record. Otherwise *record is NULL.
*
- * The returned pointer (or *errormsg) points to an internal buffer that's
- * valid until the next call to XLogReadRecord.
+ * Returns XLREAD_NEED_DATA if more data is needed to finish reading the
+ * current record. In that case, state->readPagePtr and state->readLen inform
+ * the desired position and minimum length of data needed. The caller shall
+ * read in the requested data and set state->readBuf to point to a buffer
+ * containing it. The caller must also set state->readPageTLI and
+ * state->readLen to indicate the timeline that it was read from, and the
+ * length of data that is now available (which must be >= given readLen),
+ * respectively.
+ *
+ * If invalid data is encountered, returns XLREAD_FAIL with *record being set to
+ * NULL. *errormsg is set to a string with details of the failure.
+ * The returned pointer (or *errormsg) points to an internal buffer that's valid
+ * until the next call to XLogReadRecord.
+ *
+ *
+ * This function runs a state machine consists of the following states.
+ *
+ * XLREAD_NEXT_RECORD :
+ * The initial state, if called with valid RecPtr, try to read a record at
+ * that position. If invalid RecPtr is given try to read a record just after
+ * the last one previously read.
+ * This state ens after setting ReadRecPtr. Then goes to XLREAD_TOT_LEN.
+ *
+ * XLREAD_TOT_LEN:
+ * Examining record header. Ends after reading record total
+ * length. recordRemainLen and recordGotLen are initialized.
+ *
+ * XLREAD_FIRST_FRAGMENT:
+ * Reading the first fragment. Ends with finishing reading a single
+ * record. Goes to XLREAD_NEXT_RECORD if that's all or
+ * XLREAD_CONTINUATION if we have continuation.
+
+ * XLREAD_CONTINUATION:
+ * Reading continuation of record. Ends with finishing the whole record then
+ * goes to XLREAD_NEXT_RECORD. During this state, recordRemainLen indicates
+ * how much is left and readRecordBuf holds the partially assert
+ * record.recordContRecPtr points to the beginning of the next page where to
+ * continue.
+ *
+ * If wrong data found in any state, the state machine stays at the current
+ * state. This behavior allows to continue reading a reacord switching among
+ * different souces, while streaming replication.
*/
-XLogRecord *
-XLogReadRecord(XLogReaderState *state, char **errormsg)
+XLogReadRecordResult
+XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
{
- XLogRecPtr RecPtr;
- XLogRecord *record;
- XLogRecPtr targetPagePtr;
- bool randAccess;
- uint32 len,
- total_len;
- uint32 targetRecOff;
- uint32 pageHeaderSize;
- bool gotheader;
+ XLogRecord *prec;
- /*
- * randAccess indicates whether to verify the previous-record pointer of
- * the record we're reading. We only do this if we're reading
- * sequentially, which is what we initially assume.
- */
- randAccess = false;
+ *record = NULL;
/* reset error state */
*errormsg = NULL;
state->errormsg_buf[0] = '\0';
- ResetDecoder(state);
-
- RecPtr = state->EndRecPtr;
-
- if (state->ReadRecPtr != InvalidXLogRecPtr)
- {
- /* read the record after the one we just read */
-
- /*
- * EndRecPtr is pointing to end+1 of the previous WAL record. If
- * we're at a page boundary, no more records can fit on the current
- * page. We must skip over the page header, but we can't do that until
- * we've read in the page, since the header size is variable.
- */
- }
- else
- {
- /*
- * Caller supplied a position to start at.
- *
- * In this case, EndRecPtr should already be pointing to a valid
- * record starting position.
- */
- Assert(XRecOffIsValid(RecPtr));
- randAccess = true;
- }
-
- state->currRecPtr = RecPtr;
-
- targetPagePtr = RecPtr - (RecPtr % XLOG_BLCKSZ);
- targetRecOff = RecPtr % XLOG_BLCKSZ;
-
- /*
- * Read the page containing the record into state->readBuf. Request enough
- * byte to cover the whole record header, or at least the part of it that
- * fits on the same page.
- */
- while (XLogNeedData(state, targetPagePtr,
- Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
- targetRecOff != 0))
- {
- if (!state->routine.page_read(state, state->readPagePtr, state->readLen,
- RecPtr, state->readBuf))
- break;
- }
-
- if (!state->page_verified)
- goto err;
-
- /*
- * We have at least the page header, so we can examine it now.
- */
- pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
- if (targetRecOff == 0)
- {
- /*
- * At page start, so skip over page header.
- */
- RecPtr += pageHeaderSize;
- targetRecOff = pageHeaderSize;
- }
- else if (targetRecOff < pageHeaderSize)
- {
- report_invalid_record(state, "invalid record offset at %X/%X",
- (uint32) (RecPtr >> 32), (uint32) RecPtr);
- goto err;
- }
-
- if ((((XLogPageHeader) state->readBuf)->xlp_info & XLP_FIRST_IS_CONTRECORD) &&
- targetRecOff == pageHeaderSize)
- {
- report_invalid_record(state, "contrecord is requested by %X/%X",
- (uint32) (RecPtr >> 32), (uint32) RecPtr);
- goto err;
- }
-
- /* XLogNeedData has verified the page header */
- Assert(pageHeaderSize <= state->readLen);
-
- /*
- * Read the record length.
- *
- * NB: Even though we use an XLogRecord pointer here, the whole record
- * header might not fit on this page. xl_tot_len is the first field of the
- * struct, so it must be on this page (the records are MAXALIGNed), but we
- * cannot access any other fields until we've verified that we got the
- * whole header.
- */
- record = (XLogRecord *) (state->readBuf + RecPtr % XLOG_BLCKSZ);
- total_len = record->xl_tot_len;
-
- /*
- * If the whole record header is on this page, validate it immediately.
- * Otherwise do just a basic sanity check on xl_tot_len, and validate the
- * rest of the header after reading it from the next page. The xl_tot_len
- * check is necessary here to ensure that we enter the "Need to reassemble
- * record" code path below; otherwise we might fail to apply
- * ValidXLogRecordHeader at all.
- */
- if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
- {
- if (!ValidXLogRecordHeader(state, RecPtr, state->ReadRecPtr, record,
- randAccess))
- goto err;
- gotheader = true;
- }
- else
- {
- /* XXX: more validation should be done here */
- if (total_len < SizeOfXLogRecord)
- {
- report_invalid_record(state,
- "invalid record length at %X/%X: wanted %u, got %u",
- (uint32) (RecPtr >> 32), (uint32) RecPtr,
- (uint32) SizeOfXLogRecord, total_len);
- goto err;
- }
- gotheader = false;
- }
-
- len = XLOG_BLCKSZ - RecPtr % XLOG_BLCKSZ;
- if (total_len > len)
+ switch (state->readRecordState)
{
- /* Need to reassemble record */
- char *contdata;
- XLogPageHeader pageHeader;
- char *buffer;
- uint32 gotlen;
-
- /*
- * Enlarge readRecordBuf as needed.
- */
- if (total_len > state->readRecordBufSize &&
- !allocate_recordbuf(state, total_len))
- {
- /* We treat this as a "bogus data" condition */
- report_invalid_record(state, "record length %u at %X/%X too long",
- total_len,
- (uint32) (RecPtr >> 32), (uint32) RecPtr);
- goto err;
- }
+ case XLREAD_NEXT_RECORD:
+ ResetDecoder(state);
- /* Copy the first fragment of the record from the first page. */
- memcpy(state->readRecordBuf,
- state->readBuf + RecPtr % XLOG_BLCKSZ, len);
- buffer = state->readRecordBuf + len;
- gotlen = len;
-
- do
- {
- int rest_len = total_len - gotlen;
-
- /* Calculate pointer to beginning of next page */
- targetPagePtr += XLOG_BLCKSZ;
-
- /* Wait for the next page to become available */
- while (XLogNeedData(state, targetPagePtr,
- Min(rest_len, XLOG_BLCKSZ),
- false))
+ if (state->ReadRecPtr != InvalidXLogRecPtr)
{
- if (!state->routine.page_read(state, state->readPagePtr,
- state->readLen,
- state->ReadRecPtr,
- state->readBuf))
- break;
+ /* read the record after the one we just read */
+
+ /*
+ * EndRecPtr is pointing to end+1 of the previous WAL record.
+ * If we're at a page boundary, no more records can fit on the
+ * current page. We must skip over the page header, but we
+ * can't do that until we've read in the page, since the header
+ * size is variable.
+ */
+ state->PrevRecPtr = state->ReadRecPtr;
+ state->ReadRecPtr = state->EndRecPtr;
+ }
+ else
+ {
+ /*
+ * Caller supplied a position to start at.
+ *
+ * In this case, EndRecPtr should already be pointing to a
+ * valid record starting position.
+ */
+ Assert(XRecOffIsValid(state->EndRecPtr));
+ state->ReadRecPtr = state->EndRecPtr;
+
+ /*
+ * We cannot verify the previous-record pointer when we're
+ * seeking to a particular record. Reset PrevRecPtr so that we
+ * won't try doing that.
+ */
+ state->PrevRecPtr = InvalidXLogRecPtr;
+ state->EndRecPtr = InvalidXLogRecPtr; /* to be tidy */
}
+ state->record_verified = false;
+ state->readRecordState = XLREAD_TOT_LEN;
+ /* fall through */
+
+ case XLREAD_TOT_LEN:
+ {
+ uint32 total_len;
+ uint32 pageHeaderSize;
+ XLogRecPtr targetPagePtr;
+ uint32 targetRecOff;
+ XLogPageHeader pageHeader;
+
+ targetPagePtr =
+ state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
+ targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+
+ /*
+ * Check if we have enough data. For the first record in the page,
+ * the requesting length doesn't contain page header.
+ */
+ if (XLogNeedData(state, targetPagePtr,
+ Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+ targetRecOff != 0))
+ return XLREAD_NEED_DATA;
+
+ /* error out if caller supplied bogus page */
if (!state->page_verified)
goto err;
- Assert(SizeOfXLogShortPHD <= state->readLen);
+ /* examine page header now. */
+ pageHeaderSize =
+ XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+ if (targetRecOff == 0)
+ {
+ /* At page start, so skip over page header. */
+ state->ReadRecPtr += pageHeaderSize;
+ targetRecOff = pageHeaderSize;
+ }
+ else if (targetRecOff < pageHeaderSize)
+ {
+ report_invalid_record(state, "invalid record offset at %X/%X",
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr);
+ goto err;
+ }
- /* Check that the continuation on next page looks valid */
pageHeader = (XLogPageHeader) state->readBuf;
- if (!(pageHeader->xlp_info & XLP_FIRST_IS_CONTRECORD))
+ if ((pageHeader->xlp_info & XLP_FIRST_IS_CONTRECORD) &&
+ targetRecOff == pageHeaderSize)
{
- report_invalid_record(state,
- "there is no contrecord flag at %X/%X",
- (uint32) (RecPtr >> 32), (uint32) RecPtr);
+ report_invalid_record(state, "contrecord is requested by %X/%X",
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr);
goto err;
}
- /*
- * Cross-check that xlp_rem_len agrees with how much of the record
- * we expect there to be left.
- */
- if (pageHeader->xlp_rem_len == 0 ||
- total_len != (pageHeader->xlp_rem_len + gotlen))
- {
- report_invalid_record(state,
- "invalid contrecord length %u at %X/%X",
- pageHeader->xlp_rem_len,
- (uint32) (RecPtr >> 32), (uint32) RecPtr);
- goto err;
- }
-
- /* Append the continuation from this page to the buffer */
- pageHeaderSize = XLogPageHeaderSize(pageHeader);
-
+ /* XLogNeedData has verified the page header */
Assert(pageHeaderSize <= state->readLen);
- contdata = (char *) state->readBuf + pageHeaderSize;
- len = XLOG_BLCKSZ - pageHeaderSize;
- if (pageHeader->xlp_rem_len < len)
- len = pageHeader->xlp_rem_len;
+ /*
+ * Read the record length.
+ *
+ * NB: Even though we use an XLogRecord pointer here, the whole
+ * record header might not fit on this page. xl_tot_len is the first
+ * field of the struct, so it must be on this page (the records are
+ * MAXALIGNed), but we cannot access any other fields until we've
+ * verified that we got the whole header.
+ */
+ prec = (XLogRecord *) (state->readBuf +
+ state->ReadRecPtr % XLOG_BLCKSZ);
+ total_len = prec->xl_tot_len;
- Assert (pageHeaderSize + len <= state->readLen);
- memcpy(buffer, (char *) contdata, len);
- buffer += len;
- gotlen += len;
+ /*
+ * If the whole record header is on this page, validate it
+ * immediately. Otherwise do just a basic sanity check on
+ * xl_tot_len, and validate the rest of the header after reading it
+ * from the next page. The xl_tot_len check is necessary here to
+ * ensure that we enter the XLREAD_CONTINUATION state below;
+ * otherwise we might fail to apply ValidXLogRecordHeader at all.
+ */
+ if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
+ {
+ if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
+ state->PrevRecPtr, prec))
+ goto err;
- /* If we just reassembled the record header, validate it. */
- if (!gotheader)
+ state->record_verified = true;
+ }
+ else
{
- record = (XLogRecord *) state->readRecordBuf;
- if (!ValidXLogRecordHeader(state, RecPtr, state->ReadRecPtr,
- record, randAccess))
+ /* XXX: more validation should be done here */
+ if (total_len < SizeOfXLogRecord)
+ {
+ report_invalid_record(state,
+ "invalid record length at %X/%X: wanted %u, got %u",
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr,
+ (uint32) SizeOfXLogRecord, total_len);
goto err;
- gotheader = true;
+ }
}
- } while (gotlen < total_len);
-
- Assert(gotheader);
- record = (XLogRecord *) state->readRecordBuf;
- if (!ValidXLogRecord(state, record, RecPtr))
- goto err;
+ /*
+ * Wait for the rest of the record, or the part of the record that
+ * fit on the first page if crossed a page boundary, to become
+ * available.
+ */
+ state->recordGotLen = 0;
+ state->recordRemainLen = total_len;
+ state->readRecordState = XLREAD_FIRST_FRAGMENT;
+ }
+ /* fall through */
- pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
- state->ReadRecPtr = RecPtr;
- state->EndRecPtr = targetPagePtr + pageHeaderSize
- + MAXALIGN(pageHeader->xlp_rem_len);
- }
- else
- {
- /* Wait for the record data to become available */
- while (XLogNeedData(state, targetPagePtr,
- Min(targetRecOff + total_len, XLOG_BLCKSZ), true))
+ case XLREAD_FIRST_FRAGMENT:
{
- if (!state->routine.page_read(state, state->readPagePtr,
- state->readLen,
- state->ReadRecPtr, state->readBuf))
+ uint32 total_len = state->recordRemainLen;
+ uint32 request_len;
+ uint32 record_len;
+ XLogRecPtr targetPagePtr;
+ uint32 targetRecOff;
+
+ /*
+ * Wait for the rest of the record on the first page to become
+ * available
+ */
+ targetPagePtr =
+ state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
+ targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+
+ request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ);
+ record_len = request_len - targetRecOff;
+
+ /* ReadRecPtr contains page header */
+ Assert (targetRecOff != 0);
+ if (XLogNeedData(state, targetPagePtr, request_len, true))
+ return XLREAD_NEED_DATA;
+
+ /* error out if caller supplied bogus page */
+ if (!state->page_verified)
+ goto err;
+
+ prec = (XLogRecord *) (state->readBuf + targetRecOff);
+
+ /* validate record header if not yet */
+ if (!state->record_verified && record_len >= SizeOfXLogRecord)
+ {
+ if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
+ state->PrevRecPtr, prec))
+ goto err;
+
+ state->record_verified = true;
+ }
+
+
+ if (total_len == record_len)
+ {
+ /* Record does not cross a page boundary */
+ Assert(state->record_verified);
+
+ if (!ValidXLogRecord(state, prec, state->ReadRecPtr))
+ goto err;
+
+ state->record_verified = true; /* to be tidy */
+
+ /* We already checked the header earlier */
+ state->EndRecPtr = state->ReadRecPtr + MAXALIGN(record_len);
+
+ *record = prec;
+ state->readRecordState = XLREAD_NEXT_RECORD;
break;
+ }
+
+ /*
+ * The record continues on the next page. Need to reassemble
+ * record
+ */
+ Assert(total_len > record_len);
+
+ /* Enlarge readRecordBuf as needed. */
+ if (total_len > state->readRecordBufSize &&
+ !allocate_recordbuf(state, total_len))
+ {
+ /* We treat this as a "bogus data" condition */
+ report_invalid_record(state,
+ "record length %u at %X/%X too long",
+ total_len,
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr);
+ goto err;
+ }
+
+ /* Copy the first fragment of the record from the first page. */
+ memcpy(state->readRecordBuf, state->readBuf + targetRecOff,
+ record_len);
+ state->recordGotLen += record_len;
+ state->recordRemainLen -= record_len;
+
+ /* Calculate pointer to beginning of next page */
+ state->recordContRecPtr = state->ReadRecPtr + record_len;
+ Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0);
+
+ state->readRecordState = XLREAD_CONTINUATION;
}
+ /* fall through */
+
+ case XLREAD_CONTINUATION:
+ {
+ XLogPageHeader pageHeader;
+ uint32 pageHeaderSize;
+ XLogRecPtr targetPagePtr;
+
+ /* we enter this state only if we haven't read the whole record. */
+ Assert (state->recordRemainLen > 0);
+
+ while(state->recordRemainLen > 0)
+ {
+ char *contdata;
+ uint32 request_len;
+ uint32 record_len;
+
+ /* Wait for the next page to become available */
+ targetPagePtr = state->recordContRecPtr;
+
+ /* this request contains page header */
+ Assert (targetPagePtr != 0);
+ if (XLogNeedData(state, targetPagePtr,
+ Min(state->recordRemainLen, XLOG_BLCKSZ),
+ false))
+ return XLREAD_NEED_DATA;
+
+ if (!state->page_verified)
+ goto err;
+
+ Assert(SizeOfXLogShortPHD <= state->readLen);
- if (!state->page_verified)
- goto err;
+ /* Check that the continuation on next page looks valid */
+ pageHeader = (XLogPageHeader) state->readBuf;
+ if (!(pageHeader->xlp_info & XLP_FIRST_IS_CONTRECORD))
+ {
+ report_invalid_record(
+ state,
+ "there is no contrecord flag at %X/%X reading %X/%X",
+ (uint32) (state->recordContRecPtr >> 32),
+ (uint32) state->recordContRecPtr,
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr);
+ goto err;
+ }
- /* Record does not cross a page boundary */
- if (!ValidXLogRecord(state, record, RecPtr))
- goto err;
+ /*
+ * Cross-check that xlp_rem_len agrees with how much of the
+ * record we expect there to be left.
+ */
+ if (pageHeader->xlp_rem_len == 0 ||
+ pageHeader->xlp_rem_len != state->recordRemainLen)
+ {
+ report_invalid_record(
+ state,
+ "invalid contrecord length %u at %X/%X reading %X/%X, expected %u",
+ pageHeader->xlp_rem_len,
+ (uint32) (state->recordContRecPtr >> 32),
+ (uint32) state->recordContRecPtr,
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr,
+ state->recordRemainLen);
+ goto err;
+ }
- state->EndRecPtr = RecPtr + MAXALIGN(total_len);
+ /* Append the continuation from this page to the buffer */
+ pageHeaderSize = XLogPageHeaderSize(pageHeader);
- state->ReadRecPtr = RecPtr;
+ /*
+ * XLogNeedData should have ensured that the whole page header
+ * was read
+ */
+ Assert(state->readLen >= pageHeaderSize);
+
+ contdata = (char *) state->readBuf + pageHeaderSize;
+ record_len = XLOG_BLCKSZ - pageHeaderSize;
+ if (pageHeader->xlp_rem_len < record_len)
+ record_len = pageHeader->xlp_rem_len;
+
+ request_len = record_len + pageHeaderSize;
+
+ /* XLogNeedData should have ensured all needed data was read */
+ Assert (state->readLen >= request_len);
+
+ memcpy(state->readRecordBuf + state->recordGotLen,
+ (char *) contdata, record_len);
+ state->recordGotLen += record_len;
+ state->recordRemainLen -= record_len;
+
+ /* If we just reassembled the record header, validate it. */
+ if (!state->record_verified)
+ {
+ Assert(state->recordGotLen >= SizeOfXLogRecord);
+ if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
+ state->PrevRecPtr,
+ (XLogRecord *) state->readRecordBuf))
+ goto err;
+
+ state->record_verified = true;
+ }
+
+ /* Calculate pointer to beginning of next page, and continue */
+ state->recordContRecPtr += XLOG_BLCKSZ;
+ }
+
+ /* targetPagePtr is pointing the last-read page here */
+ prec = (XLogRecord *) state->readRecordBuf;
+ if (!ValidXLogRecord(state, prec, state->ReadRecPtr))
+ goto err;
+
+ pageHeaderSize =
+ XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+ state->EndRecPtr = targetPagePtr + pageHeaderSize
+ + MAXALIGN(pageHeader->xlp_rem_len);
+
+ *record = prec;
+ state->readRecordState = XLREAD_NEXT_RECORD;
+ break;
+ }
}
/*
* Special processing if it's an XLOG SWITCH record
*/
- if (record->xl_rmid == RM_XLOG_ID &&
- (record->xl_info & ~XLR_INFO_MASK) == XLOG_SWITCH)
+ if ((*record)->xl_rmid == RM_XLOG_ID &&
+ ((*record)->xl_info & ~XLR_INFO_MASK) == XLOG_SWITCH)
{
/* Pretend it extends to end of segment */
state->EndRecPtr += state->segcxt.ws_segsize - 1;
state->EndRecPtr -= XLogSegmentOffset(state->EndRecPtr, state->segcxt.ws_segsize);
}
- if (DecodeXLogRecord(state, record, errormsg))
- return record;
- else
- return NULL;
+ Assert (!*record || state->readLen >= 0);
+ if (DecodeXLogRecord(state, *record, errormsg))
+ return XLREAD_SUCCESS;
+
+ *record = NULL;
+ return XLREAD_FAIL;
err:
/*
- * Invalidate the read state. We might read from a different source after
+ * Invalidate the read page. We might read from a different source after
* failure.
*/
XLogReaderInvalReadState(state);
@@ -572,7 +706,8 @@ err:
if (state->errormsg_buf[0] != '\0')
*errormsg = state->errormsg_buf;
- return NULL;
+ *record = NULL;
+ return XLREAD_FAIL;
}
/*
@@ -724,11 +859,12 @@ XLogReaderInvalReadState(XLogReaderState *state)
*
* This is just a convenience subroutine to avoid duplicated code in
* XLogReadRecord. It's not intended for use from anywhere else.
+ *
+ * If PrevRecPtr is valid, the xl_prev is is cross-checked with it.
*/
static bool
ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
- XLogRecPtr PrevRecPtr, XLogRecord *record,
- bool randAccess)
+ XLogRecPtr PrevRecPtr, XLogRecord *record)
{
if (record->xl_tot_len < SizeOfXLogRecord)
{
@@ -746,7 +882,7 @@ ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
(uint32) RecPtr);
return false;
}
- if (randAccess)
+ if (PrevRecPtr == InvalidXLogRecPtr)
{
/*
* We can't exactly verify the prev-link, but surely it should be less
@@ -978,11 +1114,14 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
* XLogReadRecord() will read the next valid record.
*/
XLogRecPtr
-XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
+XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
+ XLogFindNextRecordCB read_page, void *private)
{
XLogRecPtr tmpRecPtr;
XLogRecPtr found = InvalidXLogRecPtr;
XLogPageHeader header;
+ XLogRecord *record;
+ XLogReadRecordResult result;
char *errormsg;
Assert(!XLogRecPtrIsInvalid(RecPtr));
@@ -1015,9 +1154,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
while(XLogNeedData(state, targetPagePtr, targetRecOff,
targetRecOff != 0))
{
- if (!state->routine.page_read(state, state->readPagePtr,
- state->readLen,
- state->ReadRecPtr, state->readBuf))
+ if (!read_page(state, private))
break;
}
@@ -1069,8 +1206,16 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
* or we just jumped over the remaining data of a continuation.
*/
XLogBeginRead(state, tmpRecPtr);
- while (XLogReadRecord(state, &errormsg) != NULL)
+ while ((result = XLogReadRecord(state, &record, &errormsg)) !=
+ XLREAD_FAIL)
{
+ if (result == XLREAD_NEED_DATA)
+ {
+ if (!read_page(state, private))
+ goto err;
+ continue;
+ }
+
/* past the record we've found, break out */
if (RecPtr <= state->ReadRecPtr)
{
@@ -1090,9 +1235,9 @@ err:
#endif /* FRONTEND */
/*
- * Helper function to ease writing of XLogRoutine->page_read callbacks.
- * If this function is used, caller must supply a segment_open callback in
- * 'state', as that is used here.
+ * Helper function to ease writing of page_read callback.
+ * If this function is used, caller must supply a segment_open callback and
+ * segment_close callback as that is used here.
*
* Read 'count' bytes into 'buf', starting at location 'startptr', from WAL
* fetched from timeline 'tli'.
@@ -1105,6 +1250,7 @@ err:
*/
bool
WALRead(XLogReaderState *state,
+ WALSegmentOpenCB segopenfn, WALSegmentCloseCB segclosefn,
char *buf, XLogRecPtr startptr, Size count, TimeLineID tli,
WALReadError *errinfo)
{
@@ -1136,10 +1282,10 @@ WALRead(XLogReaderState *state,
XLogSegNo nextSegNo;
if (state->seg.ws_file >= 0)
- state->routine.segment_close(state);
+ segclosefn(state);
XLByteToSeg(recptr, nextSegNo, state->segcxt.ws_segsize);
- state->routine.segment_open(state, nextSegNo, &tli);
+ segopenfn(state, nextSegNo, &tli);
/* This shouldn't happen -- indicates a bug in segment_open */
Assert(state->seg.ws_file >= 0);
diff --git a/src/backend/access/transam/xlogutils.c b/src/backend/access/transam/xlogutils.c
index 1b10b6df20..303333571f 100644
--- a/src/backend/access/transam/xlogutils.c
+++ b/src/backend/access/transam/xlogutils.c
@@ -689,8 +689,7 @@ XLogTruncateRelation(RelFileNode rnode, ForkNumber forkNum,
void
XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wantLength)
{
- const XLogRecPtr lastReadPage = state->seg.ws_segno *
- state->segcxt.ws_segsize + state->readLen;
+ const XLogRecPtr lastReadPage = state->readPagePtr;
Assert(wantPage != InvalidXLogRecPtr && wantPage % XLOG_BLCKSZ == 0);
Assert(wantLength <= XLOG_BLCKSZ);
@@ -705,7 +704,7 @@ XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wa
* current TLI has since become historical.
*/
if (lastReadPage == wantPage &&
- state->readLen != 0 &&
+ state->page_verified &&
lastReadPage + state->readLen >= wantPage + Min(wantLength, XLOG_BLCKSZ - 1))
return;
@@ -792,6 +791,7 @@ wal_segment_open(XLogReaderState *state, XLogSegNo nextSegNo,
char path[MAXPGPATH];
XLogFilePath(path, tli, nextSegNo, state->segcxt.ws_segsize);
+ elog(LOG, "HOGE: %lu, %d => %s", nextSegNo, tli, path);
state->seg.ws_file = BasicOpenFile(path, O_RDONLY | PG_BINARY);
if (state->seg.ws_file >= 0)
return;
@@ -829,9 +829,11 @@ wal_segment_close(XLogReaderState *state)
* loop for now.
*/
bool
-read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *cur_page)
+read_local_xlog_page(XLogReaderState *state)
{
+ XLogRecPtr targetPagePtr = state->readPagePtr;
+ int reqLen = state->readLen;
+ char *cur_page = state->readBuf;
XLogRecPtr read_upto,
loc;
TimeLineID tli;
@@ -944,11 +946,12 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
* as 'count', read the whole page anyway. It's guaranteed to be
* zero-padded up to the page boundary if it's incomplete.
*/
- if (!WALRead(state, cur_page, targetPagePtr, XLOG_BLCKSZ, tli,
- &errinfo))
+ if (!WALRead(state, wal_segment_open, wal_segment_close,
+ cur_page, targetPagePtr, XLOG_BLCKSZ, tli, &errinfo))
WALReadRaiseError(&errinfo);
/* number of valid bytes in the buffer */
+ state->readPagePtr = targetPagePtr;
state->readLen = count;
return true;
}
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index 61902be3b0..a15b0b3355 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -120,7 +120,8 @@ StartupDecodingContext(List *output_plugin_options,
TransactionId xmin_horizon,
bool need_full_snapshot,
bool fast_forward,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress)
@@ -169,11 +170,12 @@ StartupDecodingContext(List *output_plugin_options,
ctx->slot = slot;
- ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, xl_routine, ctx);
+ ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, cleanup_cb);
if (!ctx->reader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
errmsg("out of memory")));
+ ctx->page_read = page_read;
ctx->reorder = ReorderBufferAllocate();
ctx->snapshot_builder =
@@ -231,7 +233,8 @@ CreateInitDecodingContext(char *plugin,
List *output_plugin_options,
bool need_full_snapshot,
XLogRecPtr restart_lsn,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress)
@@ -328,7 +331,7 @@ CreateInitDecodingContext(char *plugin,
ctx = StartupDecodingContext(NIL, restart_lsn, xmin_horizon,
need_full_snapshot, false,
- xl_routine, prepare_write, do_write,
+ page_read, cleanup_cb, prepare_write, do_write,
update_progress);
/* call output plugin initialization callback */
@@ -376,7 +379,8 @@ LogicalDecodingContext *
CreateDecodingContext(XLogRecPtr start_lsn,
List *output_plugin_options,
bool fast_forward,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress)
@@ -429,8 +433,8 @@ CreateDecodingContext(XLogRecPtr start_lsn,
ctx = StartupDecodingContext(output_plugin_options,
start_lsn, InvalidTransactionId, false,
- fast_forward, xl_routine, prepare_write,
- do_write, update_progress);
+ fast_forward, page_read, cleanup_cb,
+ prepare_write, do_write, update_progress);
/* call output plugin initialization callback */
old_context = MemoryContextSwitchTo(ctx->context);
@@ -483,7 +487,13 @@ DecodingContextFindStartpoint(LogicalDecodingContext *ctx)
char *err = NULL;
/* the read_page callback waits for new WAL */
- record = XLogReadRecord(ctx->reader, &err);
+ while (XLogReadRecord(ctx->reader, &record, &err) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!ctx->page_read(ctx->reader))
+ break;
+ }
+
if (err)
elog(ERROR, "%s", err);
if (!record)
diff --git a/src/backend/replication/logical/logicalfuncs.c b/src/backend/replication/logical/logicalfuncs.c
index b99c94e848..fcc81b358e 100644
--- a/src/backend/replication/logical/logicalfuncs.c
+++ b/src/backend/replication/logical/logicalfuncs.c
@@ -233,9 +233,8 @@ pg_logical_slot_get_changes_guts(FunctionCallInfo fcinfo, bool confirm, bool bin
ctx = CreateDecodingContext(InvalidXLogRecPtr,
options,
false,
- XL_ROUTINE(.page_read = read_local_xlog_page,
- .segment_open = wal_segment_open,
- .segment_close = wal_segment_close),
+ read_local_xlog_page,
+ wal_segment_close,
LogicalOutputPrepareWrite,
LogicalOutputWrite, NULL);
@@ -284,7 +283,13 @@ pg_logical_slot_get_changes_guts(FunctionCallInfo fcinfo, bool confirm, bool bin
XLogRecord *record;
char *errm = NULL;
- record = XLogReadRecord(ctx->reader, &errm);
+ while (XLogReadRecord(ctx->reader, &record, &errm) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!ctx->page_read(ctx->reader))
+ break;
+ }
+
if (errm)
elog(ERROR, "%s", errm);
diff --git a/src/backend/replication/slotfuncs.c b/src/backend/replication/slotfuncs.c
index 88033a79b2..8a678074d5 100644
--- a/src/backend/replication/slotfuncs.c
+++ b/src/backend/replication/slotfuncs.c
@@ -152,9 +152,8 @@ create_logical_replication_slot(char *name, char *plugin,
ctx = CreateInitDecodingContext(plugin, NIL,
false, /* just catalogs is OK */
restart_lsn,
- XL_ROUTINE(.page_read = read_local_xlog_page,
- .segment_open = wal_segment_open,
- .segment_close = wal_segment_close),
+ read_local_xlog_page,
+ wal_segment_close,
NULL, NULL, NULL);
/*
@@ -486,9 +485,8 @@ pg_logical_replication_slot_advance(XLogRecPtr moveto)
ctx = CreateDecodingContext(InvalidXLogRecPtr,
NIL,
true, /* fast_forward */
- XL_ROUTINE(.page_read = read_local_xlog_page,
- .segment_open = wal_segment_open,
- .segment_close = wal_segment_close),
+ read_local_xlog_page,
+ wal_segment_close,
NULL, NULL, NULL);
/*
@@ -513,7 +511,13 @@ pg_logical_replication_slot_advance(XLogRecPtr moveto)
* Read records. No changes are generated in fast_forward mode,
* but snapbuilder/slot statuses are updated properly.
*/
- record = XLogReadRecord(ctx->reader, &errm);
+ while (XLogReadRecord(ctx->reader, &record, &errm) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!ctx->page_read(ctx->reader))
+ break;
+ }
+
if (errm)
elog(ERROR, "%s", errm);
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index bf2711bddd..2fd24a9b14 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -580,10 +580,7 @@ StartReplication(StartReplicationCmd *cmd)
/* create xlogreader for physical replication */
xlogreader =
- XLogReaderAllocate(wal_segment_size, NULL,
- XL_ROUTINE(.segment_open = WalSndSegmentOpen,
- .segment_close = wal_segment_close),
- NULL);
+ XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
if (!xlogreader)
ereport(ERROR,
@@ -812,9 +809,11 @@ StartReplication(StartReplicationCmd *cmd)
* set every time WAL is flushed.
*/
static bool
-logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetRecPtr, char *cur_page)
+logical_read_xlog_page(XLogReaderState *state)
{
+ XLogRecPtr targetPagePtr = state->readPagePtr;
+ int reqLen = state->readLen;
+ char *cur_page = state->readBuf;
XLogRecPtr flushptr;
int count;
WALReadError errinfo;
@@ -842,7 +841,7 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
count = flushptr - targetPagePtr; /* part of the page available */
/* now actually read the data, we know it's there */
- if (!WALRead(state,
+ if (!WALRead(state, WalSndSegmentOpen, wal_segment_close,
cur_page,
targetPagePtr,
XLOG_BLCKSZ,
@@ -1014,9 +1013,8 @@ CreateReplicationSlot(CreateReplicationSlotCmd *cmd)
ctx = CreateInitDecodingContext(cmd->plugin, NIL, need_full_snapshot,
InvalidXLogRecPtr,
- XL_ROUTINE(.page_read = logical_read_xlog_page,
- .segment_open = WalSndSegmentOpen,
- .segment_close = wal_segment_close),
+ logical_read_xlog_page,
+ wal_segment_close,
WalSndPrepareWrite, WalSndWriteData,
WalSndUpdateProgress);
@@ -1179,9 +1177,8 @@ StartLogicalReplication(StartReplicationCmd *cmd)
*/
logical_decoding_ctx =
CreateDecodingContext(cmd->startpoint, cmd->options, false,
- XL_ROUTINE(.page_read = logical_read_xlog_page,
- .segment_open = WalSndSegmentOpen,
- .segment_close = wal_segment_close),
+ logical_read_xlog_page,
+ wal_segment_close,
WalSndPrepareWrite, WalSndWriteData,
WalSndUpdateProgress);
xlogreader = logical_decoding_ctx->reader;
@@ -2761,7 +2758,7 @@ XLogSendPhysical(void)
enlargeStringInfo(&output_message, nbytes);
retry:
- if (!WALRead(xlogreader,
+ if (!WALRead(xlogreader, WalSndSegmentOpen, wal_segment_close,
&output_message.data[output_message.len],
startptr,
nbytes,
@@ -2859,7 +2856,12 @@ XLogSendLogical(void)
*/
WalSndCaughtUp = false;
- record = XLogReadRecord(logical_decoding_ctx->reader, &errm);
+ while (XLogReadRecord(logical_decoding_ctx->reader, &record, &errm) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!logical_decoding_ctx->page_read(logical_decoding_ctx->reader))
+ break;
+ }
/* xlog record was invalid */
if (errm != NULL)
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index 0b7e73ae79..3dd6df4be6 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -40,15 +40,9 @@ static int xlogreadfd = -1;
static XLogSegNo xlogreadsegno = -1;
static char xlogfpath[MAXPGPATH];
-typedef struct XLogPageReadPrivate
-{
- const char *restoreCommand;
- int tliIndex;
-} XLogPageReadPrivate;
-
-static bool SimpleXLogPageRead(XLogReaderState *xlogreader,
- XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
+static bool SimpleXLogPageRead(XLogReaderState *xlogreader,
+ const char *datadir, int *tliIndex,
+ const char *restoreCommand);
/*
* Read WAL from the datadir/pg_wal, starting from 'startpoint' on timeline
@@ -62,20 +56,22 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
XLogRecord *record;
XLogReaderState *xlogreader;
char *errormsg;
- XLogPageReadPrivate private;
- private.tliIndex = tliIndex;
- private.restoreCommand = restoreCommand;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir,
- XL_ROUTINE(.page_read = &SimpleXLogPageRead),
- &private);
+ xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+
if (xlogreader == NULL)
pg_fatal("out of memory");
XLogBeginRead(xlogreader, startpoint);
do
{
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!SimpleXLogPageRead(xlogreader, datadir,
+ &tliIndex, restoreCommand))
+ break;
+ }
if (record == NULL)
{
@@ -113,19 +109,19 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex,
XLogRecord *record;
XLogReaderState *xlogreader;
char *errormsg;
- XLogPageReadPrivate private;
XLogRecPtr endptr;
- private.tliIndex = tliIndex;
- private.restoreCommand = restoreCommand;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir,
- XL_ROUTINE(.page_read = &SimpleXLogPageRead),
- &private);
+ xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
XLogBeginRead(xlogreader, ptr);
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!SimpleXLogPageRead(xlogreader, datadir, &tliIndex, restoreCommand))
+ break;
+ }
if (record == NULL)
{
if (errormsg)
@@ -160,7 +156,6 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
XLogRecPtr searchptr;
XLogReaderState *xlogreader;
char *errormsg;
- XLogPageReadPrivate private;
/*
* The given fork pointer points to the end of the last common record,
@@ -176,11 +171,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
forkptr += SizeOfXLogShortPHD;
}
- private.tliIndex = tliIndex;
- private.restoreCommand = restoreCommand;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir,
- XL_ROUTINE(.page_read = &SimpleXLogPageRead),
- &private);
+ xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
@@ -190,7 +181,13 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
uint8 info;
XLogBeginRead(xlogreader, searchptr);
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!SimpleXLogPageRead(xlogreader, datadir,
+ &tliIndex, restoreCommand))
+ break;
+ }
if (record == NULL)
{
@@ -237,10 +234,11 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
/* XLogReader callback function, to read a WAL page */
static bool
-SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *readBuf)
+SimpleXLogPageRead(XLogReaderState *xlogreader, const char *datadir,
+ int *tliIndex, const char *restoreCommand)
{
- XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
+ XLogRecPtr targetPagePtr = xlogreader->readPagePtr;
+ char *readBuf = xlogreader->readBuf;
uint32 targetPageOff;
XLogRecPtr targetSegEnd;
XLogSegNo targetSegNo;
@@ -273,14 +271,14 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
* be done both forward and backward, consider also switching timeline
* accordingly.
*/
- while (private->tliIndex < targetNentries - 1 &&
- targetHistory[private->tliIndex].end < targetSegEnd)
- private->tliIndex++;
- while (private->tliIndex > 0 &&
- targetHistory[private->tliIndex].begin >= targetSegEnd)
- private->tliIndex--;
+ while (*tliIndex < targetNentries - 1 &&
+ targetHistory[*tliIndex].end < targetSegEnd)
+ (*tliIndex)++;
+ while (*tliIndex > 0 &&
+ targetHistory[*tliIndex].begin >= targetSegEnd)
+ (*tliIndex)--;
- XLogFileName(xlogfname, targetHistory[private->tliIndex].tli,
+ XLogFileName(xlogfname, targetHistory[*tliIndex].tli,
xlogreadsegno, WalSegSz);
snprintf(xlogfpath, MAXPGPATH, "%s/" XLOGDIR "/%s",
@@ -293,7 +291,7 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
/*
* If we have no restore_command to execute, then exit.
*/
- if (private->restoreCommand == NULL)
+ if (restoreCommand == NULL)
{
pg_log_error("could not open file \"%s\": %m", xlogfpath);
xlogreader->readLen = -1;
@@ -307,7 +305,7 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
xlogreadfd = RestoreArchivedFile(xlogreader->segcxt.ws_dir,
xlogfname,
WalSegSz,
- private->restoreCommand);
+ restoreCommand);
if (xlogreadfd < 0)
{
@@ -349,7 +347,7 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
Assert(targetSegNo == xlogreadsegno);
- xlogreader->seg.ws_tli = targetHistory[private->tliIndex].tli;
+ xlogreader->seg.ws_tli = targetHistory[*tliIndex].tli;
xlogreader->readLen = XLOG_BLCKSZ;
return true;
}
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index ab0c28d9b7..02e71138cc 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -330,12 +330,17 @@ WALDumpCloseSegment(XLogReaderState *state)
state->seg.ws_file = -1;
}
-/* pg_waldump's XLogReaderRoutine->page_read callback */
+/*
+ * pg_waldump's WAL page rader, also used as page_read callback for
+ * XLogFindNextRecord
+ */
static bool
-WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetPtr, char *readBuff)
+WALDumpReadPage(XLogReaderState *state, void *priv)
{
- XLogDumpPrivate *private = state->private_data;
+ XLogRecPtr targetPagePtr = state->readPagePtr;
+ int reqLen = state->readLen;
+ char *readBuff = state->readBuf;
+ XLogDumpPrivate *private = (XLogDumpPrivate *) priv;
int count = XLOG_BLCKSZ;
WALReadError errinfo;
@@ -353,8 +358,8 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
}
}
- if (!WALRead(state, readBuff, targetPagePtr, count, private->timeline,
- &errinfo))
+ if (!WALRead(state, WALDumpOpenSegment, WALDumpCloseSegment,
+ readBuff, targetPagePtr, count, private->timeline, &errinfo))
{
WALOpenSegment *seg = &errinfo.wre_seg;
char fname[MAXPGPATH];
@@ -374,6 +379,7 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
(Size) errinfo.wre_req);
}
+ Assert(count >= state->readLen);
state->readLen = count;
return true;
}
@@ -1042,16 +1048,14 @@ main(int argc, char **argv)
/* we have everything we need, start reading */
xlogreader_state =
- XLogReaderAllocate(WalSegSz, waldir,
- XL_ROUTINE(.page_read = WALDumpReadPage,
- .segment_open = WALDumpOpenSegment,
- .segment_close = WALDumpCloseSegment),
- &private);
+ XLogReaderAllocate(WalSegSz, waldir, WALDumpCloseSegment);
+
if (!xlogreader_state)
fatal_error("out of memory");
/* first find a valid recptr to start from */
- first_record = XLogFindNextRecord(xlogreader_state, private.startptr);
+ first_record = XLogFindNextRecord(xlogreader_state, private.startptr,
+ &WALDumpReadPage, (void*) &private);
if (first_record == InvalidXLogRecPtr)
fatal_error("could not find a valid record after %X/%X",
@@ -1075,7 +1079,13 @@ main(int argc, char **argv)
for (;;)
{
/* try to read the next record */
- record = XLogReadRecord(xlogreader_state, &errormsg);
+ while (XLogReadRecord(xlogreader_state, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!WALDumpReadPage(xlogreader_state, (void *) &private))
+ break;
+ }
+
if (!record)
{
if (!config.follow || private.endptr_reached)
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index 6c8848e14f..6a7fe140cb 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -57,64 +57,15 @@ typedef struct WALSegmentContext
typedef struct XLogReaderState XLogReaderState;
-/* Function type definition for the read_page callback */
-typedef bool (*XLogPageReadCB) (XLogReaderState *xlogreader,
- XLogRecPtr targetPagePtr,
- int reqLen,
- XLogRecPtr targetRecPtr,
- char *readBuf);
+/* Function type definition for the segment cleanup callback */
+typedef void (*WALSegmentCleanupCB) (XLogReaderState *xlogreader);
+
+/* Function type definition for the open/close callbacks for WALRead() */
typedef void (*WALSegmentOpenCB) (XLogReaderState *xlogreader,
XLogSegNo nextSegNo,
TimeLineID *tli_p);
typedef void (*WALSegmentCloseCB) (XLogReaderState *xlogreader);
-typedef struct XLogReaderRoutine
-{
- /*
- * Data input callback
- *
- * This callback shall read at least reqLen valid bytes of the xlog page
- * starting at targetPagePtr, and store them in readBuf. The callback
- * shall return the number of bytes read (never more than XLOG_BLCKSZ), or
- * -1 on failure. The callback shall sleep, if necessary, to wait for the
- * requested bytes to become available. The callback will not be invoked
- * again for the same page unless more than the returned number of bytes
- * are needed.
- *
- * targetRecPtr is the position of the WAL record we're reading. Usually
- * it is equal to targetPagePtr + reqLen, but sometimes xlogreader needs
- * to read and verify the page or segment header, before it reads the
- * actual WAL record it's interested in. In that case, targetRecPtr can
- * be used to determine which timeline to read the page from.
- *
- * The callback shall set ->seg.ws_tli to the TLI of the file the page was
- * read from.
- */
- XLogPageReadCB page_read;
-
- /*
- * Callback to open the specified WAL segment for reading. ->seg.ws_file
- * shall be set to the file descriptor of the opened segment. In case of
- * failure, an error shall be raised by the callback and it shall not
- * return.
- *
- * "nextSegNo" is the number of the segment to be opened.
- *
- * "tli_p" is an input/output argument. WALRead() uses it to pass the
- * timeline in which the new segment should be found, but the callback can
- * use it to return the TLI that it actually opened.
- */
- WALSegmentOpenCB segment_open;
-
- /*
- * WAL segment close callback. ->seg.ws_file shall be set to a negative
- * number.
- */
- WALSegmentCloseCB segment_close;
-} XLogReaderRoutine;
-
-#define XL_ROUTINE(...) &(XLogReaderRoutine){__VA_ARGS__}
-
typedef struct
{
/* Is this block ref in use? */
@@ -144,12 +95,35 @@ typedef struct
uint16 data_bufsz;
} DecodedBkpBlock;
+/* Return code from XLogReadRecord */
+typedef enum XLogReadRecordResult
+{
+ XLREAD_SUCCESS, /* record is successfully read */
+ XLREAD_NEED_DATA, /* need more data. see XLogReadRecord. */
+ XLREAD_FAIL /* failed during reading a record */
+} XLogReadRecordResult;
+
+/*
+ * internal state of XLogReadRecord
+ *
+ * XLogReadState runs a state machine while reading a record. Theses states
+ * are not seen outside the function. Each state may repeat several times
+ * exiting requesting caller for new data. See the comment of XLogReadRecrod
+ * for details.
+ */
+typedef enum XLogReadRecordState {
+ XLREAD_NEXT_RECORD,
+ XLREAD_TOT_LEN,
+ XLREAD_FIRST_FRAGMENT,
+ XLREAD_CONTINUATION
+} XLogReadRecordState;
+
struct XLogReaderState
{
/*
* Operational callbacks
*/
- XLogReaderRoutine routine;
+ WALSegmentCleanupCB cleanup_cb;
/* ----------------------------------------
* Public parameters
@@ -162,18 +136,14 @@ struct XLogReaderState
*/
uint64 system_identifier;
- /*
- * Opaque data for callbacks to use. Not used by XLogReader.
- */
- void *private_data;
-
/*
* Start and end point of last record read. EndRecPtr is also used as the
* position to read next. Calling XLogBeginRead() sets EndRecPtr to the
* starting position and ReadRecPtr to invalid.
*/
- XLogRecPtr ReadRecPtr; /* start of last record read */
+ XLogRecPtr ReadRecPtr; /* start of last record read or being read */
XLogRecPtr EndRecPtr; /* end+1 of last record read */
+ XLogRecPtr PrevRecPtr; /* start of previous record read */
/* ----------------------------------------
* Communication with page reader
@@ -187,7 +157,9 @@ struct XLogReaderState
* the request, or -1 on error */
TimeLineID readPageTLI; /* TLI for data currently in readBuf */
char *readBuf; /* buffer to store data */
- bool page_verified; /* is the page on the buffer verified? */
+ bool page_verified; /* is the page header on the buffer verified? */
+ bool record_verified;/* is the current record header verified? */
+
/* ----------------------------------------
@@ -227,8 +199,6 @@ struct XLogReaderState
XLogRecPtr latestPagePtr;
TimeLineID latestPageTLI;
- /* beginning of the WAL record being read. */
- XLogRecPtr currRecPtr;
/* timeline to read it from, 0 if a lookup is required */
TimeLineID currTLI;
@@ -255,6 +225,15 @@ struct XLogReaderState
char *readRecordBuf;
uint32 readRecordBufSize;
+ /*
+ * XLogReadRecord() state
+ */
+ XLogReadRecordState readRecordState;/* state machine state */
+ int recordGotLen; /* amount of current record that has
+ * already been read */
+ int recordRemainLen; /* length of current record that remains */
+ XLogRecPtr recordContRecPtr; /* where the current record continues */
+
/* Buffer to hold error message */
char *errormsg_buf;
};
@@ -262,9 +241,7 @@ struct XLogReaderState
/* Get a new XLogReader */
extern XLogReaderState *XLogReaderAllocate(int wal_segment_size,
const char *waldir,
- XLogReaderRoutine *routine,
- void *private_data);
-extern XLogReaderRoutine *LocalXLogReaderRoutine(void);
+ WALSegmentCleanupCB cleanup_cb);
/* Free an XLogReader */
extern void XLogReaderFree(XLogReaderState *state);
@@ -272,12 +249,17 @@ extern void XLogReaderFree(XLogReaderState *state);
/* Position the XLogReader to given record */
extern void XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr);
#ifdef FRONTEND
-extern XLogRecPtr XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr);
+/* Function type definition for the read_page callback */
+typedef bool (*XLogFindNextRecordCB) (XLogReaderState *xlogreader,
+ void *private);
+extern XLogRecPtr XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
+ XLogFindNextRecordCB read_page, void *private);
#endif /* FRONTEND */
/* Read the next XLog record. Returns NULL on end-of-WAL or failure */
-extern struct XLogRecord *XLogReadRecord(XLogReaderState *state,
- char **errormsg);
+extern XLogReadRecordResult XLogReadRecord(XLogReaderState *state,
+ XLogRecord **record,
+ char **errormsg);
/* Validate a page */
extern bool XLogReaderValidatePageHeader(XLogReaderState *state,
@@ -297,6 +279,7 @@ typedef struct WALReadError
} WALReadError;
extern bool WALRead(XLogReaderState *state,
+ WALSegmentOpenCB segopenfn, WALSegmentCloseCB sgclosefn,
char *buf, XLogRecPtr startptr, Size count,
TimeLineID tli, WALReadError *errinfo);
diff --git a/src/include/access/xlogutils.h b/src/include/access/xlogutils.h
index c287edf206..482e4ced69 100644
--- a/src/include/access/xlogutils.h
+++ b/src/include/access/xlogutils.h
@@ -47,9 +47,7 @@ extern Buffer XLogReadBufferExtended(RelFileNode rnode, ForkNumber forknum,
extern Relation CreateFakeRelcacheEntry(RelFileNode rnode);
extern void FreeFakeRelcacheEntry(Relation fakerel);
-extern bool read_local_xlog_page(XLogReaderState *state,
- XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetRecPtr, char *cur_page);
+extern bool read_local_xlog_page(XLogReaderState *state);
extern void wal_segment_open(XLogReaderState *state,
XLogSegNo nextSegNo,
TimeLineID *tli_p);
diff --git a/src/include/replication/logical.h b/src/include/replication/logical.h
index c2f2475e5d..a743db49cf 100644
--- a/src/include/replication/logical.h
+++ b/src/include/replication/logical.h
@@ -29,6 +29,10 @@ typedef void (*LogicalOutputPluginWriterUpdateProgress) (struct LogicalDecodingC
TransactionId xid
);
+typedef struct LogicalDecodingContext LogicalDecodingContext;
+
+typedef bool (*LogicalDecodingXLogPageReadCB)(XLogReaderState *ctx);
+
typedef struct LogicalDecodingContext
{
/* memory context this is all allocated in */
@@ -39,6 +43,7 @@ typedef struct LogicalDecodingContext
/* infrastructure pieces for decoding */
XLogReaderState *reader;
+ LogicalDecodingXLogPageReadCB page_read;
struct ReorderBuffer *reorder;
struct SnapBuild *snapshot_builder;
@@ -95,14 +100,16 @@ extern LogicalDecodingContext *CreateInitDecodingContext(char *plugin,
List *output_plugin_options,
bool need_full_snapshot,
XLogRecPtr restart_lsn,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress);
extern LogicalDecodingContext *CreateDecodingContext(XLogRecPtr start_lsn,
List *output_plugin_options,
bool fast_forward,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress);
--
2.18.4
----Next_Part(Thu_Jul__2_13_53_30_2020_072)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
filename="v15-0003-Remove-globals-readOff-readLen-and-readSegNo.patch"
^ permalink raw reply [nested|flat] 70+ messages in thread
* [PATCH v15 2/4] Move page-reader out of XLogReadRecord
@ 2019-09-10 03:58 Kyotaro Horiguchi <[email protected]>
0 siblings, 0 replies; 70+ messages in thread
From: Kyotaro Horiguchi @ 2019-09-10 03:58 UTC (permalink / raw)
This is the second step of removing callbacks from WAL record reader.
Since it is essential to take in additional data while reading a
record, the function have to ask caller for new data while keeping
working state. Thus the function is turned into a state machine.
---
src/backend/access/transam/twophase.c | 15 +-
src/backend/access/transam/xlog.c | 58 +-
src/backend/access/transam/xlogreader.c | 700 +++++++++++-------
src/backend/access/transam/xlogutils.c | 17 +-
src/backend/replication/logical/logical.c | 26 +-
.../replication/logical/logicalfuncs.c | 13 +-
src/backend/replication/slotfuncs.c | 18 +-
src/backend/replication/walsender.c | 32 +-
src/bin/pg_rewind/parsexlog.c | 84 +--
src/bin/pg_waldump/pg_waldump.c | 36 +-
src/include/access/xlogreader.h | 121 ++-
src/include/access/xlogutils.h | 4 +-
src/include/replication/logical.h | 11 +-
13 files changed, 648 insertions(+), 487 deletions(-)
diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index 9b2e59bf0e..b0d60a0d0f 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -1330,11 +1330,8 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
XLogReaderState *xlogreader;
char *errormsg;
- xlogreader = XLogReaderAllocate(wal_segment_size, NULL,
- XL_ROUTINE(.page_read = &read_local_xlog_page,
- .segment_open = &wal_segment_open,
- .segment_close = &wal_segment_close),
- NULL);
+ xlogreader = XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+
if (!xlogreader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
@@ -1342,7 +1339,13 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
errdetail("Failed while allocating a WAL reading processor.")));
XLogBeginRead(xlogreader, lsn);
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!read_local_xlog_page(xlogreader))
+ break;
+ }
+
if (record == NULL)
ereport(ERROR,
(errcode_for_file_access(),
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index e570e56a24..f9b0108602 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -831,13 +831,6 @@ static XLogSource currentSource = XLOG_FROM_ANY;
static bool lastSourceFailed = false;
static bool pendingWalRcvRestart = false;
-typedef struct XLogPageReadPrivate
-{
- int emode;
- bool fetching_ckpt; /* are we fetching a checkpoint record? */
- bool randAccess;
-} XLogPageReadPrivate;
-
/*
* These variables track when we last obtained some WAL data to process,
* and where we got it from. (XLogReceiptSource is initially the same as
@@ -912,8 +905,8 @@ static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath,
static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli,
XLogSource source, bool notfoundOk);
static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, XLogSource source);
-static bool XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
+static bool XLogPageRead(XLogReaderState *xlogreader,
+ bool fetching_ckpt, int emode, bool randAccess);
static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
bool fetching_ckpt, XLogRecPtr tliRecPtr);
static int emode_for_corrupt_record(int emode, XLogRecPtr RecPtr);
@@ -1225,8 +1218,7 @@ XLogInsertRecord(XLogRecData *rdata,
appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len);
if (!debug_reader)
- debug_reader = XLogReaderAllocate(wal_segment_size, NULL,
- XL_ROUTINE(), NULL);
+ debug_reader = XLogReaderAllocate(wal_segment_size, NULL NULL);
if (!debug_reader)
{
@@ -4326,15 +4318,10 @@ CleanupBackupHistory(void)
* record is available.
*/
static XLogRecord *
-ReadRecord(XLogReaderState *xlogreader, int emode,
- bool fetching_ckpt)
+ReadRecord(XLogReaderState *xlogreader, int emode, bool fetching_ckpt)
{
XLogRecord *record;
- XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
-
- private->fetching_ckpt = fetching_ckpt;
- private->emode = emode;
- private->randAccess = (xlogreader->ReadRecPtr == InvalidXLogRecPtr);
+ bool randAccess = (xlogreader->ReadRecPtr == InvalidXLogRecPtr);
/* This is the first attempt to read this page. */
lastSourceFailed = false;
@@ -4342,8 +4329,16 @@ ReadRecord(XLogReaderState *xlogreader, int emode,
for (;;)
{
char *errormsg;
+ XLogReadRecordResult result;
+
+ while ((result = XLogReadRecord(xlogreader, &record, &errormsg))
+ == XLREAD_NEED_DATA)
+ {
+ if (!XLogPageRead(xlogreader, fetching_ckpt, emode, randAccess))
+ break;
+
+ }
- record = XLogReadRecord(xlogreader, &errormsg);
ReadRecPtr = xlogreader->ReadRecPtr;
EndRecPtr = xlogreader->EndRecPtr;
if (record == NULL)
@@ -6320,7 +6315,6 @@ StartupXLOG(void)
bool backupFromStandby = false;
DBState dbstate_at_startup;
XLogReaderState *xlogreader;
- XLogPageReadPrivate private;
bool fast_promoted = false;
struct stat st;
@@ -6480,13 +6474,9 @@ StartupXLOG(void)
OwnLatch(&XLogCtl->recoveryWakeupLatch);
/* Set up XLOG reader facility */
- MemSet(&private, 0, sizeof(XLogPageReadPrivate));
xlogreader =
- XLogReaderAllocate(wal_segment_size, NULL,
- XL_ROUTINE(.page_read = &XLogPageRead,
- .segment_open = NULL,
- .segment_close = wal_segment_close),
- &private);
+ XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+
if (!xlogreader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
@@ -11860,12 +11850,13 @@ CancelBackup(void)
* sleep and retry.
*/
static bool
-XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetRecPtr, char *readBuf)
+XLogPageRead(XLogReaderState *xlogreader,
+ bool fetching_ckpt, int emode, bool randAccess)
{
- XLogPageReadPrivate *private =
- (XLogPageReadPrivate *) xlogreader->private_data;
- int emode = private->emode;
+ char *readBuf = xlogreader->readBuf;
+ XLogRecPtr targetPagePtr = xlogreader->readPagePtr;
+ int reqLen = xlogreader->readLen;
+ XLogRecPtr targetRecPtr = xlogreader->ReadRecPtr;
uint32 targetPageOff;
XLogSegNo targetSegNo PG_USED_FOR_ASSERTS_ONLY;
int r;
@@ -11908,8 +11899,8 @@ retry:
flushedUpto < targetPagePtr + reqLen))
{
if (!WaitForWALToBecomeAvailable(targetPagePtr + reqLen,
- private->randAccess,
- private->fetching_ckpt,
+ randAccess,
+ fetching_ckpt,
targetRecPtr))
{
if (readFile >= 0)
@@ -12014,6 +12005,7 @@ retry:
goto next_record_is_invalid;
}
+ Assert(xlogreader->readPagePtr == targetPagePtr);
xlogreader->readLen = readLen;
return true;
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index 3d599325ee..9281b57379 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -40,7 +40,7 @@ static bool XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr,
int reqLen, bool header_inclusive);
static void XLogReaderInvalReadState(XLogReaderState *state);
static bool ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
- XLogRecPtr PrevRecPtr, XLogRecord *record, bool randAccess);
+ XLogRecPtr PrevRecPtr, XLogRecord *record);
static bool ValidXLogRecord(XLogReaderState *state, XLogRecord *record,
XLogRecPtr recptr);
static void ResetDecoder(XLogReaderState *state);
@@ -73,7 +73,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...)
*/
XLogReaderState *
XLogReaderAllocate(int wal_segment_size, const char *waldir,
- XLogReaderRoutine *routine, void *private_data)
+ WALSegmentCleanupCB cleanup_cb)
{
XLogReaderState *state;
@@ -84,7 +84,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
return NULL;
/* initialize caller-provided support functions */
- state->routine = *routine;
+ state->cleanup_cb = cleanup_cb;
state->max_block_id = -1;
@@ -107,8 +107,6 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
WALOpenSegmentInit(&state->seg, &state->segcxt, wal_segment_size,
waldir);
- /* system_identifier initialized to zeroes above */
- state->private_data = private_data;
/* ReadRecPtr, EndRecPtr and readLen initialized to zeroes above */
state->errormsg_buf = palloc_extended(MAX_ERRORMSG_LEN + 1,
MCXT_ALLOC_NO_OOM);
@@ -140,8 +138,8 @@ XLogReaderFree(XLogReaderState *state)
{
int block_id;
- if (state->seg.ws_file != -1)
- state->routine.segment_close(state);
+ if (state->seg.ws_file >= 0)
+ state->cleanup_cb(state);
for (block_id = 0; block_id <= XLR_MAX_BLOCK_ID; block_id++)
{
@@ -246,6 +244,7 @@ XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
/* Begin at the passed-in record pointer. */
state->EndRecPtr = RecPtr;
state->ReadRecPtr = InvalidXLogRecPtr;
+ state->readRecordState = XLREAD_NEXT_RECORD;
}
/*
@@ -254,317 +253,452 @@ XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
* XLogBeginRead() or XLogFindNextRecord() must be called before the first call
* to XLogReadRecord().
*
- * If the page_read callback fails to read the requested data, NULL is
- * returned. The callback is expected to have reported the error; errormsg
- * is set to NULL.
+ * This function may return XLREAD_NEED_DATA several times before returning a
+ * result record. The caller shall read in some new data then call this
+ * function again with the same parameters.
*
- * If the reading fails for some other reason, NULL is also returned, and
- * *errormsg is set to a string with details of the failure.
+ * When a record is successfully read, returns XLREAD_SUCCESS with result
+ * record being stored in *record. Otherwise *record is NULL.
*
- * The returned pointer (or *errormsg) points to an internal buffer that's
- * valid until the next call to XLogReadRecord.
+ * Returns XLREAD_NEED_DATA if more data is needed to finish reading the
+ * current record. In that case, state->readPagePtr and state->readLen inform
+ * the desired position and minimum length of data needed. The caller shall
+ * read in the requested data and set state->readBuf to point to a buffer
+ * containing it. The caller must also set state->readPageTLI and
+ * state->readLen to indicate the timeline that it was read from, and the
+ * length of data that is now available (which must be >= given readLen),
+ * respectively.
+ *
+ * If invalid data is encountered, returns XLREAD_FAIL with *record being set to
+ * NULL. *errormsg is set to a string with details of the failure.
+ * The returned pointer (or *errormsg) points to an internal buffer that's valid
+ * until the next call to XLogReadRecord.
+ *
+ *
+ * This function runs a state machine consists of the following states.
+ *
+ * XLREAD_NEXT_RECORD :
+ * The initial state, if called with valid RecPtr, try to read a record at
+ * that position. If invalid RecPtr is given try to read a record just after
+ * the last one previously read.
+ * This state ens after setting ReadRecPtr. Then goes to XLREAD_TOT_LEN.
+ *
+ * XLREAD_TOT_LEN:
+ * Examining record header. Ends after reading record total
+ * length. recordRemainLen and recordGotLen are initialized.
+ *
+ * XLREAD_FIRST_FRAGMENT:
+ * Reading the first fragment. Ends with finishing reading a single
+ * record. Goes to XLREAD_NEXT_RECORD if that's all or
+ * XLREAD_CONTINUATION if we have continuation.
+
+ * XLREAD_CONTINUATION:
+ * Reading continuation of record. Ends with finishing the whole record then
+ * goes to XLREAD_NEXT_RECORD. During this state, recordRemainLen indicates
+ * how much is left and readRecordBuf holds the partially assert
+ * record.recordContRecPtr points to the beginning of the next page where to
+ * continue.
+ *
+ * If wrong data found in any state, the state machine stays at the current
+ * state. This behavior allows to continue reading a reacord switching among
+ * different souces, while streaming replication.
*/
-XLogRecord *
-XLogReadRecord(XLogReaderState *state, char **errormsg)
+XLogReadRecordResult
+XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
{
- XLogRecPtr RecPtr;
- XLogRecord *record;
- XLogRecPtr targetPagePtr;
- bool randAccess;
- uint32 len,
- total_len;
- uint32 targetRecOff;
- uint32 pageHeaderSize;
- bool gotheader;
+ XLogRecord *prec;
- /*
- * randAccess indicates whether to verify the previous-record pointer of
- * the record we're reading. We only do this if we're reading
- * sequentially, which is what we initially assume.
- */
- randAccess = false;
+ *record = NULL;
/* reset error state */
*errormsg = NULL;
state->errormsg_buf[0] = '\0';
- ResetDecoder(state);
-
- RecPtr = state->EndRecPtr;
-
- if (state->ReadRecPtr != InvalidXLogRecPtr)
- {
- /* read the record after the one we just read */
-
- /*
- * EndRecPtr is pointing to end+1 of the previous WAL record. If
- * we're at a page boundary, no more records can fit on the current
- * page. We must skip over the page header, but we can't do that until
- * we've read in the page, since the header size is variable.
- */
- }
- else
- {
- /*
- * Caller supplied a position to start at.
- *
- * In this case, EndRecPtr should already be pointing to a valid
- * record starting position.
- */
- Assert(XRecOffIsValid(RecPtr));
- randAccess = true;
- }
-
- state->currRecPtr = RecPtr;
-
- targetPagePtr = RecPtr - (RecPtr % XLOG_BLCKSZ);
- targetRecOff = RecPtr % XLOG_BLCKSZ;
-
- /*
- * Read the page containing the record into state->readBuf. Request enough
- * byte to cover the whole record header, or at least the part of it that
- * fits on the same page.
- */
- while (XLogNeedData(state, targetPagePtr,
- Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
- targetRecOff != 0))
- {
- if (!state->routine.page_read(state, state->readPagePtr, state->readLen,
- RecPtr, state->readBuf))
- break;
- }
-
- if (!state->page_verified)
- goto err;
-
- /*
- * We have at least the page header, so we can examine it now.
- */
- pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
- if (targetRecOff == 0)
- {
- /*
- * At page start, so skip over page header.
- */
- RecPtr += pageHeaderSize;
- targetRecOff = pageHeaderSize;
- }
- else if (targetRecOff < pageHeaderSize)
- {
- report_invalid_record(state, "invalid record offset at %X/%X",
- (uint32) (RecPtr >> 32), (uint32) RecPtr);
- goto err;
- }
-
- if ((((XLogPageHeader) state->readBuf)->xlp_info & XLP_FIRST_IS_CONTRECORD) &&
- targetRecOff == pageHeaderSize)
- {
- report_invalid_record(state, "contrecord is requested by %X/%X",
- (uint32) (RecPtr >> 32), (uint32) RecPtr);
- goto err;
- }
-
- /* XLogNeedData has verified the page header */
- Assert(pageHeaderSize <= state->readLen);
-
- /*
- * Read the record length.
- *
- * NB: Even though we use an XLogRecord pointer here, the whole record
- * header might not fit on this page. xl_tot_len is the first field of the
- * struct, so it must be on this page (the records are MAXALIGNed), but we
- * cannot access any other fields until we've verified that we got the
- * whole header.
- */
- record = (XLogRecord *) (state->readBuf + RecPtr % XLOG_BLCKSZ);
- total_len = record->xl_tot_len;
-
- /*
- * If the whole record header is on this page, validate it immediately.
- * Otherwise do just a basic sanity check on xl_tot_len, and validate the
- * rest of the header after reading it from the next page. The xl_tot_len
- * check is necessary here to ensure that we enter the "Need to reassemble
- * record" code path below; otherwise we might fail to apply
- * ValidXLogRecordHeader at all.
- */
- if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
- {
- if (!ValidXLogRecordHeader(state, RecPtr, state->ReadRecPtr, record,
- randAccess))
- goto err;
- gotheader = true;
- }
- else
- {
- /* XXX: more validation should be done here */
- if (total_len < SizeOfXLogRecord)
- {
- report_invalid_record(state,
- "invalid record length at %X/%X: wanted %u, got %u",
- (uint32) (RecPtr >> 32), (uint32) RecPtr,
- (uint32) SizeOfXLogRecord, total_len);
- goto err;
- }
- gotheader = false;
- }
-
- len = XLOG_BLCKSZ - RecPtr % XLOG_BLCKSZ;
- if (total_len > len)
+ switch (state->readRecordState)
{
- /* Need to reassemble record */
- char *contdata;
- XLogPageHeader pageHeader;
- char *buffer;
- uint32 gotlen;
-
- /*
- * Enlarge readRecordBuf as needed.
- */
- if (total_len > state->readRecordBufSize &&
- !allocate_recordbuf(state, total_len))
- {
- /* We treat this as a "bogus data" condition */
- report_invalid_record(state, "record length %u at %X/%X too long",
- total_len,
- (uint32) (RecPtr >> 32), (uint32) RecPtr);
- goto err;
- }
+ case XLREAD_NEXT_RECORD:
+ ResetDecoder(state);
- /* Copy the first fragment of the record from the first page. */
- memcpy(state->readRecordBuf,
- state->readBuf + RecPtr % XLOG_BLCKSZ, len);
- buffer = state->readRecordBuf + len;
- gotlen = len;
-
- do
- {
- int rest_len = total_len - gotlen;
-
- /* Calculate pointer to beginning of next page */
- targetPagePtr += XLOG_BLCKSZ;
-
- /* Wait for the next page to become available */
- while (XLogNeedData(state, targetPagePtr,
- Min(rest_len, XLOG_BLCKSZ),
- false))
+ if (state->ReadRecPtr != InvalidXLogRecPtr)
{
- if (!state->routine.page_read(state, state->readPagePtr,
- state->readLen,
- state->ReadRecPtr,
- state->readBuf))
- break;
+ /* read the record after the one we just read */
+
+ /*
+ * EndRecPtr is pointing to end+1 of the previous WAL record.
+ * If we're at a page boundary, no more records can fit on the
+ * current page. We must skip over the page header, but we
+ * can't do that until we've read in the page, since the header
+ * size is variable.
+ */
+ state->PrevRecPtr = state->ReadRecPtr;
+ state->ReadRecPtr = state->EndRecPtr;
+ }
+ else
+ {
+ /*
+ * Caller supplied a position to start at.
+ *
+ * In this case, EndRecPtr should already be pointing to a
+ * valid record starting position.
+ */
+ Assert(XRecOffIsValid(state->EndRecPtr));
+ state->ReadRecPtr = state->EndRecPtr;
+
+ /*
+ * We cannot verify the previous-record pointer when we're
+ * seeking to a particular record. Reset PrevRecPtr so that we
+ * won't try doing that.
+ */
+ state->PrevRecPtr = InvalidXLogRecPtr;
+ state->EndRecPtr = InvalidXLogRecPtr; /* to be tidy */
}
+ state->record_verified = false;
+ state->readRecordState = XLREAD_TOT_LEN;
+ /* fall through */
+
+ case XLREAD_TOT_LEN:
+ {
+ uint32 total_len;
+ uint32 pageHeaderSize;
+ XLogRecPtr targetPagePtr;
+ uint32 targetRecOff;
+ XLogPageHeader pageHeader;
+
+ targetPagePtr =
+ state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
+ targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+
+ /*
+ * Check if we have enough data. For the first record in the page,
+ * the requesting length doesn't contain page header.
+ */
+ if (XLogNeedData(state, targetPagePtr,
+ Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+ targetRecOff != 0))
+ return XLREAD_NEED_DATA;
+
+ /* error out if caller supplied bogus page */
if (!state->page_verified)
goto err;
- Assert(SizeOfXLogShortPHD <= state->readLen);
+ /* examine page header now. */
+ pageHeaderSize =
+ XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+ if (targetRecOff == 0)
+ {
+ /* At page start, so skip over page header. */
+ state->ReadRecPtr += pageHeaderSize;
+ targetRecOff = pageHeaderSize;
+ }
+ else if (targetRecOff < pageHeaderSize)
+ {
+ report_invalid_record(state, "invalid record offset at %X/%X",
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr);
+ goto err;
+ }
- /* Check that the continuation on next page looks valid */
pageHeader = (XLogPageHeader) state->readBuf;
- if (!(pageHeader->xlp_info & XLP_FIRST_IS_CONTRECORD))
+ if ((pageHeader->xlp_info & XLP_FIRST_IS_CONTRECORD) &&
+ targetRecOff == pageHeaderSize)
{
- report_invalid_record(state,
- "there is no contrecord flag at %X/%X",
- (uint32) (RecPtr >> 32), (uint32) RecPtr);
+ report_invalid_record(state, "contrecord is requested by %X/%X",
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr);
goto err;
}
- /*
- * Cross-check that xlp_rem_len agrees with how much of the record
- * we expect there to be left.
- */
- if (pageHeader->xlp_rem_len == 0 ||
- total_len != (pageHeader->xlp_rem_len + gotlen))
- {
- report_invalid_record(state,
- "invalid contrecord length %u at %X/%X",
- pageHeader->xlp_rem_len,
- (uint32) (RecPtr >> 32), (uint32) RecPtr);
- goto err;
- }
-
- /* Append the continuation from this page to the buffer */
- pageHeaderSize = XLogPageHeaderSize(pageHeader);
-
+ /* XLogNeedData has verified the page header */
Assert(pageHeaderSize <= state->readLen);
- contdata = (char *) state->readBuf + pageHeaderSize;
- len = XLOG_BLCKSZ - pageHeaderSize;
- if (pageHeader->xlp_rem_len < len)
- len = pageHeader->xlp_rem_len;
+ /*
+ * Read the record length.
+ *
+ * NB: Even though we use an XLogRecord pointer here, the whole
+ * record header might not fit on this page. xl_tot_len is the first
+ * field of the struct, so it must be on this page (the records are
+ * MAXALIGNed), but we cannot access any other fields until we've
+ * verified that we got the whole header.
+ */
+ prec = (XLogRecord *) (state->readBuf +
+ state->ReadRecPtr % XLOG_BLCKSZ);
+ total_len = prec->xl_tot_len;
- Assert (pageHeaderSize + len <= state->readLen);
- memcpy(buffer, (char *) contdata, len);
- buffer += len;
- gotlen += len;
+ /*
+ * If the whole record header is on this page, validate it
+ * immediately. Otherwise do just a basic sanity check on
+ * xl_tot_len, and validate the rest of the header after reading it
+ * from the next page. The xl_tot_len check is necessary here to
+ * ensure that we enter the XLREAD_CONTINUATION state below;
+ * otherwise we might fail to apply ValidXLogRecordHeader at all.
+ */
+ if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
+ {
+ if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
+ state->PrevRecPtr, prec))
+ goto err;
- /* If we just reassembled the record header, validate it. */
- if (!gotheader)
+ state->record_verified = true;
+ }
+ else
{
- record = (XLogRecord *) state->readRecordBuf;
- if (!ValidXLogRecordHeader(state, RecPtr, state->ReadRecPtr,
- record, randAccess))
+ /* XXX: more validation should be done here */
+ if (total_len < SizeOfXLogRecord)
+ {
+ report_invalid_record(state,
+ "invalid record length at %X/%X: wanted %u, got %u",
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr,
+ (uint32) SizeOfXLogRecord, total_len);
goto err;
- gotheader = true;
+ }
}
- } while (gotlen < total_len);
-
- Assert(gotheader);
- record = (XLogRecord *) state->readRecordBuf;
- if (!ValidXLogRecord(state, record, RecPtr))
- goto err;
+ /*
+ * Wait for the rest of the record, or the part of the record that
+ * fit on the first page if crossed a page boundary, to become
+ * available.
+ */
+ state->recordGotLen = 0;
+ state->recordRemainLen = total_len;
+ state->readRecordState = XLREAD_FIRST_FRAGMENT;
+ }
+ /* fall through */
- pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
- state->ReadRecPtr = RecPtr;
- state->EndRecPtr = targetPagePtr + pageHeaderSize
- + MAXALIGN(pageHeader->xlp_rem_len);
- }
- else
- {
- /* Wait for the record data to become available */
- while (XLogNeedData(state, targetPagePtr,
- Min(targetRecOff + total_len, XLOG_BLCKSZ), true))
+ case XLREAD_FIRST_FRAGMENT:
{
- if (!state->routine.page_read(state, state->readPagePtr,
- state->readLen,
- state->ReadRecPtr, state->readBuf))
+ uint32 total_len = state->recordRemainLen;
+ uint32 request_len;
+ uint32 record_len;
+ XLogRecPtr targetPagePtr;
+ uint32 targetRecOff;
+
+ /*
+ * Wait for the rest of the record on the first page to become
+ * available
+ */
+ targetPagePtr =
+ state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
+ targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+
+ request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ);
+ record_len = request_len - targetRecOff;
+
+ /* ReadRecPtr contains page header */
+ Assert (targetRecOff != 0);
+ if (XLogNeedData(state, targetPagePtr, request_len, true))
+ return XLREAD_NEED_DATA;
+
+ /* error out if caller supplied bogus page */
+ if (!state->page_verified)
+ goto err;
+
+ prec = (XLogRecord *) (state->readBuf + targetRecOff);
+
+ /* validate record header if not yet */
+ if (!state->record_verified && record_len >= SizeOfXLogRecord)
+ {
+ if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
+ state->PrevRecPtr, prec))
+ goto err;
+
+ state->record_verified = true;
+ }
+
+
+ if (total_len == record_len)
+ {
+ /* Record does not cross a page boundary */
+ Assert(state->record_verified);
+
+ if (!ValidXLogRecord(state, prec, state->ReadRecPtr))
+ goto err;
+
+ state->record_verified = true; /* to be tidy */
+
+ /* We already checked the header earlier */
+ state->EndRecPtr = state->ReadRecPtr + MAXALIGN(record_len);
+
+ *record = prec;
+ state->readRecordState = XLREAD_NEXT_RECORD;
break;
+ }
+
+ /*
+ * The record continues on the next page. Need to reassemble
+ * record
+ */
+ Assert(total_len > record_len);
+
+ /* Enlarge readRecordBuf as needed. */
+ if (total_len > state->readRecordBufSize &&
+ !allocate_recordbuf(state, total_len))
+ {
+ /* We treat this as a "bogus data" condition */
+ report_invalid_record(state,
+ "record length %u at %X/%X too long",
+ total_len,
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr);
+ goto err;
+ }
+
+ /* Copy the first fragment of the record from the first page. */
+ memcpy(state->readRecordBuf, state->readBuf + targetRecOff,
+ record_len);
+ state->recordGotLen += record_len;
+ state->recordRemainLen -= record_len;
+
+ /* Calculate pointer to beginning of next page */
+ state->recordContRecPtr = state->ReadRecPtr + record_len;
+ Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0);
+
+ state->readRecordState = XLREAD_CONTINUATION;
}
+ /* fall through */
+
+ case XLREAD_CONTINUATION:
+ {
+ XLogPageHeader pageHeader;
+ uint32 pageHeaderSize;
+ XLogRecPtr targetPagePtr;
+
+ /* we enter this state only if we haven't read the whole record. */
+ Assert (state->recordRemainLen > 0);
+
+ while(state->recordRemainLen > 0)
+ {
+ char *contdata;
+ uint32 request_len;
+ uint32 record_len;
+
+ /* Wait for the next page to become available */
+ targetPagePtr = state->recordContRecPtr;
+
+ /* this request contains page header */
+ Assert (targetPagePtr != 0);
+ if (XLogNeedData(state, targetPagePtr,
+ Min(state->recordRemainLen, XLOG_BLCKSZ),
+ false))
+ return XLREAD_NEED_DATA;
+
+ if (!state->page_verified)
+ goto err;
+
+ Assert(SizeOfXLogShortPHD <= state->readLen);
- if (!state->page_verified)
- goto err;
+ /* Check that the continuation on next page looks valid */
+ pageHeader = (XLogPageHeader) state->readBuf;
+ if (!(pageHeader->xlp_info & XLP_FIRST_IS_CONTRECORD))
+ {
+ report_invalid_record(
+ state,
+ "there is no contrecord flag at %X/%X reading %X/%X",
+ (uint32) (state->recordContRecPtr >> 32),
+ (uint32) state->recordContRecPtr,
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr);
+ goto err;
+ }
- /* Record does not cross a page boundary */
- if (!ValidXLogRecord(state, record, RecPtr))
- goto err;
+ /*
+ * Cross-check that xlp_rem_len agrees with how much of the
+ * record we expect there to be left.
+ */
+ if (pageHeader->xlp_rem_len == 0 ||
+ pageHeader->xlp_rem_len != state->recordRemainLen)
+ {
+ report_invalid_record(
+ state,
+ "invalid contrecord length %u at %X/%X reading %X/%X, expected %u",
+ pageHeader->xlp_rem_len,
+ (uint32) (state->recordContRecPtr >> 32),
+ (uint32) state->recordContRecPtr,
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr,
+ state->recordRemainLen);
+ goto err;
+ }
- state->EndRecPtr = RecPtr + MAXALIGN(total_len);
+ /* Append the continuation from this page to the buffer */
+ pageHeaderSize = XLogPageHeaderSize(pageHeader);
- state->ReadRecPtr = RecPtr;
+ /*
+ * XLogNeedData should have ensured that the whole page header
+ * was read
+ */
+ Assert(state->readLen >= pageHeaderSize);
+
+ contdata = (char *) state->readBuf + pageHeaderSize;
+ record_len = XLOG_BLCKSZ - pageHeaderSize;
+ if (pageHeader->xlp_rem_len < record_len)
+ record_len = pageHeader->xlp_rem_len;
+
+ request_len = record_len + pageHeaderSize;
+
+ /* XLogNeedData should have ensured all needed data was read */
+ Assert (state->readLen >= request_len);
+
+ memcpy(state->readRecordBuf + state->recordGotLen,
+ (char *) contdata, record_len);
+ state->recordGotLen += record_len;
+ state->recordRemainLen -= record_len;
+
+ /* If we just reassembled the record header, validate it. */
+ if (!state->record_verified)
+ {
+ Assert(state->recordGotLen >= SizeOfXLogRecord);
+ if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
+ state->PrevRecPtr,
+ (XLogRecord *) state->readRecordBuf))
+ goto err;
+
+ state->record_verified = true;
+ }
+
+ /* Calculate pointer to beginning of next page, and continue */
+ state->recordContRecPtr += XLOG_BLCKSZ;
+ }
+
+ /* targetPagePtr is pointing the last-read page here */
+ prec = (XLogRecord *) state->readRecordBuf;
+ if (!ValidXLogRecord(state, prec, state->ReadRecPtr))
+ goto err;
+
+ pageHeaderSize =
+ XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+ state->EndRecPtr = targetPagePtr + pageHeaderSize
+ + MAXALIGN(pageHeader->xlp_rem_len);
+
+ *record = prec;
+ state->readRecordState = XLREAD_NEXT_RECORD;
+ break;
+ }
}
/*
* Special processing if it's an XLOG SWITCH record
*/
- if (record->xl_rmid == RM_XLOG_ID &&
- (record->xl_info & ~XLR_INFO_MASK) == XLOG_SWITCH)
+ if ((*record)->xl_rmid == RM_XLOG_ID &&
+ ((*record)->xl_info & ~XLR_INFO_MASK) == XLOG_SWITCH)
{
/* Pretend it extends to end of segment */
state->EndRecPtr += state->segcxt.ws_segsize - 1;
state->EndRecPtr -= XLogSegmentOffset(state->EndRecPtr, state->segcxt.ws_segsize);
}
- if (DecodeXLogRecord(state, record, errormsg))
- return record;
- else
- return NULL;
+ Assert (!*record || state->readLen >= 0);
+ if (DecodeXLogRecord(state, *record, errormsg))
+ return XLREAD_SUCCESS;
+
+ *record = NULL;
+ return XLREAD_FAIL;
err:
/*
- * Invalidate the read state. We might read from a different source after
+ * Invalidate the read page. We might read from a different source after
* failure.
*/
XLogReaderInvalReadState(state);
@@ -572,7 +706,8 @@ err:
if (state->errormsg_buf[0] != '\0')
*errormsg = state->errormsg_buf;
- return NULL;
+ *record = NULL;
+ return XLREAD_FAIL;
}
/*
@@ -724,11 +859,12 @@ XLogReaderInvalReadState(XLogReaderState *state)
*
* This is just a convenience subroutine to avoid duplicated code in
* XLogReadRecord. It's not intended for use from anywhere else.
+ *
+ * If PrevRecPtr is valid, the xl_prev is is cross-checked with it.
*/
static bool
ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
- XLogRecPtr PrevRecPtr, XLogRecord *record,
- bool randAccess)
+ XLogRecPtr PrevRecPtr, XLogRecord *record)
{
if (record->xl_tot_len < SizeOfXLogRecord)
{
@@ -746,7 +882,7 @@ ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
(uint32) RecPtr);
return false;
}
- if (randAccess)
+ if (PrevRecPtr == InvalidXLogRecPtr)
{
/*
* We can't exactly verify the prev-link, but surely it should be less
@@ -978,11 +1114,14 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
* XLogReadRecord() will read the next valid record.
*/
XLogRecPtr
-XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
+XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
+ XLogFindNextRecordCB read_page, void *private)
{
XLogRecPtr tmpRecPtr;
XLogRecPtr found = InvalidXLogRecPtr;
XLogPageHeader header;
+ XLogRecord *record;
+ XLogReadRecordResult result;
char *errormsg;
Assert(!XLogRecPtrIsInvalid(RecPtr));
@@ -1015,9 +1154,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
while(XLogNeedData(state, targetPagePtr, targetRecOff,
targetRecOff != 0))
{
- if (!state->routine.page_read(state, state->readPagePtr,
- state->readLen,
- state->ReadRecPtr, state->readBuf))
+ if (!read_page(state, private))
break;
}
@@ -1069,8 +1206,16 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
* or we just jumped over the remaining data of a continuation.
*/
XLogBeginRead(state, tmpRecPtr);
- while (XLogReadRecord(state, &errormsg) != NULL)
+ while ((result = XLogReadRecord(state, &record, &errormsg)) !=
+ XLREAD_FAIL)
{
+ if (result == XLREAD_NEED_DATA)
+ {
+ if (!read_page(state, private))
+ goto err;
+ continue;
+ }
+
/* past the record we've found, break out */
if (RecPtr <= state->ReadRecPtr)
{
@@ -1090,9 +1235,9 @@ err:
#endif /* FRONTEND */
/*
- * Helper function to ease writing of XLogRoutine->page_read callbacks.
- * If this function is used, caller must supply a segment_open callback in
- * 'state', as that is used here.
+ * Helper function to ease writing of page_read callback.
+ * If this function is used, caller must supply a segment_open callback and
+ * segment_close callback as that is used here.
*
* Read 'count' bytes into 'buf', starting at location 'startptr', from WAL
* fetched from timeline 'tli'.
@@ -1105,6 +1250,7 @@ err:
*/
bool
WALRead(XLogReaderState *state,
+ WALSegmentOpenCB segopenfn, WALSegmentCloseCB segclosefn,
char *buf, XLogRecPtr startptr, Size count, TimeLineID tli,
WALReadError *errinfo)
{
@@ -1136,10 +1282,10 @@ WALRead(XLogReaderState *state,
XLogSegNo nextSegNo;
if (state->seg.ws_file >= 0)
- state->routine.segment_close(state);
+ segclosefn(state);
XLByteToSeg(recptr, nextSegNo, state->segcxt.ws_segsize);
- state->routine.segment_open(state, nextSegNo, &tli);
+ segopenfn(state, nextSegNo, &tli);
/* This shouldn't happen -- indicates a bug in segment_open */
Assert(state->seg.ws_file >= 0);
diff --git a/src/backend/access/transam/xlogutils.c b/src/backend/access/transam/xlogutils.c
index 1b10b6df20..303333571f 100644
--- a/src/backend/access/transam/xlogutils.c
+++ b/src/backend/access/transam/xlogutils.c
@@ -689,8 +689,7 @@ XLogTruncateRelation(RelFileNode rnode, ForkNumber forkNum,
void
XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wantLength)
{
- const XLogRecPtr lastReadPage = state->seg.ws_segno *
- state->segcxt.ws_segsize + state->readLen;
+ const XLogRecPtr lastReadPage = state->readPagePtr;
Assert(wantPage != InvalidXLogRecPtr && wantPage % XLOG_BLCKSZ == 0);
Assert(wantLength <= XLOG_BLCKSZ);
@@ -705,7 +704,7 @@ XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wa
* current TLI has since become historical.
*/
if (lastReadPage == wantPage &&
- state->readLen != 0 &&
+ state->page_verified &&
lastReadPage + state->readLen >= wantPage + Min(wantLength, XLOG_BLCKSZ - 1))
return;
@@ -792,6 +791,7 @@ wal_segment_open(XLogReaderState *state, XLogSegNo nextSegNo,
char path[MAXPGPATH];
XLogFilePath(path, tli, nextSegNo, state->segcxt.ws_segsize);
+ elog(LOG, "HOGE: %lu, %d => %s", nextSegNo, tli, path);
state->seg.ws_file = BasicOpenFile(path, O_RDONLY | PG_BINARY);
if (state->seg.ws_file >= 0)
return;
@@ -829,9 +829,11 @@ wal_segment_close(XLogReaderState *state)
* loop for now.
*/
bool
-read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *cur_page)
+read_local_xlog_page(XLogReaderState *state)
{
+ XLogRecPtr targetPagePtr = state->readPagePtr;
+ int reqLen = state->readLen;
+ char *cur_page = state->readBuf;
XLogRecPtr read_upto,
loc;
TimeLineID tli;
@@ -944,11 +946,12 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
* as 'count', read the whole page anyway. It's guaranteed to be
* zero-padded up to the page boundary if it's incomplete.
*/
- if (!WALRead(state, cur_page, targetPagePtr, XLOG_BLCKSZ, tli,
- &errinfo))
+ if (!WALRead(state, wal_segment_open, wal_segment_close,
+ cur_page, targetPagePtr, XLOG_BLCKSZ, tli, &errinfo))
WALReadRaiseError(&errinfo);
/* number of valid bytes in the buffer */
+ state->readPagePtr = targetPagePtr;
state->readLen = count;
return true;
}
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index 61902be3b0..a15b0b3355 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -120,7 +120,8 @@ StartupDecodingContext(List *output_plugin_options,
TransactionId xmin_horizon,
bool need_full_snapshot,
bool fast_forward,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress)
@@ -169,11 +170,12 @@ StartupDecodingContext(List *output_plugin_options,
ctx->slot = slot;
- ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, xl_routine, ctx);
+ ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, cleanup_cb);
if (!ctx->reader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
errmsg("out of memory")));
+ ctx->page_read = page_read;
ctx->reorder = ReorderBufferAllocate();
ctx->snapshot_builder =
@@ -231,7 +233,8 @@ CreateInitDecodingContext(char *plugin,
List *output_plugin_options,
bool need_full_snapshot,
XLogRecPtr restart_lsn,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress)
@@ -328,7 +331,7 @@ CreateInitDecodingContext(char *plugin,
ctx = StartupDecodingContext(NIL, restart_lsn, xmin_horizon,
need_full_snapshot, false,
- xl_routine, prepare_write, do_write,
+ page_read, cleanup_cb, prepare_write, do_write,
update_progress);
/* call output plugin initialization callback */
@@ -376,7 +379,8 @@ LogicalDecodingContext *
CreateDecodingContext(XLogRecPtr start_lsn,
List *output_plugin_options,
bool fast_forward,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress)
@@ -429,8 +433,8 @@ CreateDecodingContext(XLogRecPtr start_lsn,
ctx = StartupDecodingContext(output_plugin_options,
start_lsn, InvalidTransactionId, false,
- fast_forward, xl_routine, prepare_write,
- do_write, update_progress);
+ fast_forward, page_read, cleanup_cb,
+ prepare_write, do_write, update_progress);
/* call output plugin initialization callback */
old_context = MemoryContextSwitchTo(ctx->context);
@@ -483,7 +487,13 @@ DecodingContextFindStartpoint(LogicalDecodingContext *ctx)
char *err = NULL;
/* the read_page callback waits for new WAL */
- record = XLogReadRecord(ctx->reader, &err);
+ while (XLogReadRecord(ctx->reader, &record, &err) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!ctx->page_read(ctx->reader))
+ break;
+ }
+
if (err)
elog(ERROR, "%s", err);
if (!record)
diff --git a/src/backend/replication/logical/logicalfuncs.c b/src/backend/replication/logical/logicalfuncs.c
index b99c94e848..fcc81b358e 100644
--- a/src/backend/replication/logical/logicalfuncs.c
+++ b/src/backend/replication/logical/logicalfuncs.c
@@ -233,9 +233,8 @@ pg_logical_slot_get_changes_guts(FunctionCallInfo fcinfo, bool confirm, bool bin
ctx = CreateDecodingContext(InvalidXLogRecPtr,
options,
false,
- XL_ROUTINE(.page_read = read_local_xlog_page,
- .segment_open = wal_segment_open,
- .segment_close = wal_segment_close),
+ read_local_xlog_page,
+ wal_segment_close,
LogicalOutputPrepareWrite,
LogicalOutputWrite, NULL);
@@ -284,7 +283,13 @@ pg_logical_slot_get_changes_guts(FunctionCallInfo fcinfo, bool confirm, bool bin
XLogRecord *record;
char *errm = NULL;
- record = XLogReadRecord(ctx->reader, &errm);
+ while (XLogReadRecord(ctx->reader, &record, &errm) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!ctx->page_read(ctx->reader))
+ break;
+ }
+
if (errm)
elog(ERROR, "%s", errm);
diff --git a/src/backend/replication/slotfuncs.c b/src/backend/replication/slotfuncs.c
index 88033a79b2..8a678074d5 100644
--- a/src/backend/replication/slotfuncs.c
+++ b/src/backend/replication/slotfuncs.c
@@ -152,9 +152,8 @@ create_logical_replication_slot(char *name, char *plugin,
ctx = CreateInitDecodingContext(plugin, NIL,
false, /* just catalogs is OK */
restart_lsn,
- XL_ROUTINE(.page_read = read_local_xlog_page,
- .segment_open = wal_segment_open,
- .segment_close = wal_segment_close),
+ read_local_xlog_page,
+ wal_segment_close,
NULL, NULL, NULL);
/*
@@ -486,9 +485,8 @@ pg_logical_replication_slot_advance(XLogRecPtr moveto)
ctx = CreateDecodingContext(InvalidXLogRecPtr,
NIL,
true, /* fast_forward */
- XL_ROUTINE(.page_read = read_local_xlog_page,
- .segment_open = wal_segment_open,
- .segment_close = wal_segment_close),
+ read_local_xlog_page,
+ wal_segment_close,
NULL, NULL, NULL);
/*
@@ -513,7 +511,13 @@ pg_logical_replication_slot_advance(XLogRecPtr moveto)
* Read records. No changes are generated in fast_forward mode,
* but snapbuilder/slot statuses are updated properly.
*/
- record = XLogReadRecord(ctx->reader, &errm);
+ while (XLogReadRecord(ctx->reader, &record, &errm) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!ctx->page_read(ctx->reader))
+ break;
+ }
+
if (errm)
elog(ERROR, "%s", errm);
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index bf2711bddd..2fd24a9b14 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -580,10 +580,7 @@ StartReplication(StartReplicationCmd *cmd)
/* create xlogreader for physical replication */
xlogreader =
- XLogReaderAllocate(wal_segment_size, NULL,
- XL_ROUTINE(.segment_open = WalSndSegmentOpen,
- .segment_close = wal_segment_close),
- NULL);
+ XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
if (!xlogreader)
ereport(ERROR,
@@ -812,9 +809,11 @@ StartReplication(StartReplicationCmd *cmd)
* set every time WAL is flushed.
*/
static bool
-logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetRecPtr, char *cur_page)
+logical_read_xlog_page(XLogReaderState *state)
{
+ XLogRecPtr targetPagePtr = state->readPagePtr;
+ int reqLen = state->readLen;
+ char *cur_page = state->readBuf;
XLogRecPtr flushptr;
int count;
WALReadError errinfo;
@@ -842,7 +841,7 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
count = flushptr - targetPagePtr; /* part of the page available */
/* now actually read the data, we know it's there */
- if (!WALRead(state,
+ if (!WALRead(state, WalSndSegmentOpen, wal_segment_close,
cur_page,
targetPagePtr,
XLOG_BLCKSZ,
@@ -1014,9 +1013,8 @@ CreateReplicationSlot(CreateReplicationSlotCmd *cmd)
ctx = CreateInitDecodingContext(cmd->plugin, NIL, need_full_snapshot,
InvalidXLogRecPtr,
- XL_ROUTINE(.page_read = logical_read_xlog_page,
- .segment_open = WalSndSegmentOpen,
- .segment_close = wal_segment_close),
+ logical_read_xlog_page,
+ wal_segment_close,
WalSndPrepareWrite, WalSndWriteData,
WalSndUpdateProgress);
@@ -1179,9 +1177,8 @@ StartLogicalReplication(StartReplicationCmd *cmd)
*/
logical_decoding_ctx =
CreateDecodingContext(cmd->startpoint, cmd->options, false,
- XL_ROUTINE(.page_read = logical_read_xlog_page,
- .segment_open = WalSndSegmentOpen,
- .segment_close = wal_segment_close),
+ logical_read_xlog_page,
+ wal_segment_close,
WalSndPrepareWrite, WalSndWriteData,
WalSndUpdateProgress);
xlogreader = logical_decoding_ctx->reader;
@@ -2761,7 +2758,7 @@ XLogSendPhysical(void)
enlargeStringInfo(&output_message, nbytes);
retry:
- if (!WALRead(xlogreader,
+ if (!WALRead(xlogreader, WalSndSegmentOpen, wal_segment_close,
&output_message.data[output_message.len],
startptr,
nbytes,
@@ -2859,7 +2856,12 @@ XLogSendLogical(void)
*/
WalSndCaughtUp = false;
- record = XLogReadRecord(logical_decoding_ctx->reader, &errm);
+ while (XLogReadRecord(logical_decoding_ctx->reader, &record, &errm) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!logical_decoding_ctx->page_read(logical_decoding_ctx->reader))
+ break;
+ }
/* xlog record was invalid */
if (errm != NULL)
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index 0b7e73ae79..3dd6df4be6 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -40,15 +40,9 @@ static int xlogreadfd = -1;
static XLogSegNo xlogreadsegno = -1;
static char xlogfpath[MAXPGPATH];
-typedef struct XLogPageReadPrivate
-{
- const char *restoreCommand;
- int tliIndex;
-} XLogPageReadPrivate;
-
-static bool SimpleXLogPageRead(XLogReaderState *xlogreader,
- XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
+static bool SimpleXLogPageRead(XLogReaderState *xlogreader,
+ const char *datadir, int *tliIndex,
+ const char *restoreCommand);
/*
* Read WAL from the datadir/pg_wal, starting from 'startpoint' on timeline
@@ -62,20 +56,22 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
XLogRecord *record;
XLogReaderState *xlogreader;
char *errormsg;
- XLogPageReadPrivate private;
- private.tliIndex = tliIndex;
- private.restoreCommand = restoreCommand;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir,
- XL_ROUTINE(.page_read = &SimpleXLogPageRead),
- &private);
+ xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+
if (xlogreader == NULL)
pg_fatal("out of memory");
XLogBeginRead(xlogreader, startpoint);
do
{
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!SimpleXLogPageRead(xlogreader, datadir,
+ &tliIndex, restoreCommand))
+ break;
+ }
if (record == NULL)
{
@@ -113,19 +109,19 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex,
XLogRecord *record;
XLogReaderState *xlogreader;
char *errormsg;
- XLogPageReadPrivate private;
XLogRecPtr endptr;
- private.tliIndex = tliIndex;
- private.restoreCommand = restoreCommand;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir,
- XL_ROUTINE(.page_read = &SimpleXLogPageRead),
- &private);
+ xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
XLogBeginRead(xlogreader, ptr);
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!SimpleXLogPageRead(xlogreader, datadir, &tliIndex, restoreCommand))
+ break;
+ }
if (record == NULL)
{
if (errormsg)
@@ -160,7 +156,6 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
XLogRecPtr searchptr;
XLogReaderState *xlogreader;
char *errormsg;
- XLogPageReadPrivate private;
/*
* The given fork pointer points to the end of the last common record,
@@ -176,11 +171,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
forkptr += SizeOfXLogShortPHD;
}
- private.tliIndex = tliIndex;
- private.restoreCommand = restoreCommand;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir,
- XL_ROUTINE(.page_read = &SimpleXLogPageRead),
- &private);
+ xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
@@ -190,7 +181,13 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
uint8 info;
XLogBeginRead(xlogreader, searchptr);
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!SimpleXLogPageRead(xlogreader, datadir,
+ &tliIndex, restoreCommand))
+ break;
+ }
if (record == NULL)
{
@@ -237,10 +234,11 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
/* XLogReader callback function, to read a WAL page */
static bool
-SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *readBuf)
+SimpleXLogPageRead(XLogReaderState *xlogreader, const char *datadir,
+ int *tliIndex, const char *restoreCommand)
{
- XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
+ XLogRecPtr targetPagePtr = xlogreader->readPagePtr;
+ char *readBuf = xlogreader->readBuf;
uint32 targetPageOff;
XLogRecPtr targetSegEnd;
XLogSegNo targetSegNo;
@@ -273,14 +271,14 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
* be done both forward and backward, consider also switching timeline
* accordingly.
*/
- while (private->tliIndex < targetNentries - 1 &&
- targetHistory[private->tliIndex].end < targetSegEnd)
- private->tliIndex++;
- while (private->tliIndex > 0 &&
- targetHistory[private->tliIndex].begin >= targetSegEnd)
- private->tliIndex--;
+ while (*tliIndex < targetNentries - 1 &&
+ targetHistory[*tliIndex].end < targetSegEnd)
+ (*tliIndex)++;
+ while (*tliIndex > 0 &&
+ targetHistory[*tliIndex].begin >= targetSegEnd)
+ (*tliIndex)--;
- XLogFileName(xlogfname, targetHistory[private->tliIndex].tli,
+ XLogFileName(xlogfname, targetHistory[*tliIndex].tli,
xlogreadsegno, WalSegSz);
snprintf(xlogfpath, MAXPGPATH, "%s/" XLOGDIR "/%s",
@@ -293,7 +291,7 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
/*
* If we have no restore_command to execute, then exit.
*/
- if (private->restoreCommand == NULL)
+ if (restoreCommand == NULL)
{
pg_log_error("could not open file \"%s\": %m", xlogfpath);
xlogreader->readLen = -1;
@@ -307,7 +305,7 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
xlogreadfd = RestoreArchivedFile(xlogreader->segcxt.ws_dir,
xlogfname,
WalSegSz,
- private->restoreCommand);
+ restoreCommand);
if (xlogreadfd < 0)
{
@@ -349,7 +347,7 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
Assert(targetSegNo == xlogreadsegno);
- xlogreader->seg.ws_tli = targetHistory[private->tliIndex].tli;
+ xlogreader->seg.ws_tli = targetHistory[*tliIndex].tli;
xlogreader->readLen = XLOG_BLCKSZ;
return true;
}
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index ab0c28d9b7..02e71138cc 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -330,12 +330,17 @@ WALDumpCloseSegment(XLogReaderState *state)
state->seg.ws_file = -1;
}
-/* pg_waldump's XLogReaderRoutine->page_read callback */
+/*
+ * pg_waldump's WAL page rader, also used as page_read callback for
+ * XLogFindNextRecord
+ */
static bool
-WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetPtr, char *readBuff)
+WALDumpReadPage(XLogReaderState *state, void *priv)
{
- XLogDumpPrivate *private = state->private_data;
+ XLogRecPtr targetPagePtr = state->readPagePtr;
+ int reqLen = state->readLen;
+ char *readBuff = state->readBuf;
+ XLogDumpPrivate *private = (XLogDumpPrivate *) priv;
int count = XLOG_BLCKSZ;
WALReadError errinfo;
@@ -353,8 +358,8 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
}
}
- if (!WALRead(state, readBuff, targetPagePtr, count, private->timeline,
- &errinfo))
+ if (!WALRead(state, WALDumpOpenSegment, WALDumpCloseSegment,
+ readBuff, targetPagePtr, count, private->timeline, &errinfo))
{
WALOpenSegment *seg = &errinfo.wre_seg;
char fname[MAXPGPATH];
@@ -374,6 +379,7 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
(Size) errinfo.wre_req);
}
+ Assert(count >= state->readLen);
state->readLen = count;
return true;
}
@@ -1042,16 +1048,14 @@ main(int argc, char **argv)
/* we have everything we need, start reading */
xlogreader_state =
- XLogReaderAllocate(WalSegSz, waldir,
- XL_ROUTINE(.page_read = WALDumpReadPage,
- .segment_open = WALDumpOpenSegment,
- .segment_close = WALDumpCloseSegment),
- &private);
+ XLogReaderAllocate(WalSegSz, waldir, WALDumpCloseSegment);
+
if (!xlogreader_state)
fatal_error("out of memory");
/* first find a valid recptr to start from */
- first_record = XLogFindNextRecord(xlogreader_state, private.startptr);
+ first_record = XLogFindNextRecord(xlogreader_state, private.startptr,
+ &WALDumpReadPage, (void*) &private);
if (first_record == InvalidXLogRecPtr)
fatal_error("could not find a valid record after %X/%X",
@@ -1075,7 +1079,13 @@ main(int argc, char **argv)
for (;;)
{
/* try to read the next record */
- record = XLogReadRecord(xlogreader_state, &errormsg);
+ while (XLogReadRecord(xlogreader_state, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!WALDumpReadPage(xlogreader_state, (void *) &private))
+ break;
+ }
+
if (!record)
{
if (!config.follow || private.endptr_reached)
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index 6c8848e14f..6a7fe140cb 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -57,64 +57,15 @@ typedef struct WALSegmentContext
typedef struct XLogReaderState XLogReaderState;
-/* Function type definition for the read_page callback */
-typedef bool (*XLogPageReadCB) (XLogReaderState *xlogreader,
- XLogRecPtr targetPagePtr,
- int reqLen,
- XLogRecPtr targetRecPtr,
- char *readBuf);
+/* Function type definition for the segment cleanup callback */
+typedef void (*WALSegmentCleanupCB) (XLogReaderState *xlogreader);
+
+/* Function type definition for the open/close callbacks for WALRead() */
typedef void (*WALSegmentOpenCB) (XLogReaderState *xlogreader,
XLogSegNo nextSegNo,
TimeLineID *tli_p);
typedef void (*WALSegmentCloseCB) (XLogReaderState *xlogreader);
-typedef struct XLogReaderRoutine
-{
- /*
- * Data input callback
- *
- * This callback shall read at least reqLen valid bytes of the xlog page
- * starting at targetPagePtr, and store them in readBuf. The callback
- * shall return the number of bytes read (never more than XLOG_BLCKSZ), or
- * -1 on failure. The callback shall sleep, if necessary, to wait for the
- * requested bytes to become available. The callback will not be invoked
- * again for the same page unless more than the returned number of bytes
- * are needed.
- *
- * targetRecPtr is the position of the WAL record we're reading. Usually
- * it is equal to targetPagePtr + reqLen, but sometimes xlogreader needs
- * to read and verify the page or segment header, before it reads the
- * actual WAL record it's interested in. In that case, targetRecPtr can
- * be used to determine which timeline to read the page from.
- *
- * The callback shall set ->seg.ws_tli to the TLI of the file the page was
- * read from.
- */
- XLogPageReadCB page_read;
-
- /*
- * Callback to open the specified WAL segment for reading. ->seg.ws_file
- * shall be set to the file descriptor of the opened segment. In case of
- * failure, an error shall be raised by the callback and it shall not
- * return.
- *
- * "nextSegNo" is the number of the segment to be opened.
- *
- * "tli_p" is an input/output argument. WALRead() uses it to pass the
- * timeline in which the new segment should be found, but the callback can
- * use it to return the TLI that it actually opened.
- */
- WALSegmentOpenCB segment_open;
-
- /*
- * WAL segment close callback. ->seg.ws_file shall be set to a negative
- * number.
- */
- WALSegmentCloseCB segment_close;
-} XLogReaderRoutine;
-
-#define XL_ROUTINE(...) &(XLogReaderRoutine){__VA_ARGS__}
-
typedef struct
{
/* Is this block ref in use? */
@@ -144,12 +95,35 @@ typedef struct
uint16 data_bufsz;
} DecodedBkpBlock;
+/* Return code from XLogReadRecord */
+typedef enum XLogReadRecordResult
+{
+ XLREAD_SUCCESS, /* record is successfully read */
+ XLREAD_NEED_DATA, /* need more data. see XLogReadRecord. */
+ XLREAD_FAIL /* failed during reading a record */
+} XLogReadRecordResult;
+
+/*
+ * internal state of XLogReadRecord
+ *
+ * XLogReadState runs a state machine while reading a record. Theses states
+ * are not seen outside the function. Each state may repeat several times
+ * exiting requesting caller for new data. See the comment of XLogReadRecrod
+ * for details.
+ */
+typedef enum XLogReadRecordState {
+ XLREAD_NEXT_RECORD,
+ XLREAD_TOT_LEN,
+ XLREAD_FIRST_FRAGMENT,
+ XLREAD_CONTINUATION
+} XLogReadRecordState;
+
struct XLogReaderState
{
/*
* Operational callbacks
*/
- XLogReaderRoutine routine;
+ WALSegmentCleanupCB cleanup_cb;
/* ----------------------------------------
* Public parameters
@@ -162,18 +136,14 @@ struct XLogReaderState
*/
uint64 system_identifier;
- /*
- * Opaque data for callbacks to use. Not used by XLogReader.
- */
- void *private_data;
-
/*
* Start and end point of last record read. EndRecPtr is also used as the
* position to read next. Calling XLogBeginRead() sets EndRecPtr to the
* starting position and ReadRecPtr to invalid.
*/
- XLogRecPtr ReadRecPtr; /* start of last record read */
+ XLogRecPtr ReadRecPtr; /* start of last record read or being read */
XLogRecPtr EndRecPtr; /* end+1 of last record read */
+ XLogRecPtr PrevRecPtr; /* start of previous record read */
/* ----------------------------------------
* Communication with page reader
@@ -187,7 +157,9 @@ struct XLogReaderState
* the request, or -1 on error */
TimeLineID readPageTLI; /* TLI for data currently in readBuf */
char *readBuf; /* buffer to store data */
- bool page_verified; /* is the page on the buffer verified? */
+ bool page_verified; /* is the page header on the buffer verified? */
+ bool record_verified;/* is the current record header verified? */
+
/* ----------------------------------------
@@ -227,8 +199,6 @@ struct XLogReaderState
XLogRecPtr latestPagePtr;
TimeLineID latestPageTLI;
- /* beginning of the WAL record being read. */
- XLogRecPtr currRecPtr;
/* timeline to read it from, 0 if a lookup is required */
TimeLineID currTLI;
@@ -255,6 +225,15 @@ struct XLogReaderState
char *readRecordBuf;
uint32 readRecordBufSize;
+ /*
+ * XLogReadRecord() state
+ */
+ XLogReadRecordState readRecordState;/* state machine state */
+ int recordGotLen; /* amount of current record that has
+ * already been read */
+ int recordRemainLen; /* length of current record that remains */
+ XLogRecPtr recordContRecPtr; /* where the current record continues */
+
/* Buffer to hold error message */
char *errormsg_buf;
};
@@ -262,9 +241,7 @@ struct XLogReaderState
/* Get a new XLogReader */
extern XLogReaderState *XLogReaderAllocate(int wal_segment_size,
const char *waldir,
- XLogReaderRoutine *routine,
- void *private_data);
-extern XLogReaderRoutine *LocalXLogReaderRoutine(void);
+ WALSegmentCleanupCB cleanup_cb);
/* Free an XLogReader */
extern void XLogReaderFree(XLogReaderState *state);
@@ -272,12 +249,17 @@ extern void XLogReaderFree(XLogReaderState *state);
/* Position the XLogReader to given record */
extern void XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr);
#ifdef FRONTEND
-extern XLogRecPtr XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr);
+/* Function type definition for the read_page callback */
+typedef bool (*XLogFindNextRecordCB) (XLogReaderState *xlogreader,
+ void *private);
+extern XLogRecPtr XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
+ XLogFindNextRecordCB read_page, void *private);
#endif /* FRONTEND */
/* Read the next XLog record. Returns NULL on end-of-WAL or failure */
-extern struct XLogRecord *XLogReadRecord(XLogReaderState *state,
- char **errormsg);
+extern XLogReadRecordResult XLogReadRecord(XLogReaderState *state,
+ XLogRecord **record,
+ char **errormsg);
/* Validate a page */
extern bool XLogReaderValidatePageHeader(XLogReaderState *state,
@@ -297,6 +279,7 @@ typedef struct WALReadError
} WALReadError;
extern bool WALRead(XLogReaderState *state,
+ WALSegmentOpenCB segopenfn, WALSegmentCloseCB sgclosefn,
char *buf, XLogRecPtr startptr, Size count,
TimeLineID tli, WALReadError *errinfo);
diff --git a/src/include/access/xlogutils.h b/src/include/access/xlogutils.h
index c287edf206..482e4ced69 100644
--- a/src/include/access/xlogutils.h
+++ b/src/include/access/xlogutils.h
@@ -47,9 +47,7 @@ extern Buffer XLogReadBufferExtended(RelFileNode rnode, ForkNumber forknum,
extern Relation CreateFakeRelcacheEntry(RelFileNode rnode);
extern void FreeFakeRelcacheEntry(Relation fakerel);
-extern bool read_local_xlog_page(XLogReaderState *state,
- XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetRecPtr, char *cur_page);
+extern bool read_local_xlog_page(XLogReaderState *state);
extern void wal_segment_open(XLogReaderState *state,
XLogSegNo nextSegNo,
TimeLineID *tli_p);
diff --git a/src/include/replication/logical.h b/src/include/replication/logical.h
index c2f2475e5d..a743db49cf 100644
--- a/src/include/replication/logical.h
+++ b/src/include/replication/logical.h
@@ -29,6 +29,10 @@ typedef void (*LogicalOutputPluginWriterUpdateProgress) (struct LogicalDecodingC
TransactionId xid
);
+typedef struct LogicalDecodingContext LogicalDecodingContext;
+
+typedef bool (*LogicalDecodingXLogPageReadCB)(XLogReaderState *ctx);
+
typedef struct LogicalDecodingContext
{
/* memory context this is all allocated in */
@@ -39,6 +43,7 @@ typedef struct LogicalDecodingContext
/* infrastructure pieces for decoding */
XLogReaderState *reader;
+ LogicalDecodingXLogPageReadCB page_read;
struct ReorderBuffer *reorder;
struct SnapBuild *snapshot_builder;
@@ -95,14 +100,16 @@ extern LogicalDecodingContext *CreateInitDecodingContext(char *plugin,
List *output_plugin_options,
bool need_full_snapshot,
XLogRecPtr restart_lsn,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress);
extern LogicalDecodingContext *CreateDecodingContext(XLogRecPtr start_lsn,
List *output_plugin_options,
bool fast_forward,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress);
--
2.18.4
----Next_Part(Thu_Jul__2_13_53_30_2020_072)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
filename="v15-0003-Remove-globals-readOff-readLen-and-readSegNo.patch"
^ permalink raw reply [nested|flat] 70+ messages in thread
* [PATCH v15 2/4] Move page-reader out of XLogReadRecord
@ 2019-09-10 03:58 Kyotaro Horiguchi <[email protected]>
0 siblings, 0 replies; 70+ messages in thread
From: Kyotaro Horiguchi @ 2019-09-10 03:58 UTC (permalink / raw)
This is the second step of removing callbacks from WAL record reader.
Since it is essential to take in additional data while reading a
record, the function have to ask caller for new data while keeping
working state. Thus the function is turned into a state machine.
---
src/backend/access/transam/twophase.c | 15 +-
src/backend/access/transam/xlog.c | 58 +-
src/backend/access/transam/xlogreader.c | 700 +++++++++++-------
src/backend/access/transam/xlogutils.c | 17 +-
src/backend/replication/logical/logical.c | 26 +-
.../replication/logical/logicalfuncs.c | 13 +-
src/backend/replication/slotfuncs.c | 18 +-
src/backend/replication/walsender.c | 32 +-
src/bin/pg_rewind/parsexlog.c | 84 +--
src/bin/pg_waldump/pg_waldump.c | 36 +-
src/include/access/xlogreader.h | 121 ++-
src/include/access/xlogutils.h | 4 +-
src/include/replication/logical.h | 11 +-
13 files changed, 648 insertions(+), 487 deletions(-)
diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index 9b2e59bf0e..b0d60a0d0f 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -1330,11 +1330,8 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
XLogReaderState *xlogreader;
char *errormsg;
- xlogreader = XLogReaderAllocate(wal_segment_size, NULL,
- XL_ROUTINE(.page_read = &read_local_xlog_page,
- .segment_open = &wal_segment_open,
- .segment_close = &wal_segment_close),
- NULL);
+ xlogreader = XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+
if (!xlogreader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
@@ -1342,7 +1339,13 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
errdetail("Failed while allocating a WAL reading processor.")));
XLogBeginRead(xlogreader, lsn);
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!read_local_xlog_page(xlogreader))
+ break;
+ }
+
if (record == NULL)
ereport(ERROR,
(errcode_for_file_access(),
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index e570e56a24..f9b0108602 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -831,13 +831,6 @@ static XLogSource currentSource = XLOG_FROM_ANY;
static bool lastSourceFailed = false;
static bool pendingWalRcvRestart = false;
-typedef struct XLogPageReadPrivate
-{
- int emode;
- bool fetching_ckpt; /* are we fetching a checkpoint record? */
- bool randAccess;
-} XLogPageReadPrivate;
-
/*
* These variables track when we last obtained some WAL data to process,
* and where we got it from. (XLogReceiptSource is initially the same as
@@ -912,8 +905,8 @@ static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath,
static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli,
XLogSource source, bool notfoundOk);
static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, XLogSource source);
-static bool XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
+static bool XLogPageRead(XLogReaderState *xlogreader,
+ bool fetching_ckpt, int emode, bool randAccess);
static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
bool fetching_ckpt, XLogRecPtr tliRecPtr);
static int emode_for_corrupt_record(int emode, XLogRecPtr RecPtr);
@@ -1225,8 +1218,7 @@ XLogInsertRecord(XLogRecData *rdata,
appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len);
if (!debug_reader)
- debug_reader = XLogReaderAllocate(wal_segment_size, NULL,
- XL_ROUTINE(), NULL);
+ debug_reader = XLogReaderAllocate(wal_segment_size, NULL NULL);
if (!debug_reader)
{
@@ -4326,15 +4318,10 @@ CleanupBackupHistory(void)
* record is available.
*/
static XLogRecord *
-ReadRecord(XLogReaderState *xlogreader, int emode,
- bool fetching_ckpt)
+ReadRecord(XLogReaderState *xlogreader, int emode, bool fetching_ckpt)
{
XLogRecord *record;
- XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
-
- private->fetching_ckpt = fetching_ckpt;
- private->emode = emode;
- private->randAccess = (xlogreader->ReadRecPtr == InvalidXLogRecPtr);
+ bool randAccess = (xlogreader->ReadRecPtr == InvalidXLogRecPtr);
/* This is the first attempt to read this page. */
lastSourceFailed = false;
@@ -4342,8 +4329,16 @@ ReadRecord(XLogReaderState *xlogreader, int emode,
for (;;)
{
char *errormsg;
+ XLogReadRecordResult result;
+
+ while ((result = XLogReadRecord(xlogreader, &record, &errormsg))
+ == XLREAD_NEED_DATA)
+ {
+ if (!XLogPageRead(xlogreader, fetching_ckpt, emode, randAccess))
+ break;
+
+ }
- record = XLogReadRecord(xlogreader, &errormsg);
ReadRecPtr = xlogreader->ReadRecPtr;
EndRecPtr = xlogreader->EndRecPtr;
if (record == NULL)
@@ -6320,7 +6315,6 @@ StartupXLOG(void)
bool backupFromStandby = false;
DBState dbstate_at_startup;
XLogReaderState *xlogreader;
- XLogPageReadPrivate private;
bool fast_promoted = false;
struct stat st;
@@ -6480,13 +6474,9 @@ StartupXLOG(void)
OwnLatch(&XLogCtl->recoveryWakeupLatch);
/* Set up XLOG reader facility */
- MemSet(&private, 0, sizeof(XLogPageReadPrivate));
xlogreader =
- XLogReaderAllocate(wal_segment_size, NULL,
- XL_ROUTINE(.page_read = &XLogPageRead,
- .segment_open = NULL,
- .segment_close = wal_segment_close),
- &private);
+ XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+
if (!xlogreader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
@@ -11860,12 +11850,13 @@ CancelBackup(void)
* sleep and retry.
*/
static bool
-XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetRecPtr, char *readBuf)
+XLogPageRead(XLogReaderState *xlogreader,
+ bool fetching_ckpt, int emode, bool randAccess)
{
- XLogPageReadPrivate *private =
- (XLogPageReadPrivate *) xlogreader->private_data;
- int emode = private->emode;
+ char *readBuf = xlogreader->readBuf;
+ XLogRecPtr targetPagePtr = xlogreader->readPagePtr;
+ int reqLen = xlogreader->readLen;
+ XLogRecPtr targetRecPtr = xlogreader->ReadRecPtr;
uint32 targetPageOff;
XLogSegNo targetSegNo PG_USED_FOR_ASSERTS_ONLY;
int r;
@@ -11908,8 +11899,8 @@ retry:
flushedUpto < targetPagePtr + reqLen))
{
if (!WaitForWALToBecomeAvailable(targetPagePtr + reqLen,
- private->randAccess,
- private->fetching_ckpt,
+ randAccess,
+ fetching_ckpt,
targetRecPtr))
{
if (readFile >= 0)
@@ -12014,6 +12005,7 @@ retry:
goto next_record_is_invalid;
}
+ Assert(xlogreader->readPagePtr == targetPagePtr);
xlogreader->readLen = readLen;
return true;
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index 3d599325ee..9281b57379 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -40,7 +40,7 @@ static bool XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr,
int reqLen, bool header_inclusive);
static void XLogReaderInvalReadState(XLogReaderState *state);
static bool ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
- XLogRecPtr PrevRecPtr, XLogRecord *record, bool randAccess);
+ XLogRecPtr PrevRecPtr, XLogRecord *record);
static bool ValidXLogRecord(XLogReaderState *state, XLogRecord *record,
XLogRecPtr recptr);
static void ResetDecoder(XLogReaderState *state);
@@ -73,7 +73,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...)
*/
XLogReaderState *
XLogReaderAllocate(int wal_segment_size, const char *waldir,
- XLogReaderRoutine *routine, void *private_data)
+ WALSegmentCleanupCB cleanup_cb)
{
XLogReaderState *state;
@@ -84,7 +84,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
return NULL;
/* initialize caller-provided support functions */
- state->routine = *routine;
+ state->cleanup_cb = cleanup_cb;
state->max_block_id = -1;
@@ -107,8 +107,6 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
WALOpenSegmentInit(&state->seg, &state->segcxt, wal_segment_size,
waldir);
- /* system_identifier initialized to zeroes above */
- state->private_data = private_data;
/* ReadRecPtr, EndRecPtr and readLen initialized to zeroes above */
state->errormsg_buf = palloc_extended(MAX_ERRORMSG_LEN + 1,
MCXT_ALLOC_NO_OOM);
@@ -140,8 +138,8 @@ XLogReaderFree(XLogReaderState *state)
{
int block_id;
- if (state->seg.ws_file != -1)
- state->routine.segment_close(state);
+ if (state->seg.ws_file >= 0)
+ state->cleanup_cb(state);
for (block_id = 0; block_id <= XLR_MAX_BLOCK_ID; block_id++)
{
@@ -246,6 +244,7 @@ XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
/* Begin at the passed-in record pointer. */
state->EndRecPtr = RecPtr;
state->ReadRecPtr = InvalidXLogRecPtr;
+ state->readRecordState = XLREAD_NEXT_RECORD;
}
/*
@@ -254,317 +253,452 @@ XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
* XLogBeginRead() or XLogFindNextRecord() must be called before the first call
* to XLogReadRecord().
*
- * If the page_read callback fails to read the requested data, NULL is
- * returned. The callback is expected to have reported the error; errormsg
- * is set to NULL.
+ * This function may return XLREAD_NEED_DATA several times before returning a
+ * result record. The caller shall read in some new data then call this
+ * function again with the same parameters.
*
- * If the reading fails for some other reason, NULL is also returned, and
- * *errormsg is set to a string with details of the failure.
+ * When a record is successfully read, returns XLREAD_SUCCESS with result
+ * record being stored in *record. Otherwise *record is NULL.
*
- * The returned pointer (or *errormsg) points to an internal buffer that's
- * valid until the next call to XLogReadRecord.
+ * Returns XLREAD_NEED_DATA if more data is needed to finish reading the
+ * current record. In that case, state->readPagePtr and state->readLen inform
+ * the desired position and minimum length of data needed. The caller shall
+ * read in the requested data and set state->readBuf to point to a buffer
+ * containing it. The caller must also set state->readPageTLI and
+ * state->readLen to indicate the timeline that it was read from, and the
+ * length of data that is now available (which must be >= given readLen),
+ * respectively.
+ *
+ * If invalid data is encountered, returns XLREAD_FAIL with *record being set to
+ * NULL. *errormsg is set to a string with details of the failure.
+ * The returned pointer (or *errormsg) points to an internal buffer that's valid
+ * until the next call to XLogReadRecord.
+ *
+ *
+ * This function runs a state machine consists of the following states.
+ *
+ * XLREAD_NEXT_RECORD :
+ * The initial state, if called with valid RecPtr, try to read a record at
+ * that position. If invalid RecPtr is given try to read a record just after
+ * the last one previously read.
+ * This state ens after setting ReadRecPtr. Then goes to XLREAD_TOT_LEN.
+ *
+ * XLREAD_TOT_LEN:
+ * Examining record header. Ends after reading record total
+ * length. recordRemainLen and recordGotLen are initialized.
+ *
+ * XLREAD_FIRST_FRAGMENT:
+ * Reading the first fragment. Ends with finishing reading a single
+ * record. Goes to XLREAD_NEXT_RECORD if that's all or
+ * XLREAD_CONTINUATION if we have continuation.
+
+ * XLREAD_CONTINUATION:
+ * Reading continuation of record. Ends with finishing the whole record then
+ * goes to XLREAD_NEXT_RECORD. During this state, recordRemainLen indicates
+ * how much is left and readRecordBuf holds the partially assert
+ * record.recordContRecPtr points to the beginning of the next page where to
+ * continue.
+ *
+ * If wrong data found in any state, the state machine stays at the current
+ * state. This behavior allows to continue reading a reacord switching among
+ * different souces, while streaming replication.
*/
-XLogRecord *
-XLogReadRecord(XLogReaderState *state, char **errormsg)
+XLogReadRecordResult
+XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
{
- XLogRecPtr RecPtr;
- XLogRecord *record;
- XLogRecPtr targetPagePtr;
- bool randAccess;
- uint32 len,
- total_len;
- uint32 targetRecOff;
- uint32 pageHeaderSize;
- bool gotheader;
+ XLogRecord *prec;
- /*
- * randAccess indicates whether to verify the previous-record pointer of
- * the record we're reading. We only do this if we're reading
- * sequentially, which is what we initially assume.
- */
- randAccess = false;
+ *record = NULL;
/* reset error state */
*errormsg = NULL;
state->errormsg_buf[0] = '\0';
- ResetDecoder(state);
-
- RecPtr = state->EndRecPtr;
-
- if (state->ReadRecPtr != InvalidXLogRecPtr)
- {
- /* read the record after the one we just read */
-
- /*
- * EndRecPtr is pointing to end+1 of the previous WAL record. If
- * we're at a page boundary, no more records can fit on the current
- * page. We must skip over the page header, but we can't do that until
- * we've read in the page, since the header size is variable.
- */
- }
- else
- {
- /*
- * Caller supplied a position to start at.
- *
- * In this case, EndRecPtr should already be pointing to a valid
- * record starting position.
- */
- Assert(XRecOffIsValid(RecPtr));
- randAccess = true;
- }
-
- state->currRecPtr = RecPtr;
-
- targetPagePtr = RecPtr - (RecPtr % XLOG_BLCKSZ);
- targetRecOff = RecPtr % XLOG_BLCKSZ;
-
- /*
- * Read the page containing the record into state->readBuf. Request enough
- * byte to cover the whole record header, or at least the part of it that
- * fits on the same page.
- */
- while (XLogNeedData(state, targetPagePtr,
- Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
- targetRecOff != 0))
- {
- if (!state->routine.page_read(state, state->readPagePtr, state->readLen,
- RecPtr, state->readBuf))
- break;
- }
-
- if (!state->page_verified)
- goto err;
-
- /*
- * We have at least the page header, so we can examine it now.
- */
- pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
- if (targetRecOff == 0)
- {
- /*
- * At page start, so skip over page header.
- */
- RecPtr += pageHeaderSize;
- targetRecOff = pageHeaderSize;
- }
- else if (targetRecOff < pageHeaderSize)
- {
- report_invalid_record(state, "invalid record offset at %X/%X",
- (uint32) (RecPtr >> 32), (uint32) RecPtr);
- goto err;
- }
-
- if ((((XLogPageHeader) state->readBuf)->xlp_info & XLP_FIRST_IS_CONTRECORD) &&
- targetRecOff == pageHeaderSize)
- {
- report_invalid_record(state, "contrecord is requested by %X/%X",
- (uint32) (RecPtr >> 32), (uint32) RecPtr);
- goto err;
- }
-
- /* XLogNeedData has verified the page header */
- Assert(pageHeaderSize <= state->readLen);
-
- /*
- * Read the record length.
- *
- * NB: Even though we use an XLogRecord pointer here, the whole record
- * header might not fit on this page. xl_tot_len is the first field of the
- * struct, so it must be on this page (the records are MAXALIGNed), but we
- * cannot access any other fields until we've verified that we got the
- * whole header.
- */
- record = (XLogRecord *) (state->readBuf + RecPtr % XLOG_BLCKSZ);
- total_len = record->xl_tot_len;
-
- /*
- * If the whole record header is on this page, validate it immediately.
- * Otherwise do just a basic sanity check on xl_tot_len, and validate the
- * rest of the header after reading it from the next page. The xl_tot_len
- * check is necessary here to ensure that we enter the "Need to reassemble
- * record" code path below; otherwise we might fail to apply
- * ValidXLogRecordHeader at all.
- */
- if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
- {
- if (!ValidXLogRecordHeader(state, RecPtr, state->ReadRecPtr, record,
- randAccess))
- goto err;
- gotheader = true;
- }
- else
- {
- /* XXX: more validation should be done here */
- if (total_len < SizeOfXLogRecord)
- {
- report_invalid_record(state,
- "invalid record length at %X/%X: wanted %u, got %u",
- (uint32) (RecPtr >> 32), (uint32) RecPtr,
- (uint32) SizeOfXLogRecord, total_len);
- goto err;
- }
- gotheader = false;
- }
-
- len = XLOG_BLCKSZ - RecPtr % XLOG_BLCKSZ;
- if (total_len > len)
+ switch (state->readRecordState)
{
- /* Need to reassemble record */
- char *contdata;
- XLogPageHeader pageHeader;
- char *buffer;
- uint32 gotlen;
-
- /*
- * Enlarge readRecordBuf as needed.
- */
- if (total_len > state->readRecordBufSize &&
- !allocate_recordbuf(state, total_len))
- {
- /* We treat this as a "bogus data" condition */
- report_invalid_record(state, "record length %u at %X/%X too long",
- total_len,
- (uint32) (RecPtr >> 32), (uint32) RecPtr);
- goto err;
- }
+ case XLREAD_NEXT_RECORD:
+ ResetDecoder(state);
- /* Copy the first fragment of the record from the first page. */
- memcpy(state->readRecordBuf,
- state->readBuf + RecPtr % XLOG_BLCKSZ, len);
- buffer = state->readRecordBuf + len;
- gotlen = len;
-
- do
- {
- int rest_len = total_len - gotlen;
-
- /* Calculate pointer to beginning of next page */
- targetPagePtr += XLOG_BLCKSZ;
-
- /* Wait for the next page to become available */
- while (XLogNeedData(state, targetPagePtr,
- Min(rest_len, XLOG_BLCKSZ),
- false))
+ if (state->ReadRecPtr != InvalidXLogRecPtr)
{
- if (!state->routine.page_read(state, state->readPagePtr,
- state->readLen,
- state->ReadRecPtr,
- state->readBuf))
- break;
+ /* read the record after the one we just read */
+
+ /*
+ * EndRecPtr is pointing to end+1 of the previous WAL record.
+ * If we're at a page boundary, no more records can fit on the
+ * current page. We must skip over the page header, but we
+ * can't do that until we've read in the page, since the header
+ * size is variable.
+ */
+ state->PrevRecPtr = state->ReadRecPtr;
+ state->ReadRecPtr = state->EndRecPtr;
+ }
+ else
+ {
+ /*
+ * Caller supplied a position to start at.
+ *
+ * In this case, EndRecPtr should already be pointing to a
+ * valid record starting position.
+ */
+ Assert(XRecOffIsValid(state->EndRecPtr));
+ state->ReadRecPtr = state->EndRecPtr;
+
+ /*
+ * We cannot verify the previous-record pointer when we're
+ * seeking to a particular record. Reset PrevRecPtr so that we
+ * won't try doing that.
+ */
+ state->PrevRecPtr = InvalidXLogRecPtr;
+ state->EndRecPtr = InvalidXLogRecPtr; /* to be tidy */
}
+ state->record_verified = false;
+ state->readRecordState = XLREAD_TOT_LEN;
+ /* fall through */
+
+ case XLREAD_TOT_LEN:
+ {
+ uint32 total_len;
+ uint32 pageHeaderSize;
+ XLogRecPtr targetPagePtr;
+ uint32 targetRecOff;
+ XLogPageHeader pageHeader;
+
+ targetPagePtr =
+ state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
+ targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+
+ /*
+ * Check if we have enough data. For the first record in the page,
+ * the requesting length doesn't contain page header.
+ */
+ if (XLogNeedData(state, targetPagePtr,
+ Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+ targetRecOff != 0))
+ return XLREAD_NEED_DATA;
+
+ /* error out if caller supplied bogus page */
if (!state->page_verified)
goto err;
- Assert(SizeOfXLogShortPHD <= state->readLen);
+ /* examine page header now. */
+ pageHeaderSize =
+ XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+ if (targetRecOff == 0)
+ {
+ /* At page start, so skip over page header. */
+ state->ReadRecPtr += pageHeaderSize;
+ targetRecOff = pageHeaderSize;
+ }
+ else if (targetRecOff < pageHeaderSize)
+ {
+ report_invalid_record(state, "invalid record offset at %X/%X",
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr);
+ goto err;
+ }
- /* Check that the continuation on next page looks valid */
pageHeader = (XLogPageHeader) state->readBuf;
- if (!(pageHeader->xlp_info & XLP_FIRST_IS_CONTRECORD))
+ if ((pageHeader->xlp_info & XLP_FIRST_IS_CONTRECORD) &&
+ targetRecOff == pageHeaderSize)
{
- report_invalid_record(state,
- "there is no contrecord flag at %X/%X",
- (uint32) (RecPtr >> 32), (uint32) RecPtr);
+ report_invalid_record(state, "contrecord is requested by %X/%X",
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr);
goto err;
}
- /*
- * Cross-check that xlp_rem_len agrees with how much of the record
- * we expect there to be left.
- */
- if (pageHeader->xlp_rem_len == 0 ||
- total_len != (pageHeader->xlp_rem_len + gotlen))
- {
- report_invalid_record(state,
- "invalid contrecord length %u at %X/%X",
- pageHeader->xlp_rem_len,
- (uint32) (RecPtr >> 32), (uint32) RecPtr);
- goto err;
- }
-
- /* Append the continuation from this page to the buffer */
- pageHeaderSize = XLogPageHeaderSize(pageHeader);
-
+ /* XLogNeedData has verified the page header */
Assert(pageHeaderSize <= state->readLen);
- contdata = (char *) state->readBuf + pageHeaderSize;
- len = XLOG_BLCKSZ - pageHeaderSize;
- if (pageHeader->xlp_rem_len < len)
- len = pageHeader->xlp_rem_len;
+ /*
+ * Read the record length.
+ *
+ * NB: Even though we use an XLogRecord pointer here, the whole
+ * record header might not fit on this page. xl_tot_len is the first
+ * field of the struct, so it must be on this page (the records are
+ * MAXALIGNed), but we cannot access any other fields until we've
+ * verified that we got the whole header.
+ */
+ prec = (XLogRecord *) (state->readBuf +
+ state->ReadRecPtr % XLOG_BLCKSZ);
+ total_len = prec->xl_tot_len;
- Assert (pageHeaderSize + len <= state->readLen);
- memcpy(buffer, (char *) contdata, len);
- buffer += len;
- gotlen += len;
+ /*
+ * If the whole record header is on this page, validate it
+ * immediately. Otherwise do just a basic sanity check on
+ * xl_tot_len, and validate the rest of the header after reading it
+ * from the next page. The xl_tot_len check is necessary here to
+ * ensure that we enter the XLREAD_CONTINUATION state below;
+ * otherwise we might fail to apply ValidXLogRecordHeader at all.
+ */
+ if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
+ {
+ if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
+ state->PrevRecPtr, prec))
+ goto err;
- /* If we just reassembled the record header, validate it. */
- if (!gotheader)
+ state->record_verified = true;
+ }
+ else
{
- record = (XLogRecord *) state->readRecordBuf;
- if (!ValidXLogRecordHeader(state, RecPtr, state->ReadRecPtr,
- record, randAccess))
+ /* XXX: more validation should be done here */
+ if (total_len < SizeOfXLogRecord)
+ {
+ report_invalid_record(state,
+ "invalid record length at %X/%X: wanted %u, got %u",
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr,
+ (uint32) SizeOfXLogRecord, total_len);
goto err;
- gotheader = true;
+ }
}
- } while (gotlen < total_len);
-
- Assert(gotheader);
- record = (XLogRecord *) state->readRecordBuf;
- if (!ValidXLogRecord(state, record, RecPtr))
- goto err;
+ /*
+ * Wait for the rest of the record, or the part of the record that
+ * fit on the first page if crossed a page boundary, to become
+ * available.
+ */
+ state->recordGotLen = 0;
+ state->recordRemainLen = total_len;
+ state->readRecordState = XLREAD_FIRST_FRAGMENT;
+ }
+ /* fall through */
- pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
- state->ReadRecPtr = RecPtr;
- state->EndRecPtr = targetPagePtr + pageHeaderSize
- + MAXALIGN(pageHeader->xlp_rem_len);
- }
- else
- {
- /* Wait for the record data to become available */
- while (XLogNeedData(state, targetPagePtr,
- Min(targetRecOff + total_len, XLOG_BLCKSZ), true))
+ case XLREAD_FIRST_FRAGMENT:
{
- if (!state->routine.page_read(state, state->readPagePtr,
- state->readLen,
- state->ReadRecPtr, state->readBuf))
+ uint32 total_len = state->recordRemainLen;
+ uint32 request_len;
+ uint32 record_len;
+ XLogRecPtr targetPagePtr;
+ uint32 targetRecOff;
+
+ /*
+ * Wait for the rest of the record on the first page to become
+ * available
+ */
+ targetPagePtr =
+ state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
+ targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+
+ request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ);
+ record_len = request_len - targetRecOff;
+
+ /* ReadRecPtr contains page header */
+ Assert (targetRecOff != 0);
+ if (XLogNeedData(state, targetPagePtr, request_len, true))
+ return XLREAD_NEED_DATA;
+
+ /* error out if caller supplied bogus page */
+ if (!state->page_verified)
+ goto err;
+
+ prec = (XLogRecord *) (state->readBuf + targetRecOff);
+
+ /* validate record header if not yet */
+ if (!state->record_verified && record_len >= SizeOfXLogRecord)
+ {
+ if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
+ state->PrevRecPtr, prec))
+ goto err;
+
+ state->record_verified = true;
+ }
+
+
+ if (total_len == record_len)
+ {
+ /* Record does not cross a page boundary */
+ Assert(state->record_verified);
+
+ if (!ValidXLogRecord(state, prec, state->ReadRecPtr))
+ goto err;
+
+ state->record_verified = true; /* to be tidy */
+
+ /* We already checked the header earlier */
+ state->EndRecPtr = state->ReadRecPtr + MAXALIGN(record_len);
+
+ *record = prec;
+ state->readRecordState = XLREAD_NEXT_RECORD;
break;
+ }
+
+ /*
+ * The record continues on the next page. Need to reassemble
+ * record
+ */
+ Assert(total_len > record_len);
+
+ /* Enlarge readRecordBuf as needed. */
+ if (total_len > state->readRecordBufSize &&
+ !allocate_recordbuf(state, total_len))
+ {
+ /* We treat this as a "bogus data" condition */
+ report_invalid_record(state,
+ "record length %u at %X/%X too long",
+ total_len,
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr);
+ goto err;
+ }
+
+ /* Copy the first fragment of the record from the first page. */
+ memcpy(state->readRecordBuf, state->readBuf + targetRecOff,
+ record_len);
+ state->recordGotLen += record_len;
+ state->recordRemainLen -= record_len;
+
+ /* Calculate pointer to beginning of next page */
+ state->recordContRecPtr = state->ReadRecPtr + record_len;
+ Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0);
+
+ state->readRecordState = XLREAD_CONTINUATION;
}
+ /* fall through */
+
+ case XLREAD_CONTINUATION:
+ {
+ XLogPageHeader pageHeader;
+ uint32 pageHeaderSize;
+ XLogRecPtr targetPagePtr;
+
+ /* we enter this state only if we haven't read the whole record. */
+ Assert (state->recordRemainLen > 0);
+
+ while(state->recordRemainLen > 0)
+ {
+ char *contdata;
+ uint32 request_len;
+ uint32 record_len;
+
+ /* Wait for the next page to become available */
+ targetPagePtr = state->recordContRecPtr;
+
+ /* this request contains page header */
+ Assert (targetPagePtr != 0);
+ if (XLogNeedData(state, targetPagePtr,
+ Min(state->recordRemainLen, XLOG_BLCKSZ),
+ false))
+ return XLREAD_NEED_DATA;
+
+ if (!state->page_verified)
+ goto err;
+
+ Assert(SizeOfXLogShortPHD <= state->readLen);
- if (!state->page_verified)
- goto err;
+ /* Check that the continuation on next page looks valid */
+ pageHeader = (XLogPageHeader) state->readBuf;
+ if (!(pageHeader->xlp_info & XLP_FIRST_IS_CONTRECORD))
+ {
+ report_invalid_record(
+ state,
+ "there is no contrecord flag at %X/%X reading %X/%X",
+ (uint32) (state->recordContRecPtr >> 32),
+ (uint32) state->recordContRecPtr,
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr);
+ goto err;
+ }
- /* Record does not cross a page boundary */
- if (!ValidXLogRecord(state, record, RecPtr))
- goto err;
+ /*
+ * Cross-check that xlp_rem_len agrees with how much of the
+ * record we expect there to be left.
+ */
+ if (pageHeader->xlp_rem_len == 0 ||
+ pageHeader->xlp_rem_len != state->recordRemainLen)
+ {
+ report_invalid_record(
+ state,
+ "invalid contrecord length %u at %X/%X reading %X/%X, expected %u",
+ pageHeader->xlp_rem_len,
+ (uint32) (state->recordContRecPtr >> 32),
+ (uint32) state->recordContRecPtr,
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr,
+ state->recordRemainLen);
+ goto err;
+ }
- state->EndRecPtr = RecPtr + MAXALIGN(total_len);
+ /* Append the continuation from this page to the buffer */
+ pageHeaderSize = XLogPageHeaderSize(pageHeader);
- state->ReadRecPtr = RecPtr;
+ /*
+ * XLogNeedData should have ensured that the whole page header
+ * was read
+ */
+ Assert(state->readLen >= pageHeaderSize);
+
+ contdata = (char *) state->readBuf + pageHeaderSize;
+ record_len = XLOG_BLCKSZ - pageHeaderSize;
+ if (pageHeader->xlp_rem_len < record_len)
+ record_len = pageHeader->xlp_rem_len;
+
+ request_len = record_len + pageHeaderSize;
+
+ /* XLogNeedData should have ensured all needed data was read */
+ Assert (state->readLen >= request_len);
+
+ memcpy(state->readRecordBuf + state->recordGotLen,
+ (char *) contdata, record_len);
+ state->recordGotLen += record_len;
+ state->recordRemainLen -= record_len;
+
+ /* If we just reassembled the record header, validate it. */
+ if (!state->record_verified)
+ {
+ Assert(state->recordGotLen >= SizeOfXLogRecord);
+ if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
+ state->PrevRecPtr,
+ (XLogRecord *) state->readRecordBuf))
+ goto err;
+
+ state->record_verified = true;
+ }
+
+ /* Calculate pointer to beginning of next page, and continue */
+ state->recordContRecPtr += XLOG_BLCKSZ;
+ }
+
+ /* targetPagePtr is pointing the last-read page here */
+ prec = (XLogRecord *) state->readRecordBuf;
+ if (!ValidXLogRecord(state, prec, state->ReadRecPtr))
+ goto err;
+
+ pageHeaderSize =
+ XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+ state->EndRecPtr = targetPagePtr + pageHeaderSize
+ + MAXALIGN(pageHeader->xlp_rem_len);
+
+ *record = prec;
+ state->readRecordState = XLREAD_NEXT_RECORD;
+ break;
+ }
}
/*
* Special processing if it's an XLOG SWITCH record
*/
- if (record->xl_rmid == RM_XLOG_ID &&
- (record->xl_info & ~XLR_INFO_MASK) == XLOG_SWITCH)
+ if ((*record)->xl_rmid == RM_XLOG_ID &&
+ ((*record)->xl_info & ~XLR_INFO_MASK) == XLOG_SWITCH)
{
/* Pretend it extends to end of segment */
state->EndRecPtr += state->segcxt.ws_segsize - 1;
state->EndRecPtr -= XLogSegmentOffset(state->EndRecPtr, state->segcxt.ws_segsize);
}
- if (DecodeXLogRecord(state, record, errormsg))
- return record;
- else
- return NULL;
+ Assert (!*record || state->readLen >= 0);
+ if (DecodeXLogRecord(state, *record, errormsg))
+ return XLREAD_SUCCESS;
+
+ *record = NULL;
+ return XLREAD_FAIL;
err:
/*
- * Invalidate the read state. We might read from a different source after
+ * Invalidate the read page. We might read from a different source after
* failure.
*/
XLogReaderInvalReadState(state);
@@ -572,7 +706,8 @@ err:
if (state->errormsg_buf[0] != '\0')
*errormsg = state->errormsg_buf;
- return NULL;
+ *record = NULL;
+ return XLREAD_FAIL;
}
/*
@@ -724,11 +859,12 @@ XLogReaderInvalReadState(XLogReaderState *state)
*
* This is just a convenience subroutine to avoid duplicated code in
* XLogReadRecord. It's not intended for use from anywhere else.
+ *
+ * If PrevRecPtr is valid, the xl_prev is is cross-checked with it.
*/
static bool
ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
- XLogRecPtr PrevRecPtr, XLogRecord *record,
- bool randAccess)
+ XLogRecPtr PrevRecPtr, XLogRecord *record)
{
if (record->xl_tot_len < SizeOfXLogRecord)
{
@@ -746,7 +882,7 @@ ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
(uint32) RecPtr);
return false;
}
- if (randAccess)
+ if (PrevRecPtr == InvalidXLogRecPtr)
{
/*
* We can't exactly verify the prev-link, but surely it should be less
@@ -978,11 +1114,14 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
* XLogReadRecord() will read the next valid record.
*/
XLogRecPtr
-XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
+XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
+ XLogFindNextRecordCB read_page, void *private)
{
XLogRecPtr tmpRecPtr;
XLogRecPtr found = InvalidXLogRecPtr;
XLogPageHeader header;
+ XLogRecord *record;
+ XLogReadRecordResult result;
char *errormsg;
Assert(!XLogRecPtrIsInvalid(RecPtr));
@@ -1015,9 +1154,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
while(XLogNeedData(state, targetPagePtr, targetRecOff,
targetRecOff != 0))
{
- if (!state->routine.page_read(state, state->readPagePtr,
- state->readLen,
- state->ReadRecPtr, state->readBuf))
+ if (!read_page(state, private))
break;
}
@@ -1069,8 +1206,16 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
* or we just jumped over the remaining data of a continuation.
*/
XLogBeginRead(state, tmpRecPtr);
- while (XLogReadRecord(state, &errormsg) != NULL)
+ while ((result = XLogReadRecord(state, &record, &errormsg)) !=
+ XLREAD_FAIL)
{
+ if (result == XLREAD_NEED_DATA)
+ {
+ if (!read_page(state, private))
+ goto err;
+ continue;
+ }
+
/* past the record we've found, break out */
if (RecPtr <= state->ReadRecPtr)
{
@@ -1090,9 +1235,9 @@ err:
#endif /* FRONTEND */
/*
- * Helper function to ease writing of XLogRoutine->page_read callbacks.
- * If this function is used, caller must supply a segment_open callback in
- * 'state', as that is used here.
+ * Helper function to ease writing of page_read callback.
+ * If this function is used, caller must supply a segment_open callback and
+ * segment_close callback as that is used here.
*
* Read 'count' bytes into 'buf', starting at location 'startptr', from WAL
* fetched from timeline 'tli'.
@@ -1105,6 +1250,7 @@ err:
*/
bool
WALRead(XLogReaderState *state,
+ WALSegmentOpenCB segopenfn, WALSegmentCloseCB segclosefn,
char *buf, XLogRecPtr startptr, Size count, TimeLineID tli,
WALReadError *errinfo)
{
@@ -1136,10 +1282,10 @@ WALRead(XLogReaderState *state,
XLogSegNo nextSegNo;
if (state->seg.ws_file >= 0)
- state->routine.segment_close(state);
+ segclosefn(state);
XLByteToSeg(recptr, nextSegNo, state->segcxt.ws_segsize);
- state->routine.segment_open(state, nextSegNo, &tli);
+ segopenfn(state, nextSegNo, &tli);
/* This shouldn't happen -- indicates a bug in segment_open */
Assert(state->seg.ws_file >= 0);
diff --git a/src/backend/access/transam/xlogutils.c b/src/backend/access/transam/xlogutils.c
index 1b10b6df20..303333571f 100644
--- a/src/backend/access/transam/xlogutils.c
+++ b/src/backend/access/transam/xlogutils.c
@@ -689,8 +689,7 @@ XLogTruncateRelation(RelFileNode rnode, ForkNumber forkNum,
void
XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wantLength)
{
- const XLogRecPtr lastReadPage = state->seg.ws_segno *
- state->segcxt.ws_segsize + state->readLen;
+ const XLogRecPtr lastReadPage = state->readPagePtr;
Assert(wantPage != InvalidXLogRecPtr && wantPage % XLOG_BLCKSZ == 0);
Assert(wantLength <= XLOG_BLCKSZ);
@@ -705,7 +704,7 @@ XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wa
* current TLI has since become historical.
*/
if (lastReadPage == wantPage &&
- state->readLen != 0 &&
+ state->page_verified &&
lastReadPage + state->readLen >= wantPage + Min(wantLength, XLOG_BLCKSZ - 1))
return;
@@ -792,6 +791,7 @@ wal_segment_open(XLogReaderState *state, XLogSegNo nextSegNo,
char path[MAXPGPATH];
XLogFilePath(path, tli, nextSegNo, state->segcxt.ws_segsize);
+ elog(LOG, "HOGE: %lu, %d => %s", nextSegNo, tli, path);
state->seg.ws_file = BasicOpenFile(path, O_RDONLY | PG_BINARY);
if (state->seg.ws_file >= 0)
return;
@@ -829,9 +829,11 @@ wal_segment_close(XLogReaderState *state)
* loop for now.
*/
bool
-read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *cur_page)
+read_local_xlog_page(XLogReaderState *state)
{
+ XLogRecPtr targetPagePtr = state->readPagePtr;
+ int reqLen = state->readLen;
+ char *cur_page = state->readBuf;
XLogRecPtr read_upto,
loc;
TimeLineID tli;
@@ -944,11 +946,12 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
* as 'count', read the whole page anyway. It's guaranteed to be
* zero-padded up to the page boundary if it's incomplete.
*/
- if (!WALRead(state, cur_page, targetPagePtr, XLOG_BLCKSZ, tli,
- &errinfo))
+ if (!WALRead(state, wal_segment_open, wal_segment_close,
+ cur_page, targetPagePtr, XLOG_BLCKSZ, tli, &errinfo))
WALReadRaiseError(&errinfo);
/* number of valid bytes in the buffer */
+ state->readPagePtr = targetPagePtr;
state->readLen = count;
return true;
}
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index 61902be3b0..a15b0b3355 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -120,7 +120,8 @@ StartupDecodingContext(List *output_plugin_options,
TransactionId xmin_horizon,
bool need_full_snapshot,
bool fast_forward,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress)
@@ -169,11 +170,12 @@ StartupDecodingContext(List *output_plugin_options,
ctx->slot = slot;
- ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, xl_routine, ctx);
+ ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, cleanup_cb);
if (!ctx->reader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
errmsg("out of memory")));
+ ctx->page_read = page_read;
ctx->reorder = ReorderBufferAllocate();
ctx->snapshot_builder =
@@ -231,7 +233,8 @@ CreateInitDecodingContext(char *plugin,
List *output_plugin_options,
bool need_full_snapshot,
XLogRecPtr restart_lsn,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress)
@@ -328,7 +331,7 @@ CreateInitDecodingContext(char *plugin,
ctx = StartupDecodingContext(NIL, restart_lsn, xmin_horizon,
need_full_snapshot, false,
- xl_routine, prepare_write, do_write,
+ page_read, cleanup_cb, prepare_write, do_write,
update_progress);
/* call output plugin initialization callback */
@@ -376,7 +379,8 @@ LogicalDecodingContext *
CreateDecodingContext(XLogRecPtr start_lsn,
List *output_plugin_options,
bool fast_forward,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress)
@@ -429,8 +433,8 @@ CreateDecodingContext(XLogRecPtr start_lsn,
ctx = StartupDecodingContext(output_plugin_options,
start_lsn, InvalidTransactionId, false,
- fast_forward, xl_routine, prepare_write,
- do_write, update_progress);
+ fast_forward, page_read, cleanup_cb,
+ prepare_write, do_write, update_progress);
/* call output plugin initialization callback */
old_context = MemoryContextSwitchTo(ctx->context);
@@ -483,7 +487,13 @@ DecodingContextFindStartpoint(LogicalDecodingContext *ctx)
char *err = NULL;
/* the read_page callback waits for new WAL */
- record = XLogReadRecord(ctx->reader, &err);
+ while (XLogReadRecord(ctx->reader, &record, &err) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!ctx->page_read(ctx->reader))
+ break;
+ }
+
if (err)
elog(ERROR, "%s", err);
if (!record)
diff --git a/src/backend/replication/logical/logicalfuncs.c b/src/backend/replication/logical/logicalfuncs.c
index b99c94e848..fcc81b358e 100644
--- a/src/backend/replication/logical/logicalfuncs.c
+++ b/src/backend/replication/logical/logicalfuncs.c
@@ -233,9 +233,8 @@ pg_logical_slot_get_changes_guts(FunctionCallInfo fcinfo, bool confirm, bool bin
ctx = CreateDecodingContext(InvalidXLogRecPtr,
options,
false,
- XL_ROUTINE(.page_read = read_local_xlog_page,
- .segment_open = wal_segment_open,
- .segment_close = wal_segment_close),
+ read_local_xlog_page,
+ wal_segment_close,
LogicalOutputPrepareWrite,
LogicalOutputWrite, NULL);
@@ -284,7 +283,13 @@ pg_logical_slot_get_changes_guts(FunctionCallInfo fcinfo, bool confirm, bool bin
XLogRecord *record;
char *errm = NULL;
- record = XLogReadRecord(ctx->reader, &errm);
+ while (XLogReadRecord(ctx->reader, &record, &errm) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!ctx->page_read(ctx->reader))
+ break;
+ }
+
if (errm)
elog(ERROR, "%s", errm);
diff --git a/src/backend/replication/slotfuncs.c b/src/backend/replication/slotfuncs.c
index 88033a79b2..8a678074d5 100644
--- a/src/backend/replication/slotfuncs.c
+++ b/src/backend/replication/slotfuncs.c
@@ -152,9 +152,8 @@ create_logical_replication_slot(char *name, char *plugin,
ctx = CreateInitDecodingContext(plugin, NIL,
false, /* just catalogs is OK */
restart_lsn,
- XL_ROUTINE(.page_read = read_local_xlog_page,
- .segment_open = wal_segment_open,
- .segment_close = wal_segment_close),
+ read_local_xlog_page,
+ wal_segment_close,
NULL, NULL, NULL);
/*
@@ -486,9 +485,8 @@ pg_logical_replication_slot_advance(XLogRecPtr moveto)
ctx = CreateDecodingContext(InvalidXLogRecPtr,
NIL,
true, /* fast_forward */
- XL_ROUTINE(.page_read = read_local_xlog_page,
- .segment_open = wal_segment_open,
- .segment_close = wal_segment_close),
+ read_local_xlog_page,
+ wal_segment_close,
NULL, NULL, NULL);
/*
@@ -513,7 +511,13 @@ pg_logical_replication_slot_advance(XLogRecPtr moveto)
* Read records. No changes are generated in fast_forward mode,
* but snapbuilder/slot statuses are updated properly.
*/
- record = XLogReadRecord(ctx->reader, &errm);
+ while (XLogReadRecord(ctx->reader, &record, &errm) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!ctx->page_read(ctx->reader))
+ break;
+ }
+
if (errm)
elog(ERROR, "%s", errm);
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index bf2711bddd..2fd24a9b14 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -580,10 +580,7 @@ StartReplication(StartReplicationCmd *cmd)
/* create xlogreader for physical replication */
xlogreader =
- XLogReaderAllocate(wal_segment_size, NULL,
- XL_ROUTINE(.segment_open = WalSndSegmentOpen,
- .segment_close = wal_segment_close),
- NULL);
+ XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
if (!xlogreader)
ereport(ERROR,
@@ -812,9 +809,11 @@ StartReplication(StartReplicationCmd *cmd)
* set every time WAL is flushed.
*/
static bool
-logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetRecPtr, char *cur_page)
+logical_read_xlog_page(XLogReaderState *state)
{
+ XLogRecPtr targetPagePtr = state->readPagePtr;
+ int reqLen = state->readLen;
+ char *cur_page = state->readBuf;
XLogRecPtr flushptr;
int count;
WALReadError errinfo;
@@ -842,7 +841,7 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
count = flushptr - targetPagePtr; /* part of the page available */
/* now actually read the data, we know it's there */
- if (!WALRead(state,
+ if (!WALRead(state, WalSndSegmentOpen, wal_segment_close,
cur_page,
targetPagePtr,
XLOG_BLCKSZ,
@@ -1014,9 +1013,8 @@ CreateReplicationSlot(CreateReplicationSlotCmd *cmd)
ctx = CreateInitDecodingContext(cmd->plugin, NIL, need_full_snapshot,
InvalidXLogRecPtr,
- XL_ROUTINE(.page_read = logical_read_xlog_page,
- .segment_open = WalSndSegmentOpen,
- .segment_close = wal_segment_close),
+ logical_read_xlog_page,
+ wal_segment_close,
WalSndPrepareWrite, WalSndWriteData,
WalSndUpdateProgress);
@@ -1179,9 +1177,8 @@ StartLogicalReplication(StartReplicationCmd *cmd)
*/
logical_decoding_ctx =
CreateDecodingContext(cmd->startpoint, cmd->options, false,
- XL_ROUTINE(.page_read = logical_read_xlog_page,
- .segment_open = WalSndSegmentOpen,
- .segment_close = wal_segment_close),
+ logical_read_xlog_page,
+ wal_segment_close,
WalSndPrepareWrite, WalSndWriteData,
WalSndUpdateProgress);
xlogreader = logical_decoding_ctx->reader;
@@ -2761,7 +2758,7 @@ XLogSendPhysical(void)
enlargeStringInfo(&output_message, nbytes);
retry:
- if (!WALRead(xlogreader,
+ if (!WALRead(xlogreader, WalSndSegmentOpen, wal_segment_close,
&output_message.data[output_message.len],
startptr,
nbytes,
@@ -2859,7 +2856,12 @@ XLogSendLogical(void)
*/
WalSndCaughtUp = false;
- record = XLogReadRecord(logical_decoding_ctx->reader, &errm);
+ while (XLogReadRecord(logical_decoding_ctx->reader, &record, &errm) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!logical_decoding_ctx->page_read(logical_decoding_ctx->reader))
+ break;
+ }
/* xlog record was invalid */
if (errm != NULL)
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index 0b7e73ae79..3dd6df4be6 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -40,15 +40,9 @@ static int xlogreadfd = -1;
static XLogSegNo xlogreadsegno = -1;
static char xlogfpath[MAXPGPATH];
-typedef struct XLogPageReadPrivate
-{
- const char *restoreCommand;
- int tliIndex;
-} XLogPageReadPrivate;
-
-static bool SimpleXLogPageRead(XLogReaderState *xlogreader,
- XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
+static bool SimpleXLogPageRead(XLogReaderState *xlogreader,
+ const char *datadir, int *tliIndex,
+ const char *restoreCommand);
/*
* Read WAL from the datadir/pg_wal, starting from 'startpoint' on timeline
@@ -62,20 +56,22 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
XLogRecord *record;
XLogReaderState *xlogreader;
char *errormsg;
- XLogPageReadPrivate private;
- private.tliIndex = tliIndex;
- private.restoreCommand = restoreCommand;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir,
- XL_ROUTINE(.page_read = &SimpleXLogPageRead),
- &private);
+ xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+
if (xlogreader == NULL)
pg_fatal("out of memory");
XLogBeginRead(xlogreader, startpoint);
do
{
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!SimpleXLogPageRead(xlogreader, datadir,
+ &tliIndex, restoreCommand))
+ break;
+ }
if (record == NULL)
{
@@ -113,19 +109,19 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex,
XLogRecord *record;
XLogReaderState *xlogreader;
char *errormsg;
- XLogPageReadPrivate private;
XLogRecPtr endptr;
- private.tliIndex = tliIndex;
- private.restoreCommand = restoreCommand;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir,
- XL_ROUTINE(.page_read = &SimpleXLogPageRead),
- &private);
+ xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
XLogBeginRead(xlogreader, ptr);
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!SimpleXLogPageRead(xlogreader, datadir, &tliIndex, restoreCommand))
+ break;
+ }
if (record == NULL)
{
if (errormsg)
@@ -160,7 +156,6 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
XLogRecPtr searchptr;
XLogReaderState *xlogreader;
char *errormsg;
- XLogPageReadPrivate private;
/*
* The given fork pointer points to the end of the last common record,
@@ -176,11 +171,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
forkptr += SizeOfXLogShortPHD;
}
- private.tliIndex = tliIndex;
- private.restoreCommand = restoreCommand;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir,
- XL_ROUTINE(.page_read = &SimpleXLogPageRead),
- &private);
+ xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
@@ -190,7 +181,13 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
uint8 info;
XLogBeginRead(xlogreader, searchptr);
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!SimpleXLogPageRead(xlogreader, datadir,
+ &tliIndex, restoreCommand))
+ break;
+ }
if (record == NULL)
{
@@ -237,10 +234,11 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
/* XLogReader callback function, to read a WAL page */
static bool
-SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *readBuf)
+SimpleXLogPageRead(XLogReaderState *xlogreader, const char *datadir,
+ int *tliIndex, const char *restoreCommand)
{
- XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
+ XLogRecPtr targetPagePtr = xlogreader->readPagePtr;
+ char *readBuf = xlogreader->readBuf;
uint32 targetPageOff;
XLogRecPtr targetSegEnd;
XLogSegNo targetSegNo;
@@ -273,14 +271,14 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
* be done both forward and backward, consider also switching timeline
* accordingly.
*/
- while (private->tliIndex < targetNentries - 1 &&
- targetHistory[private->tliIndex].end < targetSegEnd)
- private->tliIndex++;
- while (private->tliIndex > 0 &&
- targetHistory[private->tliIndex].begin >= targetSegEnd)
- private->tliIndex--;
+ while (*tliIndex < targetNentries - 1 &&
+ targetHistory[*tliIndex].end < targetSegEnd)
+ (*tliIndex)++;
+ while (*tliIndex > 0 &&
+ targetHistory[*tliIndex].begin >= targetSegEnd)
+ (*tliIndex)--;
- XLogFileName(xlogfname, targetHistory[private->tliIndex].tli,
+ XLogFileName(xlogfname, targetHistory[*tliIndex].tli,
xlogreadsegno, WalSegSz);
snprintf(xlogfpath, MAXPGPATH, "%s/" XLOGDIR "/%s",
@@ -293,7 +291,7 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
/*
* If we have no restore_command to execute, then exit.
*/
- if (private->restoreCommand == NULL)
+ if (restoreCommand == NULL)
{
pg_log_error("could not open file \"%s\": %m", xlogfpath);
xlogreader->readLen = -1;
@@ -307,7 +305,7 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
xlogreadfd = RestoreArchivedFile(xlogreader->segcxt.ws_dir,
xlogfname,
WalSegSz,
- private->restoreCommand);
+ restoreCommand);
if (xlogreadfd < 0)
{
@@ -349,7 +347,7 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
Assert(targetSegNo == xlogreadsegno);
- xlogreader->seg.ws_tli = targetHistory[private->tliIndex].tli;
+ xlogreader->seg.ws_tli = targetHistory[*tliIndex].tli;
xlogreader->readLen = XLOG_BLCKSZ;
return true;
}
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index ab0c28d9b7..02e71138cc 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -330,12 +330,17 @@ WALDumpCloseSegment(XLogReaderState *state)
state->seg.ws_file = -1;
}
-/* pg_waldump's XLogReaderRoutine->page_read callback */
+/*
+ * pg_waldump's WAL page rader, also used as page_read callback for
+ * XLogFindNextRecord
+ */
static bool
-WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetPtr, char *readBuff)
+WALDumpReadPage(XLogReaderState *state, void *priv)
{
- XLogDumpPrivate *private = state->private_data;
+ XLogRecPtr targetPagePtr = state->readPagePtr;
+ int reqLen = state->readLen;
+ char *readBuff = state->readBuf;
+ XLogDumpPrivate *private = (XLogDumpPrivate *) priv;
int count = XLOG_BLCKSZ;
WALReadError errinfo;
@@ -353,8 +358,8 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
}
}
- if (!WALRead(state, readBuff, targetPagePtr, count, private->timeline,
- &errinfo))
+ if (!WALRead(state, WALDumpOpenSegment, WALDumpCloseSegment,
+ readBuff, targetPagePtr, count, private->timeline, &errinfo))
{
WALOpenSegment *seg = &errinfo.wre_seg;
char fname[MAXPGPATH];
@@ -374,6 +379,7 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
(Size) errinfo.wre_req);
}
+ Assert(count >= state->readLen);
state->readLen = count;
return true;
}
@@ -1042,16 +1048,14 @@ main(int argc, char **argv)
/* we have everything we need, start reading */
xlogreader_state =
- XLogReaderAllocate(WalSegSz, waldir,
- XL_ROUTINE(.page_read = WALDumpReadPage,
- .segment_open = WALDumpOpenSegment,
- .segment_close = WALDumpCloseSegment),
- &private);
+ XLogReaderAllocate(WalSegSz, waldir, WALDumpCloseSegment);
+
if (!xlogreader_state)
fatal_error("out of memory");
/* first find a valid recptr to start from */
- first_record = XLogFindNextRecord(xlogreader_state, private.startptr);
+ first_record = XLogFindNextRecord(xlogreader_state, private.startptr,
+ &WALDumpReadPage, (void*) &private);
if (first_record == InvalidXLogRecPtr)
fatal_error("could not find a valid record after %X/%X",
@@ -1075,7 +1079,13 @@ main(int argc, char **argv)
for (;;)
{
/* try to read the next record */
- record = XLogReadRecord(xlogreader_state, &errormsg);
+ while (XLogReadRecord(xlogreader_state, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!WALDumpReadPage(xlogreader_state, (void *) &private))
+ break;
+ }
+
if (!record)
{
if (!config.follow || private.endptr_reached)
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index 6c8848e14f..6a7fe140cb 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -57,64 +57,15 @@ typedef struct WALSegmentContext
typedef struct XLogReaderState XLogReaderState;
-/* Function type definition for the read_page callback */
-typedef bool (*XLogPageReadCB) (XLogReaderState *xlogreader,
- XLogRecPtr targetPagePtr,
- int reqLen,
- XLogRecPtr targetRecPtr,
- char *readBuf);
+/* Function type definition for the segment cleanup callback */
+typedef void (*WALSegmentCleanupCB) (XLogReaderState *xlogreader);
+
+/* Function type definition for the open/close callbacks for WALRead() */
typedef void (*WALSegmentOpenCB) (XLogReaderState *xlogreader,
XLogSegNo nextSegNo,
TimeLineID *tli_p);
typedef void (*WALSegmentCloseCB) (XLogReaderState *xlogreader);
-typedef struct XLogReaderRoutine
-{
- /*
- * Data input callback
- *
- * This callback shall read at least reqLen valid bytes of the xlog page
- * starting at targetPagePtr, and store them in readBuf. The callback
- * shall return the number of bytes read (never more than XLOG_BLCKSZ), or
- * -1 on failure. The callback shall sleep, if necessary, to wait for the
- * requested bytes to become available. The callback will not be invoked
- * again for the same page unless more than the returned number of bytes
- * are needed.
- *
- * targetRecPtr is the position of the WAL record we're reading. Usually
- * it is equal to targetPagePtr + reqLen, but sometimes xlogreader needs
- * to read and verify the page or segment header, before it reads the
- * actual WAL record it's interested in. In that case, targetRecPtr can
- * be used to determine which timeline to read the page from.
- *
- * The callback shall set ->seg.ws_tli to the TLI of the file the page was
- * read from.
- */
- XLogPageReadCB page_read;
-
- /*
- * Callback to open the specified WAL segment for reading. ->seg.ws_file
- * shall be set to the file descriptor of the opened segment. In case of
- * failure, an error shall be raised by the callback and it shall not
- * return.
- *
- * "nextSegNo" is the number of the segment to be opened.
- *
- * "tli_p" is an input/output argument. WALRead() uses it to pass the
- * timeline in which the new segment should be found, but the callback can
- * use it to return the TLI that it actually opened.
- */
- WALSegmentOpenCB segment_open;
-
- /*
- * WAL segment close callback. ->seg.ws_file shall be set to a negative
- * number.
- */
- WALSegmentCloseCB segment_close;
-} XLogReaderRoutine;
-
-#define XL_ROUTINE(...) &(XLogReaderRoutine){__VA_ARGS__}
-
typedef struct
{
/* Is this block ref in use? */
@@ -144,12 +95,35 @@ typedef struct
uint16 data_bufsz;
} DecodedBkpBlock;
+/* Return code from XLogReadRecord */
+typedef enum XLogReadRecordResult
+{
+ XLREAD_SUCCESS, /* record is successfully read */
+ XLREAD_NEED_DATA, /* need more data. see XLogReadRecord. */
+ XLREAD_FAIL /* failed during reading a record */
+} XLogReadRecordResult;
+
+/*
+ * internal state of XLogReadRecord
+ *
+ * XLogReadState runs a state machine while reading a record. Theses states
+ * are not seen outside the function. Each state may repeat several times
+ * exiting requesting caller for new data. See the comment of XLogReadRecrod
+ * for details.
+ */
+typedef enum XLogReadRecordState {
+ XLREAD_NEXT_RECORD,
+ XLREAD_TOT_LEN,
+ XLREAD_FIRST_FRAGMENT,
+ XLREAD_CONTINUATION
+} XLogReadRecordState;
+
struct XLogReaderState
{
/*
* Operational callbacks
*/
- XLogReaderRoutine routine;
+ WALSegmentCleanupCB cleanup_cb;
/* ----------------------------------------
* Public parameters
@@ -162,18 +136,14 @@ struct XLogReaderState
*/
uint64 system_identifier;
- /*
- * Opaque data for callbacks to use. Not used by XLogReader.
- */
- void *private_data;
-
/*
* Start and end point of last record read. EndRecPtr is also used as the
* position to read next. Calling XLogBeginRead() sets EndRecPtr to the
* starting position and ReadRecPtr to invalid.
*/
- XLogRecPtr ReadRecPtr; /* start of last record read */
+ XLogRecPtr ReadRecPtr; /* start of last record read or being read */
XLogRecPtr EndRecPtr; /* end+1 of last record read */
+ XLogRecPtr PrevRecPtr; /* start of previous record read */
/* ----------------------------------------
* Communication with page reader
@@ -187,7 +157,9 @@ struct XLogReaderState
* the request, or -1 on error */
TimeLineID readPageTLI; /* TLI for data currently in readBuf */
char *readBuf; /* buffer to store data */
- bool page_verified; /* is the page on the buffer verified? */
+ bool page_verified; /* is the page header on the buffer verified? */
+ bool record_verified;/* is the current record header verified? */
+
/* ----------------------------------------
@@ -227,8 +199,6 @@ struct XLogReaderState
XLogRecPtr latestPagePtr;
TimeLineID latestPageTLI;
- /* beginning of the WAL record being read. */
- XLogRecPtr currRecPtr;
/* timeline to read it from, 0 if a lookup is required */
TimeLineID currTLI;
@@ -255,6 +225,15 @@ struct XLogReaderState
char *readRecordBuf;
uint32 readRecordBufSize;
+ /*
+ * XLogReadRecord() state
+ */
+ XLogReadRecordState readRecordState;/* state machine state */
+ int recordGotLen; /* amount of current record that has
+ * already been read */
+ int recordRemainLen; /* length of current record that remains */
+ XLogRecPtr recordContRecPtr; /* where the current record continues */
+
/* Buffer to hold error message */
char *errormsg_buf;
};
@@ -262,9 +241,7 @@ struct XLogReaderState
/* Get a new XLogReader */
extern XLogReaderState *XLogReaderAllocate(int wal_segment_size,
const char *waldir,
- XLogReaderRoutine *routine,
- void *private_data);
-extern XLogReaderRoutine *LocalXLogReaderRoutine(void);
+ WALSegmentCleanupCB cleanup_cb);
/* Free an XLogReader */
extern void XLogReaderFree(XLogReaderState *state);
@@ -272,12 +249,17 @@ extern void XLogReaderFree(XLogReaderState *state);
/* Position the XLogReader to given record */
extern void XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr);
#ifdef FRONTEND
-extern XLogRecPtr XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr);
+/* Function type definition for the read_page callback */
+typedef bool (*XLogFindNextRecordCB) (XLogReaderState *xlogreader,
+ void *private);
+extern XLogRecPtr XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
+ XLogFindNextRecordCB read_page, void *private);
#endif /* FRONTEND */
/* Read the next XLog record. Returns NULL on end-of-WAL or failure */
-extern struct XLogRecord *XLogReadRecord(XLogReaderState *state,
- char **errormsg);
+extern XLogReadRecordResult XLogReadRecord(XLogReaderState *state,
+ XLogRecord **record,
+ char **errormsg);
/* Validate a page */
extern bool XLogReaderValidatePageHeader(XLogReaderState *state,
@@ -297,6 +279,7 @@ typedef struct WALReadError
} WALReadError;
extern bool WALRead(XLogReaderState *state,
+ WALSegmentOpenCB segopenfn, WALSegmentCloseCB sgclosefn,
char *buf, XLogRecPtr startptr, Size count,
TimeLineID tli, WALReadError *errinfo);
diff --git a/src/include/access/xlogutils.h b/src/include/access/xlogutils.h
index c287edf206..482e4ced69 100644
--- a/src/include/access/xlogutils.h
+++ b/src/include/access/xlogutils.h
@@ -47,9 +47,7 @@ extern Buffer XLogReadBufferExtended(RelFileNode rnode, ForkNumber forknum,
extern Relation CreateFakeRelcacheEntry(RelFileNode rnode);
extern void FreeFakeRelcacheEntry(Relation fakerel);
-extern bool read_local_xlog_page(XLogReaderState *state,
- XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetRecPtr, char *cur_page);
+extern bool read_local_xlog_page(XLogReaderState *state);
extern void wal_segment_open(XLogReaderState *state,
XLogSegNo nextSegNo,
TimeLineID *tli_p);
diff --git a/src/include/replication/logical.h b/src/include/replication/logical.h
index c2f2475e5d..a743db49cf 100644
--- a/src/include/replication/logical.h
+++ b/src/include/replication/logical.h
@@ -29,6 +29,10 @@ typedef void (*LogicalOutputPluginWriterUpdateProgress) (struct LogicalDecodingC
TransactionId xid
);
+typedef struct LogicalDecodingContext LogicalDecodingContext;
+
+typedef bool (*LogicalDecodingXLogPageReadCB)(XLogReaderState *ctx);
+
typedef struct LogicalDecodingContext
{
/* memory context this is all allocated in */
@@ -39,6 +43,7 @@ typedef struct LogicalDecodingContext
/* infrastructure pieces for decoding */
XLogReaderState *reader;
+ LogicalDecodingXLogPageReadCB page_read;
struct ReorderBuffer *reorder;
struct SnapBuild *snapshot_builder;
@@ -95,14 +100,16 @@ extern LogicalDecodingContext *CreateInitDecodingContext(char *plugin,
List *output_plugin_options,
bool need_full_snapshot,
XLogRecPtr restart_lsn,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress);
extern LogicalDecodingContext *CreateDecodingContext(XLogRecPtr start_lsn,
List *output_plugin_options,
bool fast_forward,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress);
--
2.18.4
----Next_Part(Thu_Jul__2_13_53_30_2020_072)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
filename="v15-0003-Remove-globals-readOff-readLen-and-readSegNo.patch"
^ permalink raw reply [nested|flat] 70+ messages in thread
* [PATCH v17 2/4] Move page-reader out of XLogReadRecord
@ 2019-09-10 03:58 Kyotaro Horiguchi <[email protected]>
0 siblings, 0 replies; 70+ messages in thread
From: Kyotaro Horiguchi @ 2019-09-10 03:58 UTC (permalink / raw)
This is the second step of removing callbacks from WAL record reader.
Since it is essential to take in additional data while reading a
record, the function have to ask caller for new data while keeping
working state. Thus the function is turned into a state machine.
---
src/backend/access/transam/twophase.c | 15 +-
src/backend/access/transam/xlog.c | 58 +-
src/backend/access/transam/xlogreader.c | 697 +++++++++++-------
src/backend/access/transam/xlogutils.c | 17 +-
src/backend/replication/logical/logical.c | 26 +-
.../replication/logical/logicalfuncs.c | 13 +-
src/backend/replication/slotfuncs.c | 18 +-
src/backend/replication/walsender.c | 32 +-
src/bin/pg_rewind/parsexlog.c | 84 ++-
src/bin/pg_waldump/pg_waldump.c | 36 +-
src/include/access/xlogreader.h | 121 ++-
src/include/access/xlogutils.h | 4 +-
src/include/pg_config_manual.h | 2 +-
src/include/replication/logical.h | 11 +-
14 files changed, 646 insertions(+), 488 deletions(-)
diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index 80d2d20d6c..7d5a96a5f5 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -1325,11 +1325,8 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
XLogReaderState *xlogreader;
char *errormsg;
- xlogreader = XLogReaderAllocate(wal_segment_size, NULL,
- XL_ROUTINE(.page_read = &read_local_xlog_page,
- .segment_open = &wal_segment_open,
- .segment_close = &wal_segment_close),
- NULL);
+ xlogreader = XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+
if (!xlogreader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
@@ -1337,7 +1334,13 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
errdetail("Failed while allocating a WAL reading processor.")));
XLogBeginRead(xlogreader, lsn);
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!read_local_xlog_page(xlogreader))
+ break;
+ }
+
if (record == NULL)
ereport(ERROR,
(errcode_for_file_access(),
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 606328a65a..670996ae67 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -836,13 +836,6 @@ static XLogSource currentSource = XLOG_FROM_ANY;
static bool lastSourceFailed = false;
static bool pendingWalRcvRestart = false;
-typedef struct XLogPageReadPrivate
-{
- int emode;
- bool fetching_ckpt; /* are we fetching a checkpoint record? */
- bool randAccess;
-} XLogPageReadPrivate;
-
/*
* These variables track when we last obtained some WAL data to process,
* and where we got it from. (XLogReceiptSource is initially the same as
@@ -917,8 +910,8 @@ static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath,
static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli,
XLogSource source, bool notfoundOk);
static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, XLogSource source);
-static bool XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
+static bool XLogPageRead(XLogReaderState *xlogreader,
+ bool fetching_ckpt, int emode, bool randAccess);
static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
bool fetching_ckpt, XLogRecPtr tliRecPtr);
static int emode_for_corrupt_record(int emode, XLogRecPtr RecPtr);
@@ -1231,8 +1224,7 @@ XLogInsertRecord(XLogRecData *rdata,
appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len);
if (!debug_reader)
- debug_reader = XLogReaderAllocate(wal_segment_size, NULL,
- XL_ROUTINE(), NULL);
+ debug_reader = XLogReaderAllocate(wal_segment_size, NULL, NULL);
if (!debug_reader)
{
@@ -4344,15 +4336,10 @@ CleanupBackupHistory(void)
* record is available.
*/
static XLogRecord *
-ReadRecord(XLogReaderState *xlogreader, int emode,
- bool fetching_ckpt)
+ReadRecord(XLogReaderState *xlogreader, int emode, bool fetching_ckpt)
{
XLogRecord *record;
- XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
-
- private->fetching_ckpt = fetching_ckpt;
- private->emode = emode;
- private->randAccess = (xlogreader->ReadRecPtr == InvalidXLogRecPtr);
+ bool randAccess = (xlogreader->ReadRecPtr == InvalidXLogRecPtr);
/* This is the first attempt to read this page. */
lastSourceFailed = false;
@@ -4360,8 +4347,16 @@ ReadRecord(XLogReaderState *xlogreader, int emode,
for (;;)
{
char *errormsg;
+ XLogReadRecordResult result;
+
+ while ((result = XLogReadRecord(xlogreader, &record, &errormsg))
+ == XLREAD_NEED_DATA)
+ {
+ if (!XLogPageRead(xlogreader, fetching_ckpt, emode, randAccess))
+ break;
+
+ }
- record = XLogReadRecord(xlogreader, &errormsg);
ReadRecPtr = xlogreader->ReadRecPtr;
EndRecPtr = xlogreader->EndRecPtr;
if (record == NULL)
@@ -6380,7 +6375,6 @@ StartupXLOG(void)
bool backupFromStandby = false;
DBState dbstate_at_startup;
XLogReaderState *xlogreader;
- XLogPageReadPrivate private;
bool promoted = false;
struct stat st;
@@ -6539,13 +6533,9 @@ StartupXLOG(void)
OwnLatch(&XLogCtl->recoveryWakeupLatch);
/* Set up XLOG reader facility */
- MemSet(&private, 0, sizeof(XLogPageReadPrivate));
xlogreader =
- XLogReaderAllocate(wal_segment_size, NULL,
- XL_ROUTINE(.page_read = &XLogPageRead,
- .segment_open = NULL,
- .segment_close = wal_segment_close),
- &private);
+ XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+
if (!xlogreader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
@@ -11987,12 +11977,13 @@ CancelBackup(void)
* sleep and retry.
*/
static bool
-XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetRecPtr, char *readBuf)
+XLogPageRead(XLogReaderState *xlogreader,
+ bool fetching_ckpt, int emode, bool randAccess)
{
- XLogPageReadPrivate *private =
- (XLogPageReadPrivate *) xlogreader->private_data;
- int emode = private->emode;
+ char *readBuf = xlogreader->readBuf;
+ XLogRecPtr targetPagePtr = xlogreader->readPagePtr;
+ int reqLen = xlogreader->readLen;
+ XLogRecPtr targetRecPtr = xlogreader->ReadRecPtr;
uint32 targetPageOff;
XLogSegNo targetSegNo PG_USED_FOR_ASSERTS_ONLY;
int r;
@@ -12035,8 +12026,8 @@ retry:
flushedUpto < targetPagePtr + reqLen))
{
if (!WaitForWALToBecomeAvailable(targetPagePtr + reqLen,
- private->randAccess,
- private->fetching_ckpt,
+ randAccess,
+ fetching_ckpt,
targetRecPtr))
{
if (readFile >= 0)
@@ -12141,6 +12132,7 @@ retry:
goto next_record_is_invalid;
}
+ Assert(xlogreader->readPagePtr == targetPagePtr);
xlogreader->readLen = readLen;
return true;
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index 7d04995b7b..ba3e4868f7 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -40,7 +40,7 @@ static bool XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr,
int reqLen, bool header_inclusive);
static void XLogReaderInvalReadState(XLogReaderState *state);
static bool ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
- XLogRecPtr PrevRecPtr, XLogRecord *record, bool randAccess);
+ XLogRecPtr PrevRecPtr, XLogRecord *record);
static bool ValidXLogRecord(XLogReaderState *state, XLogRecord *record,
XLogRecPtr recptr);
static void ResetDecoder(XLogReaderState *state);
@@ -73,7 +73,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...)
*/
XLogReaderState *
XLogReaderAllocate(int wal_segment_size, const char *waldir,
- XLogReaderRoutine *routine, void *private_data)
+ WALSegmentCleanupCB cleanup_cb)
{
XLogReaderState *state;
@@ -84,7 +84,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
return NULL;
/* initialize caller-provided support functions */
- state->routine = *routine;
+ state->cleanup_cb = cleanup_cb;
state->max_block_id = -1;
@@ -107,8 +107,6 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
WALOpenSegmentInit(&state->seg, &state->segcxt, wal_segment_size,
waldir);
- /* system_identifier initialized to zeroes above */
- state->private_data = private_data;
/* ReadRecPtr, EndRecPtr and readLen initialized to zeroes above */
state->errormsg_buf = palloc_extended(MAX_ERRORMSG_LEN + 1,
MCXT_ALLOC_NO_OOM);
@@ -140,8 +138,8 @@ XLogReaderFree(XLogReaderState *state)
{
int block_id;
- if (state->seg.ws_file != -1)
- state->routine.segment_close(state);
+ if (state->seg.ws_file >= 0)
+ state->cleanup_cb(state);
for (block_id = 0; block_id <= XLR_MAX_BLOCK_ID; block_id++)
{
@@ -246,6 +244,7 @@ XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
/* Begin at the passed-in record pointer. */
state->EndRecPtr = RecPtr;
state->ReadRecPtr = InvalidXLogRecPtr;
+ state->readRecordState = XLREAD_NEXT_RECORD;
}
/*
@@ -254,317 +253,449 @@ XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
* XLogBeginRead() or XLogFindNextRecord() must be called before the first call
* to XLogReadRecord().
*
- * If the page_read callback fails to read the requested data, NULL is
- * returned. The callback is expected to have reported the error; errormsg
- * is set to NULL.
+ * This function may return XLREAD_NEED_DATA several times before returning a
+ * result record. The caller shall read in some new data then call this
+ * function again with the same parameters.
*
- * If the reading fails for some other reason, NULL is also returned, and
- * *errormsg is set to a string with details of the failure.
+ * When a record is successfully read, returns XLREAD_SUCCESS with result
+ * record being stored in *record. Otherwise *record is NULL.
*
- * The returned pointer (or *errormsg) points to an internal buffer that's
- * valid until the next call to XLogReadRecord.
+ * Returns XLREAD_NEED_DATA if more data is needed to finish reading the
+ * current record. In that case, state->readPagePtr and state->readLen inform
+ * the desired position and minimum length of data needed. The caller shall
+ * read in the requested data and set state->readBuf to point to a buffer
+ * containing it. The caller must also set state->readPageTLI and
+ * state->readLen to indicate the timeline that it was read from, and the
+ * length of data that is now available (which must be >= given readLen),
+ * respectively.
+ *
+ * If invalid data is encountered, returns XLREAD_FAIL with *record being set to
+ * NULL. *errormsg is set to a string with details of the failure.
+ * The returned pointer (or *errormsg) points to an internal buffer that's valid
+ * until the next call to XLogReadRecord.
+ *
+ *
+ * This function runs a state machine consists of the following states.
+ *
+ * XLREAD_NEXT_RECORD :
+ * The initial state, if called with valid RecPtr, try to read a record at
+ * that position. If invalid RecPtr is given try to read a record just after
+ * the last one previously read.
+ * This state ens after setting ReadRecPtr. Then goes to XLREAD_TOT_LEN.
+ *
+ * XLREAD_TOT_LEN:
+ * Examining record header. Ends after reading record total
+ * length. recordRemainLen and recordGotLen are initialized.
+ *
+ * XLREAD_FIRST_FRAGMENT:
+ * Reading the first fragment. Ends with finishing reading a single
+ * record. Goes to XLREAD_NEXT_RECORD if that's all or
+ * XLREAD_CONTINUATION if we have continuation.
+
+ * XLREAD_CONTINUATION:
+ * Reading continuation of record. Ends with finishing the whole record then
+ * goes to XLREAD_NEXT_RECORD. During this state, recordRemainLen indicates
+ * how much is left and readRecordBuf holds the partially assert
+ * record.recordContRecPtr points to the beginning of the next page where to
+ * continue.
+ *
+ * If wrong data found in any state, the state machine stays at the current
+ * state. This behavior allows to continue reading a reacord switching among
+ * different souces, while streaming replication.
*/
-XLogRecord *
-XLogReadRecord(XLogReaderState *state, char **errormsg)
+XLogReadRecordResult
+XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
{
- XLogRecPtr RecPtr;
- XLogRecord *record;
- XLogRecPtr targetPagePtr;
- bool randAccess;
- uint32 len,
- total_len;
- uint32 targetRecOff;
- uint32 pageHeaderSize;
- bool gotheader;
+ XLogRecord *prec;
- /*
- * randAccess indicates whether to verify the previous-record pointer of
- * the record we're reading. We only do this if we're reading
- * sequentially, which is what we initially assume.
- */
- randAccess = false;
+ *record = NULL;
/* reset error state */
*errormsg = NULL;
state->errormsg_buf[0] = '\0';
- ResetDecoder(state);
-
- RecPtr = state->EndRecPtr;
-
- if (state->ReadRecPtr != InvalidXLogRecPtr)
- {
- /* read the record after the one we just read */
-
- /*
- * EndRecPtr is pointing to end+1 of the previous WAL record. If
- * we're at a page boundary, no more records can fit on the current
- * page. We must skip over the page header, but we can't do that until
- * we've read in the page, since the header size is variable.
- */
- }
- else
- {
- /*
- * Caller supplied a position to start at.
- *
- * In this case, EndRecPtr should already be pointing to a valid
- * record starting position.
- */
- Assert(XRecOffIsValid(RecPtr));
- randAccess = true;
- }
-
- state->currRecPtr = RecPtr;
-
- targetPagePtr = RecPtr - (RecPtr % XLOG_BLCKSZ);
- targetRecOff = RecPtr % XLOG_BLCKSZ;
-
- /*
- * Read the page containing the record into state->readBuf. Request enough
- * byte to cover the whole record header, or at least the part of it that
- * fits on the same page.
- */
- while (XLogNeedData(state, targetPagePtr,
- Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
- targetRecOff != 0))
- {
- if (!state->routine.page_read(state, state->readPagePtr, state->readLen,
- RecPtr, state->readBuf))
- break;
- }
-
- if (!state->page_verified)
- goto err;
-
- /*
- * We have at least the page header, so we can examine it now.
- */
- pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
- if (targetRecOff == 0)
- {
- /*
- * At page start, so skip over page header.
- */
- RecPtr += pageHeaderSize;
- targetRecOff = pageHeaderSize;
- }
- else if (targetRecOff < pageHeaderSize)
- {
- report_invalid_record(state, "invalid record offset at %X/%X",
- LSN_FORMAT_ARGS(RecPtr));
- goto err;
- }
-
- if ((((XLogPageHeader) state->readBuf)->xlp_info & XLP_FIRST_IS_CONTRECORD) &&
- targetRecOff == pageHeaderSize)
- {
- report_invalid_record(state, "contrecord is requested by %X/%X",
- LSN_FORMAT_ARGS(RecPtr));
- goto err;
- }
-
- /* XLogNeedData has verified the page header */
- Assert(pageHeaderSize <= state->readLen);
-
- /*
- * Read the record length.
- *
- * NB: Even though we use an XLogRecord pointer here, the whole record
- * header might not fit on this page. xl_tot_len is the first field of the
- * struct, so it must be on this page (the records are MAXALIGNed), but we
- * cannot access any other fields until we've verified that we got the
- * whole header.
- */
- record = (XLogRecord *) (state->readBuf + RecPtr % XLOG_BLCKSZ);
- total_len = record->xl_tot_len;
-
- /*
- * If the whole record header is on this page, validate it immediately.
- * Otherwise do just a basic sanity check on xl_tot_len, and validate the
- * rest of the header after reading it from the next page. The xl_tot_len
- * check is necessary here to ensure that we enter the "Need to reassemble
- * record" code path below; otherwise we might fail to apply
- * ValidXLogRecordHeader at all.
- */
- if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
- {
- if (!ValidXLogRecordHeader(state, RecPtr, state->ReadRecPtr, record,
- randAccess))
- goto err;
- gotheader = true;
- }
- else
- {
- /* XXX: more validation should be done here */
- if (total_len < SizeOfXLogRecord)
- {
- report_invalid_record(state,
- "invalid record length at %X/%X: wanted %u, got %u",
- LSN_FORMAT_ARGS(RecPtr),
- (uint32) SizeOfXLogRecord, total_len);
- goto err;
- }
- gotheader = false;
- }
-
- len = XLOG_BLCKSZ - RecPtr % XLOG_BLCKSZ;
- if (total_len > len)
+ switch (state->readRecordState)
{
- /* Need to reassemble record */
- char *contdata;
- XLogPageHeader pageHeader;
- char *buffer;
- uint32 gotlen;
-
- /*
- * Enlarge readRecordBuf as needed.
- */
- if (total_len > state->readRecordBufSize &&
- !allocate_recordbuf(state, total_len))
- {
- /* We treat this as a "bogus data" condition */
- report_invalid_record(state, "record length %u at %X/%X too long",
- total_len, LSN_FORMAT_ARGS(RecPtr));
- goto err;
- }
+ case XLREAD_NEXT_RECORD:
+ ResetDecoder(state);
- /* Copy the first fragment of the record from the first page. */
- memcpy(state->readRecordBuf,
- state->readBuf + RecPtr % XLOG_BLCKSZ, len);
- buffer = state->readRecordBuf + len;
- gotlen = len;
-
- do
- {
- int rest_len = total_len - gotlen;
-
- /* Calculate pointer to beginning of next page */
- targetPagePtr += XLOG_BLCKSZ;
-
- /* Wait for the next page to become available */
- while (XLogNeedData(state, targetPagePtr,
- Min(rest_len, XLOG_BLCKSZ),
- false))
+ if (state->ReadRecPtr != InvalidXLogRecPtr)
{
- if (!state->routine.page_read(state, state->readPagePtr,
- state->readLen,
- state->ReadRecPtr,
- state->readBuf))
- break;
+ /* read the record after the one we just read */
+
+ /*
+ * EndRecPtr is pointing to end+1 of the previous WAL record.
+ * If we're at a page boundary, no more records can fit on the
+ * current page. We must skip over the page header, but we
+ * can't do that until we've read in the page, since the header
+ * size is variable.
+ */
+ state->PrevRecPtr = state->ReadRecPtr;
+ state->ReadRecPtr = state->EndRecPtr;
+ }
+ else
+ {
+ /*
+ * Caller supplied a position to start at.
+ *
+ * In this case, EndRecPtr should already be pointing to a
+ * valid record starting position.
+ */
+ Assert(XRecOffIsValid(state->EndRecPtr));
+ state->ReadRecPtr = state->EndRecPtr;
+
+ /*
+ * We cannot verify the previous-record pointer when we're
+ * seeking to a particular record. Reset PrevRecPtr so that we
+ * won't try doing that.
+ */
+ state->PrevRecPtr = InvalidXLogRecPtr;
+ state->EndRecPtr = InvalidXLogRecPtr; /* to be tidy */
}
+ state->record_verified = false;
+ state->readRecordState = XLREAD_TOT_LEN;
+ /* fall through */
+
+ case XLREAD_TOT_LEN:
+ {
+ uint32 total_len;
+ uint32 pageHeaderSize;
+ XLogRecPtr targetPagePtr;
+ uint32 targetRecOff;
+ XLogPageHeader pageHeader;
+
+ targetPagePtr =
+ state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
+ targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+
+ /*
+ * Check if we have enough data. For the first record in the page,
+ * the requesting length doesn't contain page header.
+ */
+ if (XLogNeedData(state, targetPagePtr,
+ Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+ targetRecOff != 0))
+ return XLREAD_NEED_DATA;
+
+ /* error out if caller supplied bogus page */
if (!state->page_verified)
goto err;
- Assert(SizeOfXLogShortPHD <= state->readLen);
+ /* examine page header now. */
+ pageHeaderSize =
+ XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+ if (targetRecOff == 0)
+ {
+ /* At page start, so skip over page header. */
+ state->ReadRecPtr += pageHeaderSize;
+ targetRecOff = pageHeaderSize;
+ }
+ else if (targetRecOff < pageHeaderSize)
+ {
+ report_invalid_record(state, "invalid record offset at %X/%X",
+ LSN_FORMAT_ARGS(state->ReadRecPtr));
+ goto err;
+ }
- /* Check that the continuation on next page looks valid */
pageHeader = (XLogPageHeader) state->readBuf;
- if (!(pageHeader->xlp_info & XLP_FIRST_IS_CONTRECORD))
+ if ((pageHeader->xlp_info & XLP_FIRST_IS_CONTRECORD) &&
+ targetRecOff == pageHeaderSize)
{
- report_invalid_record(state,
- "there is no contrecord flag at %X/%X",
- LSN_FORMAT_ARGS(RecPtr));
+ report_invalid_record(state, "contrecord is requested by %X/%X",
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr);
goto err;
}
- /*
- * Cross-check that xlp_rem_len agrees with how much of the record
- * we expect there to be left.
- */
- if (pageHeader->xlp_rem_len == 0 ||
- total_len != (pageHeader->xlp_rem_len + gotlen))
- {
- report_invalid_record(state,
- "invalid contrecord length %u (expected %lld) at %X/%X",
- pageHeader->xlp_rem_len,
- ((long long) total_len) - gotlen,
- LSN_FORMAT_ARGS(RecPtr));
- goto err;
- }
-
- /* Append the continuation from this page to the buffer */
- pageHeaderSize = XLogPageHeaderSize(pageHeader);
-
+ /* XLogNeedData has verified the page header */
Assert(pageHeaderSize <= state->readLen);
- contdata = (char *) state->readBuf + pageHeaderSize;
- len = XLOG_BLCKSZ - pageHeaderSize;
- if (pageHeader->xlp_rem_len < len)
- len = pageHeader->xlp_rem_len;
+ /*
+ * Read the record length.
+ *
+ * NB: Even though we use an XLogRecord pointer here, the whole
+ * record header might not fit on this page. xl_tot_len is the first
+ * field of the struct, so it must be on this page (the records are
+ * MAXALIGNed), but we cannot access any other fields until we've
+ * verified that we got the whole header.
+ */
+ prec = (XLogRecord *) (state->readBuf +
+ state->ReadRecPtr % XLOG_BLCKSZ);
+ total_len = prec->xl_tot_len;
- Assert (pageHeaderSize + len <= state->readLen);
- memcpy(buffer, (char *) contdata, len);
- buffer += len;
- gotlen += len;
+ /*
+ * If the whole record header is on this page, validate it
+ * immediately. Otherwise do just a basic sanity check on
+ * xl_tot_len, and validate the rest of the header after reading it
+ * from the next page. The xl_tot_len check is necessary here to
+ * ensure that we enter the XLREAD_CONTINUATION state below;
+ * otherwise we might fail to apply ValidXLogRecordHeader at all.
+ */
+ if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
+ {
+ if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
+ state->PrevRecPtr, prec))
+ goto err;
- /* If we just reassembled the record header, validate it. */
- if (!gotheader)
+ state->record_verified = true;
+ }
+ else
{
- record = (XLogRecord *) state->readRecordBuf;
- if (!ValidXLogRecordHeader(state, RecPtr, state->ReadRecPtr,
- record, randAccess))
+ /* XXX: more validation should be done here */
+ if (total_len < SizeOfXLogRecord)
+ {
+ report_invalid_record(state,
+ "invalid record length at %X/%X: wanted %u, got %u",
+ LSN_FORMAT_ARGS(state->ReadRecPtr),
+ (uint32) SizeOfXLogRecord, total_len);
goto err;
- gotheader = true;
+ }
}
- } while (gotlen < total_len);
-
- Assert(gotheader);
- record = (XLogRecord *) state->readRecordBuf;
- if (!ValidXLogRecord(state, record, RecPtr))
- goto err;
+ /*
+ * Wait for the rest of the record, or the part of the record that
+ * fit on the first page if crossed a page boundary, to become
+ * available.
+ */
+ state->recordGotLen = 0;
+ state->recordRemainLen = total_len;
+ state->readRecordState = XLREAD_FIRST_FRAGMENT;
+ }
+ /* fall through */
- pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
- state->ReadRecPtr = RecPtr;
- state->EndRecPtr = targetPagePtr + pageHeaderSize
- + MAXALIGN(pageHeader->xlp_rem_len);
- }
- else
- {
- /* Wait for the record data to become available */
- while (XLogNeedData(state, targetPagePtr,
- Min(targetRecOff + total_len, XLOG_BLCKSZ), true))
+ case XLREAD_FIRST_FRAGMENT:
{
- if (!state->routine.page_read(state, state->readPagePtr,
- state->readLen,
- state->ReadRecPtr, state->readBuf))
+ uint32 total_len = state->recordRemainLen;
+ uint32 request_len;
+ uint32 record_len;
+ XLogRecPtr targetPagePtr;
+ uint32 targetRecOff;
+
+ /*
+ * Wait for the rest of the record on the first page to become
+ * available
+ */
+ targetPagePtr =
+ state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
+ targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+
+ request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ);
+ record_len = request_len - targetRecOff;
+
+ /* ReadRecPtr contains page header */
+ Assert (targetRecOff != 0);
+ if (XLogNeedData(state, targetPagePtr, request_len, true))
+ return XLREAD_NEED_DATA;
+
+ /* error out if caller supplied bogus page */
+ if (!state->page_verified)
+ goto err;
+
+ prec = (XLogRecord *) (state->readBuf + targetRecOff);
+
+ /* validate record header if not yet */
+ if (!state->record_verified && record_len >= SizeOfXLogRecord)
+ {
+ if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
+ state->PrevRecPtr, prec))
+ goto err;
+
+ state->record_verified = true;
+ }
+
+
+ if (total_len == record_len)
+ {
+ /* Record does not cross a page boundary */
+ Assert(state->record_verified);
+
+ if (!ValidXLogRecord(state, prec, state->ReadRecPtr))
+ goto err;
+
+ state->record_verified = true; /* to be tidy */
+
+ /* We already checked the header earlier */
+ state->EndRecPtr = state->ReadRecPtr + MAXALIGN(record_len);
+
+ *record = prec;
+ state->readRecordState = XLREAD_NEXT_RECORD;
break;
+ }
+
+ /*
+ * The record continues on the next page. Need to reassemble
+ * record
+ */
+ Assert(total_len > record_len);
+
+ /* Enlarge readRecordBuf as needed. */
+ if (total_len > state->readRecordBufSize &&
+ !allocate_recordbuf(state, total_len))
+ {
+ /* We treat this as a "bogus data" condition */
+ report_invalid_record(state,
+ "record length %u at %X/%X too long",
+ total_len,
+ LSN_FORMAT_ARGS(state->ReadRecPtr));
+ goto err;
+ }
+
+ /* Copy the first fragment of the record from the first page. */
+ memcpy(state->readRecordBuf, state->readBuf + targetRecOff,
+ record_len);
+ state->recordGotLen += record_len;
+ state->recordRemainLen -= record_len;
+
+ /* Calculate pointer to beginning of next page */
+ state->recordContRecPtr = state->ReadRecPtr + record_len;
+ Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0);
+
+ state->readRecordState = XLREAD_CONTINUATION;
}
+ /* fall through */
+
+ case XLREAD_CONTINUATION:
+ {
+ XLogPageHeader pageHeader;
+ uint32 pageHeaderSize;
+ XLogRecPtr targetPagePtr;
+
+ /* we enter this state only if we haven't read the whole record. */
+ Assert (state->recordRemainLen > 0);
+
+ while(state->recordRemainLen > 0)
+ {
+ char *contdata;
+ uint32 request_len;
+ uint32 record_len;
+
+ /* Wait for the next page to become available */
+ targetPagePtr = state->recordContRecPtr;
+
+ /* this request contains page header */
+ Assert (targetPagePtr != 0);
+ if (XLogNeedData(state, targetPagePtr,
+ Min(state->recordRemainLen, XLOG_BLCKSZ),
+ false))
+ return XLREAD_NEED_DATA;
+
+ if (!state->page_verified)
+ goto err;
+
+ Assert(SizeOfXLogShortPHD <= state->readLen);
- if (!state->page_verified)
- goto err;
+ /* Check that the continuation on next page looks valid */
+ pageHeader = (XLogPageHeader) state->readBuf;
+ if (!(pageHeader->xlp_info & XLP_FIRST_IS_CONTRECORD))
+ {
+ report_invalid_record(
+ state,
+ "there is no contrecord flag at %X/%X reading %X/%X",
+ (uint32) (state->recordContRecPtr >> 32),
+ (uint32) state->recordContRecPtr,
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr);
+ goto err;
+ }
- /* Record does not cross a page boundary */
- if (!ValidXLogRecord(state, record, RecPtr))
- goto err;
+ /*
+ * Cross-check that xlp_rem_len agrees with how much of the
+ * record we expect there to be left.
+ */
+ if (pageHeader->xlp_rem_len == 0 ||
+ pageHeader->xlp_rem_len != state->recordRemainLen)
+ {
+ report_invalid_record(
+ state,
+ "invalid contrecord length %u at %X/%X reading %X/%X, expected %u",
+ pageHeader->xlp_rem_len,
+ (uint32) (state->recordContRecPtr >> 32),
+ (uint32) state->recordContRecPtr,
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr,
+ state->recordRemainLen);
+ goto err;
+ }
- state->EndRecPtr = RecPtr + MAXALIGN(total_len);
+ /* Append the continuation from this page to the buffer */
+ pageHeaderSize = XLogPageHeaderSize(pageHeader);
- state->ReadRecPtr = RecPtr;
+ /*
+ * XLogNeedData should have ensured that the whole page header
+ * was read
+ */
+ Assert(state->readLen >= pageHeaderSize);
+
+ contdata = (char *) state->readBuf + pageHeaderSize;
+ record_len = XLOG_BLCKSZ - pageHeaderSize;
+ if (pageHeader->xlp_rem_len < record_len)
+ record_len = pageHeader->xlp_rem_len;
+
+ request_len = record_len + pageHeaderSize;
+
+ /* XLogNeedData should have ensured all needed data was read */
+ Assert (state->readLen >= request_len);
+
+ memcpy(state->readRecordBuf + state->recordGotLen,
+ (char *) contdata, record_len);
+ state->recordGotLen += record_len;
+ state->recordRemainLen -= record_len;
+
+ /* If we just reassembled the record header, validate it. */
+ if (!state->record_verified)
+ {
+ Assert(state->recordGotLen >= SizeOfXLogRecord);
+ if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
+ state->PrevRecPtr,
+ (XLogRecord *) state->readRecordBuf))
+ goto err;
+
+ state->record_verified = true;
+ }
+
+ /* Calculate pointer to beginning of next page, and continue */
+ state->recordContRecPtr += XLOG_BLCKSZ;
+ }
+
+ /* targetPagePtr is pointing the last-read page here */
+ prec = (XLogRecord *) state->readRecordBuf;
+ if (!ValidXLogRecord(state, prec, state->ReadRecPtr))
+ goto err;
+
+ pageHeaderSize =
+ XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+ state->EndRecPtr = targetPagePtr + pageHeaderSize
+ + MAXALIGN(pageHeader->xlp_rem_len);
+
+ *record = prec;
+ state->readRecordState = XLREAD_NEXT_RECORD;
+ break;
+ }
}
/*
* Special processing if it's an XLOG SWITCH record
*/
- if (record->xl_rmid == RM_XLOG_ID &&
- (record->xl_info & ~XLR_INFO_MASK) == XLOG_SWITCH)
+ if ((*record)->xl_rmid == RM_XLOG_ID &&
+ ((*record)->xl_info & ~XLR_INFO_MASK) == XLOG_SWITCH)
{
/* Pretend it extends to end of segment */
state->EndRecPtr += state->segcxt.ws_segsize - 1;
state->EndRecPtr -= XLogSegmentOffset(state->EndRecPtr, state->segcxt.ws_segsize);
}
- if (DecodeXLogRecord(state, record, errormsg))
- return record;
- else
- return NULL;
+ Assert (!*record || state->readLen >= 0);
+ if (DecodeXLogRecord(state, *record, errormsg))
+ return XLREAD_SUCCESS;
+
+ *record = NULL;
+ return XLREAD_FAIL;
err:
/*
- * Invalidate the read state. We might read from a different source after
+ * Invalidate the read page. We might read from a different source after
* failure.
*/
XLogReaderInvalReadState(state);
@@ -572,7 +703,8 @@ err:
if (state->errormsg_buf[0] != '\0')
*errormsg = state->errormsg_buf;
- return NULL;
+ *record = NULL;
+ return XLREAD_FAIL;
}
/*
@@ -724,11 +856,12 @@ XLogReaderInvalReadState(XLogReaderState *state)
*
* This is just a convenience subroutine to avoid duplicated code in
* XLogReadRecord. It's not intended for use from anywhere else.
+ *
+ * If PrevRecPtr is valid, the xl_prev is is cross-checked with it.
*/
static bool
ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
- XLogRecPtr PrevRecPtr, XLogRecord *record,
- bool randAccess)
+ XLogRecPtr PrevRecPtr, XLogRecord *record)
{
if (record->xl_tot_len < SizeOfXLogRecord)
{
@@ -745,7 +878,7 @@ ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
record->xl_rmid, LSN_FORMAT_ARGS(RecPtr));
return false;
}
- if (randAccess)
+ if (PrevRecPtr == InvalidXLogRecPtr)
{
/*
* We can't exactly verify the prev-link, but surely it should be less
@@ -975,11 +1108,14 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
* XLogReadRecord() will read the next valid record.
*/
XLogRecPtr
-XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
+XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
+ XLogFindNextRecordCB read_page, void *private)
{
XLogRecPtr tmpRecPtr;
XLogRecPtr found = InvalidXLogRecPtr;
XLogPageHeader header;
+ XLogRecord *record;
+ XLogReadRecordResult result;
char *errormsg;
Assert(!XLogRecPtrIsInvalid(RecPtr));
@@ -1012,9 +1148,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
while(XLogNeedData(state, targetPagePtr, targetRecOff,
targetRecOff != 0))
{
- if (!state->routine.page_read(state, state->readPagePtr,
- state->readLen,
- state->ReadRecPtr, state->readBuf))
+ if (!read_page(state, private))
break;
}
@@ -1066,8 +1200,16 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
* or we just jumped over the remaining data of a continuation.
*/
XLogBeginRead(state, tmpRecPtr);
- while (XLogReadRecord(state, &errormsg) != NULL)
+ while ((result = XLogReadRecord(state, &record, &errormsg)) !=
+ XLREAD_FAIL)
{
+ if (result == XLREAD_NEED_DATA)
+ {
+ if (!read_page(state, private))
+ goto err;
+ continue;
+ }
+
/* past the record we've found, break out */
if (RecPtr <= state->ReadRecPtr)
{
@@ -1087,9 +1229,9 @@ err:
#endif /* FRONTEND */
/*
- * Helper function to ease writing of XLogRoutine->page_read callbacks.
- * If this function is used, caller must supply a segment_open callback in
- * 'state', as that is used here.
+ * Helper function to ease writing of page_read callback.
+ * If this function is used, caller must supply a segment_open callback and
+ * segment_close callback as that is used here.
*
* Read 'count' bytes into 'buf', starting at location 'startptr', from WAL
* fetched from timeline 'tli'.
@@ -1102,6 +1244,7 @@ err:
*/
bool
WALRead(XLogReaderState *state,
+ WALSegmentOpenCB segopenfn, WALSegmentCloseCB segclosefn,
char *buf, XLogRecPtr startptr, Size count, TimeLineID tli,
WALReadError *errinfo)
{
@@ -1133,10 +1276,10 @@ WALRead(XLogReaderState *state,
XLogSegNo nextSegNo;
if (state->seg.ws_file >= 0)
- state->routine.segment_close(state);
+ segclosefn(state);
XLByteToSeg(recptr, nextSegNo, state->segcxt.ws_segsize);
- state->routine.segment_open(state, nextSegNo, &tli);
+ segopenfn(state, nextSegNo, &tli);
/* This shouldn't happen -- indicates a bug in segment_open */
Assert(state->seg.ws_file >= 0);
diff --git a/src/backend/access/transam/xlogutils.c b/src/backend/access/transam/xlogutils.c
index 46eda33f25..b003990745 100644
--- a/src/backend/access/transam/xlogutils.c
+++ b/src/backend/access/transam/xlogutils.c
@@ -686,8 +686,7 @@ XLogTruncateRelation(RelFileNode rnode, ForkNumber forkNum,
void
XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wantLength)
{
- const XLogRecPtr lastReadPage = state->seg.ws_segno *
- state->segcxt.ws_segsize + state->readLen;
+ const XLogRecPtr lastReadPage = state->readPagePtr;
Assert(wantPage != InvalidXLogRecPtr && wantPage % XLOG_BLCKSZ == 0);
Assert(wantLength <= XLOG_BLCKSZ);
@@ -702,7 +701,7 @@ XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wa
* current TLI has since become historical.
*/
if (lastReadPage == wantPage &&
- state->readLen != 0 &&
+ state->page_verified &&
lastReadPage + state->readLen >= wantPage + Min(wantLength, XLOG_BLCKSZ - 1))
return;
@@ -788,6 +787,7 @@ wal_segment_open(XLogReaderState *state, XLogSegNo nextSegNo,
char path[MAXPGPATH];
XLogFilePath(path, tli, nextSegNo, state->segcxt.ws_segsize);
+ elog(LOG, "HOGE: %lu, %d => %s", nextSegNo, tli, path);
state->seg.ws_file = BasicOpenFile(path, O_RDONLY | PG_BINARY);
if (state->seg.ws_file >= 0)
return;
@@ -825,9 +825,11 @@ wal_segment_close(XLogReaderState *state)
* loop for now.
*/
bool
-read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *cur_page)
+read_local_xlog_page(XLogReaderState *state)
{
+ XLogRecPtr targetPagePtr = state->readPagePtr;
+ int reqLen = state->readLen;
+ char *cur_page = state->readBuf;
XLogRecPtr read_upto,
loc;
TimeLineID tli;
@@ -940,11 +942,12 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
* as 'count', read the whole page anyway. It's guaranteed to be
* zero-padded up to the page boundary if it's incomplete.
*/
- if (!WALRead(state, cur_page, targetPagePtr, XLOG_BLCKSZ, tli,
- &errinfo))
+ if (!WALRead(state, wal_segment_open, wal_segment_close,
+ cur_page, targetPagePtr, XLOG_BLCKSZ, tli, &errinfo))
WALReadRaiseError(&errinfo);
/* number of valid bytes in the buffer */
+ state->readPagePtr = targetPagePtr;
state->readLen = count;
return true;
}
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index 37b75deb72..9cbd8afa49 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -148,7 +148,8 @@ StartupDecodingContext(List *output_plugin_options,
TransactionId xmin_horizon,
bool need_full_snapshot,
bool fast_forward,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress)
@@ -198,11 +199,12 @@ StartupDecodingContext(List *output_plugin_options,
ctx->slot = slot;
- ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, xl_routine, ctx);
+ ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, cleanup_cb);
if (!ctx->reader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
errmsg("out of memory")));
+ ctx->page_read = page_read;
ctx->reorder = ReorderBufferAllocate();
ctx->snapshot_builder =
@@ -319,7 +321,8 @@ CreateInitDecodingContext(const char *plugin,
List *output_plugin_options,
bool need_full_snapshot,
XLogRecPtr restart_lsn,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress)
@@ -422,7 +425,7 @@ CreateInitDecodingContext(const char *plugin,
ctx = StartupDecodingContext(NIL, restart_lsn, xmin_horizon,
need_full_snapshot, false,
- xl_routine, prepare_write, do_write,
+ page_read, cleanup_cb, prepare_write, do_write,
update_progress);
/* call output plugin initialization callback */
@@ -476,7 +479,8 @@ LogicalDecodingContext *
CreateDecodingContext(XLogRecPtr start_lsn,
List *output_plugin_options,
bool fast_forward,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress)
@@ -528,8 +532,8 @@ CreateDecodingContext(XLogRecPtr start_lsn,
ctx = StartupDecodingContext(output_plugin_options,
start_lsn, InvalidTransactionId, false,
- fast_forward, xl_routine, prepare_write,
- do_write, update_progress);
+ fast_forward, page_read, cleanup_cb,
+ prepare_write, do_write, update_progress);
/* call output plugin initialization callback */
old_context = MemoryContextSwitchTo(ctx->context);
@@ -585,7 +589,13 @@ DecodingContextFindStartpoint(LogicalDecodingContext *ctx)
char *err = NULL;
/* the read_page callback waits for new WAL */
- record = XLogReadRecord(ctx->reader, &err);
+ while (XLogReadRecord(ctx->reader, &record, &err) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!ctx->page_read(ctx->reader))
+ break;
+ }
+
if (err)
elog(ERROR, "%s", err);
if (!record)
diff --git a/src/backend/replication/logical/logicalfuncs.c b/src/backend/replication/logical/logicalfuncs.c
index f7e055874e..b69f6911c5 100644
--- a/src/backend/replication/logical/logicalfuncs.c
+++ b/src/backend/replication/logical/logicalfuncs.c
@@ -233,9 +233,8 @@ pg_logical_slot_get_changes_guts(FunctionCallInfo fcinfo, bool confirm, bool bin
ctx = CreateDecodingContext(InvalidXLogRecPtr,
options,
false,
- XL_ROUTINE(.page_read = read_local_xlog_page,
- .segment_open = wal_segment_open,
- .segment_close = wal_segment_close),
+ read_local_xlog_page,
+ wal_segment_close,
LogicalOutputPrepareWrite,
LogicalOutputWrite, NULL);
@@ -284,7 +283,13 @@ pg_logical_slot_get_changes_guts(FunctionCallInfo fcinfo, bool confirm, bool bin
XLogRecord *record;
char *errm = NULL;
- record = XLogReadRecord(ctx->reader, &errm);
+ while (XLogReadRecord(ctx->reader, &record, &errm) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!ctx->page_read(ctx->reader))
+ break;
+ }
+
if (errm)
elog(ERROR, "%s", errm);
diff --git a/src/backend/replication/slotfuncs.c b/src/backend/replication/slotfuncs.c
index 9817b44113..deb7cf15eb 100644
--- a/src/backend/replication/slotfuncs.c
+++ b/src/backend/replication/slotfuncs.c
@@ -153,9 +153,8 @@ create_logical_replication_slot(char *name, char *plugin,
ctx = CreateInitDecodingContext(plugin, NIL,
false, /* just catalogs is OK */
restart_lsn,
- XL_ROUTINE(.page_read = read_local_xlog_page,
- .segment_open = wal_segment_open,
- .segment_close = wal_segment_close),
+ read_local_xlog_page,
+ wal_segment_close,
NULL, NULL, NULL);
/*
@@ -512,9 +511,8 @@ pg_logical_replication_slot_advance(XLogRecPtr moveto)
ctx = CreateDecodingContext(InvalidXLogRecPtr,
NIL,
true, /* fast_forward */
- XL_ROUTINE(.page_read = read_local_xlog_page,
- .segment_open = wal_segment_open,
- .segment_close = wal_segment_close),
+ read_local_xlog_page,
+ wal_segment_close,
NULL, NULL, NULL);
/*
@@ -536,7 +534,13 @@ pg_logical_replication_slot_advance(XLogRecPtr moveto)
* Read records. No changes are generated in fast_forward mode,
* but snapbuilder/slot statuses are updated properly.
*/
- record = XLogReadRecord(ctx->reader, &errm);
+ while (XLogReadRecord(ctx->reader, &record, &errm) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!ctx->page_read(ctx->reader))
+ break;
+ }
+
if (errm)
elog(ERROR, "%s", errm);
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index c36b91ae5e..062093727d 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -580,10 +580,7 @@ StartReplication(StartReplicationCmd *cmd)
/* create xlogreader for physical replication */
xlogreader =
- XLogReaderAllocate(wal_segment_size, NULL,
- XL_ROUTINE(.segment_open = WalSndSegmentOpen,
- .segment_close = wal_segment_close),
- NULL);
+ XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
if (!xlogreader)
ereport(ERROR,
@@ -807,9 +804,11 @@ StartReplication(StartReplicationCmd *cmd)
* set every time WAL is flushed.
*/
static bool
-logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetRecPtr, char *cur_page)
+logical_read_xlog_page(XLogReaderState *state)
{
+ XLogRecPtr targetPagePtr = state->readPagePtr;
+ int reqLen = state->readLen;
+ char *cur_page = state->readBuf;
XLogRecPtr flushptr;
int count;
WALReadError errinfo;
@@ -837,7 +836,7 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
count = flushptr - targetPagePtr; /* part of the page available */
/* now actually read the data, we know it's there */
- if (!WALRead(state,
+ if (!WALRead(state, WalSndSegmentOpen, wal_segment_close,
cur_page,
targetPagePtr,
XLOG_BLCKSZ,
@@ -1011,9 +1010,8 @@ CreateReplicationSlot(CreateReplicationSlotCmd *cmd)
ctx = CreateInitDecodingContext(cmd->plugin, NIL, need_full_snapshot,
InvalidXLogRecPtr,
- XL_ROUTINE(.page_read = logical_read_xlog_page,
- .segment_open = WalSndSegmentOpen,
- .segment_close = wal_segment_close),
+ logical_read_xlog_page,
+ wal_segment_close,
WalSndPrepareWrite, WalSndWriteData,
WalSndUpdateProgress);
@@ -1171,9 +1169,8 @@ StartLogicalReplication(StartReplicationCmd *cmd)
*/
logical_decoding_ctx =
CreateDecodingContext(cmd->startpoint, cmd->options, false,
- XL_ROUTINE(.page_read = logical_read_xlog_page,
- .segment_open = WalSndSegmentOpen,
- .segment_close = wal_segment_close),
+ logical_read_xlog_page,
+ wal_segment_close,
WalSndPrepareWrite, WalSndWriteData,
WalSndUpdateProgress);
xlogreader = logical_decoding_ctx->reader;
@@ -2749,7 +2746,7 @@ XLogSendPhysical(void)
enlargeStringInfo(&output_message, nbytes);
retry:
- if (!WALRead(xlogreader,
+ if (!WALRead(xlogreader, WalSndSegmentOpen, wal_segment_close,
&output_message.data[output_message.len],
startptr,
nbytes,
@@ -2847,7 +2844,12 @@ XLogSendLogical(void)
*/
WalSndCaughtUp = false;
- record = XLogReadRecord(logical_decoding_ctx->reader, &errm);
+ while (XLogReadRecord(logical_decoding_ctx->reader, &record, &errm) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!logical_decoding_ctx->page_read(logical_decoding_ctx->reader))
+ break;
+ }
/* xlog record was invalid */
if (errm != NULL)
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index cf119848b0..712c85281c 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -41,15 +41,9 @@ static int xlogreadfd = -1;
static XLogSegNo xlogreadsegno = -1;
static char xlogfpath[MAXPGPATH];
-typedef struct XLogPageReadPrivate
-{
- const char *restoreCommand;
- int tliIndex;
-} XLogPageReadPrivate;
-
-static bool SimpleXLogPageRead(XLogReaderState *xlogreader,
- XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
+static bool SimpleXLogPageRead(XLogReaderState *xlogreader,
+ const char *datadir, int *tliIndex,
+ const char *restoreCommand);
/*
* Read WAL from the datadir/pg_wal, starting from 'startpoint' on timeline
@@ -66,20 +60,22 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
XLogRecord *record;
XLogReaderState *xlogreader;
char *errormsg;
- XLogPageReadPrivate private;
- private.tliIndex = tliIndex;
- private.restoreCommand = restoreCommand;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir,
- XL_ROUTINE(.page_read = &SimpleXLogPageRead),
- &private);
+ xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+
if (xlogreader == NULL)
pg_fatal("out of memory");
XLogBeginRead(xlogreader, startpoint);
do
{
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!SimpleXLogPageRead(xlogreader, datadir,
+ &tliIndex, restoreCommand))
+ break;
+ }
if (record == NULL)
{
@@ -123,19 +119,19 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex,
XLogRecord *record;
XLogReaderState *xlogreader;
char *errormsg;
- XLogPageReadPrivate private;
XLogRecPtr endptr;
- private.tliIndex = tliIndex;
- private.restoreCommand = restoreCommand;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir,
- XL_ROUTINE(.page_read = &SimpleXLogPageRead),
- &private);
+ xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
XLogBeginRead(xlogreader, ptr);
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!SimpleXLogPageRead(xlogreader, datadir, &tliIndex, restoreCommand))
+ break;
+ }
if (record == NULL)
{
if (errormsg)
@@ -170,7 +166,6 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
XLogRecPtr searchptr;
XLogReaderState *xlogreader;
char *errormsg;
- XLogPageReadPrivate private;
/*
* The given fork pointer points to the end of the last common record,
@@ -186,11 +181,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
forkptr += SizeOfXLogShortPHD;
}
- private.tliIndex = tliIndex;
- private.restoreCommand = restoreCommand;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir,
- XL_ROUTINE(.page_read = &SimpleXLogPageRead),
- &private);
+ xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
@@ -200,7 +191,13 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
uint8 info;
XLogBeginRead(xlogreader, searchptr);
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!SimpleXLogPageRead(xlogreader, datadir,
+ &tliIndex, restoreCommand))
+ break;
+ }
if (record == NULL)
{
@@ -247,10 +244,11 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
/* XLogReader callback function, to read a WAL page */
static bool
-SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *readBuf)
+SimpleXLogPageRead(XLogReaderState *xlogreader, const char *datadir,
+ int *tliIndex, const char *restoreCommand)
{
- XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
+ XLogRecPtr targetPagePtr = xlogreader->readPagePtr;
+ char *readBuf = xlogreader->readBuf;
uint32 targetPageOff;
XLogRecPtr targetSegEnd;
XLogSegNo targetSegNo;
@@ -283,14 +281,14 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
* be done both forward and backward, consider also switching timeline
* accordingly.
*/
- while (private->tliIndex < targetNentries - 1 &&
- targetHistory[private->tliIndex].end < targetSegEnd)
- private->tliIndex++;
- while (private->tliIndex > 0 &&
- targetHistory[private->tliIndex].begin >= targetSegEnd)
- private->tliIndex--;
+ while (*tliIndex < targetNentries - 1 &&
+ targetHistory[*tliIndex].end < targetSegEnd)
+ (*tliIndex)++;
+ while (*tliIndex > 0 &&
+ targetHistory[*tliIndex].begin >= targetSegEnd)
+ (*tliIndex)--;
- XLogFileName(xlogfname, targetHistory[private->tliIndex].tli,
+ XLogFileName(xlogfname, targetHistory[*tliIndex].tli,
xlogreadsegno, WalSegSz);
snprintf(xlogfpath, MAXPGPATH, "%s/" XLOGDIR "/%s",
@@ -303,7 +301,7 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
/*
* If we have no restore_command to execute, then exit.
*/
- if (private->restoreCommand == NULL)
+ if (restoreCommand == NULL)
{
pg_log_error("could not open file \"%s\": %m", xlogfpath);
xlogreader->readLen = -1;
@@ -317,7 +315,7 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
xlogreadfd = RestoreArchivedFile(xlogreader->segcxt.ws_dir,
xlogfname,
WalSegSz,
- private->restoreCommand);
+ restoreCommand);
if (xlogreadfd < 0)
{
@@ -359,7 +357,7 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
Assert(targetSegNo == xlogreadsegno);
- xlogreader->seg.ws_tli = targetHistory[private->tliIndex].tli;
+ xlogreader->seg.ws_tli = targetHistory[*tliIndex].tli;
xlogreader->readLen = XLOG_BLCKSZ;
return true;
}
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index 0f0e0723a3..f514de6b13 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -330,12 +330,17 @@ WALDumpCloseSegment(XLogReaderState *state)
state->seg.ws_file = -1;
}
-/* pg_waldump's XLogReaderRoutine->page_read callback */
+/*
+ * pg_waldump's WAL page rader, also used as page_read callback for
+ * XLogFindNextRecord
+ */
static bool
-WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetPtr, char *readBuff)
+WALDumpReadPage(XLogReaderState *state, void *priv)
{
- XLogDumpPrivate *private = state->private_data;
+ XLogRecPtr targetPagePtr = state->readPagePtr;
+ int reqLen = state->readLen;
+ char *readBuff = state->readBuf;
+ XLogDumpPrivate *private = (XLogDumpPrivate *) priv;
int count = XLOG_BLCKSZ;
WALReadError errinfo;
@@ -353,8 +358,8 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
}
}
- if (!WALRead(state, readBuff, targetPagePtr, count, private->timeline,
- &errinfo))
+ if (!WALRead(state, WALDumpOpenSegment, WALDumpCloseSegment,
+ readBuff, targetPagePtr, count, private->timeline, &errinfo))
{
WALOpenSegment *seg = &errinfo.wre_seg;
char fname[MAXPGPATH];
@@ -374,6 +379,7 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
(Size) errinfo.wre_req);
}
+ Assert(count >= state->readLen);
state->readLen = count;
return true;
}
@@ -1035,16 +1041,14 @@ main(int argc, char **argv)
/* we have everything we need, start reading */
xlogreader_state =
- XLogReaderAllocate(WalSegSz, waldir,
- XL_ROUTINE(.page_read = WALDumpReadPage,
- .segment_open = WALDumpOpenSegment,
- .segment_close = WALDumpCloseSegment),
- &private);
+ XLogReaderAllocate(WalSegSz, waldir, WALDumpCloseSegment);
+
if (!xlogreader_state)
fatal_error("out of memory");
/* first find a valid recptr to start from */
- first_record = XLogFindNextRecord(xlogreader_state, private.startptr);
+ first_record = XLogFindNextRecord(xlogreader_state, private.startptr,
+ &WALDumpReadPage, (void*) &private);
if (first_record == InvalidXLogRecPtr)
fatal_error("could not find a valid record after %X/%X",
@@ -1067,7 +1071,13 @@ main(int argc, char **argv)
for (;;)
{
/* try to read the next record */
- record = XLogReadRecord(xlogreader_state, &errormsg);
+ while (XLogReadRecord(xlogreader_state, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!WALDumpReadPage(xlogreader_state, (void *) &private))
+ break;
+ }
+
if (!record)
{
if (!config.follow || private.endptr_reached)
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index 23d9e70a59..4e9268b553 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -57,64 +57,15 @@ typedef struct WALSegmentContext
typedef struct XLogReaderState XLogReaderState;
-/* Function type definition for the read_page callback */
-typedef bool (*XLogPageReadCB) (XLogReaderState *xlogreader,
- XLogRecPtr targetPagePtr,
- int reqLen,
- XLogRecPtr targetRecPtr,
- char *readBuf);
+/* Function type definition for the segment cleanup callback */
+typedef void (*WALSegmentCleanupCB) (XLogReaderState *xlogreader);
+
+/* Function type definition for the open/close callbacks for WALRead() */
typedef void (*WALSegmentOpenCB) (XLogReaderState *xlogreader,
XLogSegNo nextSegNo,
TimeLineID *tli_p);
typedef void (*WALSegmentCloseCB) (XLogReaderState *xlogreader);
-typedef struct XLogReaderRoutine
-{
- /*
- * Data input callback
- *
- * This callback shall read at least reqLen valid bytes of the xlog page
- * starting at targetPagePtr, and store them in readBuf. The callback
- * shall return the number of bytes read (never more than XLOG_BLCKSZ), or
- * -1 on failure. The callback shall sleep, if necessary, to wait for the
- * requested bytes to become available. The callback will not be invoked
- * again for the same page unless more than the returned number of bytes
- * are needed.
- *
- * targetRecPtr is the position of the WAL record we're reading. Usually
- * it is equal to targetPagePtr + reqLen, but sometimes xlogreader needs
- * to read and verify the page or segment header, before it reads the
- * actual WAL record it's interested in. In that case, targetRecPtr can
- * be used to determine which timeline to read the page from.
- *
- * The callback shall set ->seg.ws_tli to the TLI of the file the page was
- * read from.
- */
- XLogPageReadCB page_read;
-
- /*
- * Callback to open the specified WAL segment for reading. ->seg.ws_file
- * shall be set to the file descriptor of the opened segment. In case of
- * failure, an error shall be raised by the callback and it shall not
- * return.
- *
- * "nextSegNo" is the number of the segment to be opened.
- *
- * "tli_p" is an input/output argument. WALRead() uses it to pass the
- * timeline in which the new segment should be found, but the callback can
- * use it to return the TLI that it actually opened.
- */
- WALSegmentOpenCB segment_open;
-
- /*
- * WAL segment close callback. ->seg.ws_file shall be set to a negative
- * number.
- */
- WALSegmentCloseCB segment_close;
-} XLogReaderRoutine;
-
-#define XL_ROUTINE(...) &(XLogReaderRoutine){__VA_ARGS__}
-
typedef struct
{
/* Is this block ref in use? */
@@ -144,12 +95,35 @@ typedef struct
uint16 data_bufsz;
} DecodedBkpBlock;
+/* Return code from XLogReadRecord */
+typedef enum XLogReadRecordResult
+{
+ XLREAD_SUCCESS, /* record is successfully read */
+ XLREAD_NEED_DATA, /* need more data. see XLogReadRecord. */
+ XLREAD_FAIL /* failed during reading a record */
+} XLogReadRecordResult;
+
+/*
+ * internal state of XLogReadRecord
+ *
+ * XLogReadState runs a state machine while reading a record. Theses states
+ * are not seen outside the function. Each state may repeat several times
+ * exiting requesting caller for new data. See the comment of XLogReadRecrod
+ * for details.
+ */
+typedef enum XLogReadRecordState {
+ XLREAD_NEXT_RECORD,
+ XLREAD_TOT_LEN,
+ XLREAD_FIRST_FRAGMENT,
+ XLREAD_CONTINUATION
+} XLogReadRecordState;
+
struct XLogReaderState
{
/*
* Operational callbacks
*/
- XLogReaderRoutine routine;
+ WALSegmentCleanupCB cleanup_cb;
/* ----------------------------------------
* Public parameters
@@ -162,18 +136,14 @@ struct XLogReaderState
*/
uint64 system_identifier;
- /*
- * Opaque data for callbacks to use. Not used by XLogReader.
- */
- void *private_data;
-
/*
* Start and end point of last record read. EndRecPtr is also used as the
* position to read next. Calling XLogBeginRead() sets EndRecPtr to the
* starting position and ReadRecPtr to invalid.
*/
- XLogRecPtr ReadRecPtr; /* start of last record read */
+ XLogRecPtr ReadRecPtr; /* start of last record read or being read */
XLogRecPtr EndRecPtr; /* end+1 of last record read */
+ XLogRecPtr PrevRecPtr; /* start of previous record read */
/* ----------------------------------------
* Communication with page reader
@@ -187,7 +157,9 @@ struct XLogReaderState
* the request, or -1 on error */
TimeLineID readPageTLI; /* TLI for data currently in readBuf */
char *readBuf; /* buffer to store data */
- bool page_verified; /* is the page on the buffer verified? */
+ bool page_verified; /* is the page header on the buffer verified? */
+ bool record_verified;/* is the current record header verified? */
+
/* ----------------------------------------
@@ -229,8 +201,6 @@ struct XLogReaderState
XLogRecPtr latestPagePtr;
TimeLineID latestPageTLI;
- /* beginning of the WAL record being read. */
- XLogRecPtr currRecPtr;
/* timeline to read it from, 0 if a lookup is required */
TimeLineID currTLI;
@@ -257,6 +227,15 @@ struct XLogReaderState
char *readRecordBuf;
uint32 readRecordBufSize;
+ /*
+ * XLogReadRecord() state
+ */
+ XLogReadRecordState readRecordState;/* state machine state */
+ int recordGotLen; /* amount of current record that has
+ * already been read */
+ int recordRemainLen; /* length of current record that remains */
+ XLogRecPtr recordContRecPtr; /* where the current record continues */
+
/* Buffer to hold error message */
char *errormsg_buf;
};
@@ -264,9 +243,7 @@ struct XLogReaderState
/* Get a new XLogReader */
extern XLogReaderState *XLogReaderAllocate(int wal_segment_size,
const char *waldir,
- XLogReaderRoutine *routine,
- void *private_data);
-extern XLogReaderRoutine *LocalXLogReaderRoutine(void);
+ WALSegmentCleanupCB cleanup_cb);
/* Free an XLogReader */
extern void XLogReaderFree(XLogReaderState *state);
@@ -274,12 +251,17 @@ extern void XLogReaderFree(XLogReaderState *state);
/* Position the XLogReader to given record */
extern void XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr);
#ifdef FRONTEND
-extern XLogRecPtr XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr);
+/* Function type definition for the read_page callback */
+typedef bool (*XLogFindNextRecordCB) (XLogReaderState *xlogreader,
+ void *private);
+extern XLogRecPtr XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
+ XLogFindNextRecordCB read_page, void *private);
#endif /* FRONTEND */
/* Read the next XLog record. Returns NULL on end-of-WAL or failure */
-extern struct XLogRecord *XLogReadRecord(XLogReaderState *state,
- char **errormsg);
+extern XLogReadRecordResult XLogReadRecord(XLogReaderState *state,
+ XLogRecord **record,
+ char **errormsg);
/* Validate a page */
extern bool XLogReaderValidatePageHeader(XLogReaderState *state,
@@ -299,6 +281,7 @@ typedef struct WALReadError
} WALReadError;
extern bool WALRead(XLogReaderState *state,
+ WALSegmentOpenCB segopenfn, WALSegmentCloseCB sgclosefn,
char *buf, XLogRecPtr startptr, Size count,
TimeLineID tli, WALReadError *errinfo);
diff --git a/src/include/access/xlogutils.h b/src/include/access/xlogutils.h
index 364a21c4ea..397fb27fc2 100644
--- a/src/include/access/xlogutils.h
+++ b/src/include/access/xlogutils.h
@@ -47,9 +47,7 @@ extern Buffer XLogReadBufferExtended(RelFileNode rnode, ForkNumber forknum,
extern Relation CreateFakeRelcacheEntry(RelFileNode rnode);
extern void FreeFakeRelcacheEntry(Relation fakerel);
-extern bool read_local_xlog_page(XLogReaderState *state,
- XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetRecPtr, char *cur_page);
+extern bool read_local_xlog_page(XLogReaderState *state);
extern void wal_segment_open(XLogReaderState *state,
XLogSegNo nextSegNo,
TimeLineID *tli_p);
diff --git a/src/include/pg_config_manual.h b/src/include/pg_config_manual.h
index f10ad0acd6..94749609b4 100644
--- a/src/include/pg_config_manual.h
+++ b/src/include/pg_config_manual.h
@@ -377,7 +377,7 @@
* Enable debugging print statements for WAL-related operations; see
* also the wal_debug GUC var.
*/
-/* #define WAL_DEBUG */
+#define WAL_DEBUG
/*
* Enable tracing of resource consumption during sort operations;
diff --git a/src/include/replication/logical.h b/src/include/replication/logical.h
index c253403372..b936f31393 100644
--- a/src/include/replication/logical.h
+++ b/src/include/replication/logical.h
@@ -29,6 +29,10 @@ typedef void (*LogicalOutputPluginWriterUpdateProgress) (struct LogicalDecodingC
TransactionId xid
);
+typedef struct LogicalDecodingContext LogicalDecodingContext;
+
+typedef bool (*LogicalDecodingXLogPageReadCB)(XLogReaderState *ctx);
+
typedef struct LogicalDecodingContext
{
/* memory context this is all allocated in */
@@ -39,6 +43,7 @@ typedef struct LogicalDecodingContext
/* infrastructure pieces for decoding */
XLogReaderState *reader;
+ LogicalDecodingXLogPageReadCB page_read;
struct ReorderBuffer *reorder;
struct SnapBuild *snapshot_builder;
@@ -105,14 +110,16 @@ extern LogicalDecodingContext *CreateInitDecodingContext(const char *plugin,
List *output_plugin_options,
bool need_full_snapshot,
XLogRecPtr restart_lsn,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress);
extern LogicalDecodingContext *CreateDecodingContext(XLogRecPtr start_lsn,
List *output_plugin_options,
bool fast_forward,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress);
--
2.27.0
----Next_Part(Thu_Mar__4_11_28_53_2021_014)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
filename="v17-0003-Remove-globals-readOff-readLen-and-readSegNo.patch"
^ permalink raw reply [nested|flat] 70+ messages in thread
* [PATCH v17 2/4] Move page-reader out of XLogReadRecord
@ 2019-09-10 03:58 Kyotaro Horiguchi <[email protected]>
0 siblings, 0 replies; 70+ messages in thread
From: Kyotaro Horiguchi @ 2019-09-10 03:58 UTC (permalink / raw)
This is the second step of removing callbacks from WAL record reader.
Since it is essential to take in additional data while reading a
record, the function have to ask caller for new data while keeping
working state. Thus the function is turned into a state machine.
---
src/backend/access/transam/twophase.c | 15 +-
src/backend/access/transam/xlog.c | 58 +-
src/backend/access/transam/xlogreader.c | 697 +++++++++++-------
src/backend/access/transam/xlogutils.c | 17 +-
src/backend/replication/logical/logical.c | 26 +-
.../replication/logical/logicalfuncs.c | 13 +-
src/backend/replication/slotfuncs.c | 18 +-
src/backend/replication/walsender.c | 32 +-
src/bin/pg_rewind/parsexlog.c | 84 ++-
src/bin/pg_waldump/pg_waldump.c | 36 +-
src/include/access/xlogreader.h | 121 ++-
src/include/access/xlogutils.h | 4 +-
src/include/pg_config_manual.h | 2 +-
src/include/replication/logical.h | 11 +-
14 files changed, 646 insertions(+), 488 deletions(-)
diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index 80d2d20d6c..7d5a96a5f5 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -1325,11 +1325,8 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
XLogReaderState *xlogreader;
char *errormsg;
- xlogreader = XLogReaderAllocate(wal_segment_size, NULL,
- XL_ROUTINE(.page_read = &read_local_xlog_page,
- .segment_open = &wal_segment_open,
- .segment_close = &wal_segment_close),
- NULL);
+ xlogreader = XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+
if (!xlogreader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
@@ -1337,7 +1334,13 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
errdetail("Failed while allocating a WAL reading processor.")));
XLogBeginRead(xlogreader, lsn);
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!read_local_xlog_page(xlogreader))
+ break;
+ }
+
if (record == NULL)
ereport(ERROR,
(errcode_for_file_access(),
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 606328a65a..670996ae67 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -836,13 +836,6 @@ static XLogSource currentSource = XLOG_FROM_ANY;
static bool lastSourceFailed = false;
static bool pendingWalRcvRestart = false;
-typedef struct XLogPageReadPrivate
-{
- int emode;
- bool fetching_ckpt; /* are we fetching a checkpoint record? */
- bool randAccess;
-} XLogPageReadPrivate;
-
/*
* These variables track when we last obtained some WAL data to process,
* and where we got it from. (XLogReceiptSource is initially the same as
@@ -917,8 +910,8 @@ static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath,
static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli,
XLogSource source, bool notfoundOk);
static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, XLogSource source);
-static bool XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
+static bool XLogPageRead(XLogReaderState *xlogreader,
+ bool fetching_ckpt, int emode, bool randAccess);
static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
bool fetching_ckpt, XLogRecPtr tliRecPtr);
static int emode_for_corrupt_record(int emode, XLogRecPtr RecPtr);
@@ -1231,8 +1224,7 @@ XLogInsertRecord(XLogRecData *rdata,
appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len);
if (!debug_reader)
- debug_reader = XLogReaderAllocate(wal_segment_size, NULL,
- XL_ROUTINE(), NULL);
+ debug_reader = XLogReaderAllocate(wal_segment_size, NULL, NULL);
if (!debug_reader)
{
@@ -4344,15 +4336,10 @@ CleanupBackupHistory(void)
* record is available.
*/
static XLogRecord *
-ReadRecord(XLogReaderState *xlogreader, int emode,
- bool fetching_ckpt)
+ReadRecord(XLogReaderState *xlogreader, int emode, bool fetching_ckpt)
{
XLogRecord *record;
- XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
-
- private->fetching_ckpt = fetching_ckpt;
- private->emode = emode;
- private->randAccess = (xlogreader->ReadRecPtr == InvalidXLogRecPtr);
+ bool randAccess = (xlogreader->ReadRecPtr == InvalidXLogRecPtr);
/* This is the first attempt to read this page. */
lastSourceFailed = false;
@@ -4360,8 +4347,16 @@ ReadRecord(XLogReaderState *xlogreader, int emode,
for (;;)
{
char *errormsg;
+ XLogReadRecordResult result;
+
+ while ((result = XLogReadRecord(xlogreader, &record, &errormsg))
+ == XLREAD_NEED_DATA)
+ {
+ if (!XLogPageRead(xlogreader, fetching_ckpt, emode, randAccess))
+ break;
+
+ }
- record = XLogReadRecord(xlogreader, &errormsg);
ReadRecPtr = xlogreader->ReadRecPtr;
EndRecPtr = xlogreader->EndRecPtr;
if (record == NULL)
@@ -6380,7 +6375,6 @@ StartupXLOG(void)
bool backupFromStandby = false;
DBState dbstate_at_startup;
XLogReaderState *xlogreader;
- XLogPageReadPrivate private;
bool promoted = false;
struct stat st;
@@ -6539,13 +6533,9 @@ StartupXLOG(void)
OwnLatch(&XLogCtl->recoveryWakeupLatch);
/* Set up XLOG reader facility */
- MemSet(&private, 0, sizeof(XLogPageReadPrivate));
xlogreader =
- XLogReaderAllocate(wal_segment_size, NULL,
- XL_ROUTINE(.page_read = &XLogPageRead,
- .segment_open = NULL,
- .segment_close = wal_segment_close),
- &private);
+ XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+
if (!xlogreader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
@@ -11987,12 +11977,13 @@ CancelBackup(void)
* sleep and retry.
*/
static bool
-XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetRecPtr, char *readBuf)
+XLogPageRead(XLogReaderState *xlogreader,
+ bool fetching_ckpt, int emode, bool randAccess)
{
- XLogPageReadPrivate *private =
- (XLogPageReadPrivate *) xlogreader->private_data;
- int emode = private->emode;
+ char *readBuf = xlogreader->readBuf;
+ XLogRecPtr targetPagePtr = xlogreader->readPagePtr;
+ int reqLen = xlogreader->readLen;
+ XLogRecPtr targetRecPtr = xlogreader->ReadRecPtr;
uint32 targetPageOff;
XLogSegNo targetSegNo PG_USED_FOR_ASSERTS_ONLY;
int r;
@@ -12035,8 +12026,8 @@ retry:
flushedUpto < targetPagePtr + reqLen))
{
if (!WaitForWALToBecomeAvailable(targetPagePtr + reqLen,
- private->randAccess,
- private->fetching_ckpt,
+ randAccess,
+ fetching_ckpt,
targetRecPtr))
{
if (readFile >= 0)
@@ -12141,6 +12132,7 @@ retry:
goto next_record_is_invalid;
}
+ Assert(xlogreader->readPagePtr == targetPagePtr);
xlogreader->readLen = readLen;
return true;
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index 7d04995b7b..ba3e4868f7 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -40,7 +40,7 @@ static bool XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr,
int reqLen, bool header_inclusive);
static void XLogReaderInvalReadState(XLogReaderState *state);
static bool ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
- XLogRecPtr PrevRecPtr, XLogRecord *record, bool randAccess);
+ XLogRecPtr PrevRecPtr, XLogRecord *record);
static bool ValidXLogRecord(XLogReaderState *state, XLogRecord *record,
XLogRecPtr recptr);
static void ResetDecoder(XLogReaderState *state);
@@ -73,7 +73,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...)
*/
XLogReaderState *
XLogReaderAllocate(int wal_segment_size, const char *waldir,
- XLogReaderRoutine *routine, void *private_data)
+ WALSegmentCleanupCB cleanup_cb)
{
XLogReaderState *state;
@@ -84,7 +84,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
return NULL;
/* initialize caller-provided support functions */
- state->routine = *routine;
+ state->cleanup_cb = cleanup_cb;
state->max_block_id = -1;
@@ -107,8 +107,6 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
WALOpenSegmentInit(&state->seg, &state->segcxt, wal_segment_size,
waldir);
- /* system_identifier initialized to zeroes above */
- state->private_data = private_data;
/* ReadRecPtr, EndRecPtr and readLen initialized to zeroes above */
state->errormsg_buf = palloc_extended(MAX_ERRORMSG_LEN + 1,
MCXT_ALLOC_NO_OOM);
@@ -140,8 +138,8 @@ XLogReaderFree(XLogReaderState *state)
{
int block_id;
- if (state->seg.ws_file != -1)
- state->routine.segment_close(state);
+ if (state->seg.ws_file >= 0)
+ state->cleanup_cb(state);
for (block_id = 0; block_id <= XLR_MAX_BLOCK_ID; block_id++)
{
@@ -246,6 +244,7 @@ XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
/* Begin at the passed-in record pointer. */
state->EndRecPtr = RecPtr;
state->ReadRecPtr = InvalidXLogRecPtr;
+ state->readRecordState = XLREAD_NEXT_RECORD;
}
/*
@@ -254,317 +253,449 @@ XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
* XLogBeginRead() or XLogFindNextRecord() must be called before the first call
* to XLogReadRecord().
*
- * If the page_read callback fails to read the requested data, NULL is
- * returned. The callback is expected to have reported the error; errormsg
- * is set to NULL.
+ * This function may return XLREAD_NEED_DATA several times before returning a
+ * result record. The caller shall read in some new data then call this
+ * function again with the same parameters.
*
- * If the reading fails for some other reason, NULL is also returned, and
- * *errormsg is set to a string with details of the failure.
+ * When a record is successfully read, returns XLREAD_SUCCESS with result
+ * record being stored in *record. Otherwise *record is NULL.
*
- * The returned pointer (or *errormsg) points to an internal buffer that's
- * valid until the next call to XLogReadRecord.
+ * Returns XLREAD_NEED_DATA if more data is needed to finish reading the
+ * current record. In that case, state->readPagePtr and state->readLen inform
+ * the desired position and minimum length of data needed. The caller shall
+ * read in the requested data and set state->readBuf to point to a buffer
+ * containing it. The caller must also set state->readPageTLI and
+ * state->readLen to indicate the timeline that it was read from, and the
+ * length of data that is now available (which must be >= given readLen),
+ * respectively.
+ *
+ * If invalid data is encountered, returns XLREAD_FAIL with *record being set to
+ * NULL. *errormsg is set to a string with details of the failure.
+ * The returned pointer (or *errormsg) points to an internal buffer that's valid
+ * until the next call to XLogReadRecord.
+ *
+ *
+ * This function runs a state machine consists of the following states.
+ *
+ * XLREAD_NEXT_RECORD :
+ * The initial state, if called with valid RecPtr, try to read a record at
+ * that position. If invalid RecPtr is given try to read a record just after
+ * the last one previously read.
+ * This state ens after setting ReadRecPtr. Then goes to XLREAD_TOT_LEN.
+ *
+ * XLREAD_TOT_LEN:
+ * Examining record header. Ends after reading record total
+ * length. recordRemainLen and recordGotLen are initialized.
+ *
+ * XLREAD_FIRST_FRAGMENT:
+ * Reading the first fragment. Ends with finishing reading a single
+ * record. Goes to XLREAD_NEXT_RECORD if that's all or
+ * XLREAD_CONTINUATION if we have continuation.
+
+ * XLREAD_CONTINUATION:
+ * Reading continuation of record. Ends with finishing the whole record then
+ * goes to XLREAD_NEXT_RECORD. During this state, recordRemainLen indicates
+ * how much is left and readRecordBuf holds the partially assert
+ * record.recordContRecPtr points to the beginning of the next page where to
+ * continue.
+ *
+ * If wrong data found in any state, the state machine stays at the current
+ * state. This behavior allows to continue reading a reacord switching among
+ * different souces, while streaming replication.
*/
-XLogRecord *
-XLogReadRecord(XLogReaderState *state, char **errormsg)
+XLogReadRecordResult
+XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
{
- XLogRecPtr RecPtr;
- XLogRecord *record;
- XLogRecPtr targetPagePtr;
- bool randAccess;
- uint32 len,
- total_len;
- uint32 targetRecOff;
- uint32 pageHeaderSize;
- bool gotheader;
+ XLogRecord *prec;
- /*
- * randAccess indicates whether to verify the previous-record pointer of
- * the record we're reading. We only do this if we're reading
- * sequentially, which is what we initially assume.
- */
- randAccess = false;
+ *record = NULL;
/* reset error state */
*errormsg = NULL;
state->errormsg_buf[0] = '\0';
- ResetDecoder(state);
-
- RecPtr = state->EndRecPtr;
-
- if (state->ReadRecPtr != InvalidXLogRecPtr)
- {
- /* read the record after the one we just read */
-
- /*
- * EndRecPtr is pointing to end+1 of the previous WAL record. If
- * we're at a page boundary, no more records can fit on the current
- * page. We must skip over the page header, but we can't do that until
- * we've read in the page, since the header size is variable.
- */
- }
- else
- {
- /*
- * Caller supplied a position to start at.
- *
- * In this case, EndRecPtr should already be pointing to a valid
- * record starting position.
- */
- Assert(XRecOffIsValid(RecPtr));
- randAccess = true;
- }
-
- state->currRecPtr = RecPtr;
-
- targetPagePtr = RecPtr - (RecPtr % XLOG_BLCKSZ);
- targetRecOff = RecPtr % XLOG_BLCKSZ;
-
- /*
- * Read the page containing the record into state->readBuf. Request enough
- * byte to cover the whole record header, or at least the part of it that
- * fits on the same page.
- */
- while (XLogNeedData(state, targetPagePtr,
- Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
- targetRecOff != 0))
- {
- if (!state->routine.page_read(state, state->readPagePtr, state->readLen,
- RecPtr, state->readBuf))
- break;
- }
-
- if (!state->page_verified)
- goto err;
-
- /*
- * We have at least the page header, so we can examine it now.
- */
- pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
- if (targetRecOff == 0)
- {
- /*
- * At page start, so skip over page header.
- */
- RecPtr += pageHeaderSize;
- targetRecOff = pageHeaderSize;
- }
- else if (targetRecOff < pageHeaderSize)
- {
- report_invalid_record(state, "invalid record offset at %X/%X",
- LSN_FORMAT_ARGS(RecPtr));
- goto err;
- }
-
- if ((((XLogPageHeader) state->readBuf)->xlp_info & XLP_FIRST_IS_CONTRECORD) &&
- targetRecOff == pageHeaderSize)
- {
- report_invalid_record(state, "contrecord is requested by %X/%X",
- LSN_FORMAT_ARGS(RecPtr));
- goto err;
- }
-
- /* XLogNeedData has verified the page header */
- Assert(pageHeaderSize <= state->readLen);
-
- /*
- * Read the record length.
- *
- * NB: Even though we use an XLogRecord pointer here, the whole record
- * header might not fit on this page. xl_tot_len is the first field of the
- * struct, so it must be on this page (the records are MAXALIGNed), but we
- * cannot access any other fields until we've verified that we got the
- * whole header.
- */
- record = (XLogRecord *) (state->readBuf + RecPtr % XLOG_BLCKSZ);
- total_len = record->xl_tot_len;
-
- /*
- * If the whole record header is on this page, validate it immediately.
- * Otherwise do just a basic sanity check on xl_tot_len, and validate the
- * rest of the header after reading it from the next page. The xl_tot_len
- * check is necessary here to ensure that we enter the "Need to reassemble
- * record" code path below; otherwise we might fail to apply
- * ValidXLogRecordHeader at all.
- */
- if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
- {
- if (!ValidXLogRecordHeader(state, RecPtr, state->ReadRecPtr, record,
- randAccess))
- goto err;
- gotheader = true;
- }
- else
- {
- /* XXX: more validation should be done here */
- if (total_len < SizeOfXLogRecord)
- {
- report_invalid_record(state,
- "invalid record length at %X/%X: wanted %u, got %u",
- LSN_FORMAT_ARGS(RecPtr),
- (uint32) SizeOfXLogRecord, total_len);
- goto err;
- }
- gotheader = false;
- }
-
- len = XLOG_BLCKSZ - RecPtr % XLOG_BLCKSZ;
- if (total_len > len)
+ switch (state->readRecordState)
{
- /* Need to reassemble record */
- char *contdata;
- XLogPageHeader pageHeader;
- char *buffer;
- uint32 gotlen;
-
- /*
- * Enlarge readRecordBuf as needed.
- */
- if (total_len > state->readRecordBufSize &&
- !allocate_recordbuf(state, total_len))
- {
- /* We treat this as a "bogus data" condition */
- report_invalid_record(state, "record length %u at %X/%X too long",
- total_len, LSN_FORMAT_ARGS(RecPtr));
- goto err;
- }
+ case XLREAD_NEXT_RECORD:
+ ResetDecoder(state);
- /* Copy the first fragment of the record from the first page. */
- memcpy(state->readRecordBuf,
- state->readBuf + RecPtr % XLOG_BLCKSZ, len);
- buffer = state->readRecordBuf + len;
- gotlen = len;
-
- do
- {
- int rest_len = total_len - gotlen;
-
- /* Calculate pointer to beginning of next page */
- targetPagePtr += XLOG_BLCKSZ;
-
- /* Wait for the next page to become available */
- while (XLogNeedData(state, targetPagePtr,
- Min(rest_len, XLOG_BLCKSZ),
- false))
+ if (state->ReadRecPtr != InvalidXLogRecPtr)
{
- if (!state->routine.page_read(state, state->readPagePtr,
- state->readLen,
- state->ReadRecPtr,
- state->readBuf))
- break;
+ /* read the record after the one we just read */
+
+ /*
+ * EndRecPtr is pointing to end+1 of the previous WAL record.
+ * If we're at a page boundary, no more records can fit on the
+ * current page. We must skip over the page header, but we
+ * can't do that until we've read in the page, since the header
+ * size is variable.
+ */
+ state->PrevRecPtr = state->ReadRecPtr;
+ state->ReadRecPtr = state->EndRecPtr;
+ }
+ else
+ {
+ /*
+ * Caller supplied a position to start at.
+ *
+ * In this case, EndRecPtr should already be pointing to a
+ * valid record starting position.
+ */
+ Assert(XRecOffIsValid(state->EndRecPtr));
+ state->ReadRecPtr = state->EndRecPtr;
+
+ /*
+ * We cannot verify the previous-record pointer when we're
+ * seeking to a particular record. Reset PrevRecPtr so that we
+ * won't try doing that.
+ */
+ state->PrevRecPtr = InvalidXLogRecPtr;
+ state->EndRecPtr = InvalidXLogRecPtr; /* to be tidy */
}
+ state->record_verified = false;
+ state->readRecordState = XLREAD_TOT_LEN;
+ /* fall through */
+
+ case XLREAD_TOT_LEN:
+ {
+ uint32 total_len;
+ uint32 pageHeaderSize;
+ XLogRecPtr targetPagePtr;
+ uint32 targetRecOff;
+ XLogPageHeader pageHeader;
+
+ targetPagePtr =
+ state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
+ targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+
+ /*
+ * Check if we have enough data. For the first record in the page,
+ * the requesting length doesn't contain page header.
+ */
+ if (XLogNeedData(state, targetPagePtr,
+ Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+ targetRecOff != 0))
+ return XLREAD_NEED_DATA;
+
+ /* error out if caller supplied bogus page */
if (!state->page_verified)
goto err;
- Assert(SizeOfXLogShortPHD <= state->readLen);
+ /* examine page header now. */
+ pageHeaderSize =
+ XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+ if (targetRecOff == 0)
+ {
+ /* At page start, so skip over page header. */
+ state->ReadRecPtr += pageHeaderSize;
+ targetRecOff = pageHeaderSize;
+ }
+ else if (targetRecOff < pageHeaderSize)
+ {
+ report_invalid_record(state, "invalid record offset at %X/%X",
+ LSN_FORMAT_ARGS(state->ReadRecPtr));
+ goto err;
+ }
- /* Check that the continuation on next page looks valid */
pageHeader = (XLogPageHeader) state->readBuf;
- if (!(pageHeader->xlp_info & XLP_FIRST_IS_CONTRECORD))
+ if ((pageHeader->xlp_info & XLP_FIRST_IS_CONTRECORD) &&
+ targetRecOff == pageHeaderSize)
{
- report_invalid_record(state,
- "there is no contrecord flag at %X/%X",
- LSN_FORMAT_ARGS(RecPtr));
+ report_invalid_record(state, "contrecord is requested by %X/%X",
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr);
goto err;
}
- /*
- * Cross-check that xlp_rem_len agrees with how much of the record
- * we expect there to be left.
- */
- if (pageHeader->xlp_rem_len == 0 ||
- total_len != (pageHeader->xlp_rem_len + gotlen))
- {
- report_invalid_record(state,
- "invalid contrecord length %u (expected %lld) at %X/%X",
- pageHeader->xlp_rem_len,
- ((long long) total_len) - gotlen,
- LSN_FORMAT_ARGS(RecPtr));
- goto err;
- }
-
- /* Append the continuation from this page to the buffer */
- pageHeaderSize = XLogPageHeaderSize(pageHeader);
-
+ /* XLogNeedData has verified the page header */
Assert(pageHeaderSize <= state->readLen);
- contdata = (char *) state->readBuf + pageHeaderSize;
- len = XLOG_BLCKSZ - pageHeaderSize;
- if (pageHeader->xlp_rem_len < len)
- len = pageHeader->xlp_rem_len;
+ /*
+ * Read the record length.
+ *
+ * NB: Even though we use an XLogRecord pointer here, the whole
+ * record header might not fit on this page. xl_tot_len is the first
+ * field of the struct, so it must be on this page (the records are
+ * MAXALIGNed), but we cannot access any other fields until we've
+ * verified that we got the whole header.
+ */
+ prec = (XLogRecord *) (state->readBuf +
+ state->ReadRecPtr % XLOG_BLCKSZ);
+ total_len = prec->xl_tot_len;
- Assert (pageHeaderSize + len <= state->readLen);
- memcpy(buffer, (char *) contdata, len);
- buffer += len;
- gotlen += len;
+ /*
+ * If the whole record header is on this page, validate it
+ * immediately. Otherwise do just a basic sanity check on
+ * xl_tot_len, and validate the rest of the header after reading it
+ * from the next page. The xl_tot_len check is necessary here to
+ * ensure that we enter the XLREAD_CONTINUATION state below;
+ * otherwise we might fail to apply ValidXLogRecordHeader at all.
+ */
+ if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
+ {
+ if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
+ state->PrevRecPtr, prec))
+ goto err;
- /* If we just reassembled the record header, validate it. */
- if (!gotheader)
+ state->record_verified = true;
+ }
+ else
{
- record = (XLogRecord *) state->readRecordBuf;
- if (!ValidXLogRecordHeader(state, RecPtr, state->ReadRecPtr,
- record, randAccess))
+ /* XXX: more validation should be done here */
+ if (total_len < SizeOfXLogRecord)
+ {
+ report_invalid_record(state,
+ "invalid record length at %X/%X: wanted %u, got %u",
+ LSN_FORMAT_ARGS(state->ReadRecPtr),
+ (uint32) SizeOfXLogRecord, total_len);
goto err;
- gotheader = true;
+ }
}
- } while (gotlen < total_len);
-
- Assert(gotheader);
- record = (XLogRecord *) state->readRecordBuf;
- if (!ValidXLogRecord(state, record, RecPtr))
- goto err;
+ /*
+ * Wait for the rest of the record, or the part of the record that
+ * fit on the first page if crossed a page boundary, to become
+ * available.
+ */
+ state->recordGotLen = 0;
+ state->recordRemainLen = total_len;
+ state->readRecordState = XLREAD_FIRST_FRAGMENT;
+ }
+ /* fall through */
- pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
- state->ReadRecPtr = RecPtr;
- state->EndRecPtr = targetPagePtr + pageHeaderSize
- + MAXALIGN(pageHeader->xlp_rem_len);
- }
- else
- {
- /* Wait for the record data to become available */
- while (XLogNeedData(state, targetPagePtr,
- Min(targetRecOff + total_len, XLOG_BLCKSZ), true))
+ case XLREAD_FIRST_FRAGMENT:
{
- if (!state->routine.page_read(state, state->readPagePtr,
- state->readLen,
- state->ReadRecPtr, state->readBuf))
+ uint32 total_len = state->recordRemainLen;
+ uint32 request_len;
+ uint32 record_len;
+ XLogRecPtr targetPagePtr;
+ uint32 targetRecOff;
+
+ /*
+ * Wait for the rest of the record on the first page to become
+ * available
+ */
+ targetPagePtr =
+ state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
+ targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+
+ request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ);
+ record_len = request_len - targetRecOff;
+
+ /* ReadRecPtr contains page header */
+ Assert (targetRecOff != 0);
+ if (XLogNeedData(state, targetPagePtr, request_len, true))
+ return XLREAD_NEED_DATA;
+
+ /* error out if caller supplied bogus page */
+ if (!state->page_verified)
+ goto err;
+
+ prec = (XLogRecord *) (state->readBuf + targetRecOff);
+
+ /* validate record header if not yet */
+ if (!state->record_verified && record_len >= SizeOfXLogRecord)
+ {
+ if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
+ state->PrevRecPtr, prec))
+ goto err;
+
+ state->record_verified = true;
+ }
+
+
+ if (total_len == record_len)
+ {
+ /* Record does not cross a page boundary */
+ Assert(state->record_verified);
+
+ if (!ValidXLogRecord(state, prec, state->ReadRecPtr))
+ goto err;
+
+ state->record_verified = true; /* to be tidy */
+
+ /* We already checked the header earlier */
+ state->EndRecPtr = state->ReadRecPtr + MAXALIGN(record_len);
+
+ *record = prec;
+ state->readRecordState = XLREAD_NEXT_RECORD;
break;
+ }
+
+ /*
+ * The record continues on the next page. Need to reassemble
+ * record
+ */
+ Assert(total_len > record_len);
+
+ /* Enlarge readRecordBuf as needed. */
+ if (total_len > state->readRecordBufSize &&
+ !allocate_recordbuf(state, total_len))
+ {
+ /* We treat this as a "bogus data" condition */
+ report_invalid_record(state,
+ "record length %u at %X/%X too long",
+ total_len,
+ LSN_FORMAT_ARGS(state->ReadRecPtr));
+ goto err;
+ }
+
+ /* Copy the first fragment of the record from the first page. */
+ memcpy(state->readRecordBuf, state->readBuf + targetRecOff,
+ record_len);
+ state->recordGotLen += record_len;
+ state->recordRemainLen -= record_len;
+
+ /* Calculate pointer to beginning of next page */
+ state->recordContRecPtr = state->ReadRecPtr + record_len;
+ Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0);
+
+ state->readRecordState = XLREAD_CONTINUATION;
}
+ /* fall through */
+
+ case XLREAD_CONTINUATION:
+ {
+ XLogPageHeader pageHeader;
+ uint32 pageHeaderSize;
+ XLogRecPtr targetPagePtr;
+
+ /* we enter this state only if we haven't read the whole record. */
+ Assert (state->recordRemainLen > 0);
+
+ while(state->recordRemainLen > 0)
+ {
+ char *contdata;
+ uint32 request_len;
+ uint32 record_len;
+
+ /* Wait for the next page to become available */
+ targetPagePtr = state->recordContRecPtr;
+
+ /* this request contains page header */
+ Assert (targetPagePtr != 0);
+ if (XLogNeedData(state, targetPagePtr,
+ Min(state->recordRemainLen, XLOG_BLCKSZ),
+ false))
+ return XLREAD_NEED_DATA;
+
+ if (!state->page_verified)
+ goto err;
+
+ Assert(SizeOfXLogShortPHD <= state->readLen);
- if (!state->page_verified)
- goto err;
+ /* Check that the continuation on next page looks valid */
+ pageHeader = (XLogPageHeader) state->readBuf;
+ if (!(pageHeader->xlp_info & XLP_FIRST_IS_CONTRECORD))
+ {
+ report_invalid_record(
+ state,
+ "there is no contrecord flag at %X/%X reading %X/%X",
+ (uint32) (state->recordContRecPtr >> 32),
+ (uint32) state->recordContRecPtr,
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr);
+ goto err;
+ }
- /* Record does not cross a page boundary */
- if (!ValidXLogRecord(state, record, RecPtr))
- goto err;
+ /*
+ * Cross-check that xlp_rem_len agrees with how much of the
+ * record we expect there to be left.
+ */
+ if (pageHeader->xlp_rem_len == 0 ||
+ pageHeader->xlp_rem_len != state->recordRemainLen)
+ {
+ report_invalid_record(
+ state,
+ "invalid contrecord length %u at %X/%X reading %X/%X, expected %u",
+ pageHeader->xlp_rem_len,
+ (uint32) (state->recordContRecPtr >> 32),
+ (uint32) state->recordContRecPtr,
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr,
+ state->recordRemainLen);
+ goto err;
+ }
- state->EndRecPtr = RecPtr + MAXALIGN(total_len);
+ /* Append the continuation from this page to the buffer */
+ pageHeaderSize = XLogPageHeaderSize(pageHeader);
- state->ReadRecPtr = RecPtr;
+ /*
+ * XLogNeedData should have ensured that the whole page header
+ * was read
+ */
+ Assert(state->readLen >= pageHeaderSize);
+
+ contdata = (char *) state->readBuf + pageHeaderSize;
+ record_len = XLOG_BLCKSZ - pageHeaderSize;
+ if (pageHeader->xlp_rem_len < record_len)
+ record_len = pageHeader->xlp_rem_len;
+
+ request_len = record_len + pageHeaderSize;
+
+ /* XLogNeedData should have ensured all needed data was read */
+ Assert (state->readLen >= request_len);
+
+ memcpy(state->readRecordBuf + state->recordGotLen,
+ (char *) contdata, record_len);
+ state->recordGotLen += record_len;
+ state->recordRemainLen -= record_len;
+
+ /* If we just reassembled the record header, validate it. */
+ if (!state->record_verified)
+ {
+ Assert(state->recordGotLen >= SizeOfXLogRecord);
+ if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
+ state->PrevRecPtr,
+ (XLogRecord *) state->readRecordBuf))
+ goto err;
+
+ state->record_verified = true;
+ }
+
+ /* Calculate pointer to beginning of next page, and continue */
+ state->recordContRecPtr += XLOG_BLCKSZ;
+ }
+
+ /* targetPagePtr is pointing the last-read page here */
+ prec = (XLogRecord *) state->readRecordBuf;
+ if (!ValidXLogRecord(state, prec, state->ReadRecPtr))
+ goto err;
+
+ pageHeaderSize =
+ XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+ state->EndRecPtr = targetPagePtr + pageHeaderSize
+ + MAXALIGN(pageHeader->xlp_rem_len);
+
+ *record = prec;
+ state->readRecordState = XLREAD_NEXT_RECORD;
+ break;
+ }
}
/*
* Special processing if it's an XLOG SWITCH record
*/
- if (record->xl_rmid == RM_XLOG_ID &&
- (record->xl_info & ~XLR_INFO_MASK) == XLOG_SWITCH)
+ if ((*record)->xl_rmid == RM_XLOG_ID &&
+ ((*record)->xl_info & ~XLR_INFO_MASK) == XLOG_SWITCH)
{
/* Pretend it extends to end of segment */
state->EndRecPtr += state->segcxt.ws_segsize - 1;
state->EndRecPtr -= XLogSegmentOffset(state->EndRecPtr, state->segcxt.ws_segsize);
}
- if (DecodeXLogRecord(state, record, errormsg))
- return record;
- else
- return NULL;
+ Assert (!*record || state->readLen >= 0);
+ if (DecodeXLogRecord(state, *record, errormsg))
+ return XLREAD_SUCCESS;
+
+ *record = NULL;
+ return XLREAD_FAIL;
err:
/*
- * Invalidate the read state. We might read from a different source after
+ * Invalidate the read page. We might read from a different source after
* failure.
*/
XLogReaderInvalReadState(state);
@@ -572,7 +703,8 @@ err:
if (state->errormsg_buf[0] != '\0')
*errormsg = state->errormsg_buf;
- return NULL;
+ *record = NULL;
+ return XLREAD_FAIL;
}
/*
@@ -724,11 +856,12 @@ XLogReaderInvalReadState(XLogReaderState *state)
*
* This is just a convenience subroutine to avoid duplicated code in
* XLogReadRecord. It's not intended for use from anywhere else.
+ *
+ * If PrevRecPtr is valid, the xl_prev is is cross-checked with it.
*/
static bool
ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
- XLogRecPtr PrevRecPtr, XLogRecord *record,
- bool randAccess)
+ XLogRecPtr PrevRecPtr, XLogRecord *record)
{
if (record->xl_tot_len < SizeOfXLogRecord)
{
@@ -745,7 +878,7 @@ ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
record->xl_rmid, LSN_FORMAT_ARGS(RecPtr));
return false;
}
- if (randAccess)
+ if (PrevRecPtr == InvalidXLogRecPtr)
{
/*
* We can't exactly verify the prev-link, but surely it should be less
@@ -975,11 +1108,14 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
* XLogReadRecord() will read the next valid record.
*/
XLogRecPtr
-XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
+XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
+ XLogFindNextRecordCB read_page, void *private)
{
XLogRecPtr tmpRecPtr;
XLogRecPtr found = InvalidXLogRecPtr;
XLogPageHeader header;
+ XLogRecord *record;
+ XLogReadRecordResult result;
char *errormsg;
Assert(!XLogRecPtrIsInvalid(RecPtr));
@@ -1012,9 +1148,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
while(XLogNeedData(state, targetPagePtr, targetRecOff,
targetRecOff != 0))
{
- if (!state->routine.page_read(state, state->readPagePtr,
- state->readLen,
- state->ReadRecPtr, state->readBuf))
+ if (!read_page(state, private))
break;
}
@@ -1066,8 +1200,16 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
* or we just jumped over the remaining data of a continuation.
*/
XLogBeginRead(state, tmpRecPtr);
- while (XLogReadRecord(state, &errormsg) != NULL)
+ while ((result = XLogReadRecord(state, &record, &errormsg)) !=
+ XLREAD_FAIL)
{
+ if (result == XLREAD_NEED_DATA)
+ {
+ if (!read_page(state, private))
+ goto err;
+ continue;
+ }
+
/* past the record we've found, break out */
if (RecPtr <= state->ReadRecPtr)
{
@@ -1087,9 +1229,9 @@ err:
#endif /* FRONTEND */
/*
- * Helper function to ease writing of XLogRoutine->page_read callbacks.
- * If this function is used, caller must supply a segment_open callback in
- * 'state', as that is used here.
+ * Helper function to ease writing of page_read callback.
+ * If this function is used, caller must supply a segment_open callback and
+ * segment_close callback as that is used here.
*
* Read 'count' bytes into 'buf', starting at location 'startptr', from WAL
* fetched from timeline 'tli'.
@@ -1102,6 +1244,7 @@ err:
*/
bool
WALRead(XLogReaderState *state,
+ WALSegmentOpenCB segopenfn, WALSegmentCloseCB segclosefn,
char *buf, XLogRecPtr startptr, Size count, TimeLineID tli,
WALReadError *errinfo)
{
@@ -1133,10 +1276,10 @@ WALRead(XLogReaderState *state,
XLogSegNo nextSegNo;
if (state->seg.ws_file >= 0)
- state->routine.segment_close(state);
+ segclosefn(state);
XLByteToSeg(recptr, nextSegNo, state->segcxt.ws_segsize);
- state->routine.segment_open(state, nextSegNo, &tli);
+ segopenfn(state, nextSegNo, &tli);
/* This shouldn't happen -- indicates a bug in segment_open */
Assert(state->seg.ws_file >= 0);
diff --git a/src/backend/access/transam/xlogutils.c b/src/backend/access/transam/xlogutils.c
index 46eda33f25..b003990745 100644
--- a/src/backend/access/transam/xlogutils.c
+++ b/src/backend/access/transam/xlogutils.c
@@ -686,8 +686,7 @@ XLogTruncateRelation(RelFileNode rnode, ForkNumber forkNum,
void
XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wantLength)
{
- const XLogRecPtr lastReadPage = state->seg.ws_segno *
- state->segcxt.ws_segsize + state->readLen;
+ const XLogRecPtr lastReadPage = state->readPagePtr;
Assert(wantPage != InvalidXLogRecPtr && wantPage % XLOG_BLCKSZ == 0);
Assert(wantLength <= XLOG_BLCKSZ);
@@ -702,7 +701,7 @@ XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wa
* current TLI has since become historical.
*/
if (lastReadPage == wantPage &&
- state->readLen != 0 &&
+ state->page_verified &&
lastReadPage + state->readLen >= wantPage + Min(wantLength, XLOG_BLCKSZ - 1))
return;
@@ -788,6 +787,7 @@ wal_segment_open(XLogReaderState *state, XLogSegNo nextSegNo,
char path[MAXPGPATH];
XLogFilePath(path, tli, nextSegNo, state->segcxt.ws_segsize);
+ elog(LOG, "HOGE: %lu, %d => %s", nextSegNo, tli, path);
state->seg.ws_file = BasicOpenFile(path, O_RDONLY | PG_BINARY);
if (state->seg.ws_file >= 0)
return;
@@ -825,9 +825,11 @@ wal_segment_close(XLogReaderState *state)
* loop for now.
*/
bool
-read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *cur_page)
+read_local_xlog_page(XLogReaderState *state)
{
+ XLogRecPtr targetPagePtr = state->readPagePtr;
+ int reqLen = state->readLen;
+ char *cur_page = state->readBuf;
XLogRecPtr read_upto,
loc;
TimeLineID tli;
@@ -940,11 +942,12 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
* as 'count', read the whole page anyway. It's guaranteed to be
* zero-padded up to the page boundary if it's incomplete.
*/
- if (!WALRead(state, cur_page, targetPagePtr, XLOG_BLCKSZ, tli,
- &errinfo))
+ if (!WALRead(state, wal_segment_open, wal_segment_close,
+ cur_page, targetPagePtr, XLOG_BLCKSZ, tli, &errinfo))
WALReadRaiseError(&errinfo);
/* number of valid bytes in the buffer */
+ state->readPagePtr = targetPagePtr;
state->readLen = count;
return true;
}
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index 37b75deb72..9cbd8afa49 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -148,7 +148,8 @@ StartupDecodingContext(List *output_plugin_options,
TransactionId xmin_horizon,
bool need_full_snapshot,
bool fast_forward,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress)
@@ -198,11 +199,12 @@ StartupDecodingContext(List *output_plugin_options,
ctx->slot = slot;
- ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, xl_routine, ctx);
+ ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, cleanup_cb);
if (!ctx->reader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
errmsg("out of memory")));
+ ctx->page_read = page_read;
ctx->reorder = ReorderBufferAllocate();
ctx->snapshot_builder =
@@ -319,7 +321,8 @@ CreateInitDecodingContext(const char *plugin,
List *output_plugin_options,
bool need_full_snapshot,
XLogRecPtr restart_lsn,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress)
@@ -422,7 +425,7 @@ CreateInitDecodingContext(const char *plugin,
ctx = StartupDecodingContext(NIL, restart_lsn, xmin_horizon,
need_full_snapshot, false,
- xl_routine, prepare_write, do_write,
+ page_read, cleanup_cb, prepare_write, do_write,
update_progress);
/* call output plugin initialization callback */
@@ -476,7 +479,8 @@ LogicalDecodingContext *
CreateDecodingContext(XLogRecPtr start_lsn,
List *output_plugin_options,
bool fast_forward,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress)
@@ -528,8 +532,8 @@ CreateDecodingContext(XLogRecPtr start_lsn,
ctx = StartupDecodingContext(output_plugin_options,
start_lsn, InvalidTransactionId, false,
- fast_forward, xl_routine, prepare_write,
- do_write, update_progress);
+ fast_forward, page_read, cleanup_cb,
+ prepare_write, do_write, update_progress);
/* call output plugin initialization callback */
old_context = MemoryContextSwitchTo(ctx->context);
@@ -585,7 +589,13 @@ DecodingContextFindStartpoint(LogicalDecodingContext *ctx)
char *err = NULL;
/* the read_page callback waits for new WAL */
- record = XLogReadRecord(ctx->reader, &err);
+ while (XLogReadRecord(ctx->reader, &record, &err) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!ctx->page_read(ctx->reader))
+ break;
+ }
+
if (err)
elog(ERROR, "%s", err);
if (!record)
diff --git a/src/backend/replication/logical/logicalfuncs.c b/src/backend/replication/logical/logicalfuncs.c
index f7e055874e..b69f6911c5 100644
--- a/src/backend/replication/logical/logicalfuncs.c
+++ b/src/backend/replication/logical/logicalfuncs.c
@@ -233,9 +233,8 @@ pg_logical_slot_get_changes_guts(FunctionCallInfo fcinfo, bool confirm, bool bin
ctx = CreateDecodingContext(InvalidXLogRecPtr,
options,
false,
- XL_ROUTINE(.page_read = read_local_xlog_page,
- .segment_open = wal_segment_open,
- .segment_close = wal_segment_close),
+ read_local_xlog_page,
+ wal_segment_close,
LogicalOutputPrepareWrite,
LogicalOutputWrite, NULL);
@@ -284,7 +283,13 @@ pg_logical_slot_get_changes_guts(FunctionCallInfo fcinfo, bool confirm, bool bin
XLogRecord *record;
char *errm = NULL;
- record = XLogReadRecord(ctx->reader, &errm);
+ while (XLogReadRecord(ctx->reader, &record, &errm) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!ctx->page_read(ctx->reader))
+ break;
+ }
+
if (errm)
elog(ERROR, "%s", errm);
diff --git a/src/backend/replication/slotfuncs.c b/src/backend/replication/slotfuncs.c
index 9817b44113..deb7cf15eb 100644
--- a/src/backend/replication/slotfuncs.c
+++ b/src/backend/replication/slotfuncs.c
@@ -153,9 +153,8 @@ create_logical_replication_slot(char *name, char *plugin,
ctx = CreateInitDecodingContext(plugin, NIL,
false, /* just catalogs is OK */
restart_lsn,
- XL_ROUTINE(.page_read = read_local_xlog_page,
- .segment_open = wal_segment_open,
- .segment_close = wal_segment_close),
+ read_local_xlog_page,
+ wal_segment_close,
NULL, NULL, NULL);
/*
@@ -512,9 +511,8 @@ pg_logical_replication_slot_advance(XLogRecPtr moveto)
ctx = CreateDecodingContext(InvalidXLogRecPtr,
NIL,
true, /* fast_forward */
- XL_ROUTINE(.page_read = read_local_xlog_page,
- .segment_open = wal_segment_open,
- .segment_close = wal_segment_close),
+ read_local_xlog_page,
+ wal_segment_close,
NULL, NULL, NULL);
/*
@@ -536,7 +534,13 @@ pg_logical_replication_slot_advance(XLogRecPtr moveto)
* Read records. No changes are generated in fast_forward mode,
* but snapbuilder/slot statuses are updated properly.
*/
- record = XLogReadRecord(ctx->reader, &errm);
+ while (XLogReadRecord(ctx->reader, &record, &errm) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!ctx->page_read(ctx->reader))
+ break;
+ }
+
if (errm)
elog(ERROR, "%s", errm);
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index c36b91ae5e..062093727d 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -580,10 +580,7 @@ StartReplication(StartReplicationCmd *cmd)
/* create xlogreader for physical replication */
xlogreader =
- XLogReaderAllocate(wal_segment_size, NULL,
- XL_ROUTINE(.segment_open = WalSndSegmentOpen,
- .segment_close = wal_segment_close),
- NULL);
+ XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
if (!xlogreader)
ereport(ERROR,
@@ -807,9 +804,11 @@ StartReplication(StartReplicationCmd *cmd)
* set every time WAL is flushed.
*/
static bool
-logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetRecPtr, char *cur_page)
+logical_read_xlog_page(XLogReaderState *state)
{
+ XLogRecPtr targetPagePtr = state->readPagePtr;
+ int reqLen = state->readLen;
+ char *cur_page = state->readBuf;
XLogRecPtr flushptr;
int count;
WALReadError errinfo;
@@ -837,7 +836,7 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
count = flushptr - targetPagePtr; /* part of the page available */
/* now actually read the data, we know it's there */
- if (!WALRead(state,
+ if (!WALRead(state, WalSndSegmentOpen, wal_segment_close,
cur_page,
targetPagePtr,
XLOG_BLCKSZ,
@@ -1011,9 +1010,8 @@ CreateReplicationSlot(CreateReplicationSlotCmd *cmd)
ctx = CreateInitDecodingContext(cmd->plugin, NIL, need_full_snapshot,
InvalidXLogRecPtr,
- XL_ROUTINE(.page_read = logical_read_xlog_page,
- .segment_open = WalSndSegmentOpen,
- .segment_close = wal_segment_close),
+ logical_read_xlog_page,
+ wal_segment_close,
WalSndPrepareWrite, WalSndWriteData,
WalSndUpdateProgress);
@@ -1171,9 +1169,8 @@ StartLogicalReplication(StartReplicationCmd *cmd)
*/
logical_decoding_ctx =
CreateDecodingContext(cmd->startpoint, cmd->options, false,
- XL_ROUTINE(.page_read = logical_read_xlog_page,
- .segment_open = WalSndSegmentOpen,
- .segment_close = wal_segment_close),
+ logical_read_xlog_page,
+ wal_segment_close,
WalSndPrepareWrite, WalSndWriteData,
WalSndUpdateProgress);
xlogreader = logical_decoding_ctx->reader;
@@ -2749,7 +2746,7 @@ XLogSendPhysical(void)
enlargeStringInfo(&output_message, nbytes);
retry:
- if (!WALRead(xlogreader,
+ if (!WALRead(xlogreader, WalSndSegmentOpen, wal_segment_close,
&output_message.data[output_message.len],
startptr,
nbytes,
@@ -2847,7 +2844,12 @@ XLogSendLogical(void)
*/
WalSndCaughtUp = false;
- record = XLogReadRecord(logical_decoding_ctx->reader, &errm);
+ while (XLogReadRecord(logical_decoding_ctx->reader, &record, &errm) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!logical_decoding_ctx->page_read(logical_decoding_ctx->reader))
+ break;
+ }
/* xlog record was invalid */
if (errm != NULL)
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index cf119848b0..712c85281c 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -41,15 +41,9 @@ static int xlogreadfd = -1;
static XLogSegNo xlogreadsegno = -1;
static char xlogfpath[MAXPGPATH];
-typedef struct XLogPageReadPrivate
-{
- const char *restoreCommand;
- int tliIndex;
-} XLogPageReadPrivate;
-
-static bool SimpleXLogPageRead(XLogReaderState *xlogreader,
- XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
+static bool SimpleXLogPageRead(XLogReaderState *xlogreader,
+ const char *datadir, int *tliIndex,
+ const char *restoreCommand);
/*
* Read WAL from the datadir/pg_wal, starting from 'startpoint' on timeline
@@ -66,20 +60,22 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
XLogRecord *record;
XLogReaderState *xlogreader;
char *errormsg;
- XLogPageReadPrivate private;
- private.tliIndex = tliIndex;
- private.restoreCommand = restoreCommand;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir,
- XL_ROUTINE(.page_read = &SimpleXLogPageRead),
- &private);
+ xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+
if (xlogreader == NULL)
pg_fatal("out of memory");
XLogBeginRead(xlogreader, startpoint);
do
{
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!SimpleXLogPageRead(xlogreader, datadir,
+ &tliIndex, restoreCommand))
+ break;
+ }
if (record == NULL)
{
@@ -123,19 +119,19 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex,
XLogRecord *record;
XLogReaderState *xlogreader;
char *errormsg;
- XLogPageReadPrivate private;
XLogRecPtr endptr;
- private.tliIndex = tliIndex;
- private.restoreCommand = restoreCommand;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir,
- XL_ROUTINE(.page_read = &SimpleXLogPageRead),
- &private);
+ xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
XLogBeginRead(xlogreader, ptr);
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!SimpleXLogPageRead(xlogreader, datadir, &tliIndex, restoreCommand))
+ break;
+ }
if (record == NULL)
{
if (errormsg)
@@ -170,7 +166,6 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
XLogRecPtr searchptr;
XLogReaderState *xlogreader;
char *errormsg;
- XLogPageReadPrivate private;
/*
* The given fork pointer points to the end of the last common record,
@@ -186,11 +181,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
forkptr += SizeOfXLogShortPHD;
}
- private.tliIndex = tliIndex;
- private.restoreCommand = restoreCommand;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir,
- XL_ROUTINE(.page_read = &SimpleXLogPageRead),
- &private);
+ xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
@@ -200,7 +191,13 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
uint8 info;
XLogBeginRead(xlogreader, searchptr);
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!SimpleXLogPageRead(xlogreader, datadir,
+ &tliIndex, restoreCommand))
+ break;
+ }
if (record == NULL)
{
@@ -247,10 +244,11 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
/* XLogReader callback function, to read a WAL page */
static bool
-SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *readBuf)
+SimpleXLogPageRead(XLogReaderState *xlogreader, const char *datadir,
+ int *tliIndex, const char *restoreCommand)
{
- XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
+ XLogRecPtr targetPagePtr = xlogreader->readPagePtr;
+ char *readBuf = xlogreader->readBuf;
uint32 targetPageOff;
XLogRecPtr targetSegEnd;
XLogSegNo targetSegNo;
@@ -283,14 +281,14 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
* be done both forward and backward, consider also switching timeline
* accordingly.
*/
- while (private->tliIndex < targetNentries - 1 &&
- targetHistory[private->tliIndex].end < targetSegEnd)
- private->tliIndex++;
- while (private->tliIndex > 0 &&
- targetHistory[private->tliIndex].begin >= targetSegEnd)
- private->tliIndex--;
+ while (*tliIndex < targetNentries - 1 &&
+ targetHistory[*tliIndex].end < targetSegEnd)
+ (*tliIndex)++;
+ while (*tliIndex > 0 &&
+ targetHistory[*tliIndex].begin >= targetSegEnd)
+ (*tliIndex)--;
- XLogFileName(xlogfname, targetHistory[private->tliIndex].tli,
+ XLogFileName(xlogfname, targetHistory[*tliIndex].tli,
xlogreadsegno, WalSegSz);
snprintf(xlogfpath, MAXPGPATH, "%s/" XLOGDIR "/%s",
@@ -303,7 +301,7 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
/*
* If we have no restore_command to execute, then exit.
*/
- if (private->restoreCommand == NULL)
+ if (restoreCommand == NULL)
{
pg_log_error("could not open file \"%s\": %m", xlogfpath);
xlogreader->readLen = -1;
@@ -317,7 +315,7 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
xlogreadfd = RestoreArchivedFile(xlogreader->segcxt.ws_dir,
xlogfname,
WalSegSz,
- private->restoreCommand);
+ restoreCommand);
if (xlogreadfd < 0)
{
@@ -359,7 +357,7 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
Assert(targetSegNo == xlogreadsegno);
- xlogreader->seg.ws_tli = targetHistory[private->tliIndex].tli;
+ xlogreader->seg.ws_tli = targetHistory[*tliIndex].tli;
xlogreader->readLen = XLOG_BLCKSZ;
return true;
}
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index 0f0e0723a3..f514de6b13 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -330,12 +330,17 @@ WALDumpCloseSegment(XLogReaderState *state)
state->seg.ws_file = -1;
}
-/* pg_waldump's XLogReaderRoutine->page_read callback */
+/*
+ * pg_waldump's WAL page rader, also used as page_read callback for
+ * XLogFindNextRecord
+ */
static bool
-WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetPtr, char *readBuff)
+WALDumpReadPage(XLogReaderState *state, void *priv)
{
- XLogDumpPrivate *private = state->private_data;
+ XLogRecPtr targetPagePtr = state->readPagePtr;
+ int reqLen = state->readLen;
+ char *readBuff = state->readBuf;
+ XLogDumpPrivate *private = (XLogDumpPrivate *) priv;
int count = XLOG_BLCKSZ;
WALReadError errinfo;
@@ -353,8 +358,8 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
}
}
- if (!WALRead(state, readBuff, targetPagePtr, count, private->timeline,
- &errinfo))
+ if (!WALRead(state, WALDumpOpenSegment, WALDumpCloseSegment,
+ readBuff, targetPagePtr, count, private->timeline, &errinfo))
{
WALOpenSegment *seg = &errinfo.wre_seg;
char fname[MAXPGPATH];
@@ -374,6 +379,7 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
(Size) errinfo.wre_req);
}
+ Assert(count >= state->readLen);
state->readLen = count;
return true;
}
@@ -1035,16 +1041,14 @@ main(int argc, char **argv)
/* we have everything we need, start reading */
xlogreader_state =
- XLogReaderAllocate(WalSegSz, waldir,
- XL_ROUTINE(.page_read = WALDumpReadPage,
- .segment_open = WALDumpOpenSegment,
- .segment_close = WALDumpCloseSegment),
- &private);
+ XLogReaderAllocate(WalSegSz, waldir, WALDumpCloseSegment);
+
if (!xlogreader_state)
fatal_error("out of memory");
/* first find a valid recptr to start from */
- first_record = XLogFindNextRecord(xlogreader_state, private.startptr);
+ first_record = XLogFindNextRecord(xlogreader_state, private.startptr,
+ &WALDumpReadPage, (void*) &private);
if (first_record == InvalidXLogRecPtr)
fatal_error("could not find a valid record after %X/%X",
@@ -1067,7 +1071,13 @@ main(int argc, char **argv)
for (;;)
{
/* try to read the next record */
- record = XLogReadRecord(xlogreader_state, &errormsg);
+ while (XLogReadRecord(xlogreader_state, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!WALDumpReadPage(xlogreader_state, (void *) &private))
+ break;
+ }
+
if (!record)
{
if (!config.follow || private.endptr_reached)
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index 23d9e70a59..4e9268b553 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -57,64 +57,15 @@ typedef struct WALSegmentContext
typedef struct XLogReaderState XLogReaderState;
-/* Function type definition for the read_page callback */
-typedef bool (*XLogPageReadCB) (XLogReaderState *xlogreader,
- XLogRecPtr targetPagePtr,
- int reqLen,
- XLogRecPtr targetRecPtr,
- char *readBuf);
+/* Function type definition for the segment cleanup callback */
+typedef void (*WALSegmentCleanupCB) (XLogReaderState *xlogreader);
+
+/* Function type definition for the open/close callbacks for WALRead() */
typedef void (*WALSegmentOpenCB) (XLogReaderState *xlogreader,
XLogSegNo nextSegNo,
TimeLineID *tli_p);
typedef void (*WALSegmentCloseCB) (XLogReaderState *xlogreader);
-typedef struct XLogReaderRoutine
-{
- /*
- * Data input callback
- *
- * This callback shall read at least reqLen valid bytes of the xlog page
- * starting at targetPagePtr, and store them in readBuf. The callback
- * shall return the number of bytes read (never more than XLOG_BLCKSZ), or
- * -1 on failure. The callback shall sleep, if necessary, to wait for the
- * requested bytes to become available. The callback will not be invoked
- * again for the same page unless more than the returned number of bytes
- * are needed.
- *
- * targetRecPtr is the position of the WAL record we're reading. Usually
- * it is equal to targetPagePtr + reqLen, but sometimes xlogreader needs
- * to read and verify the page or segment header, before it reads the
- * actual WAL record it's interested in. In that case, targetRecPtr can
- * be used to determine which timeline to read the page from.
- *
- * The callback shall set ->seg.ws_tli to the TLI of the file the page was
- * read from.
- */
- XLogPageReadCB page_read;
-
- /*
- * Callback to open the specified WAL segment for reading. ->seg.ws_file
- * shall be set to the file descriptor of the opened segment. In case of
- * failure, an error shall be raised by the callback and it shall not
- * return.
- *
- * "nextSegNo" is the number of the segment to be opened.
- *
- * "tli_p" is an input/output argument. WALRead() uses it to pass the
- * timeline in which the new segment should be found, but the callback can
- * use it to return the TLI that it actually opened.
- */
- WALSegmentOpenCB segment_open;
-
- /*
- * WAL segment close callback. ->seg.ws_file shall be set to a negative
- * number.
- */
- WALSegmentCloseCB segment_close;
-} XLogReaderRoutine;
-
-#define XL_ROUTINE(...) &(XLogReaderRoutine){__VA_ARGS__}
-
typedef struct
{
/* Is this block ref in use? */
@@ -144,12 +95,35 @@ typedef struct
uint16 data_bufsz;
} DecodedBkpBlock;
+/* Return code from XLogReadRecord */
+typedef enum XLogReadRecordResult
+{
+ XLREAD_SUCCESS, /* record is successfully read */
+ XLREAD_NEED_DATA, /* need more data. see XLogReadRecord. */
+ XLREAD_FAIL /* failed during reading a record */
+} XLogReadRecordResult;
+
+/*
+ * internal state of XLogReadRecord
+ *
+ * XLogReadState runs a state machine while reading a record. Theses states
+ * are not seen outside the function. Each state may repeat several times
+ * exiting requesting caller for new data. See the comment of XLogReadRecrod
+ * for details.
+ */
+typedef enum XLogReadRecordState {
+ XLREAD_NEXT_RECORD,
+ XLREAD_TOT_LEN,
+ XLREAD_FIRST_FRAGMENT,
+ XLREAD_CONTINUATION
+} XLogReadRecordState;
+
struct XLogReaderState
{
/*
* Operational callbacks
*/
- XLogReaderRoutine routine;
+ WALSegmentCleanupCB cleanup_cb;
/* ----------------------------------------
* Public parameters
@@ -162,18 +136,14 @@ struct XLogReaderState
*/
uint64 system_identifier;
- /*
- * Opaque data for callbacks to use. Not used by XLogReader.
- */
- void *private_data;
-
/*
* Start and end point of last record read. EndRecPtr is also used as the
* position to read next. Calling XLogBeginRead() sets EndRecPtr to the
* starting position and ReadRecPtr to invalid.
*/
- XLogRecPtr ReadRecPtr; /* start of last record read */
+ XLogRecPtr ReadRecPtr; /* start of last record read or being read */
XLogRecPtr EndRecPtr; /* end+1 of last record read */
+ XLogRecPtr PrevRecPtr; /* start of previous record read */
/* ----------------------------------------
* Communication with page reader
@@ -187,7 +157,9 @@ struct XLogReaderState
* the request, or -1 on error */
TimeLineID readPageTLI; /* TLI for data currently in readBuf */
char *readBuf; /* buffer to store data */
- bool page_verified; /* is the page on the buffer verified? */
+ bool page_verified; /* is the page header on the buffer verified? */
+ bool record_verified;/* is the current record header verified? */
+
/* ----------------------------------------
@@ -229,8 +201,6 @@ struct XLogReaderState
XLogRecPtr latestPagePtr;
TimeLineID latestPageTLI;
- /* beginning of the WAL record being read. */
- XLogRecPtr currRecPtr;
/* timeline to read it from, 0 if a lookup is required */
TimeLineID currTLI;
@@ -257,6 +227,15 @@ struct XLogReaderState
char *readRecordBuf;
uint32 readRecordBufSize;
+ /*
+ * XLogReadRecord() state
+ */
+ XLogReadRecordState readRecordState;/* state machine state */
+ int recordGotLen; /* amount of current record that has
+ * already been read */
+ int recordRemainLen; /* length of current record that remains */
+ XLogRecPtr recordContRecPtr; /* where the current record continues */
+
/* Buffer to hold error message */
char *errormsg_buf;
};
@@ -264,9 +243,7 @@ struct XLogReaderState
/* Get a new XLogReader */
extern XLogReaderState *XLogReaderAllocate(int wal_segment_size,
const char *waldir,
- XLogReaderRoutine *routine,
- void *private_data);
-extern XLogReaderRoutine *LocalXLogReaderRoutine(void);
+ WALSegmentCleanupCB cleanup_cb);
/* Free an XLogReader */
extern void XLogReaderFree(XLogReaderState *state);
@@ -274,12 +251,17 @@ extern void XLogReaderFree(XLogReaderState *state);
/* Position the XLogReader to given record */
extern void XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr);
#ifdef FRONTEND
-extern XLogRecPtr XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr);
+/* Function type definition for the read_page callback */
+typedef bool (*XLogFindNextRecordCB) (XLogReaderState *xlogreader,
+ void *private);
+extern XLogRecPtr XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
+ XLogFindNextRecordCB read_page, void *private);
#endif /* FRONTEND */
/* Read the next XLog record. Returns NULL on end-of-WAL or failure */
-extern struct XLogRecord *XLogReadRecord(XLogReaderState *state,
- char **errormsg);
+extern XLogReadRecordResult XLogReadRecord(XLogReaderState *state,
+ XLogRecord **record,
+ char **errormsg);
/* Validate a page */
extern bool XLogReaderValidatePageHeader(XLogReaderState *state,
@@ -299,6 +281,7 @@ typedef struct WALReadError
} WALReadError;
extern bool WALRead(XLogReaderState *state,
+ WALSegmentOpenCB segopenfn, WALSegmentCloseCB sgclosefn,
char *buf, XLogRecPtr startptr, Size count,
TimeLineID tli, WALReadError *errinfo);
diff --git a/src/include/access/xlogutils.h b/src/include/access/xlogutils.h
index 364a21c4ea..397fb27fc2 100644
--- a/src/include/access/xlogutils.h
+++ b/src/include/access/xlogutils.h
@@ -47,9 +47,7 @@ extern Buffer XLogReadBufferExtended(RelFileNode rnode, ForkNumber forknum,
extern Relation CreateFakeRelcacheEntry(RelFileNode rnode);
extern void FreeFakeRelcacheEntry(Relation fakerel);
-extern bool read_local_xlog_page(XLogReaderState *state,
- XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetRecPtr, char *cur_page);
+extern bool read_local_xlog_page(XLogReaderState *state);
extern void wal_segment_open(XLogReaderState *state,
XLogSegNo nextSegNo,
TimeLineID *tli_p);
diff --git a/src/include/pg_config_manual.h b/src/include/pg_config_manual.h
index f10ad0acd6..94749609b4 100644
--- a/src/include/pg_config_manual.h
+++ b/src/include/pg_config_manual.h
@@ -377,7 +377,7 @@
* Enable debugging print statements for WAL-related operations; see
* also the wal_debug GUC var.
*/
-/* #define WAL_DEBUG */
+#define WAL_DEBUG
/*
* Enable tracing of resource consumption during sort operations;
diff --git a/src/include/replication/logical.h b/src/include/replication/logical.h
index c253403372..b936f31393 100644
--- a/src/include/replication/logical.h
+++ b/src/include/replication/logical.h
@@ -29,6 +29,10 @@ typedef void (*LogicalOutputPluginWriterUpdateProgress) (struct LogicalDecodingC
TransactionId xid
);
+typedef struct LogicalDecodingContext LogicalDecodingContext;
+
+typedef bool (*LogicalDecodingXLogPageReadCB)(XLogReaderState *ctx);
+
typedef struct LogicalDecodingContext
{
/* memory context this is all allocated in */
@@ -39,6 +43,7 @@ typedef struct LogicalDecodingContext
/* infrastructure pieces for decoding */
XLogReaderState *reader;
+ LogicalDecodingXLogPageReadCB page_read;
struct ReorderBuffer *reorder;
struct SnapBuild *snapshot_builder;
@@ -105,14 +110,16 @@ extern LogicalDecodingContext *CreateInitDecodingContext(const char *plugin,
List *output_plugin_options,
bool need_full_snapshot,
XLogRecPtr restart_lsn,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress);
extern LogicalDecodingContext *CreateDecodingContext(XLogRecPtr start_lsn,
List *output_plugin_options,
bool fast_forward,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress);
--
2.27.0
----Next_Part(Thu_Mar__4_11_28_53_2021_014)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
filename="v17-0003-Remove-globals-readOff-readLen-and-readSegNo.patch"
^ permalink raw reply [nested|flat] 70+ messages in thread
* [PATCH v17 2/4] Move page-reader out of XLogReadRecord
@ 2019-09-10 03:58 Kyotaro Horiguchi <[email protected]>
0 siblings, 0 replies; 70+ messages in thread
From: Kyotaro Horiguchi @ 2019-09-10 03:58 UTC (permalink / raw)
This is the second step of removing callbacks from WAL record reader.
Since it is essential to take in additional data while reading a
record, the function have to ask caller for new data while keeping
working state. Thus the function is turned into a state machine.
---
src/backend/access/transam/twophase.c | 15 +-
src/backend/access/transam/xlog.c | 58 +-
src/backend/access/transam/xlogreader.c | 697 +++++++++++-------
src/backend/access/transam/xlogutils.c | 17 +-
src/backend/replication/logical/logical.c | 26 +-
.../replication/logical/logicalfuncs.c | 13 +-
src/backend/replication/slotfuncs.c | 18 +-
src/backend/replication/walsender.c | 32 +-
src/bin/pg_rewind/parsexlog.c | 84 ++-
src/bin/pg_waldump/pg_waldump.c | 36 +-
src/include/access/xlogreader.h | 121 ++-
src/include/access/xlogutils.h | 4 +-
src/include/pg_config_manual.h | 2 +-
src/include/replication/logical.h | 11 +-
14 files changed, 646 insertions(+), 488 deletions(-)
diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index 80d2d20d6c..7d5a96a5f5 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -1325,11 +1325,8 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
XLogReaderState *xlogreader;
char *errormsg;
- xlogreader = XLogReaderAllocate(wal_segment_size, NULL,
- XL_ROUTINE(.page_read = &read_local_xlog_page,
- .segment_open = &wal_segment_open,
- .segment_close = &wal_segment_close),
- NULL);
+ xlogreader = XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+
if (!xlogreader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
@@ -1337,7 +1334,13 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
errdetail("Failed while allocating a WAL reading processor.")));
XLogBeginRead(xlogreader, lsn);
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!read_local_xlog_page(xlogreader))
+ break;
+ }
+
if (record == NULL)
ereport(ERROR,
(errcode_for_file_access(),
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 606328a65a..670996ae67 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -836,13 +836,6 @@ static XLogSource currentSource = XLOG_FROM_ANY;
static bool lastSourceFailed = false;
static bool pendingWalRcvRestart = false;
-typedef struct XLogPageReadPrivate
-{
- int emode;
- bool fetching_ckpt; /* are we fetching a checkpoint record? */
- bool randAccess;
-} XLogPageReadPrivate;
-
/*
* These variables track when we last obtained some WAL data to process,
* and where we got it from. (XLogReceiptSource is initially the same as
@@ -917,8 +910,8 @@ static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath,
static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli,
XLogSource source, bool notfoundOk);
static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, XLogSource source);
-static bool XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
+static bool XLogPageRead(XLogReaderState *xlogreader,
+ bool fetching_ckpt, int emode, bool randAccess);
static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
bool fetching_ckpt, XLogRecPtr tliRecPtr);
static int emode_for_corrupt_record(int emode, XLogRecPtr RecPtr);
@@ -1231,8 +1224,7 @@ XLogInsertRecord(XLogRecData *rdata,
appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len);
if (!debug_reader)
- debug_reader = XLogReaderAllocate(wal_segment_size, NULL,
- XL_ROUTINE(), NULL);
+ debug_reader = XLogReaderAllocate(wal_segment_size, NULL, NULL);
if (!debug_reader)
{
@@ -4344,15 +4336,10 @@ CleanupBackupHistory(void)
* record is available.
*/
static XLogRecord *
-ReadRecord(XLogReaderState *xlogreader, int emode,
- bool fetching_ckpt)
+ReadRecord(XLogReaderState *xlogreader, int emode, bool fetching_ckpt)
{
XLogRecord *record;
- XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
-
- private->fetching_ckpt = fetching_ckpt;
- private->emode = emode;
- private->randAccess = (xlogreader->ReadRecPtr == InvalidXLogRecPtr);
+ bool randAccess = (xlogreader->ReadRecPtr == InvalidXLogRecPtr);
/* This is the first attempt to read this page. */
lastSourceFailed = false;
@@ -4360,8 +4347,16 @@ ReadRecord(XLogReaderState *xlogreader, int emode,
for (;;)
{
char *errormsg;
+ XLogReadRecordResult result;
+
+ while ((result = XLogReadRecord(xlogreader, &record, &errormsg))
+ == XLREAD_NEED_DATA)
+ {
+ if (!XLogPageRead(xlogreader, fetching_ckpt, emode, randAccess))
+ break;
+
+ }
- record = XLogReadRecord(xlogreader, &errormsg);
ReadRecPtr = xlogreader->ReadRecPtr;
EndRecPtr = xlogreader->EndRecPtr;
if (record == NULL)
@@ -6380,7 +6375,6 @@ StartupXLOG(void)
bool backupFromStandby = false;
DBState dbstate_at_startup;
XLogReaderState *xlogreader;
- XLogPageReadPrivate private;
bool promoted = false;
struct stat st;
@@ -6539,13 +6533,9 @@ StartupXLOG(void)
OwnLatch(&XLogCtl->recoveryWakeupLatch);
/* Set up XLOG reader facility */
- MemSet(&private, 0, sizeof(XLogPageReadPrivate));
xlogreader =
- XLogReaderAllocate(wal_segment_size, NULL,
- XL_ROUTINE(.page_read = &XLogPageRead,
- .segment_open = NULL,
- .segment_close = wal_segment_close),
- &private);
+ XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+
if (!xlogreader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
@@ -11987,12 +11977,13 @@ CancelBackup(void)
* sleep and retry.
*/
static bool
-XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetRecPtr, char *readBuf)
+XLogPageRead(XLogReaderState *xlogreader,
+ bool fetching_ckpt, int emode, bool randAccess)
{
- XLogPageReadPrivate *private =
- (XLogPageReadPrivate *) xlogreader->private_data;
- int emode = private->emode;
+ char *readBuf = xlogreader->readBuf;
+ XLogRecPtr targetPagePtr = xlogreader->readPagePtr;
+ int reqLen = xlogreader->readLen;
+ XLogRecPtr targetRecPtr = xlogreader->ReadRecPtr;
uint32 targetPageOff;
XLogSegNo targetSegNo PG_USED_FOR_ASSERTS_ONLY;
int r;
@@ -12035,8 +12026,8 @@ retry:
flushedUpto < targetPagePtr + reqLen))
{
if (!WaitForWALToBecomeAvailable(targetPagePtr + reqLen,
- private->randAccess,
- private->fetching_ckpt,
+ randAccess,
+ fetching_ckpt,
targetRecPtr))
{
if (readFile >= 0)
@@ -12141,6 +12132,7 @@ retry:
goto next_record_is_invalid;
}
+ Assert(xlogreader->readPagePtr == targetPagePtr);
xlogreader->readLen = readLen;
return true;
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index 7d04995b7b..ba3e4868f7 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -40,7 +40,7 @@ static bool XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr,
int reqLen, bool header_inclusive);
static void XLogReaderInvalReadState(XLogReaderState *state);
static bool ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
- XLogRecPtr PrevRecPtr, XLogRecord *record, bool randAccess);
+ XLogRecPtr PrevRecPtr, XLogRecord *record);
static bool ValidXLogRecord(XLogReaderState *state, XLogRecord *record,
XLogRecPtr recptr);
static void ResetDecoder(XLogReaderState *state);
@@ -73,7 +73,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...)
*/
XLogReaderState *
XLogReaderAllocate(int wal_segment_size, const char *waldir,
- XLogReaderRoutine *routine, void *private_data)
+ WALSegmentCleanupCB cleanup_cb)
{
XLogReaderState *state;
@@ -84,7 +84,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
return NULL;
/* initialize caller-provided support functions */
- state->routine = *routine;
+ state->cleanup_cb = cleanup_cb;
state->max_block_id = -1;
@@ -107,8 +107,6 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
WALOpenSegmentInit(&state->seg, &state->segcxt, wal_segment_size,
waldir);
- /* system_identifier initialized to zeroes above */
- state->private_data = private_data;
/* ReadRecPtr, EndRecPtr and readLen initialized to zeroes above */
state->errormsg_buf = palloc_extended(MAX_ERRORMSG_LEN + 1,
MCXT_ALLOC_NO_OOM);
@@ -140,8 +138,8 @@ XLogReaderFree(XLogReaderState *state)
{
int block_id;
- if (state->seg.ws_file != -1)
- state->routine.segment_close(state);
+ if (state->seg.ws_file >= 0)
+ state->cleanup_cb(state);
for (block_id = 0; block_id <= XLR_MAX_BLOCK_ID; block_id++)
{
@@ -246,6 +244,7 @@ XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
/* Begin at the passed-in record pointer. */
state->EndRecPtr = RecPtr;
state->ReadRecPtr = InvalidXLogRecPtr;
+ state->readRecordState = XLREAD_NEXT_RECORD;
}
/*
@@ -254,317 +253,449 @@ XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
* XLogBeginRead() or XLogFindNextRecord() must be called before the first call
* to XLogReadRecord().
*
- * If the page_read callback fails to read the requested data, NULL is
- * returned. The callback is expected to have reported the error; errormsg
- * is set to NULL.
+ * This function may return XLREAD_NEED_DATA several times before returning a
+ * result record. The caller shall read in some new data then call this
+ * function again with the same parameters.
*
- * If the reading fails for some other reason, NULL is also returned, and
- * *errormsg is set to a string with details of the failure.
+ * When a record is successfully read, returns XLREAD_SUCCESS with result
+ * record being stored in *record. Otherwise *record is NULL.
*
- * The returned pointer (or *errormsg) points to an internal buffer that's
- * valid until the next call to XLogReadRecord.
+ * Returns XLREAD_NEED_DATA if more data is needed to finish reading the
+ * current record. In that case, state->readPagePtr and state->readLen inform
+ * the desired position and minimum length of data needed. The caller shall
+ * read in the requested data and set state->readBuf to point to a buffer
+ * containing it. The caller must also set state->readPageTLI and
+ * state->readLen to indicate the timeline that it was read from, and the
+ * length of data that is now available (which must be >= given readLen),
+ * respectively.
+ *
+ * If invalid data is encountered, returns XLREAD_FAIL with *record being set to
+ * NULL. *errormsg is set to a string with details of the failure.
+ * The returned pointer (or *errormsg) points to an internal buffer that's valid
+ * until the next call to XLogReadRecord.
+ *
+ *
+ * This function runs a state machine consists of the following states.
+ *
+ * XLREAD_NEXT_RECORD :
+ * The initial state, if called with valid RecPtr, try to read a record at
+ * that position. If invalid RecPtr is given try to read a record just after
+ * the last one previously read.
+ * This state ens after setting ReadRecPtr. Then goes to XLREAD_TOT_LEN.
+ *
+ * XLREAD_TOT_LEN:
+ * Examining record header. Ends after reading record total
+ * length. recordRemainLen and recordGotLen are initialized.
+ *
+ * XLREAD_FIRST_FRAGMENT:
+ * Reading the first fragment. Ends with finishing reading a single
+ * record. Goes to XLREAD_NEXT_RECORD if that's all or
+ * XLREAD_CONTINUATION if we have continuation.
+
+ * XLREAD_CONTINUATION:
+ * Reading continuation of record. Ends with finishing the whole record then
+ * goes to XLREAD_NEXT_RECORD. During this state, recordRemainLen indicates
+ * how much is left and readRecordBuf holds the partially assert
+ * record.recordContRecPtr points to the beginning of the next page where to
+ * continue.
+ *
+ * If wrong data found in any state, the state machine stays at the current
+ * state. This behavior allows to continue reading a reacord switching among
+ * different souces, while streaming replication.
*/
-XLogRecord *
-XLogReadRecord(XLogReaderState *state, char **errormsg)
+XLogReadRecordResult
+XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
{
- XLogRecPtr RecPtr;
- XLogRecord *record;
- XLogRecPtr targetPagePtr;
- bool randAccess;
- uint32 len,
- total_len;
- uint32 targetRecOff;
- uint32 pageHeaderSize;
- bool gotheader;
+ XLogRecord *prec;
- /*
- * randAccess indicates whether to verify the previous-record pointer of
- * the record we're reading. We only do this if we're reading
- * sequentially, which is what we initially assume.
- */
- randAccess = false;
+ *record = NULL;
/* reset error state */
*errormsg = NULL;
state->errormsg_buf[0] = '\0';
- ResetDecoder(state);
-
- RecPtr = state->EndRecPtr;
-
- if (state->ReadRecPtr != InvalidXLogRecPtr)
- {
- /* read the record after the one we just read */
-
- /*
- * EndRecPtr is pointing to end+1 of the previous WAL record. If
- * we're at a page boundary, no more records can fit on the current
- * page. We must skip over the page header, but we can't do that until
- * we've read in the page, since the header size is variable.
- */
- }
- else
- {
- /*
- * Caller supplied a position to start at.
- *
- * In this case, EndRecPtr should already be pointing to a valid
- * record starting position.
- */
- Assert(XRecOffIsValid(RecPtr));
- randAccess = true;
- }
-
- state->currRecPtr = RecPtr;
-
- targetPagePtr = RecPtr - (RecPtr % XLOG_BLCKSZ);
- targetRecOff = RecPtr % XLOG_BLCKSZ;
-
- /*
- * Read the page containing the record into state->readBuf. Request enough
- * byte to cover the whole record header, or at least the part of it that
- * fits on the same page.
- */
- while (XLogNeedData(state, targetPagePtr,
- Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
- targetRecOff != 0))
- {
- if (!state->routine.page_read(state, state->readPagePtr, state->readLen,
- RecPtr, state->readBuf))
- break;
- }
-
- if (!state->page_verified)
- goto err;
-
- /*
- * We have at least the page header, so we can examine it now.
- */
- pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
- if (targetRecOff == 0)
- {
- /*
- * At page start, so skip over page header.
- */
- RecPtr += pageHeaderSize;
- targetRecOff = pageHeaderSize;
- }
- else if (targetRecOff < pageHeaderSize)
- {
- report_invalid_record(state, "invalid record offset at %X/%X",
- LSN_FORMAT_ARGS(RecPtr));
- goto err;
- }
-
- if ((((XLogPageHeader) state->readBuf)->xlp_info & XLP_FIRST_IS_CONTRECORD) &&
- targetRecOff == pageHeaderSize)
- {
- report_invalid_record(state, "contrecord is requested by %X/%X",
- LSN_FORMAT_ARGS(RecPtr));
- goto err;
- }
-
- /* XLogNeedData has verified the page header */
- Assert(pageHeaderSize <= state->readLen);
-
- /*
- * Read the record length.
- *
- * NB: Even though we use an XLogRecord pointer here, the whole record
- * header might not fit on this page. xl_tot_len is the first field of the
- * struct, so it must be on this page (the records are MAXALIGNed), but we
- * cannot access any other fields until we've verified that we got the
- * whole header.
- */
- record = (XLogRecord *) (state->readBuf + RecPtr % XLOG_BLCKSZ);
- total_len = record->xl_tot_len;
-
- /*
- * If the whole record header is on this page, validate it immediately.
- * Otherwise do just a basic sanity check on xl_tot_len, and validate the
- * rest of the header after reading it from the next page. The xl_tot_len
- * check is necessary here to ensure that we enter the "Need to reassemble
- * record" code path below; otherwise we might fail to apply
- * ValidXLogRecordHeader at all.
- */
- if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
- {
- if (!ValidXLogRecordHeader(state, RecPtr, state->ReadRecPtr, record,
- randAccess))
- goto err;
- gotheader = true;
- }
- else
- {
- /* XXX: more validation should be done here */
- if (total_len < SizeOfXLogRecord)
- {
- report_invalid_record(state,
- "invalid record length at %X/%X: wanted %u, got %u",
- LSN_FORMAT_ARGS(RecPtr),
- (uint32) SizeOfXLogRecord, total_len);
- goto err;
- }
- gotheader = false;
- }
-
- len = XLOG_BLCKSZ - RecPtr % XLOG_BLCKSZ;
- if (total_len > len)
+ switch (state->readRecordState)
{
- /* Need to reassemble record */
- char *contdata;
- XLogPageHeader pageHeader;
- char *buffer;
- uint32 gotlen;
-
- /*
- * Enlarge readRecordBuf as needed.
- */
- if (total_len > state->readRecordBufSize &&
- !allocate_recordbuf(state, total_len))
- {
- /* We treat this as a "bogus data" condition */
- report_invalid_record(state, "record length %u at %X/%X too long",
- total_len, LSN_FORMAT_ARGS(RecPtr));
- goto err;
- }
+ case XLREAD_NEXT_RECORD:
+ ResetDecoder(state);
- /* Copy the first fragment of the record from the first page. */
- memcpy(state->readRecordBuf,
- state->readBuf + RecPtr % XLOG_BLCKSZ, len);
- buffer = state->readRecordBuf + len;
- gotlen = len;
-
- do
- {
- int rest_len = total_len - gotlen;
-
- /* Calculate pointer to beginning of next page */
- targetPagePtr += XLOG_BLCKSZ;
-
- /* Wait for the next page to become available */
- while (XLogNeedData(state, targetPagePtr,
- Min(rest_len, XLOG_BLCKSZ),
- false))
+ if (state->ReadRecPtr != InvalidXLogRecPtr)
{
- if (!state->routine.page_read(state, state->readPagePtr,
- state->readLen,
- state->ReadRecPtr,
- state->readBuf))
- break;
+ /* read the record after the one we just read */
+
+ /*
+ * EndRecPtr is pointing to end+1 of the previous WAL record.
+ * If we're at a page boundary, no more records can fit on the
+ * current page. We must skip over the page header, but we
+ * can't do that until we've read in the page, since the header
+ * size is variable.
+ */
+ state->PrevRecPtr = state->ReadRecPtr;
+ state->ReadRecPtr = state->EndRecPtr;
+ }
+ else
+ {
+ /*
+ * Caller supplied a position to start at.
+ *
+ * In this case, EndRecPtr should already be pointing to a
+ * valid record starting position.
+ */
+ Assert(XRecOffIsValid(state->EndRecPtr));
+ state->ReadRecPtr = state->EndRecPtr;
+
+ /*
+ * We cannot verify the previous-record pointer when we're
+ * seeking to a particular record. Reset PrevRecPtr so that we
+ * won't try doing that.
+ */
+ state->PrevRecPtr = InvalidXLogRecPtr;
+ state->EndRecPtr = InvalidXLogRecPtr; /* to be tidy */
}
+ state->record_verified = false;
+ state->readRecordState = XLREAD_TOT_LEN;
+ /* fall through */
+
+ case XLREAD_TOT_LEN:
+ {
+ uint32 total_len;
+ uint32 pageHeaderSize;
+ XLogRecPtr targetPagePtr;
+ uint32 targetRecOff;
+ XLogPageHeader pageHeader;
+
+ targetPagePtr =
+ state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
+ targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+
+ /*
+ * Check if we have enough data. For the first record in the page,
+ * the requesting length doesn't contain page header.
+ */
+ if (XLogNeedData(state, targetPagePtr,
+ Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+ targetRecOff != 0))
+ return XLREAD_NEED_DATA;
+
+ /* error out if caller supplied bogus page */
if (!state->page_verified)
goto err;
- Assert(SizeOfXLogShortPHD <= state->readLen);
+ /* examine page header now. */
+ pageHeaderSize =
+ XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+ if (targetRecOff == 0)
+ {
+ /* At page start, so skip over page header. */
+ state->ReadRecPtr += pageHeaderSize;
+ targetRecOff = pageHeaderSize;
+ }
+ else if (targetRecOff < pageHeaderSize)
+ {
+ report_invalid_record(state, "invalid record offset at %X/%X",
+ LSN_FORMAT_ARGS(state->ReadRecPtr));
+ goto err;
+ }
- /* Check that the continuation on next page looks valid */
pageHeader = (XLogPageHeader) state->readBuf;
- if (!(pageHeader->xlp_info & XLP_FIRST_IS_CONTRECORD))
+ if ((pageHeader->xlp_info & XLP_FIRST_IS_CONTRECORD) &&
+ targetRecOff == pageHeaderSize)
{
- report_invalid_record(state,
- "there is no contrecord flag at %X/%X",
- LSN_FORMAT_ARGS(RecPtr));
+ report_invalid_record(state, "contrecord is requested by %X/%X",
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr);
goto err;
}
- /*
- * Cross-check that xlp_rem_len agrees with how much of the record
- * we expect there to be left.
- */
- if (pageHeader->xlp_rem_len == 0 ||
- total_len != (pageHeader->xlp_rem_len + gotlen))
- {
- report_invalid_record(state,
- "invalid contrecord length %u (expected %lld) at %X/%X",
- pageHeader->xlp_rem_len,
- ((long long) total_len) - gotlen,
- LSN_FORMAT_ARGS(RecPtr));
- goto err;
- }
-
- /* Append the continuation from this page to the buffer */
- pageHeaderSize = XLogPageHeaderSize(pageHeader);
-
+ /* XLogNeedData has verified the page header */
Assert(pageHeaderSize <= state->readLen);
- contdata = (char *) state->readBuf + pageHeaderSize;
- len = XLOG_BLCKSZ - pageHeaderSize;
- if (pageHeader->xlp_rem_len < len)
- len = pageHeader->xlp_rem_len;
+ /*
+ * Read the record length.
+ *
+ * NB: Even though we use an XLogRecord pointer here, the whole
+ * record header might not fit on this page. xl_tot_len is the first
+ * field of the struct, so it must be on this page (the records are
+ * MAXALIGNed), but we cannot access any other fields until we've
+ * verified that we got the whole header.
+ */
+ prec = (XLogRecord *) (state->readBuf +
+ state->ReadRecPtr % XLOG_BLCKSZ);
+ total_len = prec->xl_tot_len;
- Assert (pageHeaderSize + len <= state->readLen);
- memcpy(buffer, (char *) contdata, len);
- buffer += len;
- gotlen += len;
+ /*
+ * If the whole record header is on this page, validate it
+ * immediately. Otherwise do just a basic sanity check on
+ * xl_tot_len, and validate the rest of the header after reading it
+ * from the next page. The xl_tot_len check is necessary here to
+ * ensure that we enter the XLREAD_CONTINUATION state below;
+ * otherwise we might fail to apply ValidXLogRecordHeader at all.
+ */
+ if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
+ {
+ if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
+ state->PrevRecPtr, prec))
+ goto err;
- /* If we just reassembled the record header, validate it. */
- if (!gotheader)
+ state->record_verified = true;
+ }
+ else
{
- record = (XLogRecord *) state->readRecordBuf;
- if (!ValidXLogRecordHeader(state, RecPtr, state->ReadRecPtr,
- record, randAccess))
+ /* XXX: more validation should be done here */
+ if (total_len < SizeOfXLogRecord)
+ {
+ report_invalid_record(state,
+ "invalid record length at %X/%X: wanted %u, got %u",
+ LSN_FORMAT_ARGS(state->ReadRecPtr),
+ (uint32) SizeOfXLogRecord, total_len);
goto err;
- gotheader = true;
+ }
}
- } while (gotlen < total_len);
-
- Assert(gotheader);
- record = (XLogRecord *) state->readRecordBuf;
- if (!ValidXLogRecord(state, record, RecPtr))
- goto err;
+ /*
+ * Wait for the rest of the record, or the part of the record that
+ * fit on the first page if crossed a page boundary, to become
+ * available.
+ */
+ state->recordGotLen = 0;
+ state->recordRemainLen = total_len;
+ state->readRecordState = XLREAD_FIRST_FRAGMENT;
+ }
+ /* fall through */
- pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
- state->ReadRecPtr = RecPtr;
- state->EndRecPtr = targetPagePtr + pageHeaderSize
- + MAXALIGN(pageHeader->xlp_rem_len);
- }
- else
- {
- /* Wait for the record data to become available */
- while (XLogNeedData(state, targetPagePtr,
- Min(targetRecOff + total_len, XLOG_BLCKSZ), true))
+ case XLREAD_FIRST_FRAGMENT:
{
- if (!state->routine.page_read(state, state->readPagePtr,
- state->readLen,
- state->ReadRecPtr, state->readBuf))
+ uint32 total_len = state->recordRemainLen;
+ uint32 request_len;
+ uint32 record_len;
+ XLogRecPtr targetPagePtr;
+ uint32 targetRecOff;
+
+ /*
+ * Wait for the rest of the record on the first page to become
+ * available
+ */
+ targetPagePtr =
+ state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
+ targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+
+ request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ);
+ record_len = request_len - targetRecOff;
+
+ /* ReadRecPtr contains page header */
+ Assert (targetRecOff != 0);
+ if (XLogNeedData(state, targetPagePtr, request_len, true))
+ return XLREAD_NEED_DATA;
+
+ /* error out if caller supplied bogus page */
+ if (!state->page_verified)
+ goto err;
+
+ prec = (XLogRecord *) (state->readBuf + targetRecOff);
+
+ /* validate record header if not yet */
+ if (!state->record_verified && record_len >= SizeOfXLogRecord)
+ {
+ if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
+ state->PrevRecPtr, prec))
+ goto err;
+
+ state->record_verified = true;
+ }
+
+
+ if (total_len == record_len)
+ {
+ /* Record does not cross a page boundary */
+ Assert(state->record_verified);
+
+ if (!ValidXLogRecord(state, prec, state->ReadRecPtr))
+ goto err;
+
+ state->record_verified = true; /* to be tidy */
+
+ /* We already checked the header earlier */
+ state->EndRecPtr = state->ReadRecPtr + MAXALIGN(record_len);
+
+ *record = prec;
+ state->readRecordState = XLREAD_NEXT_RECORD;
break;
+ }
+
+ /*
+ * The record continues on the next page. Need to reassemble
+ * record
+ */
+ Assert(total_len > record_len);
+
+ /* Enlarge readRecordBuf as needed. */
+ if (total_len > state->readRecordBufSize &&
+ !allocate_recordbuf(state, total_len))
+ {
+ /* We treat this as a "bogus data" condition */
+ report_invalid_record(state,
+ "record length %u at %X/%X too long",
+ total_len,
+ LSN_FORMAT_ARGS(state->ReadRecPtr));
+ goto err;
+ }
+
+ /* Copy the first fragment of the record from the first page. */
+ memcpy(state->readRecordBuf, state->readBuf + targetRecOff,
+ record_len);
+ state->recordGotLen += record_len;
+ state->recordRemainLen -= record_len;
+
+ /* Calculate pointer to beginning of next page */
+ state->recordContRecPtr = state->ReadRecPtr + record_len;
+ Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0);
+
+ state->readRecordState = XLREAD_CONTINUATION;
}
+ /* fall through */
+
+ case XLREAD_CONTINUATION:
+ {
+ XLogPageHeader pageHeader;
+ uint32 pageHeaderSize;
+ XLogRecPtr targetPagePtr;
+
+ /* we enter this state only if we haven't read the whole record. */
+ Assert (state->recordRemainLen > 0);
+
+ while(state->recordRemainLen > 0)
+ {
+ char *contdata;
+ uint32 request_len;
+ uint32 record_len;
+
+ /* Wait for the next page to become available */
+ targetPagePtr = state->recordContRecPtr;
+
+ /* this request contains page header */
+ Assert (targetPagePtr != 0);
+ if (XLogNeedData(state, targetPagePtr,
+ Min(state->recordRemainLen, XLOG_BLCKSZ),
+ false))
+ return XLREAD_NEED_DATA;
+
+ if (!state->page_verified)
+ goto err;
+
+ Assert(SizeOfXLogShortPHD <= state->readLen);
- if (!state->page_verified)
- goto err;
+ /* Check that the continuation on next page looks valid */
+ pageHeader = (XLogPageHeader) state->readBuf;
+ if (!(pageHeader->xlp_info & XLP_FIRST_IS_CONTRECORD))
+ {
+ report_invalid_record(
+ state,
+ "there is no contrecord flag at %X/%X reading %X/%X",
+ (uint32) (state->recordContRecPtr >> 32),
+ (uint32) state->recordContRecPtr,
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr);
+ goto err;
+ }
- /* Record does not cross a page boundary */
- if (!ValidXLogRecord(state, record, RecPtr))
- goto err;
+ /*
+ * Cross-check that xlp_rem_len agrees with how much of the
+ * record we expect there to be left.
+ */
+ if (pageHeader->xlp_rem_len == 0 ||
+ pageHeader->xlp_rem_len != state->recordRemainLen)
+ {
+ report_invalid_record(
+ state,
+ "invalid contrecord length %u at %X/%X reading %X/%X, expected %u",
+ pageHeader->xlp_rem_len,
+ (uint32) (state->recordContRecPtr >> 32),
+ (uint32) state->recordContRecPtr,
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr,
+ state->recordRemainLen);
+ goto err;
+ }
- state->EndRecPtr = RecPtr + MAXALIGN(total_len);
+ /* Append the continuation from this page to the buffer */
+ pageHeaderSize = XLogPageHeaderSize(pageHeader);
- state->ReadRecPtr = RecPtr;
+ /*
+ * XLogNeedData should have ensured that the whole page header
+ * was read
+ */
+ Assert(state->readLen >= pageHeaderSize);
+
+ contdata = (char *) state->readBuf + pageHeaderSize;
+ record_len = XLOG_BLCKSZ - pageHeaderSize;
+ if (pageHeader->xlp_rem_len < record_len)
+ record_len = pageHeader->xlp_rem_len;
+
+ request_len = record_len + pageHeaderSize;
+
+ /* XLogNeedData should have ensured all needed data was read */
+ Assert (state->readLen >= request_len);
+
+ memcpy(state->readRecordBuf + state->recordGotLen,
+ (char *) contdata, record_len);
+ state->recordGotLen += record_len;
+ state->recordRemainLen -= record_len;
+
+ /* If we just reassembled the record header, validate it. */
+ if (!state->record_verified)
+ {
+ Assert(state->recordGotLen >= SizeOfXLogRecord);
+ if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
+ state->PrevRecPtr,
+ (XLogRecord *) state->readRecordBuf))
+ goto err;
+
+ state->record_verified = true;
+ }
+
+ /* Calculate pointer to beginning of next page, and continue */
+ state->recordContRecPtr += XLOG_BLCKSZ;
+ }
+
+ /* targetPagePtr is pointing the last-read page here */
+ prec = (XLogRecord *) state->readRecordBuf;
+ if (!ValidXLogRecord(state, prec, state->ReadRecPtr))
+ goto err;
+
+ pageHeaderSize =
+ XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+ state->EndRecPtr = targetPagePtr + pageHeaderSize
+ + MAXALIGN(pageHeader->xlp_rem_len);
+
+ *record = prec;
+ state->readRecordState = XLREAD_NEXT_RECORD;
+ break;
+ }
}
/*
* Special processing if it's an XLOG SWITCH record
*/
- if (record->xl_rmid == RM_XLOG_ID &&
- (record->xl_info & ~XLR_INFO_MASK) == XLOG_SWITCH)
+ if ((*record)->xl_rmid == RM_XLOG_ID &&
+ ((*record)->xl_info & ~XLR_INFO_MASK) == XLOG_SWITCH)
{
/* Pretend it extends to end of segment */
state->EndRecPtr += state->segcxt.ws_segsize - 1;
state->EndRecPtr -= XLogSegmentOffset(state->EndRecPtr, state->segcxt.ws_segsize);
}
- if (DecodeXLogRecord(state, record, errormsg))
- return record;
- else
- return NULL;
+ Assert (!*record || state->readLen >= 0);
+ if (DecodeXLogRecord(state, *record, errormsg))
+ return XLREAD_SUCCESS;
+
+ *record = NULL;
+ return XLREAD_FAIL;
err:
/*
- * Invalidate the read state. We might read from a different source after
+ * Invalidate the read page. We might read from a different source after
* failure.
*/
XLogReaderInvalReadState(state);
@@ -572,7 +703,8 @@ err:
if (state->errormsg_buf[0] != '\0')
*errormsg = state->errormsg_buf;
- return NULL;
+ *record = NULL;
+ return XLREAD_FAIL;
}
/*
@@ -724,11 +856,12 @@ XLogReaderInvalReadState(XLogReaderState *state)
*
* This is just a convenience subroutine to avoid duplicated code in
* XLogReadRecord. It's not intended for use from anywhere else.
+ *
+ * If PrevRecPtr is valid, the xl_prev is is cross-checked with it.
*/
static bool
ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
- XLogRecPtr PrevRecPtr, XLogRecord *record,
- bool randAccess)
+ XLogRecPtr PrevRecPtr, XLogRecord *record)
{
if (record->xl_tot_len < SizeOfXLogRecord)
{
@@ -745,7 +878,7 @@ ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
record->xl_rmid, LSN_FORMAT_ARGS(RecPtr));
return false;
}
- if (randAccess)
+ if (PrevRecPtr == InvalidXLogRecPtr)
{
/*
* We can't exactly verify the prev-link, but surely it should be less
@@ -975,11 +1108,14 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
* XLogReadRecord() will read the next valid record.
*/
XLogRecPtr
-XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
+XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
+ XLogFindNextRecordCB read_page, void *private)
{
XLogRecPtr tmpRecPtr;
XLogRecPtr found = InvalidXLogRecPtr;
XLogPageHeader header;
+ XLogRecord *record;
+ XLogReadRecordResult result;
char *errormsg;
Assert(!XLogRecPtrIsInvalid(RecPtr));
@@ -1012,9 +1148,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
while(XLogNeedData(state, targetPagePtr, targetRecOff,
targetRecOff != 0))
{
- if (!state->routine.page_read(state, state->readPagePtr,
- state->readLen,
- state->ReadRecPtr, state->readBuf))
+ if (!read_page(state, private))
break;
}
@@ -1066,8 +1200,16 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
* or we just jumped over the remaining data of a continuation.
*/
XLogBeginRead(state, tmpRecPtr);
- while (XLogReadRecord(state, &errormsg) != NULL)
+ while ((result = XLogReadRecord(state, &record, &errormsg)) !=
+ XLREAD_FAIL)
{
+ if (result == XLREAD_NEED_DATA)
+ {
+ if (!read_page(state, private))
+ goto err;
+ continue;
+ }
+
/* past the record we've found, break out */
if (RecPtr <= state->ReadRecPtr)
{
@@ -1087,9 +1229,9 @@ err:
#endif /* FRONTEND */
/*
- * Helper function to ease writing of XLogRoutine->page_read callbacks.
- * If this function is used, caller must supply a segment_open callback in
- * 'state', as that is used here.
+ * Helper function to ease writing of page_read callback.
+ * If this function is used, caller must supply a segment_open callback and
+ * segment_close callback as that is used here.
*
* Read 'count' bytes into 'buf', starting at location 'startptr', from WAL
* fetched from timeline 'tli'.
@@ -1102,6 +1244,7 @@ err:
*/
bool
WALRead(XLogReaderState *state,
+ WALSegmentOpenCB segopenfn, WALSegmentCloseCB segclosefn,
char *buf, XLogRecPtr startptr, Size count, TimeLineID tli,
WALReadError *errinfo)
{
@@ -1133,10 +1276,10 @@ WALRead(XLogReaderState *state,
XLogSegNo nextSegNo;
if (state->seg.ws_file >= 0)
- state->routine.segment_close(state);
+ segclosefn(state);
XLByteToSeg(recptr, nextSegNo, state->segcxt.ws_segsize);
- state->routine.segment_open(state, nextSegNo, &tli);
+ segopenfn(state, nextSegNo, &tli);
/* This shouldn't happen -- indicates a bug in segment_open */
Assert(state->seg.ws_file >= 0);
diff --git a/src/backend/access/transam/xlogutils.c b/src/backend/access/transam/xlogutils.c
index 46eda33f25..b003990745 100644
--- a/src/backend/access/transam/xlogutils.c
+++ b/src/backend/access/transam/xlogutils.c
@@ -686,8 +686,7 @@ XLogTruncateRelation(RelFileNode rnode, ForkNumber forkNum,
void
XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wantLength)
{
- const XLogRecPtr lastReadPage = state->seg.ws_segno *
- state->segcxt.ws_segsize + state->readLen;
+ const XLogRecPtr lastReadPage = state->readPagePtr;
Assert(wantPage != InvalidXLogRecPtr && wantPage % XLOG_BLCKSZ == 0);
Assert(wantLength <= XLOG_BLCKSZ);
@@ -702,7 +701,7 @@ XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wa
* current TLI has since become historical.
*/
if (lastReadPage == wantPage &&
- state->readLen != 0 &&
+ state->page_verified &&
lastReadPage + state->readLen >= wantPage + Min(wantLength, XLOG_BLCKSZ - 1))
return;
@@ -788,6 +787,7 @@ wal_segment_open(XLogReaderState *state, XLogSegNo nextSegNo,
char path[MAXPGPATH];
XLogFilePath(path, tli, nextSegNo, state->segcxt.ws_segsize);
+ elog(LOG, "HOGE: %lu, %d => %s", nextSegNo, tli, path);
state->seg.ws_file = BasicOpenFile(path, O_RDONLY | PG_BINARY);
if (state->seg.ws_file >= 0)
return;
@@ -825,9 +825,11 @@ wal_segment_close(XLogReaderState *state)
* loop for now.
*/
bool
-read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *cur_page)
+read_local_xlog_page(XLogReaderState *state)
{
+ XLogRecPtr targetPagePtr = state->readPagePtr;
+ int reqLen = state->readLen;
+ char *cur_page = state->readBuf;
XLogRecPtr read_upto,
loc;
TimeLineID tli;
@@ -940,11 +942,12 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
* as 'count', read the whole page anyway. It's guaranteed to be
* zero-padded up to the page boundary if it's incomplete.
*/
- if (!WALRead(state, cur_page, targetPagePtr, XLOG_BLCKSZ, tli,
- &errinfo))
+ if (!WALRead(state, wal_segment_open, wal_segment_close,
+ cur_page, targetPagePtr, XLOG_BLCKSZ, tli, &errinfo))
WALReadRaiseError(&errinfo);
/* number of valid bytes in the buffer */
+ state->readPagePtr = targetPagePtr;
state->readLen = count;
return true;
}
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index 37b75deb72..9cbd8afa49 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -148,7 +148,8 @@ StartupDecodingContext(List *output_plugin_options,
TransactionId xmin_horizon,
bool need_full_snapshot,
bool fast_forward,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress)
@@ -198,11 +199,12 @@ StartupDecodingContext(List *output_plugin_options,
ctx->slot = slot;
- ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, xl_routine, ctx);
+ ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, cleanup_cb);
if (!ctx->reader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
errmsg("out of memory")));
+ ctx->page_read = page_read;
ctx->reorder = ReorderBufferAllocate();
ctx->snapshot_builder =
@@ -319,7 +321,8 @@ CreateInitDecodingContext(const char *plugin,
List *output_plugin_options,
bool need_full_snapshot,
XLogRecPtr restart_lsn,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress)
@@ -422,7 +425,7 @@ CreateInitDecodingContext(const char *plugin,
ctx = StartupDecodingContext(NIL, restart_lsn, xmin_horizon,
need_full_snapshot, false,
- xl_routine, prepare_write, do_write,
+ page_read, cleanup_cb, prepare_write, do_write,
update_progress);
/* call output plugin initialization callback */
@@ -476,7 +479,8 @@ LogicalDecodingContext *
CreateDecodingContext(XLogRecPtr start_lsn,
List *output_plugin_options,
bool fast_forward,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress)
@@ -528,8 +532,8 @@ CreateDecodingContext(XLogRecPtr start_lsn,
ctx = StartupDecodingContext(output_plugin_options,
start_lsn, InvalidTransactionId, false,
- fast_forward, xl_routine, prepare_write,
- do_write, update_progress);
+ fast_forward, page_read, cleanup_cb,
+ prepare_write, do_write, update_progress);
/* call output plugin initialization callback */
old_context = MemoryContextSwitchTo(ctx->context);
@@ -585,7 +589,13 @@ DecodingContextFindStartpoint(LogicalDecodingContext *ctx)
char *err = NULL;
/* the read_page callback waits for new WAL */
- record = XLogReadRecord(ctx->reader, &err);
+ while (XLogReadRecord(ctx->reader, &record, &err) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!ctx->page_read(ctx->reader))
+ break;
+ }
+
if (err)
elog(ERROR, "%s", err);
if (!record)
diff --git a/src/backend/replication/logical/logicalfuncs.c b/src/backend/replication/logical/logicalfuncs.c
index f7e055874e..b69f6911c5 100644
--- a/src/backend/replication/logical/logicalfuncs.c
+++ b/src/backend/replication/logical/logicalfuncs.c
@@ -233,9 +233,8 @@ pg_logical_slot_get_changes_guts(FunctionCallInfo fcinfo, bool confirm, bool bin
ctx = CreateDecodingContext(InvalidXLogRecPtr,
options,
false,
- XL_ROUTINE(.page_read = read_local_xlog_page,
- .segment_open = wal_segment_open,
- .segment_close = wal_segment_close),
+ read_local_xlog_page,
+ wal_segment_close,
LogicalOutputPrepareWrite,
LogicalOutputWrite, NULL);
@@ -284,7 +283,13 @@ pg_logical_slot_get_changes_guts(FunctionCallInfo fcinfo, bool confirm, bool bin
XLogRecord *record;
char *errm = NULL;
- record = XLogReadRecord(ctx->reader, &errm);
+ while (XLogReadRecord(ctx->reader, &record, &errm) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!ctx->page_read(ctx->reader))
+ break;
+ }
+
if (errm)
elog(ERROR, "%s", errm);
diff --git a/src/backend/replication/slotfuncs.c b/src/backend/replication/slotfuncs.c
index 9817b44113..deb7cf15eb 100644
--- a/src/backend/replication/slotfuncs.c
+++ b/src/backend/replication/slotfuncs.c
@@ -153,9 +153,8 @@ create_logical_replication_slot(char *name, char *plugin,
ctx = CreateInitDecodingContext(plugin, NIL,
false, /* just catalogs is OK */
restart_lsn,
- XL_ROUTINE(.page_read = read_local_xlog_page,
- .segment_open = wal_segment_open,
- .segment_close = wal_segment_close),
+ read_local_xlog_page,
+ wal_segment_close,
NULL, NULL, NULL);
/*
@@ -512,9 +511,8 @@ pg_logical_replication_slot_advance(XLogRecPtr moveto)
ctx = CreateDecodingContext(InvalidXLogRecPtr,
NIL,
true, /* fast_forward */
- XL_ROUTINE(.page_read = read_local_xlog_page,
- .segment_open = wal_segment_open,
- .segment_close = wal_segment_close),
+ read_local_xlog_page,
+ wal_segment_close,
NULL, NULL, NULL);
/*
@@ -536,7 +534,13 @@ pg_logical_replication_slot_advance(XLogRecPtr moveto)
* Read records. No changes are generated in fast_forward mode,
* but snapbuilder/slot statuses are updated properly.
*/
- record = XLogReadRecord(ctx->reader, &errm);
+ while (XLogReadRecord(ctx->reader, &record, &errm) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!ctx->page_read(ctx->reader))
+ break;
+ }
+
if (errm)
elog(ERROR, "%s", errm);
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index c36b91ae5e..062093727d 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -580,10 +580,7 @@ StartReplication(StartReplicationCmd *cmd)
/* create xlogreader for physical replication */
xlogreader =
- XLogReaderAllocate(wal_segment_size, NULL,
- XL_ROUTINE(.segment_open = WalSndSegmentOpen,
- .segment_close = wal_segment_close),
- NULL);
+ XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
if (!xlogreader)
ereport(ERROR,
@@ -807,9 +804,11 @@ StartReplication(StartReplicationCmd *cmd)
* set every time WAL is flushed.
*/
static bool
-logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetRecPtr, char *cur_page)
+logical_read_xlog_page(XLogReaderState *state)
{
+ XLogRecPtr targetPagePtr = state->readPagePtr;
+ int reqLen = state->readLen;
+ char *cur_page = state->readBuf;
XLogRecPtr flushptr;
int count;
WALReadError errinfo;
@@ -837,7 +836,7 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
count = flushptr - targetPagePtr; /* part of the page available */
/* now actually read the data, we know it's there */
- if (!WALRead(state,
+ if (!WALRead(state, WalSndSegmentOpen, wal_segment_close,
cur_page,
targetPagePtr,
XLOG_BLCKSZ,
@@ -1011,9 +1010,8 @@ CreateReplicationSlot(CreateReplicationSlotCmd *cmd)
ctx = CreateInitDecodingContext(cmd->plugin, NIL, need_full_snapshot,
InvalidXLogRecPtr,
- XL_ROUTINE(.page_read = logical_read_xlog_page,
- .segment_open = WalSndSegmentOpen,
- .segment_close = wal_segment_close),
+ logical_read_xlog_page,
+ wal_segment_close,
WalSndPrepareWrite, WalSndWriteData,
WalSndUpdateProgress);
@@ -1171,9 +1169,8 @@ StartLogicalReplication(StartReplicationCmd *cmd)
*/
logical_decoding_ctx =
CreateDecodingContext(cmd->startpoint, cmd->options, false,
- XL_ROUTINE(.page_read = logical_read_xlog_page,
- .segment_open = WalSndSegmentOpen,
- .segment_close = wal_segment_close),
+ logical_read_xlog_page,
+ wal_segment_close,
WalSndPrepareWrite, WalSndWriteData,
WalSndUpdateProgress);
xlogreader = logical_decoding_ctx->reader;
@@ -2749,7 +2746,7 @@ XLogSendPhysical(void)
enlargeStringInfo(&output_message, nbytes);
retry:
- if (!WALRead(xlogreader,
+ if (!WALRead(xlogreader, WalSndSegmentOpen, wal_segment_close,
&output_message.data[output_message.len],
startptr,
nbytes,
@@ -2847,7 +2844,12 @@ XLogSendLogical(void)
*/
WalSndCaughtUp = false;
- record = XLogReadRecord(logical_decoding_ctx->reader, &errm);
+ while (XLogReadRecord(logical_decoding_ctx->reader, &record, &errm) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!logical_decoding_ctx->page_read(logical_decoding_ctx->reader))
+ break;
+ }
/* xlog record was invalid */
if (errm != NULL)
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index cf119848b0..712c85281c 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -41,15 +41,9 @@ static int xlogreadfd = -1;
static XLogSegNo xlogreadsegno = -1;
static char xlogfpath[MAXPGPATH];
-typedef struct XLogPageReadPrivate
-{
- const char *restoreCommand;
- int tliIndex;
-} XLogPageReadPrivate;
-
-static bool SimpleXLogPageRead(XLogReaderState *xlogreader,
- XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
+static bool SimpleXLogPageRead(XLogReaderState *xlogreader,
+ const char *datadir, int *tliIndex,
+ const char *restoreCommand);
/*
* Read WAL from the datadir/pg_wal, starting from 'startpoint' on timeline
@@ -66,20 +60,22 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
XLogRecord *record;
XLogReaderState *xlogreader;
char *errormsg;
- XLogPageReadPrivate private;
- private.tliIndex = tliIndex;
- private.restoreCommand = restoreCommand;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir,
- XL_ROUTINE(.page_read = &SimpleXLogPageRead),
- &private);
+ xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+
if (xlogreader == NULL)
pg_fatal("out of memory");
XLogBeginRead(xlogreader, startpoint);
do
{
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!SimpleXLogPageRead(xlogreader, datadir,
+ &tliIndex, restoreCommand))
+ break;
+ }
if (record == NULL)
{
@@ -123,19 +119,19 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex,
XLogRecord *record;
XLogReaderState *xlogreader;
char *errormsg;
- XLogPageReadPrivate private;
XLogRecPtr endptr;
- private.tliIndex = tliIndex;
- private.restoreCommand = restoreCommand;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir,
- XL_ROUTINE(.page_read = &SimpleXLogPageRead),
- &private);
+ xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
XLogBeginRead(xlogreader, ptr);
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!SimpleXLogPageRead(xlogreader, datadir, &tliIndex, restoreCommand))
+ break;
+ }
if (record == NULL)
{
if (errormsg)
@@ -170,7 +166,6 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
XLogRecPtr searchptr;
XLogReaderState *xlogreader;
char *errormsg;
- XLogPageReadPrivate private;
/*
* The given fork pointer points to the end of the last common record,
@@ -186,11 +181,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
forkptr += SizeOfXLogShortPHD;
}
- private.tliIndex = tliIndex;
- private.restoreCommand = restoreCommand;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir,
- XL_ROUTINE(.page_read = &SimpleXLogPageRead),
- &private);
+ xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
@@ -200,7 +191,13 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
uint8 info;
XLogBeginRead(xlogreader, searchptr);
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!SimpleXLogPageRead(xlogreader, datadir,
+ &tliIndex, restoreCommand))
+ break;
+ }
if (record == NULL)
{
@@ -247,10 +244,11 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
/* XLogReader callback function, to read a WAL page */
static bool
-SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *readBuf)
+SimpleXLogPageRead(XLogReaderState *xlogreader, const char *datadir,
+ int *tliIndex, const char *restoreCommand)
{
- XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
+ XLogRecPtr targetPagePtr = xlogreader->readPagePtr;
+ char *readBuf = xlogreader->readBuf;
uint32 targetPageOff;
XLogRecPtr targetSegEnd;
XLogSegNo targetSegNo;
@@ -283,14 +281,14 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
* be done both forward and backward, consider also switching timeline
* accordingly.
*/
- while (private->tliIndex < targetNentries - 1 &&
- targetHistory[private->tliIndex].end < targetSegEnd)
- private->tliIndex++;
- while (private->tliIndex > 0 &&
- targetHistory[private->tliIndex].begin >= targetSegEnd)
- private->tliIndex--;
+ while (*tliIndex < targetNentries - 1 &&
+ targetHistory[*tliIndex].end < targetSegEnd)
+ (*tliIndex)++;
+ while (*tliIndex > 0 &&
+ targetHistory[*tliIndex].begin >= targetSegEnd)
+ (*tliIndex)--;
- XLogFileName(xlogfname, targetHistory[private->tliIndex].tli,
+ XLogFileName(xlogfname, targetHistory[*tliIndex].tli,
xlogreadsegno, WalSegSz);
snprintf(xlogfpath, MAXPGPATH, "%s/" XLOGDIR "/%s",
@@ -303,7 +301,7 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
/*
* If we have no restore_command to execute, then exit.
*/
- if (private->restoreCommand == NULL)
+ if (restoreCommand == NULL)
{
pg_log_error("could not open file \"%s\": %m", xlogfpath);
xlogreader->readLen = -1;
@@ -317,7 +315,7 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
xlogreadfd = RestoreArchivedFile(xlogreader->segcxt.ws_dir,
xlogfname,
WalSegSz,
- private->restoreCommand);
+ restoreCommand);
if (xlogreadfd < 0)
{
@@ -359,7 +357,7 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
Assert(targetSegNo == xlogreadsegno);
- xlogreader->seg.ws_tli = targetHistory[private->tliIndex].tli;
+ xlogreader->seg.ws_tli = targetHistory[*tliIndex].tli;
xlogreader->readLen = XLOG_BLCKSZ;
return true;
}
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index 0f0e0723a3..f514de6b13 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -330,12 +330,17 @@ WALDumpCloseSegment(XLogReaderState *state)
state->seg.ws_file = -1;
}
-/* pg_waldump's XLogReaderRoutine->page_read callback */
+/*
+ * pg_waldump's WAL page rader, also used as page_read callback for
+ * XLogFindNextRecord
+ */
static bool
-WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetPtr, char *readBuff)
+WALDumpReadPage(XLogReaderState *state, void *priv)
{
- XLogDumpPrivate *private = state->private_data;
+ XLogRecPtr targetPagePtr = state->readPagePtr;
+ int reqLen = state->readLen;
+ char *readBuff = state->readBuf;
+ XLogDumpPrivate *private = (XLogDumpPrivate *) priv;
int count = XLOG_BLCKSZ;
WALReadError errinfo;
@@ -353,8 +358,8 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
}
}
- if (!WALRead(state, readBuff, targetPagePtr, count, private->timeline,
- &errinfo))
+ if (!WALRead(state, WALDumpOpenSegment, WALDumpCloseSegment,
+ readBuff, targetPagePtr, count, private->timeline, &errinfo))
{
WALOpenSegment *seg = &errinfo.wre_seg;
char fname[MAXPGPATH];
@@ -374,6 +379,7 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
(Size) errinfo.wre_req);
}
+ Assert(count >= state->readLen);
state->readLen = count;
return true;
}
@@ -1035,16 +1041,14 @@ main(int argc, char **argv)
/* we have everything we need, start reading */
xlogreader_state =
- XLogReaderAllocate(WalSegSz, waldir,
- XL_ROUTINE(.page_read = WALDumpReadPage,
- .segment_open = WALDumpOpenSegment,
- .segment_close = WALDumpCloseSegment),
- &private);
+ XLogReaderAllocate(WalSegSz, waldir, WALDumpCloseSegment);
+
if (!xlogreader_state)
fatal_error("out of memory");
/* first find a valid recptr to start from */
- first_record = XLogFindNextRecord(xlogreader_state, private.startptr);
+ first_record = XLogFindNextRecord(xlogreader_state, private.startptr,
+ &WALDumpReadPage, (void*) &private);
if (first_record == InvalidXLogRecPtr)
fatal_error("could not find a valid record after %X/%X",
@@ -1067,7 +1071,13 @@ main(int argc, char **argv)
for (;;)
{
/* try to read the next record */
- record = XLogReadRecord(xlogreader_state, &errormsg);
+ while (XLogReadRecord(xlogreader_state, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!WALDumpReadPage(xlogreader_state, (void *) &private))
+ break;
+ }
+
if (!record)
{
if (!config.follow || private.endptr_reached)
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index 23d9e70a59..4e9268b553 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -57,64 +57,15 @@ typedef struct WALSegmentContext
typedef struct XLogReaderState XLogReaderState;
-/* Function type definition for the read_page callback */
-typedef bool (*XLogPageReadCB) (XLogReaderState *xlogreader,
- XLogRecPtr targetPagePtr,
- int reqLen,
- XLogRecPtr targetRecPtr,
- char *readBuf);
+/* Function type definition for the segment cleanup callback */
+typedef void (*WALSegmentCleanupCB) (XLogReaderState *xlogreader);
+
+/* Function type definition for the open/close callbacks for WALRead() */
typedef void (*WALSegmentOpenCB) (XLogReaderState *xlogreader,
XLogSegNo nextSegNo,
TimeLineID *tli_p);
typedef void (*WALSegmentCloseCB) (XLogReaderState *xlogreader);
-typedef struct XLogReaderRoutine
-{
- /*
- * Data input callback
- *
- * This callback shall read at least reqLen valid bytes of the xlog page
- * starting at targetPagePtr, and store them in readBuf. The callback
- * shall return the number of bytes read (never more than XLOG_BLCKSZ), or
- * -1 on failure. The callback shall sleep, if necessary, to wait for the
- * requested bytes to become available. The callback will not be invoked
- * again for the same page unless more than the returned number of bytes
- * are needed.
- *
- * targetRecPtr is the position of the WAL record we're reading. Usually
- * it is equal to targetPagePtr + reqLen, but sometimes xlogreader needs
- * to read and verify the page or segment header, before it reads the
- * actual WAL record it's interested in. In that case, targetRecPtr can
- * be used to determine which timeline to read the page from.
- *
- * The callback shall set ->seg.ws_tli to the TLI of the file the page was
- * read from.
- */
- XLogPageReadCB page_read;
-
- /*
- * Callback to open the specified WAL segment for reading. ->seg.ws_file
- * shall be set to the file descriptor of the opened segment. In case of
- * failure, an error shall be raised by the callback and it shall not
- * return.
- *
- * "nextSegNo" is the number of the segment to be opened.
- *
- * "tli_p" is an input/output argument. WALRead() uses it to pass the
- * timeline in which the new segment should be found, but the callback can
- * use it to return the TLI that it actually opened.
- */
- WALSegmentOpenCB segment_open;
-
- /*
- * WAL segment close callback. ->seg.ws_file shall be set to a negative
- * number.
- */
- WALSegmentCloseCB segment_close;
-} XLogReaderRoutine;
-
-#define XL_ROUTINE(...) &(XLogReaderRoutine){__VA_ARGS__}
-
typedef struct
{
/* Is this block ref in use? */
@@ -144,12 +95,35 @@ typedef struct
uint16 data_bufsz;
} DecodedBkpBlock;
+/* Return code from XLogReadRecord */
+typedef enum XLogReadRecordResult
+{
+ XLREAD_SUCCESS, /* record is successfully read */
+ XLREAD_NEED_DATA, /* need more data. see XLogReadRecord. */
+ XLREAD_FAIL /* failed during reading a record */
+} XLogReadRecordResult;
+
+/*
+ * internal state of XLogReadRecord
+ *
+ * XLogReadState runs a state machine while reading a record. Theses states
+ * are not seen outside the function. Each state may repeat several times
+ * exiting requesting caller for new data. See the comment of XLogReadRecrod
+ * for details.
+ */
+typedef enum XLogReadRecordState {
+ XLREAD_NEXT_RECORD,
+ XLREAD_TOT_LEN,
+ XLREAD_FIRST_FRAGMENT,
+ XLREAD_CONTINUATION
+} XLogReadRecordState;
+
struct XLogReaderState
{
/*
* Operational callbacks
*/
- XLogReaderRoutine routine;
+ WALSegmentCleanupCB cleanup_cb;
/* ----------------------------------------
* Public parameters
@@ -162,18 +136,14 @@ struct XLogReaderState
*/
uint64 system_identifier;
- /*
- * Opaque data for callbacks to use. Not used by XLogReader.
- */
- void *private_data;
-
/*
* Start and end point of last record read. EndRecPtr is also used as the
* position to read next. Calling XLogBeginRead() sets EndRecPtr to the
* starting position and ReadRecPtr to invalid.
*/
- XLogRecPtr ReadRecPtr; /* start of last record read */
+ XLogRecPtr ReadRecPtr; /* start of last record read or being read */
XLogRecPtr EndRecPtr; /* end+1 of last record read */
+ XLogRecPtr PrevRecPtr; /* start of previous record read */
/* ----------------------------------------
* Communication with page reader
@@ -187,7 +157,9 @@ struct XLogReaderState
* the request, or -1 on error */
TimeLineID readPageTLI; /* TLI for data currently in readBuf */
char *readBuf; /* buffer to store data */
- bool page_verified; /* is the page on the buffer verified? */
+ bool page_verified; /* is the page header on the buffer verified? */
+ bool record_verified;/* is the current record header verified? */
+
/* ----------------------------------------
@@ -229,8 +201,6 @@ struct XLogReaderState
XLogRecPtr latestPagePtr;
TimeLineID latestPageTLI;
- /* beginning of the WAL record being read. */
- XLogRecPtr currRecPtr;
/* timeline to read it from, 0 if a lookup is required */
TimeLineID currTLI;
@@ -257,6 +227,15 @@ struct XLogReaderState
char *readRecordBuf;
uint32 readRecordBufSize;
+ /*
+ * XLogReadRecord() state
+ */
+ XLogReadRecordState readRecordState;/* state machine state */
+ int recordGotLen; /* amount of current record that has
+ * already been read */
+ int recordRemainLen; /* length of current record that remains */
+ XLogRecPtr recordContRecPtr; /* where the current record continues */
+
/* Buffer to hold error message */
char *errormsg_buf;
};
@@ -264,9 +243,7 @@ struct XLogReaderState
/* Get a new XLogReader */
extern XLogReaderState *XLogReaderAllocate(int wal_segment_size,
const char *waldir,
- XLogReaderRoutine *routine,
- void *private_data);
-extern XLogReaderRoutine *LocalXLogReaderRoutine(void);
+ WALSegmentCleanupCB cleanup_cb);
/* Free an XLogReader */
extern void XLogReaderFree(XLogReaderState *state);
@@ -274,12 +251,17 @@ extern void XLogReaderFree(XLogReaderState *state);
/* Position the XLogReader to given record */
extern void XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr);
#ifdef FRONTEND
-extern XLogRecPtr XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr);
+/* Function type definition for the read_page callback */
+typedef bool (*XLogFindNextRecordCB) (XLogReaderState *xlogreader,
+ void *private);
+extern XLogRecPtr XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
+ XLogFindNextRecordCB read_page, void *private);
#endif /* FRONTEND */
/* Read the next XLog record. Returns NULL on end-of-WAL or failure */
-extern struct XLogRecord *XLogReadRecord(XLogReaderState *state,
- char **errormsg);
+extern XLogReadRecordResult XLogReadRecord(XLogReaderState *state,
+ XLogRecord **record,
+ char **errormsg);
/* Validate a page */
extern bool XLogReaderValidatePageHeader(XLogReaderState *state,
@@ -299,6 +281,7 @@ typedef struct WALReadError
} WALReadError;
extern bool WALRead(XLogReaderState *state,
+ WALSegmentOpenCB segopenfn, WALSegmentCloseCB sgclosefn,
char *buf, XLogRecPtr startptr, Size count,
TimeLineID tli, WALReadError *errinfo);
diff --git a/src/include/access/xlogutils.h b/src/include/access/xlogutils.h
index 364a21c4ea..397fb27fc2 100644
--- a/src/include/access/xlogutils.h
+++ b/src/include/access/xlogutils.h
@@ -47,9 +47,7 @@ extern Buffer XLogReadBufferExtended(RelFileNode rnode, ForkNumber forknum,
extern Relation CreateFakeRelcacheEntry(RelFileNode rnode);
extern void FreeFakeRelcacheEntry(Relation fakerel);
-extern bool read_local_xlog_page(XLogReaderState *state,
- XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetRecPtr, char *cur_page);
+extern bool read_local_xlog_page(XLogReaderState *state);
extern void wal_segment_open(XLogReaderState *state,
XLogSegNo nextSegNo,
TimeLineID *tli_p);
diff --git a/src/include/pg_config_manual.h b/src/include/pg_config_manual.h
index f10ad0acd6..94749609b4 100644
--- a/src/include/pg_config_manual.h
+++ b/src/include/pg_config_manual.h
@@ -377,7 +377,7 @@
* Enable debugging print statements for WAL-related operations; see
* also the wal_debug GUC var.
*/
-/* #define WAL_DEBUG */
+#define WAL_DEBUG
/*
* Enable tracing of resource consumption during sort operations;
diff --git a/src/include/replication/logical.h b/src/include/replication/logical.h
index c253403372..b936f31393 100644
--- a/src/include/replication/logical.h
+++ b/src/include/replication/logical.h
@@ -29,6 +29,10 @@ typedef void (*LogicalOutputPluginWriterUpdateProgress) (struct LogicalDecodingC
TransactionId xid
);
+typedef struct LogicalDecodingContext LogicalDecodingContext;
+
+typedef bool (*LogicalDecodingXLogPageReadCB)(XLogReaderState *ctx);
+
typedef struct LogicalDecodingContext
{
/* memory context this is all allocated in */
@@ -39,6 +43,7 @@ typedef struct LogicalDecodingContext
/* infrastructure pieces for decoding */
XLogReaderState *reader;
+ LogicalDecodingXLogPageReadCB page_read;
struct ReorderBuffer *reorder;
struct SnapBuild *snapshot_builder;
@@ -105,14 +110,16 @@ extern LogicalDecodingContext *CreateInitDecodingContext(const char *plugin,
List *output_plugin_options,
bool need_full_snapshot,
XLogRecPtr restart_lsn,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress);
extern LogicalDecodingContext *CreateDecodingContext(XLogRecPtr start_lsn,
List *output_plugin_options,
bool fast_forward,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress);
--
2.27.0
----Next_Part(Thu_Mar__4_11_28_53_2021_014)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
filename="v17-0003-Remove-globals-readOff-readLen-and-readSegNo.patch"
^ permalink raw reply [nested|flat] 70+ messages in thread
* [PATCH v17 2/4] Move page-reader out of XLogReadRecord
@ 2019-09-10 03:58 Kyotaro Horiguchi <[email protected]>
0 siblings, 0 replies; 70+ messages in thread
From: Kyotaro Horiguchi @ 2019-09-10 03:58 UTC (permalink / raw)
This is the second step of removing callbacks from WAL record reader.
Since it is essential to take in additional data while reading a
record, the function have to ask caller for new data while keeping
working state. Thus the function is turned into a state machine.
---
src/backend/access/transam/twophase.c | 15 +-
src/backend/access/transam/xlog.c | 58 +-
src/backend/access/transam/xlogreader.c | 697 +++++++++++-------
src/backend/access/transam/xlogutils.c | 17 +-
src/backend/replication/logical/logical.c | 26 +-
.../replication/logical/logicalfuncs.c | 13 +-
src/backend/replication/slotfuncs.c | 18 +-
src/backend/replication/walsender.c | 32 +-
src/bin/pg_rewind/parsexlog.c | 84 ++-
src/bin/pg_waldump/pg_waldump.c | 36 +-
src/include/access/xlogreader.h | 121 ++-
src/include/access/xlogutils.h | 4 +-
src/include/pg_config_manual.h | 2 +-
src/include/replication/logical.h | 11 +-
14 files changed, 646 insertions(+), 488 deletions(-)
diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index 80d2d20d6c..7d5a96a5f5 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -1325,11 +1325,8 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
XLogReaderState *xlogreader;
char *errormsg;
- xlogreader = XLogReaderAllocate(wal_segment_size, NULL,
- XL_ROUTINE(.page_read = &read_local_xlog_page,
- .segment_open = &wal_segment_open,
- .segment_close = &wal_segment_close),
- NULL);
+ xlogreader = XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+
if (!xlogreader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
@@ -1337,7 +1334,13 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
errdetail("Failed while allocating a WAL reading processor.")));
XLogBeginRead(xlogreader, lsn);
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!read_local_xlog_page(xlogreader))
+ break;
+ }
+
if (record == NULL)
ereport(ERROR,
(errcode_for_file_access(),
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 606328a65a..670996ae67 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -836,13 +836,6 @@ static XLogSource currentSource = XLOG_FROM_ANY;
static bool lastSourceFailed = false;
static bool pendingWalRcvRestart = false;
-typedef struct XLogPageReadPrivate
-{
- int emode;
- bool fetching_ckpt; /* are we fetching a checkpoint record? */
- bool randAccess;
-} XLogPageReadPrivate;
-
/*
* These variables track when we last obtained some WAL data to process,
* and where we got it from. (XLogReceiptSource is initially the same as
@@ -917,8 +910,8 @@ static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath,
static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli,
XLogSource source, bool notfoundOk);
static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, XLogSource source);
-static bool XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
+static bool XLogPageRead(XLogReaderState *xlogreader,
+ bool fetching_ckpt, int emode, bool randAccess);
static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
bool fetching_ckpt, XLogRecPtr tliRecPtr);
static int emode_for_corrupt_record(int emode, XLogRecPtr RecPtr);
@@ -1231,8 +1224,7 @@ XLogInsertRecord(XLogRecData *rdata,
appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len);
if (!debug_reader)
- debug_reader = XLogReaderAllocate(wal_segment_size, NULL,
- XL_ROUTINE(), NULL);
+ debug_reader = XLogReaderAllocate(wal_segment_size, NULL, NULL);
if (!debug_reader)
{
@@ -4344,15 +4336,10 @@ CleanupBackupHistory(void)
* record is available.
*/
static XLogRecord *
-ReadRecord(XLogReaderState *xlogreader, int emode,
- bool fetching_ckpt)
+ReadRecord(XLogReaderState *xlogreader, int emode, bool fetching_ckpt)
{
XLogRecord *record;
- XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
-
- private->fetching_ckpt = fetching_ckpt;
- private->emode = emode;
- private->randAccess = (xlogreader->ReadRecPtr == InvalidXLogRecPtr);
+ bool randAccess = (xlogreader->ReadRecPtr == InvalidXLogRecPtr);
/* This is the first attempt to read this page. */
lastSourceFailed = false;
@@ -4360,8 +4347,16 @@ ReadRecord(XLogReaderState *xlogreader, int emode,
for (;;)
{
char *errormsg;
+ XLogReadRecordResult result;
+
+ while ((result = XLogReadRecord(xlogreader, &record, &errormsg))
+ == XLREAD_NEED_DATA)
+ {
+ if (!XLogPageRead(xlogreader, fetching_ckpt, emode, randAccess))
+ break;
+
+ }
- record = XLogReadRecord(xlogreader, &errormsg);
ReadRecPtr = xlogreader->ReadRecPtr;
EndRecPtr = xlogreader->EndRecPtr;
if (record == NULL)
@@ -6380,7 +6375,6 @@ StartupXLOG(void)
bool backupFromStandby = false;
DBState dbstate_at_startup;
XLogReaderState *xlogreader;
- XLogPageReadPrivate private;
bool promoted = false;
struct stat st;
@@ -6539,13 +6533,9 @@ StartupXLOG(void)
OwnLatch(&XLogCtl->recoveryWakeupLatch);
/* Set up XLOG reader facility */
- MemSet(&private, 0, sizeof(XLogPageReadPrivate));
xlogreader =
- XLogReaderAllocate(wal_segment_size, NULL,
- XL_ROUTINE(.page_read = &XLogPageRead,
- .segment_open = NULL,
- .segment_close = wal_segment_close),
- &private);
+ XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+
if (!xlogreader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
@@ -11987,12 +11977,13 @@ CancelBackup(void)
* sleep and retry.
*/
static bool
-XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetRecPtr, char *readBuf)
+XLogPageRead(XLogReaderState *xlogreader,
+ bool fetching_ckpt, int emode, bool randAccess)
{
- XLogPageReadPrivate *private =
- (XLogPageReadPrivate *) xlogreader->private_data;
- int emode = private->emode;
+ char *readBuf = xlogreader->readBuf;
+ XLogRecPtr targetPagePtr = xlogreader->readPagePtr;
+ int reqLen = xlogreader->readLen;
+ XLogRecPtr targetRecPtr = xlogreader->ReadRecPtr;
uint32 targetPageOff;
XLogSegNo targetSegNo PG_USED_FOR_ASSERTS_ONLY;
int r;
@@ -12035,8 +12026,8 @@ retry:
flushedUpto < targetPagePtr + reqLen))
{
if (!WaitForWALToBecomeAvailable(targetPagePtr + reqLen,
- private->randAccess,
- private->fetching_ckpt,
+ randAccess,
+ fetching_ckpt,
targetRecPtr))
{
if (readFile >= 0)
@@ -12141,6 +12132,7 @@ retry:
goto next_record_is_invalid;
}
+ Assert(xlogreader->readPagePtr == targetPagePtr);
xlogreader->readLen = readLen;
return true;
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index 7d04995b7b..ba3e4868f7 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -40,7 +40,7 @@ static bool XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr,
int reqLen, bool header_inclusive);
static void XLogReaderInvalReadState(XLogReaderState *state);
static bool ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
- XLogRecPtr PrevRecPtr, XLogRecord *record, bool randAccess);
+ XLogRecPtr PrevRecPtr, XLogRecord *record);
static bool ValidXLogRecord(XLogReaderState *state, XLogRecord *record,
XLogRecPtr recptr);
static void ResetDecoder(XLogReaderState *state);
@@ -73,7 +73,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...)
*/
XLogReaderState *
XLogReaderAllocate(int wal_segment_size, const char *waldir,
- XLogReaderRoutine *routine, void *private_data)
+ WALSegmentCleanupCB cleanup_cb)
{
XLogReaderState *state;
@@ -84,7 +84,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
return NULL;
/* initialize caller-provided support functions */
- state->routine = *routine;
+ state->cleanup_cb = cleanup_cb;
state->max_block_id = -1;
@@ -107,8 +107,6 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
WALOpenSegmentInit(&state->seg, &state->segcxt, wal_segment_size,
waldir);
- /* system_identifier initialized to zeroes above */
- state->private_data = private_data;
/* ReadRecPtr, EndRecPtr and readLen initialized to zeroes above */
state->errormsg_buf = palloc_extended(MAX_ERRORMSG_LEN + 1,
MCXT_ALLOC_NO_OOM);
@@ -140,8 +138,8 @@ XLogReaderFree(XLogReaderState *state)
{
int block_id;
- if (state->seg.ws_file != -1)
- state->routine.segment_close(state);
+ if (state->seg.ws_file >= 0)
+ state->cleanup_cb(state);
for (block_id = 0; block_id <= XLR_MAX_BLOCK_ID; block_id++)
{
@@ -246,6 +244,7 @@ XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
/* Begin at the passed-in record pointer. */
state->EndRecPtr = RecPtr;
state->ReadRecPtr = InvalidXLogRecPtr;
+ state->readRecordState = XLREAD_NEXT_RECORD;
}
/*
@@ -254,317 +253,449 @@ XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
* XLogBeginRead() or XLogFindNextRecord() must be called before the first call
* to XLogReadRecord().
*
- * If the page_read callback fails to read the requested data, NULL is
- * returned. The callback is expected to have reported the error; errormsg
- * is set to NULL.
+ * This function may return XLREAD_NEED_DATA several times before returning a
+ * result record. The caller shall read in some new data then call this
+ * function again with the same parameters.
*
- * If the reading fails for some other reason, NULL is also returned, and
- * *errormsg is set to a string with details of the failure.
+ * When a record is successfully read, returns XLREAD_SUCCESS with result
+ * record being stored in *record. Otherwise *record is NULL.
*
- * The returned pointer (or *errormsg) points to an internal buffer that's
- * valid until the next call to XLogReadRecord.
+ * Returns XLREAD_NEED_DATA if more data is needed to finish reading the
+ * current record. In that case, state->readPagePtr and state->readLen inform
+ * the desired position and minimum length of data needed. The caller shall
+ * read in the requested data and set state->readBuf to point to a buffer
+ * containing it. The caller must also set state->readPageTLI and
+ * state->readLen to indicate the timeline that it was read from, and the
+ * length of data that is now available (which must be >= given readLen),
+ * respectively.
+ *
+ * If invalid data is encountered, returns XLREAD_FAIL with *record being set to
+ * NULL. *errormsg is set to a string with details of the failure.
+ * The returned pointer (or *errormsg) points to an internal buffer that's valid
+ * until the next call to XLogReadRecord.
+ *
+ *
+ * This function runs a state machine consists of the following states.
+ *
+ * XLREAD_NEXT_RECORD :
+ * The initial state, if called with valid RecPtr, try to read a record at
+ * that position. If invalid RecPtr is given try to read a record just after
+ * the last one previously read.
+ * This state ens after setting ReadRecPtr. Then goes to XLREAD_TOT_LEN.
+ *
+ * XLREAD_TOT_LEN:
+ * Examining record header. Ends after reading record total
+ * length. recordRemainLen and recordGotLen are initialized.
+ *
+ * XLREAD_FIRST_FRAGMENT:
+ * Reading the first fragment. Ends with finishing reading a single
+ * record. Goes to XLREAD_NEXT_RECORD if that's all or
+ * XLREAD_CONTINUATION if we have continuation.
+
+ * XLREAD_CONTINUATION:
+ * Reading continuation of record. Ends with finishing the whole record then
+ * goes to XLREAD_NEXT_RECORD. During this state, recordRemainLen indicates
+ * how much is left and readRecordBuf holds the partially assert
+ * record.recordContRecPtr points to the beginning of the next page where to
+ * continue.
+ *
+ * If wrong data found in any state, the state machine stays at the current
+ * state. This behavior allows to continue reading a reacord switching among
+ * different souces, while streaming replication.
*/
-XLogRecord *
-XLogReadRecord(XLogReaderState *state, char **errormsg)
+XLogReadRecordResult
+XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
{
- XLogRecPtr RecPtr;
- XLogRecord *record;
- XLogRecPtr targetPagePtr;
- bool randAccess;
- uint32 len,
- total_len;
- uint32 targetRecOff;
- uint32 pageHeaderSize;
- bool gotheader;
+ XLogRecord *prec;
- /*
- * randAccess indicates whether to verify the previous-record pointer of
- * the record we're reading. We only do this if we're reading
- * sequentially, which is what we initially assume.
- */
- randAccess = false;
+ *record = NULL;
/* reset error state */
*errormsg = NULL;
state->errormsg_buf[0] = '\0';
- ResetDecoder(state);
-
- RecPtr = state->EndRecPtr;
-
- if (state->ReadRecPtr != InvalidXLogRecPtr)
- {
- /* read the record after the one we just read */
-
- /*
- * EndRecPtr is pointing to end+1 of the previous WAL record. If
- * we're at a page boundary, no more records can fit on the current
- * page. We must skip over the page header, but we can't do that until
- * we've read in the page, since the header size is variable.
- */
- }
- else
- {
- /*
- * Caller supplied a position to start at.
- *
- * In this case, EndRecPtr should already be pointing to a valid
- * record starting position.
- */
- Assert(XRecOffIsValid(RecPtr));
- randAccess = true;
- }
-
- state->currRecPtr = RecPtr;
-
- targetPagePtr = RecPtr - (RecPtr % XLOG_BLCKSZ);
- targetRecOff = RecPtr % XLOG_BLCKSZ;
-
- /*
- * Read the page containing the record into state->readBuf. Request enough
- * byte to cover the whole record header, or at least the part of it that
- * fits on the same page.
- */
- while (XLogNeedData(state, targetPagePtr,
- Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
- targetRecOff != 0))
- {
- if (!state->routine.page_read(state, state->readPagePtr, state->readLen,
- RecPtr, state->readBuf))
- break;
- }
-
- if (!state->page_verified)
- goto err;
-
- /*
- * We have at least the page header, so we can examine it now.
- */
- pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
- if (targetRecOff == 0)
- {
- /*
- * At page start, so skip over page header.
- */
- RecPtr += pageHeaderSize;
- targetRecOff = pageHeaderSize;
- }
- else if (targetRecOff < pageHeaderSize)
- {
- report_invalid_record(state, "invalid record offset at %X/%X",
- LSN_FORMAT_ARGS(RecPtr));
- goto err;
- }
-
- if ((((XLogPageHeader) state->readBuf)->xlp_info & XLP_FIRST_IS_CONTRECORD) &&
- targetRecOff == pageHeaderSize)
- {
- report_invalid_record(state, "contrecord is requested by %X/%X",
- LSN_FORMAT_ARGS(RecPtr));
- goto err;
- }
-
- /* XLogNeedData has verified the page header */
- Assert(pageHeaderSize <= state->readLen);
-
- /*
- * Read the record length.
- *
- * NB: Even though we use an XLogRecord pointer here, the whole record
- * header might not fit on this page. xl_tot_len is the first field of the
- * struct, so it must be on this page (the records are MAXALIGNed), but we
- * cannot access any other fields until we've verified that we got the
- * whole header.
- */
- record = (XLogRecord *) (state->readBuf + RecPtr % XLOG_BLCKSZ);
- total_len = record->xl_tot_len;
-
- /*
- * If the whole record header is on this page, validate it immediately.
- * Otherwise do just a basic sanity check on xl_tot_len, and validate the
- * rest of the header after reading it from the next page. The xl_tot_len
- * check is necessary here to ensure that we enter the "Need to reassemble
- * record" code path below; otherwise we might fail to apply
- * ValidXLogRecordHeader at all.
- */
- if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
- {
- if (!ValidXLogRecordHeader(state, RecPtr, state->ReadRecPtr, record,
- randAccess))
- goto err;
- gotheader = true;
- }
- else
- {
- /* XXX: more validation should be done here */
- if (total_len < SizeOfXLogRecord)
- {
- report_invalid_record(state,
- "invalid record length at %X/%X: wanted %u, got %u",
- LSN_FORMAT_ARGS(RecPtr),
- (uint32) SizeOfXLogRecord, total_len);
- goto err;
- }
- gotheader = false;
- }
-
- len = XLOG_BLCKSZ - RecPtr % XLOG_BLCKSZ;
- if (total_len > len)
+ switch (state->readRecordState)
{
- /* Need to reassemble record */
- char *contdata;
- XLogPageHeader pageHeader;
- char *buffer;
- uint32 gotlen;
-
- /*
- * Enlarge readRecordBuf as needed.
- */
- if (total_len > state->readRecordBufSize &&
- !allocate_recordbuf(state, total_len))
- {
- /* We treat this as a "bogus data" condition */
- report_invalid_record(state, "record length %u at %X/%X too long",
- total_len, LSN_FORMAT_ARGS(RecPtr));
- goto err;
- }
+ case XLREAD_NEXT_RECORD:
+ ResetDecoder(state);
- /* Copy the first fragment of the record from the first page. */
- memcpy(state->readRecordBuf,
- state->readBuf + RecPtr % XLOG_BLCKSZ, len);
- buffer = state->readRecordBuf + len;
- gotlen = len;
-
- do
- {
- int rest_len = total_len - gotlen;
-
- /* Calculate pointer to beginning of next page */
- targetPagePtr += XLOG_BLCKSZ;
-
- /* Wait for the next page to become available */
- while (XLogNeedData(state, targetPagePtr,
- Min(rest_len, XLOG_BLCKSZ),
- false))
+ if (state->ReadRecPtr != InvalidXLogRecPtr)
{
- if (!state->routine.page_read(state, state->readPagePtr,
- state->readLen,
- state->ReadRecPtr,
- state->readBuf))
- break;
+ /* read the record after the one we just read */
+
+ /*
+ * EndRecPtr is pointing to end+1 of the previous WAL record.
+ * If we're at a page boundary, no more records can fit on the
+ * current page. We must skip over the page header, but we
+ * can't do that until we've read in the page, since the header
+ * size is variable.
+ */
+ state->PrevRecPtr = state->ReadRecPtr;
+ state->ReadRecPtr = state->EndRecPtr;
+ }
+ else
+ {
+ /*
+ * Caller supplied a position to start at.
+ *
+ * In this case, EndRecPtr should already be pointing to a
+ * valid record starting position.
+ */
+ Assert(XRecOffIsValid(state->EndRecPtr));
+ state->ReadRecPtr = state->EndRecPtr;
+
+ /*
+ * We cannot verify the previous-record pointer when we're
+ * seeking to a particular record. Reset PrevRecPtr so that we
+ * won't try doing that.
+ */
+ state->PrevRecPtr = InvalidXLogRecPtr;
+ state->EndRecPtr = InvalidXLogRecPtr; /* to be tidy */
}
+ state->record_verified = false;
+ state->readRecordState = XLREAD_TOT_LEN;
+ /* fall through */
+
+ case XLREAD_TOT_LEN:
+ {
+ uint32 total_len;
+ uint32 pageHeaderSize;
+ XLogRecPtr targetPagePtr;
+ uint32 targetRecOff;
+ XLogPageHeader pageHeader;
+
+ targetPagePtr =
+ state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
+ targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+
+ /*
+ * Check if we have enough data. For the first record in the page,
+ * the requesting length doesn't contain page header.
+ */
+ if (XLogNeedData(state, targetPagePtr,
+ Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+ targetRecOff != 0))
+ return XLREAD_NEED_DATA;
+
+ /* error out if caller supplied bogus page */
if (!state->page_verified)
goto err;
- Assert(SizeOfXLogShortPHD <= state->readLen);
+ /* examine page header now. */
+ pageHeaderSize =
+ XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+ if (targetRecOff == 0)
+ {
+ /* At page start, so skip over page header. */
+ state->ReadRecPtr += pageHeaderSize;
+ targetRecOff = pageHeaderSize;
+ }
+ else if (targetRecOff < pageHeaderSize)
+ {
+ report_invalid_record(state, "invalid record offset at %X/%X",
+ LSN_FORMAT_ARGS(state->ReadRecPtr));
+ goto err;
+ }
- /* Check that the continuation on next page looks valid */
pageHeader = (XLogPageHeader) state->readBuf;
- if (!(pageHeader->xlp_info & XLP_FIRST_IS_CONTRECORD))
+ if ((pageHeader->xlp_info & XLP_FIRST_IS_CONTRECORD) &&
+ targetRecOff == pageHeaderSize)
{
- report_invalid_record(state,
- "there is no contrecord flag at %X/%X",
- LSN_FORMAT_ARGS(RecPtr));
+ report_invalid_record(state, "contrecord is requested by %X/%X",
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr);
goto err;
}
- /*
- * Cross-check that xlp_rem_len agrees with how much of the record
- * we expect there to be left.
- */
- if (pageHeader->xlp_rem_len == 0 ||
- total_len != (pageHeader->xlp_rem_len + gotlen))
- {
- report_invalid_record(state,
- "invalid contrecord length %u (expected %lld) at %X/%X",
- pageHeader->xlp_rem_len,
- ((long long) total_len) - gotlen,
- LSN_FORMAT_ARGS(RecPtr));
- goto err;
- }
-
- /* Append the continuation from this page to the buffer */
- pageHeaderSize = XLogPageHeaderSize(pageHeader);
-
+ /* XLogNeedData has verified the page header */
Assert(pageHeaderSize <= state->readLen);
- contdata = (char *) state->readBuf + pageHeaderSize;
- len = XLOG_BLCKSZ - pageHeaderSize;
- if (pageHeader->xlp_rem_len < len)
- len = pageHeader->xlp_rem_len;
+ /*
+ * Read the record length.
+ *
+ * NB: Even though we use an XLogRecord pointer here, the whole
+ * record header might not fit on this page. xl_tot_len is the first
+ * field of the struct, so it must be on this page (the records are
+ * MAXALIGNed), but we cannot access any other fields until we've
+ * verified that we got the whole header.
+ */
+ prec = (XLogRecord *) (state->readBuf +
+ state->ReadRecPtr % XLOG_BLCKSZ);
+ total_len = prec->xl_tot_len;
- Assert (pageHeaderSize + len <= state->readLen);
- memcpy(buffer, (char *) contdata, len);
- buffer += len;
- gotlen += len;
+ /*
+ * If the whole record header is on this page, validate it
+ * immediately. Otherwise do just a basic sanity check on
+ * xl_tot_len, and validate the rest of the header after reading it
+ * from the next page. The xl_tot_len check is necessary here to
+ * ensure that we enter the XLREAD_CONTINUATION state below;
+ * otherwise we might fail to apply ValidXLogRecordHeader at all.
+ */
+ if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
+ {
+ if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
+ state->PrevRecPtr, prec))
+ goto err;
- /* If we just reassembled the record header, validate it. */
- if (!gotheader)
+ state->record_verified = true;
+ }
+ else
{
- record = (XLogRecord *) state->readRecordBuf;
- if (!ValidXLogRecordHeader(state, RecPtr, state->ReadRecPtr,
- record, randAccess))
+ /* XXX: more validation should be done here */
+ if (total_len < SizeOfXLogRecord)
+ {
+ report_invalid_record(state,
+ "invalid record length at %X/%X: wanted %u, got %u",
+ LSN_FORMAT_ARGS(state->ReadRecPtr),
+ (uint32) SizeOfXLogRecord, total_len);
goto err;
- gotheader = true;
+ }
}
- } while (gotlen < total_len);
-
- Assert(gotheader);
- record = (XLogRecord *) state->readRecordBuf;
- if (!ValidXLogRecord(state, record, RecPtr))
- goto err;
+ /*
+ * Wait for the rest of the record, or the part of the record that
+ * fit on the first page if crossed a page boundary, to become
+ * available.
+ */
+ state->recordGotLen = 0;
+ state->recordRemainLen = total_len;
+ state->readRecordState = XLREAD_FIRST_FRAGMENT;
+ }
+ /* fall through */
- pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
- state->ReadRecPtr = RecPtr;
- state->EndRecPtr = targetPagePtr + pageHeaderSize
- + MAXALIGN(pageHeader->xlp_rem_len);
- }
- else
- {
- /* Wait for the record data to become available */
- while (XLogNeedData(state, targetPagePtr,
- Min(targetRecOff + total_len, XLOG_BLCKSZ), true))
+ case XLREAD_FIRST_FRAGMENT:
{
- if (!state->routine.page_read(state, state->readPagePtr,
- state->readLen,
- state->ReadRecPtr, state->readBuf))
+ uint32 total_len = state->recordRemainLen;
+ uint32 request_len;
+ uint32 record_len;
+ XLogRecPtr targetPagePtr;
+ uint32 targetRecOff;
+
+ /*
+ * Wait for the rest of the record on the first page to become
+ * available
+ */
+ targetPagePtr =
+ state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
+ targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+
+ request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ);
+ record_len = request_len - targetRecOff;
+
+ /* ReadRecPtr contains page header */
+ Assert (targetRecOff != 0);
+ if (XLogNeedData(state, targetPagePtr, request_len, true))
+ return XLREAD_NEED_DATA;
+
+ /* error out if caller supplied bogus page */
+ if (!state->page_verified)
+ goto err;
+
+ prec = (XLogRecord *) (state->readBuf + targetRecOff);
+
+ /* validate record header if not yet */
+ if (!state->record_verified && record_len >= SizeOfXLogRecord)
+ {
+ if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
+ state->PrevRecPtr, prec))
+ goto err;
+
+ state->record_verified = true;
+ }
+
+
+ if (total_len == record_len)
+ {
+ /* Record does not cross a page boundary */
+ Assert(state->record_verified);
+
+ if (!ValidXLogRecord(state, prec, state->ReadRecPtr))
+ goto err;
+
+ state->record_verified = true; /* to be tidy */
+
+ /* We already checked the header earlier */
+ state->EndRecPtr = state->ReadRecPtr + MAXALIGN(record_len);
+
+ *record = prec;
+ state->readRecordState = XLREAD_NEXT_RECORD;
break;
+ }
+
+ /*
+ * The record continues on the next page. Need to reassemble
+ * record
+ */
+ Assert(total_len > record_len);
+
+ /* Enlarge readRecordBuf as needed. */
+ if (total_len > state->readRecordBufSize &&
+ !allocate_recordbuf(state, total_len))
+ {
+ /* We treat this as a "bogus data" condition */
+ report_invalid_record(state,
+ "record length %u at %X/%X too long",
+ total_len,
+ LSN_FORMAT_ARGS(state->ReadRecPtr));
+ goto err;
+ }
+
+ /* Copy the first fragment of the record from the first page. */
+ memcpy(state->readRecordBuf, state->readBuf + targetRecOff,
+ record_len);
+ state->recordGotLen += record_len;
+ state->recordRemainLen -= record_len;
+
+ /* Calculate pointer to beginning of next page */
+ state->recordContRecPtr = state->ReadRecPtr + record_len;
+ Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0);
+
+ state->readRecordState = XLREAD_CONTINUATION;
}
+ /* fall through */
+
+ case XLREAD_CONTINUATION:
+ {
+ XLogPageHeader pageHeader;
+ uint32 pageHeaderSize;
+ XLogRecPtr targetPagePtr;
+
+ /* we enter this state only if we haven't read the whole record. */
+ Assert (state->recordRemainLen > 0);
+
+ while(state->recordRemainLen > 0)
+ {
+ char *contdata;
+ uint32 request_len;
+ uint32 record_len;
+
+ /* Wait for the next page to become available */
+ targetPagePtr = state->recordContRecPtr;
+
+ /* this request contains page header */
+ Assert (targetPagePtr != 0);
+ if (XLogNeedData(state, targetPagePtr,
+ Min(state->recordRemainLen, XLOG_BLCKSZ),
+ false))
+ return XLREAD_NEED_DATA;
+
+ if (!state->page_verified)
+ goto err;
+
+ Assert(SizeOfXLogShortPHD <= state->readLen);
- if (!state->page_verified)
- goto err;
+ /* Check that the continuation on next page looks valid */
+ pageHeader = (XLogPageHeader) state->readBuf;
+ if (!(pageHeader->xlp_info & XLP_FIRST_IS_CONTRECORD))
+ {
+ report_invalid_record(
+ state,
+ "there is no contrecord flag at %X/%X reading %X/%X",
+ (uint32) (state->recordContRecPtr >> 32),
+ (uint32) state->recordContRecPtr,
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr);
+ goto err;
+ }
- /* Record does not cross a page boundary */
- if (!ValidXLogRecord(state, record, RecPtr))
- goto err;
+ /*
+ * Cross-check that xlp_rem_len agrees with how much of the
+ * record we expect there to be left.
+ */
+ if (pageHeader->xlp_rem_len == 0 ||
+ pageHeader->xlp_rem_len != state->recordRemainLen)
+ {
+ report_invalid_record(
+ state,
+ "invalid contrecord length %u at %X/%X reading %X/%X, expected %u",
+ pageHeader->xlp_rem_len,
+ (uint32) (state->recordContRecPtr >> 32),
+ (uint32) state->recordContRecPtr,
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr,
+ state->recordRemainLen);
+ goto err;
+ }
- state->EndRecPtr = RecPtr + MAXALIGN(total_len);
+ /* Append the continuation from this page to the buffer */
+ pageHeaderSize = XLogPageHeaderSize(pageHeader);
- state->ReadRecPtr = RecPtr;
+ /*
+ * XLogNeedData should have ensured that the whole page header
+ * was read
+ */
+ Assert(state->readLen >= pageHeaderSize);
+
+ contdata = (char *) state->readBuf + pageHeaderSize;
+ record_len = XLOG_BLCKSZ - pageHeaderSize;
+ if (pageHeader->xlp_rem_len < record_len)
+ record_len = pageHeader->xlp_rem_len;
+
+ request_len = record_len + pageHeaderSize;
+
+ /* XLogNeedData should have ensured all needed data was read */
+ Assert (state->readLen >= request_len);
+
+ memcpy(state->readRecordBuf + state->recordGotLen,
+ (char *) contdata, record_len);
+ state->recordGotLen += record_len;
+ state->recordRemainLen -= record_len;
+
+ /* If we just reassembled the record header, validate it. */
+ if (!state->record_verified)
+ {
+ Assert(state->recordGotLen >= SizeOfXLogRecord);
+ if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
+ state->PrevRecPtr,
+ (XLogRecord *) state->readRecordBuf))
+ goto err;
+
+ state->record_verified = true;
+ }
+
+ /* Calculate pointer to beginning of next page, and continue */
+ state->recordContRecPtr += XLOG_BLCKSZ;
+ }
+
+ /* targetPagePtr is pointing the last-read page here */
+ prec = (XLogRecord *) state->readRecordBuf;
+ if (!ValidXLogRecord(state, prec, state->ReadRecPtr))
+ goto err;
+
+ pageHeaderSize =
+ XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+ state->EndRecPtr = targetPagePtr + pageHeaderSize
+ + MAXALIGN(pageHeader->xlp_rem_len);
+
+ *record = prec;
+ state->readRecordState = XLREAD_NEXT_RECORD;
+ break;
+ }
}
/*
* Special processing if it's an XLOG SWITCH record
*/
- if (record->xl_rmid == RM_XLOG_ID &&
- (record->xl_info & ~XLR_INFO_MASK) == XLOG_SWITCH)
+ if ((*record)->xl_rmid == RM_XLOG_ID &&
+ ((*record)->xl_info & ~XLR_INFO_MASK) == XLOG_SWITCH)
{
/* Pretend it extends to end of segment */
state->EndRecPtr += state->segcxt.ws_segsize - 1;
state->EndRecPtr -= XLogSegmentOffset(state->EndRecPtr, state->segcxt.ws_segsize);
}
- if (DecodeXLogRecord(state, record, errormsg))
- return record;
- else
- return NULL;
+ Assert (!*record || state->readLen >= 0);
+ if (DecodeXLogRecord(state, *record, errormsg))
+ return XLREAD_SUCCESS;
+
+ *record = NULL;
+ return XLREAD_FAIL;
err:
/*
- * Invalidate the read state. We might read from a different source after
+ * Invalidate the read page. We might read from a different source after
* failure.
*/
XLogReaderInvalReadState(state);
@@ -572,7 +703,8 @@ err:
if (state->errormsg_buf[0] != '\0')
*errormsg = state->errormsg_buf;
- return NULL;
+ *record = NULL;
+ return XLREAD_FAIL;
}
/*
@@ -724,11 +856,12 @@ XLogReaderInvalReadState(XLogReaderState *state)
*
* This is just a convenience subroutine to avoid duplicated code in
* XLogReadRecord. It's not intended for use from anywhere else.
+ *
+ * If PrevRecPtr is valid, the xl_prev is is cross-checked with it.
*/
static bool
ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
- XLogRecPtr PrevRecPtr, XLogRecord *record,
- bool randAccess)
+ XLogRecPtr PrevRecPtr, XLogRecord *record)
{
if (record->xl_tot_len < SizeOfXLogRecord)
{
@@ -745,7 +878,7 @@ ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
record->xl_rmid, LSN_FORMAT_ARGS(RecPtr));
return false;
}
- if (randAccess)
+ if (PrevRecPtr == InvalidXLogRecPtr)
{
/*
* We can't exactly verify the prev-link, but surely it should be less
@@ -975,11 +1108,14 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
* XLogReadRecord() will read the next valid record.
*/
XLogRecPtr
-XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
+XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
+ XLogFindNextRecordCB read_page, void *private)
{
XLogRecPtr tmpRecPtr;
XLogRecPtr found = InvalidXLogRecPtr;
XLogPageHeader header;
+ XLogRecord *record;
+ XLogReadRecordResult result;
char *errormsg;
Assert(!XLogRecPtrIsInvalid(RecPtr));
@@ -1012,9 +1148,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
while(XLogNeedData(state, targetPagePtr, targetRecOff,
targetRecOff != 0))
{
- if (!state->routine.page_read(state, state->readPagePtr,
- state->readLen,
- state->ReadRecPtr, state->readBuf))
+ if (!read_page(state, private))
break;
}
@@ -1066,8 +1200,16 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
* or we just jumped over the remaining data of a continuation.
*/
XLogBeginRead(state, tmpRecPtr);
- while (XLogReadRecord(state, &errormsg) != NULL)
+ while ((result = XLogReadRecord(state, &record, &errormsg)) !=
+ XLREAD_FAIL)
{
+ if (result == XLREAD_NEED_DATA)
+ {
+ if (!read_page(state, private))
+ goto err;
+ continue;
+ }
+
/* past the record we've found, break out */
if (RecPtr <= state->ReadRecPtr)
{
@@ -1087,9 +1229,9 @@ err:
#endif /* FRONTEND */
/*
- * Helper function to ease writing of XLogRoutine->page_read callbacks.
- * If this function is used, caller must supply a segment_open callback in
- * 'state', as that is used here.
+ * Helper function to ease writing of page_read callback.
+ * If this function is used, caller must supply a segment_open callback and
+ * segment_close callback as that is used here.
*
* Read 'count' bytes into 'buf', starting at location 'startptr', from WAL
* fetched from timeline 'tli'.
@@ -1102,6 +1244,7 @@ err:
*/
bool
WALRead(XLogReaderState *state,
+ WALSegmentOpenCB segopenfn, WALSegmentCloseCB segclosefn,
char *buf, XLogRecPtr startptr, Size count, TimeLineID tli,
WALReadError *errinfo)
{
@@ -1133,10 +1276,10 @@ WALRead(XLogReaderState *state,
XLogSegNo nextSegNo;
if (state->seg.ws_file >= 0)
- state->routine.segment_close(state);
+ segclosefn(state);
XLByteToSeg(recptr, nextSegNo, state->segcxt.ws_segsize);
- state->routine.segment_open(state, nextSegNo, &tli);
+ segopenfn(state, nextSegNo, &tli);
/* This shouldn't happen -- indicates a bug in segment_open */
Assert(state->seg.ws_file >= 0);
diff --git a/src/backend/access/transam/xlogutils.c b/src/backend/access/transam/xlogutils.c
index 46eda33f25..b003990745 100644
--- a/src/backend/access/transam/xlogutils.c
+++ b/src/backend/access/transam/xlogutils.c
@@ -686,8 +686,7 @@ XLogTruncateRelation(RelFileNode rnode, ForkNumber forkNum,
void
XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wantLength)
{
- const XLogRecPtr lastReadPage = state->seg.ws_segno *
- state->segcxt.ws_segsize + state->readLen;
+ const XLogRecPtr lastReadPage = state->readPagePtr;
Assert(wantPage != InvalidXLogRecPtr && wantPage % XLOG_BLCKSZ == 0);
Assert(wantLength <= XLOG_BLCKSZ);
@@ -702,7 +701,7 @@ XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wa
* current TLI has since become historical.
*/
if (lastReadPage == wantPage &&
- state->readLen != 0 &&
+ state->page_verified &&
lastReadPage + state->readLen >= wantPage + Min(wantLength, XLOG_BLCKSZ - 1))
return;
@@ -788,6 +787,7 @@ wal_segment_open(XLogReaderState *state, XLogSegNo nextSegNo,
char path[MAXPGPATH];
XLogFilePath(path, tli, nextSegNo, state->segcxt.ws_segsize);
+ elog(LOG, "HOGE: %lu, %d => %s", nextSegNo, tli, path);
state->seg.ws_file = BasicOpenFile(path, O_RDONLY | PG_BINARY);
if (state->seg.ws_file >= 0)
return;
@@ -825,9 +825,11 @@ wal_segment_close(XLogReaderState *state)
* loop for now.
*/
bool
-read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *cur_page)
+read_local_xlog_page(XLogReaderState *state)
{
+ XLogRecPtr targetPagePtr = state->readPagePtr;
+ int reqLen = state->readLen;
+ char *cur_page = state->readBuf;
XLogRecPtr read_upto,
loc;
TimeLineID tli;
@@ -940,11 +942,12 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
* as 'count', read the whole page anyway. It's guaranteed to be
* zero-padded up to the page boundary if it's incomplete.
*/
- if (!WALRead(state, cur_page, targetPagePtr, XLOG_BLCKSZ, tli,
- &errinfo))
+ if (!WALRead(state, wal_segment_open, wal_segment_close,
+ cur_page, targetPagePtr, XLOG_BLCKSZ, tli, &errinfo))
WALReadRaiseError(&errinfo);
/* number of valid bytes in the buffer */
+ state->readPagePtr = targetPagePtr;
state->readLen = count;
return true;
}
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index 37b75deb72..9cbd8afa49 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -148,7 +148,8 @@ StartupDecodingContext(List *output_plugin_options,
TransactionId xmin_horizon,
bool need_full_snapshot,
bool fast_forward,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress)
@@ -198,11 +199,12 @@ StartupDecodingContext(List *output_plugin_options,
ctx->slot = slot;
- ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, xl_routine, ctx);
+ ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, cleanup_cb);
if (!ctx->reader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
errmsg("out of memory")));
+ ctx->page_read = page_read;
ctx->reorder = ReorderBufferAllocate();
ctx->snapshot_builder =
@@ -319,7 +321,8 @@ CreateInitDecodingContext(const char *plugin,
List *output_plugin_options,
bool need_full_snapshot,
XLogRecPtr restart_lsn,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress)
@@ -422,7 +425,7 @@ CreateInitDecodingContext(const char *plugin,
ctx = StartupDecodingContext(NIL, restart_lsn, xmin_horizon,
need_full_snapshot, false,
- xl_routine, prepare_write, do_write,
+ page_read, cleanup_cb, prepare_write, do_write,
update_progress);
/* call output plugin initialization callback */
@@ -476,7 +479,8 @@ LogicalDecodingContext *
CreateDecodingContext(XLogRecPtr start_lsn,
List *output_plugin_options,
bool fast_forward,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress)
@@ -528,8 +532,8 @@ CreateDecodingContext(XLogRecPtr start_lsn,
ctx = StartupDecodingContext(output_plugin_options,
start_lsn, InvalidTransactionId, false,
- fast_forward, xl_routine, prepare_write,
- do_write, update_progress);
+ fast_forward, page_read, cleanup_cb,
+ prepare_write, do_write, update_progress);
/* call output plugin initialization callback */
old_context = MemoryContextSwitchTo(ctx->context);
@@ -585,7 +589,13 @@ DecodingContextFindStartpoint(LogicalDecodingContext *ctx)
char *err = NULL;
/* the read_page callback waits for new WAL */
- record = XLogReadRecord(ctx->reader, &err);
+ while (XLogReadRecord(ctx->reader, &record, &err) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!ctx->page_read(ctx->reader))
+ break;
+ }
+
if (err)
elog(ERROR, "%s", err);
if (!record)
diff --git a/src/backend/replication/logical/logicalfuncs.c b/src/backend/replication/logical/logicalfuncs.c
index f7e055874e..b69f6911c5 100644
--- a/src/backend/replication/logical/logicalfuncs.c
+++ b/src/backend/replication/logical/logicalfuncs.c
@@ -233,9 +233,8 @@ pg_logical_slot_get_changes_guts(FunctionCallInfo fcinfo, bool confirm, bool bin
ctx = CreateDecodingContext(InvalidXLogRecPtr,
options,
false,
- XL_ROUTINE(.page_read = read_local_xlog_page,
- .segment_open = wal_segment_open,
- .segment_close = wal_segment_close),
+ read_local_xlog_page,
+ wal_segment_close,
LogicalOutputPrepareWrite,
LogicalOutputWrite, NULL);
@@ -284,7 +283,13 @@ pg_logical_slot_get_changes_guts(FunctionCallInfo fcinfo, bool confirm, bool bin
XLogRecord *record;
char *errm = NULL;
- record = XLogReadRecord(ctx->reader, &errm);
+ while (XLogReadRecord(ctx->reader, &record, &errm) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!ctx->page_read(ctx->reader))
+ break;
+ }
+
if (errm)
elog(ERROR, "%s", errm);
diff --git a/src/backend/replication/slotfuncs.c b/src/backend/replication/slotfuncs.c
index 9817b44113..deb7cf15eb 100644
--- a/src/backend/replication/slotfuncs.c
+++ b/src/backend/replication/slotfuncs.c
@@ -153,9 +153,8 @@ create_logical_replication_slot(char *name, char *plugin,
ctx = CreateInitDecodingContext(plugin, NIL,
false, /* just catalogs is OK */
restart_lsn,
- XL_ROUTINE(.page_read = read_local_xlog_page,
- .segment_open = wal_segment_open,
- .segment_close = wal_segment_close),
+ read_local_xlog_page,
+ wal_segment_close,
NULL, NULL, NULL);
/*
@@ -512,9 +511,8 @@ pg_logical_replication_slot_advance(XLogRecPtr moveto)
ctx = CreateDecodingContext(InvalidXLogRecPtr,
NIL,
true, /* fast_forward */
- XL_ROUTINE(.page_read = read_local_xlog_page,
- .segment_open = wal_segment_open,
- .segment_close = wal_segment_close),
+ read_local_xlog_page,
+ wal_segment_close,
NULL, NULL, NULL);
/*
@@ -536,7 +534,13 @@ pg_logical_replication_slot_advance(XLogRecPtr moveto)
* Read records. No changes are generated in fast_forward mode,
* but snapbuilder/slot statuses are updated properly.
*/
- record = XLogReadRecord(ctx->reader, &errm);
+ while (XLogReadRecord(ctx->reader, &record, &errm) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!ctx->page_read(ctx->reader))
+ break;
+ }
+
if (errm)
elog(ERROR, "%s", errm);
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index c36b91ae5e..062093727d 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -580,10 +580,7 @@ StartReplication(StartReplicationCmd *cmd)
/* create xlogreader for physical replication */
xlogreader =
- XLogReaderAllocate(wal_segment_size, NULL,
- XL_ROUTINE(.segment_open = WalSndSegmentOpen,
- .segment_close = wal_segment_close),
- NULL);
+ XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
if (!xlogreader)
ereport(ERROR,
@@ -807,9 +804,11 @@ StartReplication(StartReplicationCmd *cmd)
* set every time WAL is flushed.
*/
static bool
-logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetRecPtr, char *cur_page)
+logical_read_xlog_page(XLogReaderState *state)
{
+ XLogRecPtr targetPagePtr = state->readPagePtr;
+ int reqLen = state->readLen;
+ char *cur_page = state->readBuf;
XLogRecPtr flushptr;
int count;
WALReadError errinfo;
@@ -837,7 +836,7 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
count = flushptr - targetPagePtr; /* part of the page available */
/* now actually read the data, we know it's there */
- if (!WALRead(state,
+ if (!WALRead(state, WalSndSegmentOpen, wal_segment_close,
cur_page,
targetPagePtr,
XLOG_BLCKSZ,
@@ -1011,9 +1010,8 @@ CreateReplicationSlot(CreateReplicationSlotCmd *cmd)
ctx = CreateInitDecodingContext(cmd->plugin, NIL, need_full_snapshot,
InvalidXLogRecPtr,
- XL_ROUTINE(.page_read = logical_read_xlog_page,
- .segment_open = WalSndSegmentOpen,
- .segment_close = wal_segment_close),
+ logical_read_xlog_page,
+ wal_segment_close,
WalSndPrepareWrite, WalSndWriteData,
WalSndUpdateProgress);
@@ -1171,9 +1169,8 @@ StartLogicalReplication(StartReplicationCmd *cmd)
*/
logical_decoding_ctx =
CreateDecodingContext(cmd->startpoint, cmd->options, false,
- XL_ROUTINE(.page_read = logical_read_xlog_page,
- .segment_open = WalSndSegmentOpen,
- .segment_close = wal_segment_close),
+ logical_read_xlog_page,
+ wal_segment_close,
WalSndPrepareWrite, WalSndWriteData,
WalSndUpdateProgress);
xlogreader = logical_decoding_ctx->reader;
@@ -2749,7 +2746,7 @@ XLogSendPhysical(void)
enlargeStringInfo(&output_message, nbytes);
retry:
- if (!WALRead(xlogreader,
+ if (!WALRead(xlogreader, WalSndSegmentOpen, wal_segment_close,
&output_message.data[output_message.len],
startptr,
nbytes,
@@ -2847,7 +2844,12 @@ XLogSendLogical(void)
*/
WalSndCaughtUp = false;
- record = XLogReadRecord(logical_decoding_ctx->reader, &errm);
+ while (XLogReadRecord(logical_decoding_ctx->reader, &record, &errm) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!logical_decoding_ctx->page_read(logical_decoding_ctx->reader))
+ break;
+ }
/* xlog record was invalid */
if (errm != NULL)
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index cf119848b0..712c85281c 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -41,15 +41,9 @@ static int xlogreadfd = -1;
static XLogSegNo xlogreadsegno = -1;
static char xlogfpath[MAXPGPATH];
-typedef struct XLogPageReadPrivate
-{
- const char *restoreCommand;
- int tliIndex;
-} XLogPageReadPrivate;
-
-static bool SimpleXLogPageRead(XLogReaderState *xlogreader,
- XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
+static bool SimpleXLogPageRead(XLogReaderState *xlogreader,
+ const char *datadir, int *tliIndex,
+ const char *restoreCommand);
/*
* Read WAL from the datadir/pg_wal, starting from 'startpoint' on timeline
@@ -66,20 +60,22 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
XLogRecord *record;
XLogReaderState *xlogreader;
char *errormsg;
- XLogPageReadPrivate private;
- private.tliIndex = tliIndex;
- private.restoreCommand = restoreCommand;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir,
- XL_ROUTINE(.page_read = &SimpleXLogPageRead),
- &private);
+ xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+
if (xlogreader == NULL)
pg_fatal("out of memory");
XLogBeginRead(xlogreader, startpoint);
do
{
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!SimpleXLogPageRead(xlogreader, datadir,
+ &tliIndex, restoreCommand))
+ break;
+ }
if (record == NULL)
{
@@ -123,19 +119,19 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex,
XLogRecord *record;
XLogReaderState *xlogreader;
char *errormsg;
- XLogPageReadPrivate private;
XLogRecPtr endptr;
- private.tliIndex = tliIndex;
- private.restoreCommand = restoreCommand;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir,
- XL_ROUTINE(.page_read = &SimpleXLogPageRead),
- &private);
+ xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
XLogBeginRead(xlogreader, ptr);
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!SimpleXLogPageRead(xlogreader, datadir, &tliIndex, restoreCommand))
+ break;
+ }
if (record == NULL)
{
if (errormsg)
@@ -170,7 +166,6 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
XLogRecPtr searchptr;
XLogReaderState *xlogreader;
char *errormsg;
- XLogPageReadPrivate private;
/*
* The given fork pointer points to the end of the last common record,
@@ -186,11 +181,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
forkptr += SizeOfXLogShortPHD;
}
- private.tliIndex = tliIndex;
- private.restoreCommand = restoreCommand;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir,
- XL_ROUTINE(.page_read = &SimpleXLogPageRead),
- &private);
+ xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
@@ -200,7 +191,13 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
uint8 info;
XLogBeginRead(xlogreader, searchptr);
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!SimpleXLogPageRead(xlogreader, datadir,
+ &tliIndex, restoreCommand))
+ break;
+ }
if (record == NULL)
{
@@ -247,10 +244,11 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
/* XLogReader callback function, to read a WAL page */
static bool
-SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *readBuf)
+SimpleXLogPageRead(XLogReaderState *xlogreader, const char *datadir,
+ int *tliIndex, const char *restoreCommand)
{
- XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
+ XLogRecPtr targetPagePtr = xlogreader->readPagePtr;
+ char *readBuf = xlogreader->readBuf;
uint32 targetPageOff;
XLogRecPtr targetSegEnd;
XLogSegNo targetSegNo;
@@ -283,14 +281,14 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
* be done both forward and backward, consider also switching timeline
* accordingly.
*/
- while (private->tliIndex < targetNentries - 1 &&
- targetHistory[private->tliIndex].end < targetSegEnd)
- private->tliIndex++;
- while (private->tliIndex > 0 &&
- targetHistory[private->tliIndex].begin >= targetSegEnd)
- private->tliIndex--;
+ while (*tliIndex < targetNentries - 1 &&
+ targetHistory[*tliIndex].end < targetSegEnd)
+ (*tliIndex)++;
+ while (*tliIndex > 0 &&
+ targetHistory[*tliIndex].begin >= targetSegEnd)
+ (*tliIndex)--;
- XLogFileName(xlogfname, targetHistory[private->tliIndex].tli,
+ XLogFileName(xlogfname, targetHistory[*tliIndex].tli,
xlogreadsegno, WalSegSz);
snprintf(xlogfpath, MAXPGPATH, "%s/" XLOGDIR "/%s",
@@ -303,7 +301,7 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
/*
* If we have no restore_command to execute, then exit.
*/
- if (private->restoreCommand == NULL)
+ if (restoreCommand == NULL)
{
pg_log_error("could not open file \"%s\": %m", xlogfpath);
xlogreader->readLen = -1;
@@ -317,7 +315,7 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
xlogreadfd = RestoreArchivedFile(xlogreader->segcxt.ws_dir,
xlogfname,
WalSegSz,
- private->restoreCommand);
+ restoreCommand);
if (xlogreadfd < 0)
{
@@ -359,7 +357,7 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
Assert(targetSegNo == xlogreadsegno);
- xlogreader->seg.ws_tli = targetHistory[private->tliIndex].tli;
+ xlogreader->seg.ws_tli = targetHistory[*tliIndex].tli;
xlogreader->readLen = XLOG_BLCKSZ;
return true;
}
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index 0f0e0723a3..f514de6b13 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -330,12 +330,17 @@ WALDumpCloseSegment(XLogReaderState *state)
state->seg.ws_file = -1;
}
-/* pg_waldump's XLogReaderRoutine->page_read callback */
+/*
+ * pg_waldump's WAL page rader, also used as page_read callback for
+ * XLogFindNextRecord
+ */
static bool
-WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetPtr, char *readBuff)
+WALDumpReadPage(XLogReaderState *state, void *priv)
{
- XLogDumpPrivate *private = state->private_data;
+ XLogRecPtr targetPagePtr = state->readPagePtr;
+ int reqLen = state->readLen;
+ char *readBuff = state->readBuf;
+ XLogDumpPrivate *private = (XLogDumpPrivate *) priv;
int count = XLOG_BLCKSZ;
WALReadError errinfo;
@@ -353,8 +358,8 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
}
}
- if (!WALRead(state, readBuff, targetPagePtr, count, private->timeline,
- &errinfo))
+ if (!WALRead(state, WALDumpOpenSegment, WALDumpCloseSegment,
+ readBuff, targetPagePtr, count, private->timeline, &errinfo))
{
WALOpenSegment *seg = &errinfo.wre_seg;
char fname[MAXPGPATH];
@@ -374,6 +379,7 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
(Size) errinfo.wre_req);
}
+ Assert(count >= state->readLen);
state->readLen = count;
return true;
}
@@ -1035,16 +1041,14 @@ main(int argc, char **argv)
/* we have everything we need, start reading */
xlogreader_state =
- XLogReaderAllocate(WalSegSz, waldir,
- XL_ROUTINE(.page_read = WALDumpReadPage,
- .segment_open = WALDumpOpenSegment,
- .segment_close = WALDumpCloseSegment),
- &private);
+ XLogReaderAllocate(WalSegSz, waldir, WALDumpCloseSegment);
+
if (!xlogreader_state)
fatal_error("out of memory");
/* first find a valid recptr to start from */
- first_record = XLogFindNextRecord(xlogreader_state, private.startptr);
+ first_record = XLogFindNextRecord(xlogreader_state, private.startptr,
+ &WALDumpReadPage, (void*) &private);
if (first_record == InvalidXLogRecPtr)
fatal_error("could not find a valid record after %X/%X",
@@ -1067,7 +1071,13 @@ main(int argc, char **argv)
for (;;)
{
/* try to read the next record */
- record = XLogReadRecord(xlogreader_state, &errormsg);
+ while (XLogReadRecord(xlogreader_state, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!WALDumpReadPage(xlogreader_state, (void *) &private))
+ break;
+ }
+
if (!record)
{
if (!config.follow || private.endptr_reached)
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index 23d9e70a59..4e9268b553 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -57,64 +57,15 @@ typedef struct WALSegmentContext
typedef struct XLogReaderState XLogReaderState;
-/* Function type definition for the read_page callback */
-typedef bool (*XLogPageReadCB) (XLogReaderState *xlogreader,
- XLogRecPtr targetPagePtr,
- int reqLen,
- XLogRecPtr targetRecPtr,
- char *readBuf);
+/* Function type definition for the segment cleanup callback */
+typedef void (*WALSegmentCleanupCB) (XLogReaderState *xlogreader);
+
+/* Function type definition for the open/close callbacks for WALRead() */
typedef void (*WALSegmentOpenCB) (XLogReaderState *xlogreader,
XLogSegNo nextSegNo,
TimeLineID *tli_p);
typedef void (*WALSegmentCloseCB) (XLogReaderState *xlogreader);
-typedef struct XLogReaderRoutine
-{
- /*
- * Data input callback
- *
- * This callback shall read at least reqLen valid bytes of the xlog page
- * starting at targetPagePtr, and store them in readBuf. The callback
- * shall return the number of bytes read (never more than XLOG_BLCKSZ), or
- * -1 on failure. The callback shall sleep, if necessary, to wait for the
- * requested bytes to become available. The callback will not be invoked
- * again for the same page unless more than the returned number of bytes
- * are needed.
- *
- * targetRecPtr is the position of the WAL record we're reading. Usually
- * it is equal to targetPagePtr + reqLen, but sometimes xlogreader needs
- * to read and verify the page or segment header, before it reads the
- * actual WAL record it's interested in. In that case, targetRecPtr can
- * be used to determine which timeline to read the page from.
- *
- * The callback shall set ->seg.ws_tli to the TLI of the file the page was
- * read from.
- */
- XLogPageReadCB page_read;
-
- /*
- * Callback to open the specified WAL segment for reading. ->seg.ws_file
- * shall be set to the file descriptor of the opened segment. In case of
- * failure, an error shall be raised by the callback and it shall not
- * return.
- *
- * "nextSegNo" is the number of the segment to be opened.
- *
- * "tli_p" is an input/output argument. WALRead() uses it to pass the
- * timeline in which the new segment should be found, but the callback can
- * use it to return the TLI that it actually opened.
- */
- WALSegmentOpenCB segment_open;
-
- /*
- * WAL segment close callback. ->seg.ws_file shall be set to a negative
- * number.
- */
- WALSegmentCloseCB segment_close;
-} XLogReaderRoutine;
-
-#define XL_ROUTINE(...) &(XLogReaderRoutine){__VA_ARGS__}
-
typedef struct
{
/* Is this block ref in use? */
@@ -144,12 +95,35 @@ typedef struct
uint16 data_bufsz;
} DecodedBkpBlock;
+/* Return code from XLogReadRecord */
+typedef enum XLogReadRecordResult
+{
+ XLREAD_SUCCESS, /* record is successfully read */
+ XLREAD_NEED_DATA, /* need more data. see XLogReadRecord. */
+ XLREAD_FAIL /* failed during reading a record */
+} XLogReadRecordResult;
+
+/*
+ * internal state of XLogReadRecord
+ *
+ * XLogReadState runs a state machine while reading a record. Theses states
+ * are not seen outside the function. Each state may repeat several times
+ * exiting requesting caller for new data. See the comment of XLogReadRecrod
+ * for details.
+ */
+typedef enum XLogReadRecordState {
+ XLREAD_NEXT_RECORD,
+ XLREAD_TOT_LEN,
+ XLREAD_FIRST_FRAGMENT,
+ XLREAD_CONTINUATION
+} XLogReadRecordState;
+
struct XLogReaderState
{
/*
* Operational callbacks
*/
- XLogReaderRoutine routine;
+ WALSegmentCleanupCB cleanup_cb;
/* ----------------------------------------
* Public parameters
@@ -162,18 +136,14 @@ struct XLogReaderState
*/
uint64 system_identifier;
- /*
- * Opaque data for callbacks to use. Not used by XLogReader.
- */
- void *private_data;
-
/*
* Start and end point of last record read. EndRecPtr is also used as the
* position to read next. Calling XLogBeginRead() sets EndRecPtr to the
* starting position and ReadRecPtr to invalid.
*/
- XLogRecPtr ReadRecPtr; /* start of last record read */
+ XLogRecPtr ReadRecPtr; /* start of last record read or being read */
XLogRecPtr EndRecPtr; /* end+1 of last record read */
+ XLogRecPtr PrevRecPtr; /* start of previous record read */
/* ----------------------------------------
* Communication with page reader
@@ -187,7 +157,9 @@ struct XLogReaderState
* the request, or -1 on error */
TimeLineID readPageTLI; /* TLI for data currently in readBuf */
char *readBuf; /* buffer to store data */
- bool page_verified; /* is the page on the buffer verified? */
+ bool page_verified; /* is the page header on the buffer verified? */
+ bool record_verified;/* is the current record header verified? */
+
/* ----------------------------------------
@@ -229,8 +201,6 @@ struct XLogReaderState
XLogRecPtr latestPagePtr;
TimeLineID latestPageTLI;
- /* beginning of the WAL record being read. */
- XLogRecPtr currRecPtr;
/* timeline to read it from, 0 if a lookup is required */
TimeLineID currTLI;
@@ -257,6 +227,15 @@ struct XLogReaderState
char *readRecordBuf;
uint32 readRecordBufSize;
+ /*
+ * XLogReadRecord() state
+ */
+ XLogReadRecordState readRecordState;/* state machine state */
+ int recordGotLen; /* amount of current record that has
+ * already been read */
+ int recordRemainLen; /* length of current record that remains */
+ XLogRecPtr recordContRecPtr; /* where the current record continues */
+
/* Buffer to hold error message */
char *errormsg_buf;
};
@@ -264,9 +243,7 @@ struct XLogReaderState
/* Get a new XLogReader */
extern XLogReaderState *XLogReaderAllocate(int wal_segment_size,
const char *waldir,
- XLogReaderRoutine *routine,
- void *private_data);
-extern XLogReaderRoutine *LocalXLogReaderRoutine(void);
+ WALSegmentCleanupCB cleanup_cb);
/* Free an XLogReader */
extern void XLogReaderFree(XLogReaderState *state);
@@ -274,12 +251,17 @@ extern void XLogReaderFree(XLogReaderState *state);
/* Position the XLogReader to given record */
extern void XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr);
#ifdef FRONTEND
-extern XLogRecPtr XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr);
+/* Function type definition for the read_page callback */
+typedef bool (*XLogFindNextRecordCB) (XLogReaderState *xlogreader,
+ void *private);
+extern XLogRecPtr XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
+ XLogFindNextRecordCB read_page, void *private);
#endif /* FRONTEND */
/* Read the next XLog record. Returns NULL on end-of-WAL or failure */
-extern struct XLogRecord *XLogReadRecord(XLogReaderState *state,
- char **errormsg);
+extern XLogReadRecordResult XLogReadRecord(XLogReaderState *state,
+ XLogRecord **record,
+ char **errormsg);
/* Validate a page */
extern bool XLogReaderValidatePageHeader(XLogReaderState *state,
@@ -299,6 +281,7 @@ typedef struct WALReadError
} WALReadError;
extern bool WALRead(XLogReaderState *state,
+ WALSegmentOpenCB segopenfn, WALSegmentCloseCB sgclosefn,
char *buf, XLogRecPtr startptr, Size count,
TimeLineID tli, WALReadError *errinfo);
diff --git a/src/include/access/xlogutils.h b/src/include/access/xlogutils.h
index 364a21c4ea..397fb27fc2 100644
--- a/src/include/access/xlogutils.h
+++ b/src/include/access/xlogutils.h
@@ -47,9 +47,7 @@ extern Buffer XLogReadBufferExtended(RelFileNode rnode, ForkNumber forknum,
extern Relation CreateFakeRelcacheEntry(RelFileNode rnode);
extern void FreeFakeRelcacheEntry(Relation fakerel);
-extern bool read_local_xlog_page(XLogReaderState *state,
- XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetRecPtr, char *cur_page);
+extern bool read_local_xlog_page(XLogReaderState *state);
extern void wal_segment_open(XLogReaderState *state,
XLogSegNo nextSegNo,
TimeLineID *tli_p);
diff --git a/src/include/pg_config_manual.h b/src/include/pg_config_manual.h
index f10ad0acd6..94749609b4 100644
--- a/src/include/pg_config_manual.h
+++ b/src/include/pg_config_manual.h
@@ -377,7 +377,7 @@
* Enable debugging print statements for WAL-related operations; see
* also the wal_debug GUC var.
*/
-/* #define WAL_DEBUG */
+#define WAL_DEBUG
/*
* Enable tracing of resource consumption during sort operations;
diff --git a/src/include/replication/logical.h b/src/include/replication/logical.h
index c253403372..b936f31393 100644
--- a/src/include/replication/logical.h
+++ b/src/include/replication/logical.h
@@ -29,6 +29,10 @@ typedef void (*LogicalOutputPluginWriterUpdateProgress) (struct LogicalDecodingC
TransactionId xid
);
+typedef struct LogicalDecodingContext LogicalDecodingContext;
+
+typedef bool (*LogicalDecodingXLogPageReadCB)(XLogReaderState *ctx);
+
typedef struct LogicalDecodingContext
{
/* memory context this is all allocated in */
@@ -39,6 +43,7 @@ typedef struct LogicalDecodingContext
/* infrastructure pieces for decoding */
XLogReaderState *reader;
+ LogicalDecodingXLogPageReadCB page_read;
struct ReorderBuffer *reorder;
struct SnapBuild *snapshot_builder;
@@ -105,14 +110,16 @@ extern LogicalDecodingContext *CreateInitDecodingContext(const char *plugin,
List *output_plugin_options,
bool need_full_snapshot,
XLogRecPtr restart_lsn,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress);
extern LogicalDecodingContext *CreateDecodingContext(XLogRecPtr start_lsn,
List *output_plugin_options,
bool fast_forward,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress);
--
2.27.0
----Next_Part(Thu_Mar__4_11_28_53_2021_014)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
filename="v17-0003-Remove-globals-readOff-readLen-and-readSegNo.patch"
^ permalink raw reply [nested|flat] 70+ messages in thread
* [PATCH v17 2/4] Move page-reader out of XLogReadRecord
@ 2019-09-10 03:58 Kyotaro Horiguchi <[email protected]>
0 siblings, 0 replies; 70+ messages in thread
From: Kyotaro Horiguchi @ 2019-09-10 03:58 UTC (permalink / raw)
This is the second step of removing callbacks from WAL record reader.
Since it is essential to take in additional data while reading a
record, the function have to ask caller for new data while keeping
working state. Thus the function is turned into a state machine.
---
src/backend/access/transam/twophase.c | 15 +-
src/backend/access/transam/xlog.c | 58 +-
src/backend/access/transam/xlogreader.c | 697 +++++++++++-------
src/backend/access/transam/xlogutils.c | 17 +-
src/backend/replication/logical/logical.c | 26 +-
.../replication/logical/logicalfuncs.c | 13 +-
src/backend/replication/slotfuncs.c | 18 +-
src/backend/replication/walsender.c | 32 +-
src/bin/pg_rewind/parsexlog.c | 84 ++-
src/bin/pg_waldump/pg_waldump.c | 36 +-
src/include/access/xlogreader.h | 121 ++-
src/include/access/xlogutils.h | 4 +-
src/include/pg_config_manual.h | 2 +-
src/include/replication/logical.h | 11 +-
14 files changed, 646 insertions(+), 488 deletions(-)
diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index 80d2d20d6c..7d5a96a5f5 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -1325,11 +1325,8 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
XLogReaderState *xlogreader;
char *errormsg;
- xlogreader = XLogReaderAllocate(wal_segment_size, NULL,
- XL_ROUTINE(.page_read = &read_local_xlog_page,
- .segment_open = &wal_segment_open,
- .segment_close = &wal_segment_close),
- NULL);
+ xlogreader = XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+
if (!xlogreader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
@@ -1337,7 +1334,13 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
errdetail("Failed while allocating a WAL reading processor.")));
XLogBeginRead(xlogreader, lsn);
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!read_local_xlog_page(xlogreader))
+ break;
+ }
+
if (record == NULL)
ereport(ERROR,
(errcode_for_file_access(),
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 606328a65a..670996ae67 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -836,13 +836,6 @@ static XLogSource currentSource = XLOG_FROM_ANY;
static bool lastSourceFailed = false;
static bool pendingWalRcvRestart = false;
-typedef struct XLogPageReadPrivate
-{
- int emode;
- bool fetching_ckpt; /* are we fetching a checkpoint record? */
- bool randAccess;
-} XLogPageReadPrivate;
-
/*
* These variables track when we last obtained some WAL data to process,
* and where we got it from. (XLogReceiptSource is initially the same as
@@ -917,8 +910,8 @@ static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath,
static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli,
XLogSource source, bool notfoundOk);
static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, XLogSource source);
-static bool XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
+static bool XLogPageRead(XLogReaderState *xlogreader,
+ bool fetching_ckpt, int emode, bool randAccess);
static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
bool fetching_ckpt, XLogRecPtr tliRecPtr);
static int emode_for_corrupt_record(int emode, XLogRecPtr RecPtr);
@@ -1231,8 +1224,7 @@ XLogInsertRecord(XLogRecData *rdata,
appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len);
if (!debug_reader)
- debug_reader = XLogReaderAllocate(wal_segment_size, NULL,
- XL_ROUTINE(), NULL);
+ debug_reader = XLogReaderAllocate(wal_segment_size, NULL, NULL);
if (!debug_reader)
{
@@ -4344,15 +4336,10 @@ CleanupBackupHistory(void)
* record is available.
*/
static XLogRecord *
-ReadRecord(XLogReaderState *xlogreader, int emode,
- bool fetching_ckpt)
+ReadRecord(XLogReaderState *xlogreader, int emode, bool fetching_ckpt)
{
XLogRecord *record;
- XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
-
- private->fetching_ckpt = fetching_ckpt;
- private->emode = emode;
- private->randAccess = (xlogreader->ReadRecPtr == InvalidXLogRecPtr);
+ bool randAccess = (xlogreader->ReadRecPtr == InvalidXLogRecPtr);
/* This is the first attempt to read this page. */
lastSourceFailed = false;
@@ -4360,8 +4347,16 @@ ReadRecord(XLogReaderState *xlogreader, int emode,
for (;;)
{
char *errormsg;
+ XLogReadRecordResult result;
+
+ while ((result = XLogReadRecord(xlogreader, &record, &errormsg))
+ == XLREAD_NEED_DATA)
+ {
+ if (!XLogPageRead(xlogreader, fetching_ckpt, emode, randAccess))
+ break;
+
+ }
- record = XLogReadRecord(xlogreader, &errormsg);
ReadRecPtr = xlogreader->ReadRecPtr;
EndRecPtr = xlogreader->EndRecPtr;
if (record == NULL)
@@ -6380,7 +6375,6 @@ StartupXLOG(void)
bool backupFromStandby = false;
DBState dbstate_at_startup;
XLogReaderState *xlogreader;
- XLogPageReadPrivate private;
bool promoted = false;
struct stat st;
@@ -6539,13 +6533,9 @@ StartupXLOG(void)
OwnLatch(&XLogCtl->recoveryWakeupLatch);
/* Set up XLOG reader facility */
- MemSet(&private, 0, sizeof(XLogPageReadPrivate));
xlogreader =
- XLogReaderAllocate(wal_segment_size, NULL,
- XL_ROUTINE(.page_read = &XLogPageRead,
- .segment_open = NULL,
- .segment_close = wal_segment_close),
- &private);
+ XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+
if (!xlogreader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
@@ -11987,12 +11977,13 @@ CancelBackup(void)
* sleep and retry.
*/
static bool
-XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetRecPtr, char *readBuf)
+XLogPageRead(XLogReaderState *xlogreader,
+ bool fetching_ckpt, int emode, bool randAccess)
{
- XLogPageReadPrivate *private =
- (XLogPageReadPrivate *) xlogreader->private_data;
- int emode = private->emode;
+ char *readBuf = xlogreader->readBuf;
+ XLogRecPtr targetPagePtr = xlogreader->readPagePtr;
+ int reqLen = xlogreader->readLen;
+ XLogRecPtr targetRecPtr = xlogreader->ReadRecPtr;
uint32 targetPageOff;
XLogSegNo targetSegNo PG_USED_FOR_ASSERTS_ONLY;
int r;
@@ -12035,8 +12026,8 @@ retry:
flushedUpto < targetPagePtr + reqLen))
{
if (!WaitForWALToBecomeAvailable(targetPagePtr + reqLen,
- private->randAccess,
- private->fetching_ckpt,
+ randAccess,
+ fetching_ckpt,
targetRecPtr))
{
if (readFile >= 0)
@@ -12141,6 +12132,7 @@ retry:
goto next_record_is_invalid;
}
+ Assert(xlogreader->readPagePtr == targetPagePtr);
xlogreader->readLen = readLen;
return true;
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index 7d04995b7b..ba3e4868f7 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -40,7 +40,7 @@ static bool XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr,
int reqLen, bool header_inclusive);
static void XLogReaderInvalReadState(XLogReaderState *state);
static bool ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
- XLogRecPtr PrevRecPtr, XLogRecord *record, bool randAccess);
+ XLogRecPtr PrevRecPtr, XLogRecord *record);
static bool ValidXLogRecord(XLogReaderState *state, XLogRecord *record,
XLogRecPtr recptr);
static void ResetDecoder(XLogReaderState *state);
@@ -73,7 +73,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...)
*/
XLogReaderState *
XLogReaderAllocate(int wal_segment_size, const char *waldir,
- XLogReaderRoutine *routine, void *private_data)
+ WALSegmentCleanupCB cleanup_cb)
{
XLogReaderState *state;
@@ -84,7 +84,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
return NULL;
/* initialize caller-provided support functions */
- state->routine = *routine;
+ state->cleanup_cb = cleanup_cb;
state->max_block_id = -1;
@@ -107,8 +107,6 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
WALOpenSegmentInit(&state->seg, &state->segcxt, wal_segment_size,
waldir);
- /* system_identifier initialized to zeroes above */
- state->private_data = private_data;
/* ReadRecPtr, EndRecPtr and readLen initialized to zeroes above */
state->errormsg_buf = palloc_extended(MAX_ERRORMSG_LEN + 1,
MCXT_ALLOC_NO_OOM);
@@ -140,8 +138,8 @@ XLogReaderFree(XLogReaderState *state)
{
int block_id;
- if (state->seg.ws_file != -1)
- state->routine.segment_close(state);
+ if (state->seg.ws_file >= 0)
+ state->cleanup_cb(state);
for (block_id = 0; block_id <= XLR_MAX_BLOCK_ID; block_id++)
{
@@ -246,6 +244,7 @@ XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
/* Begin at the passed-in record pointer. */
state->EndRecPtr = RecPtr;
state->ReadRecPtr = InvalidXLogRecPtr;
+ state->readRecordState = XLREAD_NEXT_RECORD;
}
/*
@@ -254,317 +253,449 @@ XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
* XLogBeginRead() or XLogFindNextRecord() must be called before the first call
* to XLogReadRecord().
*
- * If the page_read callback fails to read the requested data, NULL is
- * returned. The callback is expected to have reported the error; errormsg
- * is set to NULL.
+ * This function may return XLREAD_NEED_DATA several times before returning a
+ * result record. The caller shall read in some new data then call this
+ * function again with the same parameters.
*
- * If the reading fails for some other reason, NULL is also returned, and
- * *errormsg is set to a string with details of the failure.
+ * When a record is successfully read, returns XLREAD_SUCCESS with result
+ * record being stored in *record. Otherwise *record is NULL.
*
- * The returned pointer (or *errormsg) points to an internal buffer that's
- * valid until the next call to XLogReadRecord.
+ * Returns XLREAD_NEED_DATA if more data is needed to finish reading the
+ * current record. In that case, state->readPagePtr and state->readLen inform
+ * the desired position and minimum length of data needed. The caller shall
+ * read in the requested data and set state->readBuf to point to a buffer
+ * containing it. The caller must also set state->readPageTLI and
+ * state->readLen to indicate the timeline that it was read from, and the
+ * length of data that is now available (which must be >= given readLen),
+ * respectively.
+ *
+ * If invalid data is encountered, returns XLREAD_FAIL with *record being set to
+ * NULL. *errormsg is set to a string with details of the failure.
+ * The returned pointer (or *errormsg) points to an internal buffer that's valid
+ * until the next call to XLogReadRecord.
+ *
+ *
+ * This function runs a state machine consists of the following states.
+ *
+ * XLREAD_NEXT_RECORD :
+ * The initial state, if called with valid RecPtr, try to read a record at
+ * that position. If invalid RecPtr is given try to read a record just after
+ * the last one previously read.
+ * This state ens after setting ReadRecPtr. Then goes to XLREAD_TOT_LEN.
+ *
+ * XLREAD_TOT_LEN:
+ * Examining record header. Ends after reading record total
+ * length. recordRemainLen and recordGotLen are initialized.
+ *
+ * XLREAD_FIRST_FRAGMENT:
+ * Reading the first fragment. Ends with finishing reading a single
+ * record. Goes to XLREAD_NEXT_RECORD if that's all or
+ * XLREAD_CONTINUATION if we have continuation.
+
+ * XLREAD_CONTINUATION:
+ * Reading continuation of record. Ends with finishing the whole record then
+ * goes to XLREAD_NEXT_RECORD. During this state, recordRemainLen indicates
+ * how much is left and readRecordBuf holds the partially assert
+ * record.recordContRecPtr points to the beginning of the next page where to
+ * continue.
+ *
+ * If wrong data found in any state, the state machine stays at the current
+ * state. This behavior allows to continue reading a reacord switching among
+ * different souces, while streaming replication.
*/
-XLogRecord *
-XLogReadRecord(XLogReaderState *state, char **errormsg)
+XLogReadRecordResult
+XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
{
- XLogRecPtr RecPtr;
- XLogRecord *record;
- XLogRecPtr targetPagePtr;
- bool randAccess;
- uint32 len,
- total_len;
- uint32 targetRecOff;
- uint32 pageHeaderSize;
- bool gotheader;
+ XLogRecord *prec;
- /*
- * randAccess indicates whether to verify the previous-record pointer of
- * the record we're reading. We only do this if we're reading
- * sequentially, which is what we initially assume.
- */
- randAccess = false;
+ *record = NULL;
/* reset error state */
*errormsg = NULL;
state->errormsg_buf[0] = '\0';
- ResetDecoder(state);
-
- RecPtr = state->EndRecPtr;
-
- if (state->ReadRecPtr != InvalidXLogRecPtr)
- {
- /* read the record after the one we just read */
-
- /*
- * EndRecPtr is pointing to end+1 of the previous WAL record. If
- * we're at a page boundary, no more records can fit on the current
- * page. We must skip over the page header, but we can't do that until
- * we've read in the page, since the header size is variable.
- */
- }
- else
- {
- /*
- * Caller supplied a position to start at.
- *
- * In this case, EndRecPtr should already be pointing to a valid
- * record starting position.
- */
- Assert(XRecOffIsValid(RecPtr));
- randAccess = true;
- }
-
- state->currRecPtr = RecPtr;
-
- targetPagePtr = RecPtr - (RecPtr % XLOG_BLCKSZ);
- targetRecOff = RecPtr % XLOG_BLCKSZ;
-
- /*
- * Read the page containing the record into state->readBuf. Request enough
- * byte to cover the whole record header, or at least the part of it that
- * fits on the same page.
- */
- while (XLogNeedData(state, targetPagePtr,
- Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
- targetRecOff != 0))
- {
- if (!state->routine.page_read(state, state->readPagePtr, state->readLen,
- RecPtr, state->readBuf))
- break;
- }
-
- if (!state->page_verified)
- goto err;
-
- /*
- * We have at least the page header, so we can examine it now.
- */
- pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
- if (targetRecOff == 0)
- {
- /*
- * At page start, so skip over page header.
- */
- RecPtr += pageHeaderSize;
- targetRecOff = pageHeaderSize;
- }
- else if (targetRecOff < pageHeaderSize)
- {
- report_invalid_record(state, "invalid record offset at %X/%X",
- LSN_FORMAT_ARGS(RecPtr));
- goto err;
- }
-
- if ((((XLogPageHeader) state->readBuf)->xlp_info & XLP_FIRST_IS_CONTRECORD) &&
- targetRecOff == pageHeaderSize)
- {
- report_invalid_record(state, "contrecord is requested by %X/%X",
- LSN_FORMAT_ARGS(RecPtr));
- goto err;
- }
-
- /* XLogNeedData has verified the page header */
- Assert(pageHeaderSize <= state->readLen);
-
- /*
- * Read the record length.
- *
- * NB: Even though we use an XLogRecord pointer here, the whole record
- * header might not fit on this page. xl_tot_len is the first field of the
- * struct, so it must be on this page (the records are MAXALIGNed), but we
- * cannot access any other fields until we've verified that we got the
- * whole header.
- */
- record = (XLogRecord *) (state->readBuf + RecPtr % XLOG_BLCKSZ);
- total_len = record->xl_tot_len;
-
- /*
- * If the whole record header is on this page, validate it immediately.
- * Otherwise do just a basic sanity check on xl_tot_len, and validate the
- * rest of the header after reading it from the next page. The xl_tot_len
- * check is necessary here to ensure that we enter the "Need to reassemble
- * record" code path below; otherwise we might fail to apply
- * ValidXLogRecordHeader at all.
- */
- if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
- {
- if (!ValidXLogRecordHeader(state, RecPtr, state->ReadRecPtr, record,
- randAccess))
- goto err;
- gotheader = true;
- }
- else
- {
- /* XXX: more validation should be done here */
- if (total_len < SizeOfXLogRecord)
- {
- report_invalid_record(state,
- "invalid record length at %X/%X: wanted %u, got %u",
- LSN_FORMAT_ARGS(RecPtr),
- (uint32) SizeOfXLogRecord, total_len);
- goto err;
- }
- gotheader = false;
- }
-
- len = XLOG_BLCKSZ - RecPtr % XLOG_BLCKSZ;
- if (total_len > len)
+ switch (state->readRecordState)
{
- /* Need to reassemble record */
- char *contdata;
- XLogPageHeader pageHeader;
- char *buffer;
- uint32 gotlen;
-
- /*
- * Enlarge readRecordBuf as needed.
- */
- if (total_len > state->readRecordBufSize &&
- !allocate_recordbuf(state, total_len))
- {
- /* We treat this as a "bogus data" condition */
- report_invalid_record(state, "record length %u at %X/%X too long",
- total_len, LSN_FORMAT_ARGS(RecPtr));
- goto err;
- }
+ case XLREAD_NEXT_RECORD:
+ ResetDecoder(state);
- /* Copy the first fragment of the record from the first page. */
- memcpy(state->readRecordBuf,
- state->readBuf + RecPtr % XLOG_BLCKSZ, len);
- buffer = state->readRecordBuf + len;
- gotlen = len;
-
- do
- {
- int rest_len = total_len - gotlen;
-
- /* Calculate pointer to beginning of next page */
- targetPagePtr += XLOG_BLCKSZ;
-
- /* Wait for the next page to become available */
- while (XLogNeedData(state, targetPagePtr,
- Min(rest_len, XLOG_BLCKSZ),
- false))
+ if (state->ReadRecPtr != InvalidXLogRecPtr)
{
- if (!state->routine.page_read(state, state->readPagePtr,
- state->readLen,
- state->ReadRecPtr,
- state->readBuf))
- break;
+ /* read the record after the one we just read */
+
+ /*
+ * EndRecPtr is pointing to end+1 of the previous WAL record.
+ * If we're at a page boundary, no more records can fit on the
+ * current page. We must skip over the page header, but we
+ * can't do that until we've read in the page, since the header
+ * size is variable.
+ */
+ state->PrevRecPtr = state->ReadRecPtr;
+ state->ReadRecPtr = state->EndRecPtr;
+ }
+ else
+ {
+ /*
+ * Caller supplied a position to start at.
+ *
+ * In this case, EndRecPtr should already be pointing to a
+ * valid record starting position.
+ */
+ Assert(XRecOffIsValid(state->EndRecPtr));
+ state->ReadRecPtr = state->EndRecPtr;
+
+ /*
+ * We cannot verify the previous-record pointer when we're
+ * seeking to a particular record. Reset PrevRecPtr so that we
+ * won't try doing that.
+ */
+ state->PrevRecPtr = InvalidXLogRecPtr;
+ state->EndRecPtr = InvalidXLogRecPtr; /* to be tidy */
}
+ state->record_verified = false;
+ state->readRecordState = XLREAD_TOT_LEN;
+ /* fall through */
+
+ case XLREAD_TOT_LEN:
+ {
+ uint32 total_len;
+ uint32 pageHeaderSize;
+ XLogRecPtr targetPagePtr;
+ uint32 targetRecOff;
+ XLogPageHeader pageHeader;
+
+ targetPagePtr =
+ state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
+ targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+
+ /*
+ * Check if we have enough data. For the first record in the page,
+ * the requesting length doesn't contain page header.
+ */
+ if (XLogNeedData(state, targetPagePtr,
+ Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+ targetRecOff != 0))
+ return XLREAD_NEED_DATA;
+
+ /* error out if caller supplied bogus page */
if (!state->page_verified)
goto err;
- Assert(SizeOfXLogShortPHD <= state->readLen);
+ /* examine page header now. */
+ pageHeaderSize =
+ XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+ if (targetRecOff == 0)
+ {
+ /* At page start, so skip over page header. */
+ state->ReadRecPtr += pageHeaderSize;
+ targetRecOff = pageHeaderSize;
+ }
+ else if (targetRecOff < pageHeaderSize)
+ {
+ report_invalid_record(state, "invalid record offset at %X/%X",
+ LSN_FORMAT_ARGS(state->ReadRecPtr));
+ goto err;
+ }
- /* Check that the continuation on next page looks valid */
pageHeader = (XLogPageHeader) state->readBuf;
- if (!(pageHeader->xlp_info & XLP_FIRST_IS_CONTRECORD))
+ if ((pageHeader->xlp_info & XLP_FIRST_IS_CONTRECORD) &&
+ targetRecOff == pageHeaderSize)
{
- report_invalid_record(state,
- "there is no contrecord flag at %X/%X",
- LSN_FORMAT_ARGS(RecPtr));
+ report_invalid_record(state, "contrecord is requested by %X/%X",
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr);
goto err;
}
- /*
- * Cross-check that xlp_rem_len agrees with how much of the record
- * we expect there to be left.
- */
- if (pageHeader->xlp_rem_len == 0 ||
- total_len != (pageHeader->xlp_rem_len + gotlen))
- {
- report_invalid_record(state,
- "invalid contrecord length %u (expected %lld) at %X/%X",
- pageHeader->xlp_rem_len,
- ((long long) total_len) - gotlen,
- LSN_FORMAT_ARGS(RecPtr));
- goto err;
- }
-
- /* Append the continuation from this page to the buffer */
- pageHeaderSize = XLogPageHeaderSize(pageHeader);
-
+ /* XLogNeedData has verified the page header */
Assert(pageHeaderSize <= state->readLen);
- contdata = (char *) state->readBuf + pageHeaderSize;
- len = XLOG_BLCKSZ - pageHeaderSize;
- if (pageHeader->xlp_rem_len < len)
- len = pageHeader->xlp_rem_len;
+ /*
+ * Read the record length.
+ *
+ * NB: Even though we use an XLogRecord pointer here, the whole
+ * record header might not fit on this page. xl_tot_len is the first
+ * field of the struct, so it must be on this page (the records are
+ * MAXALIGNed), but we cannot access any other fields until we've
+ * verified that we got the whole header.
+ */
+ prec = (XLogRecord *) (state->readBuf +
+ state->ReadRecPtr % XLOG_BLCKSZ);
+ total_len = prec->xl_tot_len;
- Assert (pageHeaderSize + len <= state->readLen);
- memcpy(buffer, (char *) contdata, len);
- buffer += len;
- gotlen += len;
+ /*
+ * If the whole record header is on this page, validate it
+ * immediately. Otherwise do just a basic sanity check on
+ * xl_tot_len, and validate the rest of the header after reading it
+ * from the next page. The xl_tot_len check is necessary here to
+ * ensure that we enter the XLREAD_CONTINUATION state below;
+ * otherwise we might fail to apply ValidXLogRecordHeader at all.
+ */
+ if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
+ {
+ if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
+ state->PrevRecPtr, prec))
+ goto err;
- /* If we just reassembled the record header, validate it. */
- if (!gotheader)
+ state->record_verified = true;
+ }
+ else
{
- record = (XLogRecord *) state->readRecordBuf;
- if (!ValidXLogRecordHeader(state, RecPtr, state->ReadRecPtr,
- record, randAccess))
+ /* XXX: more validation should be done here */
+ if (total_len < SizeOfXLogRecord)
+ {
+ report_invalid_record(state,
+ "invalid record length at %X/%X: wanted %u, got %u",
+ LSN_FORMAT_ARGS(state->ReadRecPtr),
+ (uint32) SizeOfXLogRecord, total_len);
goto err;
- gotheader = true;
+ }
}
- } while (gotlen < total_len);
-
- Assert(gotheader);
- record = (XLogRecord *) state->readRecordBuf;
- if (!ValidXLogRecord(state, record, RecPtr))
- goto err;
+ /*
+ * Wait for the rest of the record, or the part of the record that
+ * fit on the first page if crossed a page boundary, to become
+ * available.
+ */
+ state->recordGotLen = 0;
+ state->recordRemainLen = total_len;
+ state->readRecordState = XLREAD_FIRST_FRAGMENT;
+ }
+ /* fall through */
- pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
- state->ReadRecPtr = RecPtr;
- state->EndRecPtr = targetPagePtr + pageHeaderSize
- + MAXALIGN(pageHeader->xlp_rem_len);
- }
- else
- {
- /* Wait for the record data to become available */
- while (XLogNeedData(state, targetPagePtr,
- Min(targetRecOff + total_len, XLOG_BLCKSZ), true))
+ case XLREAD_FIRST_FRAGMENT:
{
- if (!state->routine.page_read(state, state->readPagePtr,
- state->readLen,
- state->ReadRecPtr, state->readBuf))
+ uint32 total_len = state->recordRemainLen;
+ uint32 request_len;
+ uint32 record_len;
+ XLogRecPtr targetPagePtr;
+ uint32 targetRecOff;
+
+ /*
+ * Wait for the rest of the record on the first page to become
+ * available
+ */
+ targetPagePtr =
+ state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
+ targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+
+ request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ);
+ record_len = request_len - targetRecOff;
+
+ /* ReadRecPtr contains page header */
+ Assert (targetRecOff != 0);
+ if (XLogNeedData(state, targetPagePtr, request_len, true))
+ return XLREAD_NEED_DATA;
+
+ /* error out if caller supplied bogus page */
+ if (!state->page_verified)
+ goto err;
+
+ prec = (XLogRecord *) (state->readBuf + targetRecOff);
+
+ /* validate record header if not yet */
+ if (!state->record_verified && record_len >= SizeOfXLogRecord)
+ {
+ if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
+ state->PrevRecPtr, prec))
+ goto err;
+
+ state->record_verified = true;
+ }
+
+
+ if (total_len == record_len)
+ {
+ /* Record does not cross a page boundary */
+ Assert(state->record_verified);
+
+ if (!ValidXLogRecord(state, prec, state->ReadRecPtr))
+ goto err;
+
+ state->record_verified = true; /* to be tidy */
+
+ /* We already checked the header earlier */
+ state->EndRecPtr = state->ReadRecPtr + MAXALIGN(record_len);
+
+ *record = prec;
+ state->readRecordState = XLREAD_NEXT_RECORD;
break;
+ }
+
+ /*
+ * The record continues on the next page. Need to reassemble
+ * record
+ */
+ Assert(total_len > record_len);
+
+ /* Enlarge readRecordBuf as needed. */
+ if (total_len > state->readRecordBufSize &&
+ !allocate_recordbuf(state, total_len))
+ {
+ /* We treat this as a "bogus data" condition */
+ report_invalid_record(state,
+ "record length %u at %X/%X too long",
+ total_len,
+ LSN_FORMAT_ARGS(state->ReadRecPtr));
+ goto err;
+ }
+
+ /* Copy the first fragment of the record from the first page. */
+ memcpy(state->readRecordBuf, state->readBuf + targetRecOff,
+ record_len);
+ state->recordGotLen += record_len;
+ state->recordRemainLen -= record_len;
+
+ /* Calculate pointer to beginning of next page */
+ state->recordContRecPtr = state->ReadRecPtr + record_len;
+ Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0);
+
+ state->readRecordState = XLREAD_CONTINUATION;
}
+ /* fall through */
+
+ case XLREAD_CONTINUATION:
+ {
+ XLogPageHeader pageHeader;
+ uint32 pageHeaderSize;
+ XLogRecPtr targetPagePtr;
+
+ /* we enter this state only if we haven't read the whole record. */
+ Assert (state->recordRemainLen > 0);
+
+ while(state->recordRemainLen > 0)
+ {
+ char *contdata;
+ uint32 request_len;
+ uint32 record_len;
+
+ /* Wait for the next page to become available */
+ targetPagePtr = state->recordContRecPtr;
+
+ /* this request contains page header */
+ Assert (targetPagePtr != 0);
+ if (XLogNeedData(state, targetPagePtr,
+ Min(state->recordRemainLen, XLOG_BLCKSZ),
+ false))
+ return XLREAD_NEED_DATA;
+
+ if (!state->page_verified)
+ goto err;
+
+ Assert(SizeOfXLogShortPHD <= state->readLen);
- if (!state->page_verified)
- goto err;
+ /* Check that the continuation on next page looks valid */
+ pageHeader = (XLogPageHeader) state->readBuf;
+ if (!(pageHeader->xlp_info & XLP_FIRST_IS_CONTRECORD))
+ {
+ report_invalid_record(
+ state,
+ "there is no contrecord flag at %X/%X reading %X/%X",
+ (uint32) (state->recordContRecPtr >> 32),
+ (uint32) state->recordContRecPtr,
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr);
+ goto err;
+ }
- /* Record does not cross a page boundary */
- if (!ValidXLogRecord(state, record, RecPtr))
- goto err;
+ /*
+ * Cross-check that xlp_rem_len agrees with how much of the
+ * record we expect there to be left.
+ */
+ if (pageHeader->xlp_rem_len == 0 ||
+ pageHeader->xlp_rem_len != state->recordRemainLen)
+ {
+ report_invalid_record(
+ state,
+ "invalid contrecord length %u at %X/%X reading %X/%X, expected %u",
+ pageHeader->xlp_rem_len,
+ (uint32) (state->recordContRecPtr >> 32),
+ (uint32) state->recordContRecPtr,
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr,
+ state->recordRemainLen);
+ goto err;
+ }
- state->EndRecPtr = RecPtr + MAXALIGN(total_len);
+ /* Append the continuation from this page to the buffer */
+ pageHeaderSize = XLogPageHeaderSize(pageHeader);
- state->ReadRecPtr = RecPtr;
+ /*
+ * XLogNeedData should have ensured that the whole page header
+ * was read
+ */
+ Assert(state->readLen >= pageHeaderSize);
+
+ contdata = (char *) state->readBuf + pageHeaderSize;
+ record_len = XLOG_BLCKSZ - pageHeaderSize;
+ if (pageHeader->xlp_rem_len < record_len)
+ record_len = pageHeader->xlp_rem_len;
+
+ request_len = record_len + pageHeaderSize;
+
+ /* XLogNeedData should have ensured all needed data was read */
+ Assert (state->readLen >= request_len);
+
+ memcpy(state->readRecordBuf + state->recordGotLen,
+ (char *) contdata, record_len);
+ state->recordGotLen += record_len;
+ state->recordRemainLen -= record_len;
+
+ /* If we just reassembled the record header, validate it. */
+ if (!state->record_verified)
+ {
+ Assert(state->recordGotLen >= SizeOfXLogRecord);
+ if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
+ state->PrevRecPtr,
+ (XLogRecord *) state->readRecordBuf))
+ goto err;
+
+ state->record_verified = true;
+ }
+
+ /* Calculate pointer to beginning of next page, and continue */
+ state->recordContRecPtr += XLOG_BLCKSZ;
+ }
+
+ /* targetPagePtr is pointing the last-read page here */
+ prec = (XLogRecord *) state->readRecordBuf;
+ if (!ValidXLogRecord(state, prec, state->ReadRecPtr))
+ goto err;
+
+ pageHeaderSize =
+ XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+ state->EndRecPtr = targetPagePtr + pageHeaderSize
+ + MAXALIGN(pageHeader->xlp_rem_len);
+
+ *record = prec;
+ state->readRecordState = XLREAD_NEXT_RECORD;
+ break;
+ }
}
/*
* Special processing if it's an XLOG SWITCH record
*/
- if (record->xl_rmid == RM_XLOG_ID &&
- (record->xl_info & ~XLR_INFO_MASK) == XLOG_SWITCH)
+ if ((*record)->xl_rmid == RM_XLOG_ID &&
+ ((*record)->xl_info & ~XLR_INFO_MASK) == XLOG_SWITCH)
{
/* Pretend it extends to end of segment */
state->EndRecPtr += state->segcxt.ws_segsize - 1;
state->EndRecPtr -= XLogSegmentOffset(state->EndRecPtr, state->segcxt.ws_segsize);
}
- if (DecodeXLogRecord(state, record, errormsg))
- return record;
- else
- return NULL;
+ Assert (!*record || state->readLen >= 0);
+ if (DecodeXLogRecord(state, *record, errormsg))
+ return XLREAD_SUCCESS;
+
+ *record = NULL;
+ return XLREAD_FAIL;
err:
/*
- * Invalidate the read state. We might read from a different source after
+ * Invalidate the read page. We might read from a different source after
* failure.
*/
XLogReaderInvalReadState(state);
@@ -572,7 +703,8 @@ err:
if (state->errormsg_buf[0] != '\0')
*errormsg = state->errormsg_buf;
- return NULL;
+ *record = NULL;
+ return XLREAD_FAIL;
}
/*
@@ -724,11 +856,12 @@ XLogReaderInvalReadState(XLogReaderState *state)
*
* This is just a convenience subroutine to avoid duplicated code in
* XLogReadRecord. It's not intended for use from anywhere else.
+ *
+ * If PrevRecPtr is valid, the xl_prev is is cross-checked with it.
*/
static bool
ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
- XLogRecPtr PrevRecPtr, XLogRecord *record,
- bool randAccess)
+ XLogRecPtr PrevRecPtr, XLogRecord *record)
{
if (record->xl_tot_len < SizeOfXLogRecord)
{
@@ -745,7 +878,7 @@ ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
record->xl_rmid, LSN_FORMAT_ARGS(RecPtr));
return false;
}
- if (randAccess)
+ if (PrevRecPtr == InvalidXLogRecPtr)
{
/*
* We can't exactly verify the prev-link, but surely it should be less
@@ -975,11 +1108,14 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
* XLogReadRecord() will read the next valid record.
*/
XLogRecPtr
-XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
+XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
+ XLogFindNextRecordCB read_page, void *private)
{
XLogRecPtr tmpRecPtr;
XLogRecPtr found = InvalidXLogRecPtr;
XLogPageHeader header;
+ XLogRecord *record;
+ XLogReadRecordResult result;
char *errormsg;
Assert(!XLogRecPtrIsInvalid(RecPtr));
@@ -1012,9 +1148,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
while(XLogNeedData(state, targetPagePtr, targetRecOff,
targetRecOff != 0))
{
- if (!state->routine.page_read(state, state->readPagePtr,
- state->readLen,
- state->ReadRecPtr, state->readBuf))
+ if (!read_page(state, private))
break;
}
@@ -1066,8 +1200,16 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
* or we just jumped over the remaining data of a continuation.
*/
XLogBeginRead(state, tmpRecPtr);
- while (XLogReadRecord(state, &errormsg) != NULL)
+ while ((result = XLogReadRecord(state, &record, &errormsg)) !=
+ XLREAD_FAIL)
{
+ if (result == XLREAD_NEED_DATA)
+ {
+ if (!read_page(state, private))
+ goto err;
+ continue;
+ }
+
/* past the record we've found, break out */
if (RecPtr <= state->ReadRecPtr)
{
@@ -1087,9 +1229,9 @@ err:
#endif /* FRONTEND */
/*
- * Helper function to ease writing of XLogRoutine->page_read callbacks.
- * If this function is used, caller must supply a segment_open callback in
- * 'state', as that is used here.
+ * Helper function to ease writing of page_read callback.
+ * If this function is used, caller must supply a segment_open callback and
+ * segment_close callback as that is used here.
*
* Read 'count' bytes into 'buf', starting at location 'startptr', from WAL
* fetched from timeline 'tli'.
@@ -1102,6 +1244,7 @@ err:
*/
bool
WALRead(XLogReaderState *state,
+ WALSegmentOpenCB segopenfn, WALSegmentCloseCB segclosefn,
char *buf, XLogRecPtr startptr, Size count, TimeLineID tli,
WALReadError *errinfo)
{
@@ -1133,10 +1276,10 @@ WALRead(XLogReaderState *state,
XLogSegNo nextSegNo;
if (state->seg.ws_file >= 0)
- state->routine.segment_close(state);
+ segclosefn(state);
XLByteToSeg(recptr, nextSegNo, state->segcxt.ws_segsize);
- state->routine.segment_open(state, nextSegNo, &tli);
+ segopenfn(state, nextSegNo, &tli);
/* This shouldn't happen -- indicates a bug in segment_open */
Assert(state->seg.ws_file >= 0);
diff --git a/src/backend/access/transam/xlogutils.c b/src/backend/access/transam/xlogutils.c
index 46eda33f25..b003990745 100644
--- a/src/backend/access/transam/xlogutils.c
+++ b/src/backend/access/transam/xlogutils.c
@@ -686,8 +686,7 @@ XLogTruncateRelation(RelFileNode rnode, ForkNumber forkNum,
void
XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wantLength)
{
- const XLogRecPtr lastReadPage = state->seg.ws_segno *
- state->segcxt.ws_segsize + state->readLen;
+ const XLogRecPtr lastReadPage = state->readPagePtr;
Assert(wantPage != InvalidXLogRecPtr && wantPage % XLOG_BLCKSZ == 0);
Assert(wantLength <= XLOG_BLCKSZ);
@@ -702,7 +701,7 @@ XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wa
* current TLI has since become historical.
*/
if (lastReadPage == wantPage &&
- state->readLen != 0 &&
+ state->page_verified &&
lastReadPage + state->readLen >= wantPage + Min(wantLength, XLOG_BLCKSZ - 1))
return;
@@ -788,6 +787,7 @@ wal_segment_open(XLogReaderState *state, XLogSegNo nextSegNo,
char path[MAXPGPATH];
XLogFilePath(path, tli, nextSegNo, state->segcxt.ws_segsize);
+ elog(LOG, "HOGE: %lu, %d => %s", nextSegNo, tli, path);
state->seg.ws_file = BasicOpenFile(path, O_RDONLY | PG_BINARY);
if (state->seg.ws_file >= 0)
return;
@@ -825,9 +825,11 @@ wal_segment_close(XLogReaderState *state)
* loop for now.
*/
bool
-read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *cur_page)
+read_local_xlog_page(XLogReaderState *state)
{
+ XLogRecPtr targetPagePtr = state->readPagePtr;
+ int reqLen = state->readLen;
+ char *cur_page = state->readBuf;
XLogRecPtr read_upto,
loc;
TimeLineID tli;
@@ -940,11 +942,12 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
* as 'count', read the whole page anyway. It's guaranteed to be
* zero-padded up to the page boundary if it's incomplete.
*/
- if (!WALRead(state, cur_page, targetPagePtr, XLOG_BLCKSZ, tli,
- &errinfo))
+ if (!WALRead(state, wal_segment_open, wal_segment_close,
+ cur_page, targetPagePtr, XLOG_BLCKSZ, tli, &errinfo))
WALReadRaiseError(&errinfo);
/* number of valid bytes in the buffer */
+ state->readPagePtr = targetPagePtr;
state->readLen = count;
return true;
}
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index 37b75deb72..9cbd8afa49 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -148,7 +148,8 @@ StartupDecodingContext(List *output_plugin_options,
TransactionId xmin_horizon,
bool need_full_snapshot,
bool fast_forward,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress)
@@ -198,11 +199,12 @@ StartupDecodingContext(List *output_plugin_options,
ctx->slot = slot;
- ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, xl_routine, ctx);
+ ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, cleanup_cb);
if (!ctx->reader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
errmsg("out of memory")));
+ ctx->page_read = page_read;
ctx->reorder = ReorderBufferAllocate();
ctx->snapshot_builder =
@@ -319,7 +321,8 @@ CreateInitDecodingContext(const char *plugin,
List *output_plugin_options,
bool need_full_snapshot,
XLogRecPtr restart_lsn,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress)
@@ -422,7 +425,7 @@ CreateInitDecodingContext(const char *plugin,
ctx = StartupDecodingContext(NIL, restart_lsn, xmin_horizon,
need_full_snapshot, false,
- xl_routine, prepare_write, do_write,
+ page_read, cleanup_cb, prepare_write, do_write,
update_progress);
/* call output plugin initialization callback */
@@ -476,7 +479,8 @@ LogicalDecodingContext *
CreateDecodingContext(XLogRecPtr start_lsn,
List *output_plugin_options,
bool fast_forward,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress)
@@ -528,8 +532,8 @@ CreateDecodingContext(XLogRecPtr start_lsn,
ctx = StartupDecodingContext(output_plugin_options,
start_lsn, InvalidTransactionId, false,
- fast_forward, xl_routine, prepare_write,
- do_write, update_progress);
+ fast_forward, page_read, cleanup_cb,
+ prepare_write, do_write, update_progress);
/* call output plugin initialization callback */
old_context = MemoryContextSwitchTo(ctx->context);
@@ -585,7 +589,13 @@ DecodingContextFindStartpoint(LogicalDecodingContext *ctx)
char *err = NULL;
/* the read_page callback waits for new WAL */
- record = XLogReadRecord(ctx->reader, &err);
+ while (XLogReadRecord(ctx->reader, &record, &err) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!ctx->page_read(ctx->reader))
+ break;
+ }
+
if (err)
elog(ERROR, "%s", err);
if (!record)
diff --git a/src/backend/replication/logical/logicalfuncs.c b/src/backend/replication/logical/logicalfuncs.c
index f7e055874e..b69f6911c5 100644
--- a/src/backend/replication/logical/logicalfuncs.c
+++ b/src/backend/replication/logical/logicalfuncs.c
@@ -233,9 +233,8 @@ pg_logical_slot_get_changes_guts(FunctionCallInfo fcinfo, bool confirm, bool bin
ctx = CreateDecodingContext(InvalidXLogRecPtr,
options,
false,
- XL_ROUTINE(.page_read = read_local_xlog_page,
- .segment_open = wal_segment_open,
- .segment_close = wal_segment_close),
+ read_local_xlog_page,
+ wal_segment_close,
LogicalOutputPrepareWrite,
LogicalOutputWrite, NULL);
@@ -284,7 +283,13 @@ pg_logical_slot_get_changes_guts(FunctionCallInfo fcinfo, bool confirm, bool bin
XLogRecord *record;
char *errm = NULL;
- record = XLogReadRecord(ctx->reader, &errm);
+ while (XLogReadRecord(ctx->reader, &record, &errm) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!ctx->page_read(ctx->reader))
+ break;
+ }
+
if (errm)
elog(ERROR, "%s", errm);
diff --git a/src/backend/replication/slotfuncs.c b/src/backend/replication/slotfuncs.c
index 9817b44113..deb7cf15eb 100644
--- a/src/backend/replication/slotfuncs.c
+++ b/src/backend/replication/slotfuncs.c
@@ -153,9 +153,8 @@ create_logical_replication_slot(char *name, char *plugin,
ctx = CreateInitDecodingContext(plugin, NIL,
false, /* just catalogs is OK */
restart_lsn,
- XL_ROUTINE(.page_read = read_local_xlog_page,
- .segment_open = wal_segment_open,
- .segment_close = wal_segment_close),
+ read_local_xlog_page,
+ wal_segment_close,
NULL, NULL, NULL);
/*
@@ -512,9 +511,8 @@ pg_logical_replication_slot_advance(XLogRecPtr moveto)
ctx = CreateDecodingContext(InvalidXLogRecPtr,
NIL,
true, /* fast_forward */
- XL_ROUTINE(.page_read = read_local_xlog_page,
- .segment_open = wal_segment_open,
- .segment_close = wal_segment_close),
+ read_local_xlog_page,
+ wal_segment_close,
NULL, NULL, NULL);
/*
@@ -536,7 +534,13 @@ pg_logical_replication_slot_advance(XLogRecPtr moveto)
* Read records. No changes are generated in fast_forward mode,
* but snapbuilder/slot statuses are updated properly.
*/
- record = XLogReadRecord(ctx->reader, &errm);
+ while (XLogReadRecord(ctx->reader, &record, &errm) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!ctx->page_read(ctx->reader))
+ break;
+ }
+
if (errm)
elog(ERROR, "%s", errm);
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index c36b91ae5e..062093727d 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -580,10 +580,7 @@ StartReplication(StartReplicationCmd *cmd)
/* create xlogreader for physical replication */
xlogreader =
- XLogReaderAllocate(wal_segment_size, NULL,
- XL_ROUTINE(.segment_open = WalSndSegmentOpen,
- .segment_close = wal_segment_close),
- NULL);
+ XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
if (!xlogreader)
ereport(ERROR,
@@ -807,9 +804,11 @@ StartReplication(StartReplicationCmd *cmd)
* set every time WAL is flushed.
*/
static bool
-logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetRecPtr, char *cur_page)
+logical_read_xlog_page(XLogReaderState *state)
{
+ XLogRecPtr targetPagePtr = state->readPagePtr;
+ int reqLen = state->readLen;
+ char *cur_page = state->readBuf;
XLogRecPtr flushptr;
int count;
WALReadError errinfo;
@@ -837,7 +836,7 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
count = flushptr - targetPagePtr; /* part of the page available */
/* now actually read the data, we know it's there */
- if (!WALRead(state,
+ if (!WALRead(state, WalSndSegmentOpen, wal_segment_close,
cur_page,
targetPagePtr,
XLOG_BLCKSZ,
@@ -1011,9 +1010,8 @@ CreateReplicationSlot(CreateReplicationSlotCmd *cmd)
ctx = CreateInitDecodingContext(cmd->plugin, NIL, need_full_snapshot,
InvalidXLogRecPtr,
- XL_ROUTINE(.page_read = logical_read_xlog_page,
- .segment_open = WalSndSegmentOpen,
- .segment_close = wal_segment_close),
+ logical_read_xlog_page,
+ wal_segment_close,
WalSndPrepareWrite, WalSndWriteData,
WalSndUpdateProgress);
@@ -1171,9 +1169,8 @@ StartLogicalReplication(StartReplicationCmd *cmd)
*/
logical_decoding_ctx =
CreateDecodingContext(cmd->startpoint, cmd->options, false,
- XL_ROUTINE(.page_read = logical_read_xlog_page,
- .segment_open = WalSndSegmentOpen,
- .segment_close = wal_segment_close),
+ logical_read_xlog_page,
+ wal_segment_close,
WalSndPrepareWrite, WalSndWriteData,
WalSndUpdateProgress);
xlogreader = logical_decoding_ctx->reader;
@@ -2749,7 +2746,7 @@ XLogSendPhysical(void)
enlargeStringInfo(&output_message, nbytes);
retry:
- if (!WALRead(xlogreader,
+ if (!WALRead(xlogreader, WalSndSegmentOpen, wal_segment_close,
&output_message.data[output_message.len],
startptr,
nbytes,
@@ -2847,7 +2844,12 @@ XLogSendLogical(void)
*/
WalSndCaughtUp = false;
- record = XLogReadRecord(logical_decoding_ctx->reader, &errm);
+ while (XLogReadRecord(logical_decoding_ctx->reader, &record, &errm) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!logical_decoding_ctx->page_read(logical_decoding_ctx->reader))
+ break;
+ }
/* xlog record was invalid */
if (errm != NULL)
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index cf119848b0..712c85281c 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -41,15 +41,9 @@ static int xlogreadfd = -1;
static XLogSegNo xlogreadsegno = -1;
static char xlogfpath[MAXPGPATH];
-typedef struct XLogPageReadPrivate
-{
- const char *restoreCommand;
- int tliIndex;
-} XLogPageReadPrivate;
-
-static bool SimpleXLogPageRead(XLogReaderState *xlogreader,
- XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
+static bool SimpleXLogPageRead(XLogReaderState *xlogreader,
+ const char *datadir, int *tliIndex,
+ const char *restoreCommand);
/*
* Read WAL from the datadir/pg_wal, starting from 'startpoint' on timeline
@@ -66,20 +60,22 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
XLogRecord *record;
XLogReaderState *xlogreader;
char *errormsg;
- XLogPageReadPrivate private;
- private.tliIndex = tliIndex;
- private.restoreCommand = restoreCommand;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir,
- XL_ROUTINE(.page_read = &SimpleXLogPageRead),
- &private);
+ xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+
if (xlogreader == NULL)
pg_fatal("out of memory");
XLogBeginRead(xlogreader, startpoint);
do
{
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!SimpleXLogPageRead(xlogreader, datadir,
+ &tliIndex, restoreCommand))
+ break;
+ }
if (record == NULL)
{
@@ -123,19 +119,19 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex,
XLogRecord *record;
XLogReaderState *xlogreader;
char *errormsg;
- XLogPageReadPrivate private;
XLogRecPtr endptr;
- private.tliIndex = tliIndex;
- private.restoreCommand = restoreCommand;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir,
- XL_ROUTINE(.page_read = &SimpleXLogPageRead),
- &private);
+ xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
XLogBeginRead(xlogreader, ptr);
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!SimpleXLogPageRead(xlogreader, datadir, &tliIndex, restoreCommand))
+ break;
+ }
if (record == NULL)
{
if (errormsg)
@@ -170,7 +166,6 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
XLogRecPtr searchptr;
XLogReaderState *xlogreader;
char *errormsg;
- XLogPageReadPrivate private;
/*
* The given fork pointer points to the end of the last common record,
@@ -186,11 +181,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
forkptr += SizeOfXLogShortPHD;
}
- private.tliIndex = tliIndex;
- private.restoreCommand = restoreCommand;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir,
- XL_ROUTINE(.page_read = &SimpleXLogPageRead),
- &private);
+ xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
@@ -200,7 +191,13 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
uint8 info;
XLogBeginRead(xlogreader, searchptr);
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!SimpleXLogPageRead(xlogreader, datadir,
+ &tliIndex, restoreCommand))
+ break;
+ }
if (record == NULL)
{
@@ -247,10 +244,11 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
/* XLogReader callback function, to read a WAL page */
static bool
-SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *readBuf)
+SimpleXLogPageRead(XLogReaderState *xlogreader, const char *datadir,
+ int *tliIndex, const char *restoreCommand)
{
- XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
+ XLogRecPtr targetPagePtr = xlogreader->readPagePtr;
+ char *readBuf = xlogreader->readBuf;
uint32 targetPageOff;
XLogRecPtr targetSegEnd;
XLogSegNo targetSegNo;
@@ -283,14 +281,14 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
* be done both forward and backward, consider also switching timeline
* accordingly.
*/
- while (private->tliIndex < targetNentries - 1 &&
- targetHistory[private->tliIndex].end < targetSegEnd)
- private->tliIndex++;
- while (private->tliIndex > 0 &&
- targetHistory[private->tliIndex].begin >= targetSegEnd)
- private->tliIndex--;
+ while (*tliIndex < targetNentries - 1 &&
+ targetHistory[*tliIndex].end < targetSegEnd)
+ (*tliIndex)++;
+ while (*tliIndex > 0 &&
+ targetHistory[*tliIndex].begin >= targetSegEnd)
+ (*tliIndex)--;
- XLogFileName(xlogfname, targetHistory[private->tliIndex].tli,
+ XLogFileName(xlogfname, targetHistory[*tliIndex].tli,
xlogreadsegno, WalSegSz);
snprintf(xlogfpath, MAXPGPATH, "%s/" XLOGDIR "/%s",
@@ -303,7 +301,7 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
/*
* If we have no restore_command to execute, then exit.
*/
- if (private->restoreCommand == NULL)
+ if (restoreCommand == NULL)
{
pg_log_error("could not open file \"%s\": %m", xlogfpath);
xlogreader->readLen = -1;
@@ -317,7 +315,7 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
xlogreadfd = RestoreArchivedFile(xlogreader->segcxt.ws_dir,
xlogfname,
WalSegSz,
- private->restoreCommand);
+ restoreCommand);
if (xlogreadfd < 0)
{
@@ -359,7 +357,7 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
Assert(targetSegNo == xlogreadsegno);
- xlogreader->seg.ws_tli = targetHistory[private->tliIndex].tli;
+ xlogreader->seg.ws_tli = targetHistory[*tliIndex].tli;
xlogreader->readLen = XLOG_BLCKSZ;
return true;
}
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index 0f0e0723a3..f514de6b13 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -330,12 +330,17 @@ WALDumpCloseSegment(XLogReaderState *state)
state->seg.ws_file = -1;
}
-/* pg_waldump's XLogReaderRoutine->page_read callback */
+/*
+ * pg_waldump's WAL page rader, also used as page_read callback for
+ * XLogFindNextRecord
+ */
static bool
-WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetPtr, char *readBuff)
+WALDumpReadPage(XLogReaderState *state, void *priv)
{
- XLogDumpPrivate *private = state->private_data;
+ XLogRecPtr targetPagePtr = state->readPagePtr;
+ int reqLen = state->readLen;
+ char *readBuff = state->readBuf;
+ XLogDumpPrivate *private = (XLogDumpPrivate *) priv;
int count = XLOG_BLCKSZ;
WALReadError errinfo;
@@ -353,8 +358,8 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
}
}
- if (!WALRead(state, readBuff, targetPagePtr, count, private->timeline,
- &errinfo))
+ if (!WALRead(state, WALDumpOpenSegment, WALDumpCloseSegment,
+ readBuff, targetPagePtr, count, private->timeline, &errinfo))
{
WALOpenSegment *seg = &errinfo.wre_seg;
char fname[MAXPGPATH];
@@ -374,6 +379,7 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
(Size) errinfo.wre_req);
}
+ Assert(count >= state->readLen);
state->readLen = count;
return true;
}
@@ -1035,16 +1041,14 @@ main(int argc, char **argv)
/* we have everything we need, start reading */
xlogreader_state =
- XLogReaderAllocate(WalSegSz, waldir,
- XL_ROUTINE(.page_read = WALDumpReadPage,
- .segment_open = WALDumpOpenSegment,
- .segment_close = WALDumpCloseSegment),
- &private);
+ XLogReaderAllocate(WalSegSz, waldir, WALDumpCloseSegment);
+
if (!xlogreader_state)
fatal_error("out of memory");
/* first find a valid recptr to start from */
- first_record = XLogFindNextRecord(xlogreader_state, private.startptr);
+ first_record = XLogFindNextRecord(xlogreader_state, private.startptr,
+ &WALDumpReadPage, (void*) &private);
if (first_record == InvalidXLogRecPtr)
fatal_error("could not find a valid record after %X/%X",
@@ -1067,7 +1071,13 @@ main(int argc, char **argv)
for (;;)
{
/* try to read the next record */
- record = XLogReadRecord(xlogreader_state, &errormsg);
+ while (XLogReadRecord(xlogreader_state, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!WALDumpReadPage(xlogreader_state, (void *) &private))
+ break;
+ }
+
if (!record)
{
if (!config.follow || private.endptr_reached)
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index 23d9e70a59..4e9268b553 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -57,64 +57,15 @@ typedef struct WALSegmentContext
typedef struct XLogReaderState XLogReaderState;
-/* Function type definition for the read_page callback */
-typedef bool (*XLogPageReadCB) (XLogReaderState *xlogreader,
- XLogRecPtr targetPagePtr,
- int reqLen,
- XLogRecPtr targetRecPtr,
- char *readBuf);
+/* Function type definition for the segment cleanup callback */
+typedef void (*WALSegmentCleanupCB) (XLogReaderState *xlogreader);
+
+/* Function type definition for the open/close callbacks for WALRead() */
typedef void (*WALSegmentOpenCB) (XLogReaderState *xlogreader,
XLogSegNo nextSegNo,
TimeLineID *tli_p);
typedef void (*WALSegmentCloseCB) (XLogReaderState *xlogreader);
-typedef struct XLogReaderRoutine
-{
- /*
- * Data input callback
- *
- * This callback shall read at least reqLen valid bytes of the xlog page
- * starting at targetPagePtr, and store them in readBuf. The callback
- * shall return the number of bytes read (never more than XLOG_BLCKSZ), or
- * -1 on failure. The callback shall sleep, if necessary, to wait for the
- * requested bytes to become available. The callback will not be invoked
- * again for the same page unless more than the returned number of bytes
- * are needed.
- *
- * targetRecPtr is the position of the WAL record we're reading. Usually
- * it is equal to targetPagePtr + reqLen, but sometimes xlogreader needs
- * to read and verify the page or segment header, before it reads the
- * actual WAL record it's interested in. In that case, targetRecPtr can
- * be used to determine which timeline to read the page from.
- *
- * The callback shall set ->seg.ws_tli to the TLI of the file the page was
- * read from.
- */
- XLogPageReadCB page_read;
-
- /*
- * Callback to open the specified WAL segment for reading. ->seg.ws_file
- * shall be set to the file descriptor of the opened segment. In case of
- * failure, an error shall be raised by the callback and it shall not
- * return.
- *
- * "nextSegNo" is the number of the segment to be opened.
- *
- * "tli_p" is an input/output argument. WALRead() uses it to pass the
- * timeline in which the new segment should be found, but the callback can
- * use it to return the TLI that it actually opened.
- */
- WALSegmentOpenCB segment_open;
-
- /*
- * WAL segment close callback. ->seg.ws_file shall be set to a negative
- * number.
- */
- WALSegmentCloseCB segment_close;
-} XLogReaderRoutine;
-
-#define XL_ROUTINE(...) &(XLogReaderRoutine){__VA_ARGS__}
-
typedef struct
{
/* Is this block ref in use? */
@@ -144,12 +95,35 @@ typedef struct
uint16 data_bufsz;
} DecodedBkpBlock;
+/* Return code from XLogReadRecord */
+typedef enum XLogReadRecordResult
+{
+ XLREAD_SUCCESS, /* record is successfully read */
+ XLREAD_NEED_DATA, /* need more data. see XLogReadRecord. */
+ XLREAD_FAIL /* failed during reading a record */
+} XLogReadRecordResult;
+
+/*
+ * internal state of XLogReadRecord
+ *
+ * XLogReadState runs a state machine while reading a record. Theses states
+ * are not seen outside the function. Each state may repeat several times
+ * exiting requesting caller for new data. See the comment of XLogReadRecrod
+ * for details.
+ */
+typedef enum XLogReadRecordState {
+ XLREAD_NEXT_RECORD,
+ XLREAD_TOT_LEN,
+ XLREAD_FIRST_FRAGMENT,
+ XLREAD_CONTINUATION
+} XLogReadRecordState;
+
struct XLogReaderState
{
/*
* Operational callbacks
*/
- XLogReaderRoutine routine;
+ WALSegmentCleanupCB cleanup_cb;
/* ----------------------------------------
* Public parameters
@@ -162,18 +136,14 @@ struct XLogReaderState
*/
uint64 system_identifier;
- /*
- * Opaque data for callbacks to use. Not used by XLogReader.
- */
- void *private_data;
-
/*
* Start and end point of last record read. EndRecPtr is also used as the
* position to read next. Calling XLogBeginRead() sets EndRecPtr to the
* starting position and ReadRecPtr to invalid.
*/
- XLogRecPtr ReadRecPtr; /* start of last record read */
+ XLogRecPtr ReadRecPtr; /* start of last record read or being read */
XLogRecPtr EndRecPtr; /* end+1 of last record read */
+ XLogRecPtr PrevRecPtr; /* start of previous record read */
/* ----------------------------------------
* Communication with page reader
@@ -187,7 +157,9 @@ struct XLogReaderState
* the request, or -1 on error */
TimeLineID readPageTLI; /* TLI for data currently in readBuf */
char *readBuf; /* buffer to store data */
- bool page_verified; /* is the page on the buffer verified? */
+ bool page_verified; /* is the page header on the buffer verified? */
+ bool record_verified;/* is the current record header verified? */
+
/* ----------------------------------------
@@ -229,8 +201,6 @@ struct XLogReaderState
XLogRecPtr latestPagePtr;
TimeLineID latestPageTLI;
- /* beginning of the WAL record being read. */
- XLogRecPtr currRecPtr;
/* timeline to read it from, 0 if a lookup is required */
TimeLineID currTLI;
@@ -257,6 +227,15 @@ struct XLogReaderState
char *readRecordBuf;
uint32 readRecordBufSize;
+ /*
+ * XLogReadRecord() state
+ */
+ XLogReadRecordState readRecordState;/* state machine state */
+ int recordGotLen; /* amount of current record that has
+ * already been read */
+ int recordRemainLen; /* length of current record that remains */
+ XLogRecPtr recordContRecPtr; /* where the current record continues */
+
/* Buffer to hold error message */
char *errormsg_buf;
};
@@ -264,9 +243,7 @@ struct XLogReaderState
/* Get a new XLogReader */
extern XLogReaderState *XLogReaderAllocate(int wal_segment_size,
const char *waldir,
- XLogReaderRoutine *routine,
- void *private_data);
-extern XLogReaderRoutine *LocalXLogReaderRoutine(void);
+ WALSegmentCleanupCB cleanup_cb);
/* Free an XLogReader */
extern void XLogReaderFree(XLogReaderState *state);
@@ -274,12 +251,17 @@ extern void XLogReaderFree(XLogReaderState *state);
/* Position the XLogReader to given record */
extern void XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr);
#ifdef FRONTEND
-extern XLogRecPtr XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr);
+/* Function type definition for the read_page callback */
+typedef bool (*XLogFindNextRecordCB) (XLogReaderState *xlogreader,
+ void *private);
+extern XLogRecPtr XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
+ XLogFindNextRecordCB read_page, void *private);
#endif /* FRONTEND */
/* Read the next XLog record. Returns NULL on end-of-WAL or failure */
-extern struct XLogRecord *XLogReadRecord(XLogReaderState *state,
- char **errormsg);
+extern XLogReadRecordResult XLogReadRecord(XLogReaderState *state,
+ XLogRecord **record,
+ char **errormsg);
/* Validate a page */
extern bool XLogReaderValidatePageHeader(XLogReaderState *state,
@@ -299,6 +281,7 @@ typedef struct WALReadError
} WALReadError;
extern bool WALRead(XLogReaderState *state,
+ WALSegmentOpenCB segopenfn, WALSegmentCloseCB sgclosefn,
char *buf, XLogRecPtr startptr, Size count,
TimeLineID tli, WALReadError *errinfo);
diff --git a/src/include/access/xlogutils.h b/src/include/access/xlogutils.h
index 364a21c4ea..397fb27fc2 100644
--- a/src/include/access/xlogutils.h
+++ b/src/include/access/xlogutils.h
@@ -47,9 +47,7 @@ extern Buffer XLogReadBufferExtended(RelFileNode rnode, ForkNumber forknum,
extern Relation CreateFakeRelcacheEntry(RelFileNode rnode);
extern void FreeFakeRelcacheEntry(Relation fakerel);
-extern bool read_local_xlog_page(XLogReaderState *state,
- XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetRecPtr, char *cur_page);
+extern bool read_local_xlog_page(XLogReaderState *state);
extern void wal_segment_open(XLogReaderState *state,
XLogSegNo nextSegNo,
TimeLineID *tli_p);
diff --git a/src/include/pg_config_manual.h b/src/include/pg_config_manual.h
index f10ad0acd6..94749609b4 100644
--- a/src/include/pg_config_manual.h
+++ b/src/include/pg_config_manual.h
@@ -377,7 +377,7 @@
* Enable debugging print statements for WAL-related operations; see
* also the wal_debug GUC var.
*/
-/* #define WAL_DEBUG */
+#define WAL_DEBUG
/*
* Enable tracing of resource consumption during sort operations;
diff --git a/src/include/replication/logical.h b/src/include/replication/logical.h
index c253403372..b936f31393 100644
--- a/src/include/replication/logical.h
+++ b/src/include/replication/logical.h
@@ -29,6 +29,10 @@ typedef void (*LogicalOutputPluginWriterUpdateProgress) (struct LogicalDecodingC
TransactionId xid
);
+typedef struct LogicalDecodingContext LogicalDecodingContext;
+
+typedef bool (*LogicalDecodingXLogPageReadCB)(XLogReaderState *ctx);
+
typedef struct LogicalDecodingContext
{
/* memory context this is all allocated in */
@@ -39,6 +43,7 @@ typedef struct LogicalDecodingContext
/* infrastructure pieces for decoding */
XLogReaderState *reader;
+ LogicalDecodingXLogPageReadCB page_read;
struct ReorderBuffer *reorder;
struct SnapBuild *snapshot_builder;
@@ -105,14 +110,16 @@ extern LogicalDecodingContext *CreateInitDecodingContext(const char *plugin,
List *output_plugin_options,
bool need_full_snapshot,
XLogRecPtr restart_lsn,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress);
extern LogicalDecodingContext *CreateDecodingContext(XLogRecPtr start_lsn,
List *output_plugin_options,
bool fast_forward,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress);
--
2.27.0
----Next_Part(Thu_Mar__4_11_28_53_2021_014)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
filename="v17-0003-Remove-globals-readOff-readLen-and-readSegNo.patch"
^ permalink raw reply [nested|flat] 70+ messages in thread
* [PATCH v17 2/4] Move page-reader out of XLogReadRecord
@ 2019-09-10 03:58 Kyotaro Horiguchi <[email protected]>
0 siblings, 0 replies; 70+ messages in thread
From: Kyotaro Horiguchi @ 2019-09-10 03:58 UTC (permalink / raw)
This is the second step of removing callbacks from WAL record reader.
Since it is essential to take in additional data while reading a
record, the function have to ask caller for new data while keeping
working state. Thus the function is turned into a state machine.
---
src/backend/access/transam/twophase.c | 15 +-
src/backend/access/transam/xlog.c | 58 +-
src/backend/access/transam/xlogreader.c | 697 +++++++++++-------
src/backend/access/transam/xlogutils.c | 17 +-
src/backend/replication/logical/logical.c | 26 +-
.../replication/logical/logicalfuncs.c | 13 +-
src/backend/replication/slotfuncs.c | 18 +-
src/backend/replication/walsender.c | 32 +-
src/bin/pg_rewind/parsexlog.c | 84 ++-
src/bin/pg_waldump/pg_waldump.c | 36 +-
src/include/access/xlogreader.h | 121 ++-
src/include/access/xlogutils.h | 4 +-
src/include/pg_config_manual.h | 2 +-
src/include/replication/logical.h | 11 +-
14 files changed, 646 insertions(+), 488 deletions(-)
diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index 80d2d20d6c..7d5a96a5f5 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -1325,11 +1325,8 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
XLogReaderState *xlogreader;
char *errormsg;
- xlogreader = XLogReaderAllocate(wal_segment_size, NULL,
- XL_ROUTINE(.page_read = &read_local_xlog_page,
- .segment_open = &wal_segment_open,
- .segment_close = &wal_segment_close),
- NULL);
+ xlogreader = XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+
if (!xlogreader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
@@ -1337,7 +1334,13 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
errdetail("Failed while allocating a WAL reading processor.")));
XLogBeginRead(xlogreader, lsn);
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!read_local_xlog_page(xlogreader))
+ break;
+ }
+
if (record == NULL)
ereport(ERROR,
(errcode_for_file_access(),
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 606328a65a..670996ae67 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -836,13 +836,6 @@ static XLogSource currentSource = XLOG_FROM_ANY;
static bool lastSourceFailed = false;
static bool pendingWalRcvRestart = false;
-typedef struct XLogPageReadPrivate
-{
- int emode;
- bool fetching_ckpt; /* are we fetching a checkpoint record? */
- bool randAccess;
-} XLogPageReadPrivate;
-
/*
* These variables track when we last obtained some WAL data to process,
* and where we got it from. (XLogReceiptSource is initially the same as
@@ -917,8 +910,8 @@ static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath,
static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli,
XLogSource source, bool notfoundOk);
static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, XLogSource source);
-static bool XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
+static bool XLogPageRead(XLogReaderState *xlogreader,
+ bool fetching_ckpt, int emode, bool randAccess);
static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
bool fetching_ckpt, XLogRecPtr tliRecPtr);
static int emode_for_corrupt_record(int emode, XLogRecPtr RecPtr);
@@ -1231,8 +1224,7 @@ XLogInsertRecord(XLogRecData *rdata,
appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len);
if (!debug_reader)
- debug_reader = XLogReaderAllocate(wal_segment_size, NULL,
- XL_ROUTINE(), NULL);
+ debug_reader = XLogReaderAllocate(wal_segment_size, NULL, NULL);
if (!debug_reader)
{
@@ -4344,15 +4336,10 @@ CleanupBackupHistory(void)
* record is available.
*/
static XLogRecord *
-ReadRecord(XLogReaderState *xlogreader, int emode,
- bool fetching_ckpt)
+ReadRecord(XLogReaderState *xlogreader, int emode, bool fetching_ckpt)
{
XLogRecord *record;
- XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
-
- private->fetching_ckpt = fetching_ckpt;
- private->emode = emode;
- private->randAccess = (xlogreader->ReadRecPtr == InvalidXLogRecPtr);
+ bool randAccess = (xlogreader->ReadRecPtr == InvalidXLogRecPtr);
/* This is the first attempt to read this page. */
lastSourceFailed = false;
@@ -4360,8 +4347,16 @@ ReadRecord(XLogReaderState *xlogreader, int emode,
for (;;)
{
char *errormsg;
+ XLogReadRecordResult result;
+
+ while ((result = XLogReadRecord(xlogreader, &record, &errormsg))
+ == XLREAD_NEED_DATA)
+ {
+ if (!XLogPageRead(xlogreader, fetching_ckpt, emode, randAccess))
+ break;
+
+ }
- record = XLogReadRecord(xlogreader, &errormsg);
ReadRecPtr = xlogreader->ReadRecPtr;
EndRecPtr = xlogreader->EndRecPtr;
if (record == NULL)
@@ -6380,7 +6375,6 @@ StartupXLOG(void)
bool backupFromStandby = false;
DBState dbstate_at_startup;
XLogReaderState *xlogreader;
- XLogPageReadPrivate private;
bool promoted = false;
struct stat st;
@@ -6539,13 +6533,9 @@ StartupXLOG(void)
OwnLatch(&XLogCtl->recoveryWakeupLatch);
/* Set up XLOG reader facility */
- MemSet(&private, 0, sizeof(XLogPageReadPrivate));
xlogreader =
- XLogReaderAllocate(wal_segment_size, NULL,
- XL_ROUTINE(.page_read = &XLogPageRead,
- .segment_open = NULL,
- .segment_close = wal_segment_close),
- &private);
+ XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+
if (!xlogreader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
@@ -11987,12 +11977,13 @@ CancelBackup(void)
* sleep and retry.
*/
static bool
-XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetRecPtr, char *readBuf)
+XLogPageRead(XLogReaderState *xlogreader,
+ bool fetching_ckpt, int emode, bool randAccess)
{
- XLogPageReadPrivate *private =
- (XLogPageReadPrivate *) xlogreader->private_data;
- int emode = private->emode;
+ char *readBuf = xlogreader->readBuf;
+ XLogRecPtr targetPagePtr = xlogreader->readPagePtr;
+ int reqLen = xlogreader->readLen;
+ XLogRecPtr targetRecPtr = xlogreader->ReadRecPtr;
uint32 targetPageOff;
XLogSegNo targetSegNo PG_USED_FOR_ASSERTS_ONLY;
int r;
@@ -12035,8 +12026,8 @@ retry:
flushedUpto < targetPagePtr + reqLen))
{
if (!WaitForWALToBecomeAvailable(targetPagePtr + reqLen,
- private->randAccess,
- private->fetching_ckpt,
+ randAccess,
+ fetching_ckpt,
targetRecPtr))
{
if (readFile >= 0)
@@ -12141,6 +12132,7 @@ retry:
goto next_record_is_invalid;
}
+ Assert(xlogreader->readPagePtr == targetPagePtr);
xlogreader->readLen = readLen;
return true;
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index 7d04995b7b..ba3e4868f7 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -40,7 +40,7 @@ static bool XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr,
int reqLen, bool header_inclusive);
static void XLogReaderInvalReadState(XLogReaderState *state);
static bool ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
- XLogRecPtr PrevRecPtr, XLogRecord *record, bool randAccess);
+ XLogRecPtr PrevRecPtr, XLogRecord *record);
static bool ValidXLogRecord(XLogReaderState *state, XLogRecord *record,
XLogRecPtr recptr);
static void ResetDecoder(XLogReaderState *state);
@@ -73,7 +73,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...)
*/
XLogReaderState *
XLogReaderAllocate(int wal_segment_size, const char *waldir,
- XLogReaderRoutine *routine, void *private_data)
+ WALSegmentCleanupCB cleanup_cb)
{
XLogReaderState *state;
@@ -84,7 +84,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
return NULL;
/* initialize caller-provided support functions */
- state->routine = *routine;
+ state->cleanup_cb = cleanup_cb;
state->max_block_id = -1;
@@ -107,8 +107,6 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
WALOpenSegmentInit(&state->seg, &state->segcxt, wal_segment_size,
waldir);
- /* system_identifier initialized to zeroes above */
- state->private_data = private_data;
/* ReadRecPtr, EndRecPtr and readLen initialized to zeroes above */
state->errormsg_buf = palloc_extended(MAX_ERRORMSG_LEN + 1,
MCXT_ALLOC_NO_OOM);
@@ -140,8 +138,8 @@ XLogReaderFree(XLogReaderState *state)
{
int block_id;
- if (state->seg.ws_file != -1)
- state->routine.segment_close(state);
+ if (state->seg.ws_file >= 0)
+ state->cleanup_cb(state);
for (block_id = 0; block_id <= XLR_MAX_BLOCK_ID; block_id++)
{
@@ -246,6 +244,7 @@ XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
/* Begin at the passed-in record pointer. */
state->EndRecPtr = RecPtr;
state->ReadRecPtr = InvalidXLogRecPtr;
+ state->readRecordState = XLREAD_NEXT_RECORD;
}
/*
@@ -254,317 +253,449 @@ XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
* XLogBeginRead() or XLogFindNextRecord() must be called before the first call
* to XLogReadRecord().
*
- * If the page_read callback fails to read the requested data, NULL is
- * returned. The callback is expected to have reported the error; errormsg
- * is set to NULL.
+ * This function may return XLREAD_NEED_DATA several times before returning a
+ * result record. The caller shall read in some new data then call this
+ * function again with the same parameters.
*
- * If the reading fails for some other reason, NULL is also returned, and
- * *errormsg is set to a string with details of the failure.
+ * When a record is successfully read, returns XLREAD_SUCCESS with result
+ * record being stored in *record. Otherwise *record is NULL.
*
- * The returned pointer (or *errormsg) points to an internal buffer that's
- * valid until the next call to XLogReadRecord.
+ * Returns XLREAD_NEED_DATA if more data is needed to finish reading the
+ * current record. In that case, state->readPagePtr and state->readLen inform
+ * the desired position and minimum length of data needed. The caller shall
+ * read in the requested data and set state->readBuf to point to a buffer
+ * containing it. The caller must also set state->readPageTLI and
+ * state->readLen to indicate the timeline that it was read from, and the
+ * length of data that is now available (which must be >= given readLen),
+ * respectively.
+ *
+ * If invalid data is encountered, returns XLREAD_FAIL with *record being set to
+ * NULL. *errormsg is set to a string with details of the failure.
+ * The returned pointer (or *errormsg) points to an internal buffer that's valid
+ * until the next call to XLogReadRecord.
+ *
+ *
+ * This function runs a state machine consists of the following states.
+ *
+ * XLREAD_NEXT_RECORD :
+ * The initial state, if called with valid RecPtr, try to read a record at
+ * that position. If invalid RecPtr is given try to read a record just after
+ * the last one previously read.
+ * This state ens after setting ReadRecPtr. Then goes to XLREAD_TOT_LEN.
+ *
+ * XLREAD_TOT_LEN:
+ * Examining record header. Ends after reading record total
+ * length. recordRemainLen and recordGotLen are initialized.
+ *
+ * XLREAD_FIRST_FRAGMENT:
+ * Reading the first fragment. Ends with finishing reading a single
+ * record. Goes to XLREAD_NEXT_RECORD if that's all or
+ * XLREAD_CONTINUATION if we have continuation.
+
+ * XLREAD_CONTINUATION:
+ * Reading continuation of record. Ends with finishing the whole record then
+ * goes to XLREAD_NEXT_RECORD. During this state, recordRemainLen indicates
+ * how much is left and readRecordBuf holds the partially assert
+ * record.recordContRecPtr points to the beginning of the next page where to
+ * continue.
+ *
+ * If wrong data found in any state, the state machine stays at the current
+ * state. This behavior allows to continue reading a reacord switching among
+ * different souces, while streaming replication.
*/
-XLogRecord *
-XLogReadRecord(XLogReaderState *state, char **errormsg)
+XLogReadRecordResult
+XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
{
- XLogRecPtr RecPtr;
- XLogRecord *record;
- XLogRecPtr targetPagePtr;
- bool randAccess;
- uint32 len,
- total_len;
- uint32 targetRecOff;
- uint32 pageHeaderSize;
- bool gotheader;
+ XLogRecord *prec;
- /*
- * randAccess indicates whether to verify the previous-record pointer of
- * the record we're reading. We only do this if we're reading
- * sequentially, which is what we initially assume.
- */
- randAccess = false;
+ *record = NULL;
/* reset error state */
*errormsg = NULL;
state->errormsg_buf[0] = '\0';
- ResetDecoder(state);
-
- RecPtr = state->EndRecPtr;
-
- if (state->ReadRecPtr != InvalidXLogRecPtr)
- {
- /* read the record after the one we just read */
-
- /*
- * EndRecPtr is pointing to end+1 of the previous WAL record. If
- * we're at a page boundary, no more records can fit on the current
- * page. We must skip over the page header, but we can't do that until
- * we've read in the page, since the header size is variable.
- */
- }
- else
- {
- /*
- * Caller supplied a position to start at.
- *
- * In this case, EndRecPtr should already be pointing to a valid
- * record starting position.
- */
- Assert(XRecOffIsValid(RecPtr));
- randAccess = true;
- }
-
- state->currRecPtr = RecPtr;
-
- targetPagePtr = RecPtr - (RecPtr % XLOG_BLCKSZ);
- targetRecOff = RecPtr % XLOG_BLCKSZ;
-
- /*
- * Read the page containing the record into state->readBuf. Request enough
- * byte to cover the whole record header, or at least the part of it that
- * fits on the same page.
- */
- while (XLogNeedData(state, targetPagePtr,
- Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
- targetRecOff != 0))
- {
- if (!state->routine.page_read(state, state->readPagePtr, state->readLen,
- RecPtr, state->readBuf))
- break;
- }
-
- if (!state->page_verified)
- goto err;
-
- /*
- * We have at least the page header, so we can examine it now.
- */
- pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
- if (targetRecOff == 0)
- {
- /*
- * At page start, so skip over page header.
- */
- RecPtr += pageHeaderSize;
- targetRecOff = pageHeaderSize;
- }
- else if (targetRecOff < pageHeaderSize)
- {
- report_invalid_record(state, "invalid record offset at %X/%X",
- LSN_FORMAT_ARGS(RecPtr));
- goto err;
- }
-
- if ((((XLogPageHeader) state->readBuf)->xlp_info & XLP_FIRST_IS_CONTRECORD) &&
- targetRecOff == pageHeaderSize)
- {
- report_invalid_record(state, "contrecord is requested by %X/%X",
- LSN_FORMAT_ARGS(RecPtr));
- goto err;
- }
-
- /* XLogNeedData has verified the page header */
- Assert(pageHeaderSize <= state->readLen);
-
- /*
- * Read the record length.
- *
- * NB: Even though we use an XLogRecord pointer here, the whole record
- * header might not fit on this page. xl_tot_len is the first field of the
- * struct, so it must be on this page (the records are MAXALIGNed), but we
- * cannot access any other fields until we've verified that we got the
- * whole header.
- */
- record = (XLogRecord *) (state->readBuf + RecPtr % XLOG_BLCKSZ);
- total_len = record->xl_tot_len;
-
- /*
- * If the whole record header is on this page, validate it immediately.
- * Otherwise do just a basic sanity check on xl_tot_len, and validate the
- * rest of the header after reading it from the next page. The xl_tot_len
- * check is necessary here to ensure that we enter the "Need to reassemble
- * record" code path below; otherwise we might fail to apply
- * ValidXLogRecordHeader at all.
- */
- if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
- {
- if (!ValidXLogRecordHeader(state, RecPtr, state->ReadRecPtr, record,
- randAccess))
- goto err;
- gotheader = true;
- }
- else
- {
- /* XXX: more validation should be done here */
- if (total_len < SizeOfXLogRecord)
- {
- report_invalid_record(state,
- "invalid record length at %X/%X: wanted %u, got %u",
- LSN_FORMAT_ARGS(RecPtr),
- (uint32) SizeOfXLogRecord, total_len);
- goto err;
- }
- gotheader = false;
- }
-
- len = XLOG_BLCKSZ - RecPtr % XLOG_BLCKSZ;
- if (total_len > len)
+ switch (state->readRecordState)
{
- /* Need to reassemble record */
- char *contdata;
- XLogPageHeader pageHeader;
- char *buffer;
- uint32 gotlen;
-
- /*
- * Enlarge readRecordBuf as needed.
- */
- if (total_len > state->readRecordBufSize &&
- !allocate_recordbuf(state, total_len))
- {
- /* We treat this as a "bogus data" condition */
- report_invalid_record(state, "record length %u at %X/%X too long",
- total_len, LSN_FORMAT_ARGS(RecPtr));
- goto err;
- }
+ case XLREAD_NEXT_RECORD:
+ ResetDecoder(state);
- /* Copy the first fragment of the record from the first page. */
- memcpy(state->readRecordBuf,
- state->readBuf + RecPtr % XLOG_BLCKSZ, len);
- buffer = state->readRecordBuf + len;
- gotlen = len;
-
- do
- {
- int rest_len = total_len - gotlen;
-
- /* Calculate pointer to beginning of next page */
- targetPagePtr += XLOG_BLCKSZ;
-
- /* Wait for the next page to become available */
- while (XLogNeedData(state, targetPagePtr,
- Min(rest_len, XLOG_BLCKSZ),
- false))
+ if (state->ReadRecPtr != InvalidXLogRecPtr)
{
- if (!state->routine.page_read(state, state->readPagePtr,
- state->readLen,
- state->ReadRecPtr,
- state->readBuf))
- break;
+ /* read the record after the one we just read */
+
+ /*
+ * EndRecPtr is pointing to end+1 of the previous WAL record.
+ * If we're at a page boundary, no more records can fit on the
+ * current page. We must skip over the page header, but we
+ * can't do that until we've read in the page, since the header
+ * size is variable.
+ */
+ state->PrevRecPtr = state->ReadRecPtr;
+ state->ReadRecPtr = state->EndRecPtr;
+ }
+ else
+ {
+ /*
+ * Caller supplied a position to start at.
+ *
+ * In this case, EndRecPtr should already be pointing to a
+ * valid record starting position.
+ */
+ Assert(XRecOffIsValid(state->EndRecPtr));
+ state->ReadRecPtr = state->EndRecPtr;
+
+ /*
+ * We cannot verify the previous-record pointer when we're
+ * seeking to a particular record. Reset PrevRecPtr so that we
+ * won't try doing that.
+ */
+ state->PrevRecPtr = InvalidXLogRecPtr;
+ state->EndRecPtr = InvalidXLogRecPtr; /* to be tidy */
}
+ state->record_verified = false;
+ state->readRecordState = XLREAD_TOT_LEN;
+ /* fall through */
+
+ case XLREAD_TOT_LEN:
+ {
+ uint32 total_len;
+ uint32 pageHeaderSize;
+ XLogRecPtr targetPagePtr;
+ uint32 targetRecOff;
+ XLogPageHeader pageHeader;
+
+ targetPagePtr =
+ state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
+ targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+
+ /*
+ * Check if we have enough data. For the first record in the page,
+ * the requesting length doesn't contain page header.
+ */
+ if (XLogNeedData(state, targetPagePtr,
+ Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+ targetRecOff != 0))
+ return XLREAD_NEED_DATA;
+
+ /* error out if caller supplied bogus page */
if (!state->page_verified)
goto err;
- Assert(SizeOfXLogShortPHD <= state->readLen);
+ /* examine page header now. */
+ pageHeaderSize =
+ XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+ if (targetRecOff == 0)
+ {
+ /* At page start, so skip over page header. */
+ state->ReadRecPtr += pageHeaderSize;
+ targetRecOff = pageHeaderSize;
+ }
+ else if (targetRecOff < pageHeaderSize)
+ {
+ report_invalid_record(state, "invalid record offset at %X/%X",
+ LSN_FORMAT_ARGS(state->ReadRecPtr));
+ goto err;
+ }
- /* Check that the continuation on next page looks valid */
pageHeader = (XLogPageHeader) state->readBuf;
- if (!(pageHeader->xlp_info & XLP_FIRST_IS_CONTRECORD))
+ if ((pageHeader->xlp_info & XLP_FIRST_IS_CONTRECORD) &&
+ targetRecOff == pageHeaderSize)
{
- report_invalid_record(state,
- "there is no contrecord flag at %X/%X",
- LSN_FORMAT_ARGS(RecPtr));
+ report_invalid_record(state, "contrecord is requested by %X/%X",
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr);
goto err;
}
- /*
- * Cross-check that xlp_rem_len agrees with how much of the record
- * we expect there to be left.
- */
- if (pageHeader->xlp_rem_len == 0 ||
- total_len != (pageHeader->xlp_rem_len + gotlen))
- {
- report_invalid_record(state,
- "invalid contrecord length %u (expected %lld) at %X/%X",
- pageHeader->xlp_rem_len,
- ((long long) total_len) - gotlen,
- LSN_FORMAT_ARGS(RecPtr));
- goto err;
- }
-
- /* Append the continuation from this page to the buffer */
- pageHeaderSize = XLogPageHeaderSize(pageHeader);
-
+ /* XLogNeedData has verified the page header */
Assert(pageHeaderSize <= state->readLen);
- contdata = (char *) state->readBuf + pageHeaderSize;
- len = XLOG_BLCKSZ - pageHeaderSize;
- if (pageHeader->xlp_rem_len < len)
- len = pageHeader->xlp_rem_len;
+ /*
+ * Read the record length.
+ *
+ * NB: Even though we use an XLogRecord pointer here, the whole
+ * record header might not fit on this page. xl_tot_len is the first
+ * field of the struct, so it must be on this page (the records are
+ * MAXALIGNed), but we cannot access any other fields until we've
+ * verified that we got the whole header.
+ */
+ prec = (XLogRecord *) (state->readBuf +
+ state->ReadRecPtr % XLOG_BLCKSZ);
+ total_len = prec->xl_tot_len;
- Assert (pageHeaderSize + len <= state->readLen);
- memcpy(buffer, (char *) contdata, len);
- buffer += len;
- gotlen += len;
+ /*
+ * If the whole record header is on this page, validate it
+ * immediately. Otherwise do just a basic sanity check on
+ * xl_tot_len, and validate the rest of the header after reading it
+ * from the next page. The xl_tot_len check is necessary here to
+ * ensure that we enter the XLREAD_CONTINUATION state below;
+ * otherwise we might fail to apply ValidXLogRecordHeader at all.
+ */
+ if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
+ {
+ if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
+ state->PrevRecPtr, prec))
+ goto err;
- /* If we just reassembled the record header, validate it. */
- if (!gotheader)
+ state->record_verified = true;
+ }
+ else
{
- record = (XLogRecord *) state->readRecordBuf;
- if (!ValidXLogRecordHeader(state, RecPtr, state->ReadRecPtr,
- record, randAccess))
+ /* XXX: more validation should be done here */
+ if (total_len < SizeOfXLogRecord)
+ {
+ report_invalid_record(state,
+ "invalid record length at %X/%X: wanted %u, got %u",
+ LSN_FORMAT_ARGS(state->ReadRecPtr),
+ (uint32) SizeOfXLogRecord, total_len);
goto err;
- gotheader = true;
+ }
}
- } while (gotlen < total_len);
-
- Assert(gotheader);
- record = (XLogRecord *) state->readRecordBuf;
- if (!ValidXLogRecord(state, record, RecPtr))
- goto err;
+ /*
+ * Wait for the rest of the record, or the part of the record that
+ * fit on the first page if crossed a page boundary, to become
+ * available.
+ */
+ state->recordGotLen = 0;
+ state->recordRemainLen = total_len;
+ state->readRecordState = XLREAD_FIRST_FRAGMENT;
+ }
+ /* fall through */
- pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
- state->ReadRecPtr = RecPtr;
- state->EndRecPtr = targetPagePtr + pageHeaderSize
- + MAXALIGN(pageHeader->xlp_rem_len);
- }
- else
- {
- /* Wait for the record data to become available */
- while (XLogNeedData(state, targetPagePtr,
- Min(targetRecOff + total_len, XLOG_BLCKSZ), true))
+ case XLREAD_FIRST_FRAGMENT:
{
- if (!state->routine.page_read(state, state->readPagePtr,
- state->readLen,
- state->ReadRecPtr, state->readBuf))
+ uint32 total_len = state->recordRemainLen;
+ uint32 request_len;
+ uint32 record_len;
+ XLogRecPtr targetPagePtr;
+ uint32 targetRecOff;
+
+ /*
+ * Wait for the rest of the record on the first page to become
+ * available
+ */
+ targetPagePtr =
+ state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
+ targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+
+ request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ);
+ record_len = request_len - targetRecOff;
+
+ /* ReadRecPtr contains page header */
+ Assert (targetRecOff != 0);
+ if (XLogNeedData(state, targetPagePtr, request_len, true))
+ return XLREAD_NEED_DATA;
+
+ /* error out if caller supplied bogus page */
+ if (!state->page_verified)
+ goto err;
+
+ prec = (XLogRecord *) (state->readBuf + targetRecOff);
+
+ /* validate record header if not yet */
+ if (!state->record_verified && record_len >= SizeOfXLogRecord)
+ {
+ if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
+ state->PrevRecPtr, prec))
+ goto err;
+
+ state->record_verified = true;
+ }
+
+
+ if (total_len == record_len)
+ {
+ /* Record does not cross a page boundary */
+ Assert(state->record_verified);
+
+ if (!ValidXLogRecord(state, prec, state->ReadRecPtr))
+ goto err;
+
+ state->record_verified = true; /* to be tidy */
+
+ /* We already checked the header earlier */
+ state->EndRecPtr = state->ReadRecPtr + MAXALIGN(record_len);
+
+ *record = prec;
+ state->readRecordState = XLREAD_NEXT_RECORD;
break;
+ }
+
+ /*
+ * The record continues on the next page. Need to reassemble
+ * record
+ */
+ Assert(total_len > record_len);
+
+ /* Enlarge readRecordBuf as needed. */
+ if (total_len > state->readRecordBufSize &&
+ !allocate_recordbuf(state, total_len))
+ {
+ /* We treat this as a "bogus data" condition */
+ report_invalid_record(state,
+ "record length %u at %X/%X too long",
+ total_len,
+ LSN_FORMAT_ARGS(state->ReadRecPtr));
+ goto err;
+ }
+
+ /* Copy the first fragment of the record from the first page. */
+ memcpy(state->readRecordBuf, state->readBuf + targetRecOff,
+ record_len);
+ state->recordGotLen += record_len;
+ state->recordRemainLen -= record_len;
+
+ /* Calculate pointer to beginning of next page */
+ state->recordContRecPtr = state->ReadRecPtr + record_len;
+ Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0);
+
+ state->readRecordState = XLREAD_CONTINUATION;
}
+ /* fall through */
+
+ case XLREAD_CONTINUATION:
+ {
+ XLogPageHeader pageHeader;
+ uint32 pageHeaderSize;
+ XLogRecPtr targetPagePtr;
+
+ /* we enter this state only if we haven't read the whole record. */
+ Assert (state->recordRemainLen > 0);
+
+ while(state->recordRemainLen > 0)
+ {
+ char *contdata;
+ uint32 request_len;
+ uint32 record_len;
+
+ /* Wait for the next page to become available */
+ targetPagePtr = state->recordContRecPtr;
+
+ /* this request contains page header */
+ Assert (targetPagePtr != 0);
+ if (XLogNeedData(state, targetPagePtr,
+ Min(state->recordRemainLen, XLOG_BLCKSZ),
+ false))
+ return XLREAD_NEED_DATA;
+
+ if (!state->page_verified)
+ goto err;
+
+ Assert(SizeOfXLogShortPHD <= state->readLen);
- if (!state->page_verified)
- goto err;
+ /* Check that the continuation on next page looks valid */
+ pageHeader = (XLogPageHeader) state->readBuf;
+ if (!(pageHeader->xlp_info & XLP_FIRST_IS_CONTRECORD))
+ {
+ report_invalid_record(
+ state,
+ "there is no contrecord flag at %X/%X reading %X/%X",
+ (uint32) (state->recordContRecPtr >> 32),
+ (uint32) state->recordContRecPtr,
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr);
+ goto err;
+ }
- /* Record does not cross a page boundary */
- if (!ValidXLogRecord(state, record, RecPtr))
- goto err;
+ /*
+ * Cross-check that xlp_rem_len agrees with how much of the
+ * record we expect there to be left.
+ */
+ if (pageHeader->xlp_rem_len == 0 ||
+ pageHeader->xlp_rem_len != state->recordRemainLen)
+ {
+ report_invalid_record(
+ state,
+ "invalid contrecord length %u at %X/%X reading %X/%X, expected %u",
+ pageHeader->xlp_rem_len,
+ (uint32) (state->recordContRecPtr >> 32),
+ (uint32) state->recordContRecPtr,
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr,
+ state->recordRemainLen);
+ goto err;
+ }
- state->EndRecPtr = RecPtr + MAXALIGN(total_len);
+ /* Append the continuation from this page to the buffer */
+ pageHeaderSize = XLogPageHeaderSize(pageHeader);
- state->ReadRecPtr = RecPtr;
+ /*
+ * XLogNeedData should have ensured that the whole page header
+ * was read
+ */
+ Assert(state->readLen >= pageHeaderSize);
+
+ contdata = (char *) state->readBuf + pageHeaderSize;
+ record_len = XLOG_BLCKSZ - pageHeaderSize;
+ if (pageHeader->xlp_rem_len < record_len)
+ record_len = pageHeader->xlp_rem_len;
+
+ request_len = record_len + pageHeaderSize;
+
+ /* XLogNeedData should have ensured all needed data was read */
+ Assert (state->readLen >= request_len);
+
+ memcpy(state->readRecordBuf + state->recordGotLen,
+ (char *) contdata, record_len);
+ state->recordGotLen += record_len;
+ state->recordRemainLen -= record_len;
+
+ /* If we just reassembled the record header, validate it. */
+ if (!state->record_verified)
+ {
+ Assert(state->recordGotLen >= SizeOfXLogRecord);
+ if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
+ state->PrevRecPtr,
+ (XLogRecord *) state->readRecordBuf))
+ goto err;
+
+ state->record_verified = true;
+ }
+
+ /* Calculate pointer to beginning of next page, and continue */
+ state->recordContRecPtr += XLOG_BLCKSZ;
+ }
+
+ /* targetPagePtr is pointing the last-read page here */
+ prec = (XLogRecord *) state->readRecordBuf;
+ if (!ValidXLogRecord(state, prec, state->ReadRecPtr))
+ goto err;
+
+ pageHeaderSize =
+ XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+ state->EndRecPtr = targetPagePtr + pageHeaderSize
+ + MAXALIGN(pageHeader->xlp_rem_len);
+
+ *record = prec;
+ state->readRecordState = XLREAD_NEXT_RECORD;
+ break;
+ }
}
/*
* Special processing if it's an XLOG SWITCH record
*/
- if (record->xl_rmid == RM_XLOG_ID &&
- (record->xl_info & ~XLR_INFO_MASK) == XLOG_SWITCH)
+ if ((*record)->xl_rmid == RM_XLOG_ID &&
+ ((*record)->xl_info & ~XLR_INFO_MASK) == XLOG_SWITCH)
{
/* Pretend it extends to end of segment */
state->EndRecPtr += state->segcxt.ws_segsize - 1;
state->EndRecPtr -= XLogSegmentOffset(state->EndRecPtr, state->segcxt.ws_segsize);
}
- if (DecodeXLogRecord(state, record, errormsg))
- return record;
- else
- return NULL;
+ Assert (!*record || state->readLen >= 0);
+ if (DecodeXLogRecord(state, *record, errormsg))
+ return XLREAD_SUCCESS;
+
+ *record = NULL;
+ return XLREAD_FAIL;
err:
/*
- * Invalidate the read state. We might read from a different source after
+ * Invalidate the read page. We might read from a different source after
* failure.
*/
XLogReaderInvalReadState(state);
@@ -572,7 +703,8 @@ err:
if (state->errormsg_buf[0] != '\0')
*errormsg = state->errormsg_buf;
- return NULL;
+ *record = NULL;
+ return XLREAD_FAIL;
}
/*
@@ -724,11 +856,12 @@ XLogReaderInvalReadState(XLogReaderState *state)
*
* This is just a convenience subroutine to avoid duplicated code in
* XLogReadRecord. It's not intended for use from anywhere else.
+ *
+ * If PrevRecPtr is valid, the xl_prev is is cross-checked with it.
*/
static bool
ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
- XLogRecPtr PrevRecPtr, XLogRecord *record,
- bool randAccess)
+ XLogRecPtr PrevRecPtr, XLogRecord *record)
{
if (record->xl_tot_len < SizeOfXLogRecord)
{
@@ -745,7 +878,7 @@ ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
record->xl_rmid, LSN_FORMAT_ARGS(RecPtr));
return false;
}
- if (randAccess)
+ if (PrevRecPtr == InvalidXLogRecPtr)
{
/*
* We can't exactly verify the prev-link, but surely it should be less
@@ -975,11 +1108,14 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
* XLogReadRecord() will read the next valid record.
*/
XLogRecPtr
-XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
+XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
+ XLogFindNextRecordCB read_page, void *private)
{
XLogRecPtr tmpRecPtr;
XLogRecPtr found = InvalidXLogRecPtr;
XLogPageHeader header;
+ XLogRecord *record;
+ XLogReadRecordResult result;
char *errormsg;
Assert(!XLogRecPtrIsInvalid(RecPtr));
@@ -1012,9 +1148,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
while(XLogNeedData(state, targetPagePtr, targetRecOff,
targetRecOff != 0))
{
- if (!state->routine.page_read(state, state->readPagePtr,
- state->readLen,
- state->ReadRecPtr, state->readBuf))
+ if (!read_page(state, private))
break;
}
@@ -1066,8 +1200,16 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
* or we just jumped over the remaining data of a continuation.
*/
XLogBeginRead(state, tmpRecPtr);
- while (XLogReadRecord(state, &errormsg) != NULL)
+ while ((result = XLogReadRecord(state, &record, &errormsg)) !=
+ XLREAD_FAIL)
{
+ if (result == XLREAD_NEED_DATA)
+ {
+ if (!read_page(state, private))
+ goto err;
+ continue;
+ }
+
/* past the record we've found, break out */
if (RecPtr <= state->ReadRecPtr)
{
@@ -1087,9 +1229,9 @@ err:
#endif /* FRONTEND */
/*
- * Helper function to ease writing of XLogRoutine->page_read callbacks.
- * If this function is used, caller must supply a segment_open callback in
- * 'state', as that is used here.
+ * Helper function to ease writing of page_read callback.
+ * If this function is used, caller must supply a segment_open callback and
+ * segment_close callback as that is used here.
*
* Read 'count' bytes into 'buf', starting at location 'startptr', from WAL
* fetched from timeline 'tli'.
@@ -1102,6 +1244,7 @@ err:
*/
bool
WALRead(XLogReaderState *state,
+ WALSegmentOpenCB segopenfn, WALSegmentCloseCB segclosefn,
char *buf, XLogRecPtr startptr, Size count, TimeLineID tli,
WALReadError *errinfo)
{
@@ -1133,10 +1276,10 @@ WALRead(XLogReaderState *state,
XLogSegNo nextSegNo;
if (state->seg.ws_file >= 0)
- state->routine.segment_close(state);
+ segclosefn(state);
XLByteToSeg(recptr, nextSegNo, state->segcxt.ws_segsize);
- state->routine.segment_open(state, nextSegNo, &tli);
+ segopenfn(state, nextSegNo, &tli);
/* This shouldn't happen -- indicates a bug in segment_open */
Assert(state->seg.ws_file >= 0);
diff --git a/src/backend/access/transam/xlogutils.c b/src/backend/access/transam/xlogutils.c
index 46eda33f25..b003990745 100644
--- a/src/backend/access/transam/xlogutils.c
+++ b/src/backend/access/transam/xlogutils.c
@@ -686,8 +686,7 @@ XLogTruncateRelation(RelFileNode rnode, ForkNumber forkNum,
void
XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wantLength)
{
- const XLogRecPtr lastReadPage = state->seg.ws_segno *
- state->segcxt.ws_segsize + state->readLen;
+ const XLogRecPtr lastReadPage = state->readPagePtr;
Assert(wantPage != InvalidXLogRecPtr && wantPage % XLOG_BLCKSZ == 0);
Assert(wantLength <= XLOG_BLCKSZ);
@@ -702,7 +701,7 @@ XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wa
* current TLI has since become historical.
*/
if (lastReadPage == wantPage &&
- state->readLen != 0 &&
+ state->page_verified &&
lastReadPage + state->readLen >= wantPage + Min(wantLength, XLOG_BLCKSZ - 1))
return;
@@ -788,6 +787,7 @@ wal_segment_open(XLogReaderState *state, XLogSegNo nextSegNo,
char path[MAXPGPATH];
XLogFilePath(path, tli, nextSegNo, state->segcxt.ws_segsize);
+ elog(LOG, "HOGE: %lu, %d => %s", nextSegNo, tli, path);
state->seg.ws_file = BasicOpenFile(path, O_RDONLY | PG_BINARY);
if (state->seg.ws_file >= 0)
return;
@@ -825,9 +825,11 @@ wal_segment_close(XLogReaderState *state)
* loop for now.
*/
bool
-read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *cur_page)
+read_local_xlog_page(XLogReaderState *state)
{
+ XLogRecPtr targetPagePtr = state->readPagePtr;
+ int reqLen = state->readLen;
+ char *cur_page = state->readBuf;
XLogRecPtr read_upto,
loc;
TimeLineID tli;
@@ -940,11 +942,12 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
* as 'count', read the whole page anyway. It's guaranteed to be
* zero-padded up to the page boundary if it's incomplete.
*/
- if (!WALRead(state, cur_page, targetPagePtr, XLOG_BLCKSZ, tli,
- &errinfo))
+ if (!WALRead(state, wal_segment_open, wal_segment_close,
+ cur_page, targetPagePtr, XLOG_BLCKSZ, tli, &errinfo))
WALReadRaiseError(&errinfo);
/* number of valid bytes in the buffer */
+ state->readPagePtr = targetPagePtr;
state->readLen = count;
return true;
}
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index 37b75deb72..9cbd8afa49 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -148,7 +148,8 @@ StartupDecodingContext(List *output_plugin_options,
TransactionId xmin_horizon,
bool need_full_snapshot,
bool fast_forward,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress)
@@ -198,11 +199,12 @@ StartupDecodingContext(List *output_plugin_options,
ctx->slot = slot;
- ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, xl_routine, ctx);
+ ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, cleanup_cb);
if (!ctx->reader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
errmsg("out of memory")));
+ ctx->page_read = page_read;
ctx->reorder = ReorderBufferAllocate();
ctx->snapshot_builder =
@@ -319,7 +321,8 @@ CreateInitDecodingContext(const char *plugin,
List *output_plugin_options,
bool need_full_snapshot,
XLogRecPtr restart_lsn,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress)
@@ -422,7 +425,7 @@ CreateInitDecodingContext(const char *plugin,
ctx = StartupDecodingContext(NIL, restart_lsn, xmin_horizon,
need_full_snapshot, false,
- xl_routine, prepare_write, do_write,
+ page_read, cleanup_cb, prepare_write, do_write,
update_progress);
/* call output plugin initialization callback */
@@ -476,7 +479,8 @@ LogicalDecodingContext *
CreateDecodingContext(XLogRecPtr start_lsn,
List *output_plugin_options,
bool fast_forward,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress)
@@ -528,8 +532,8 @@ CreateDecodingContext(XLogRecPtr start_lsn,
ctx = StartupDecodingContext(output_plugin_options,
start_lsn, InvalidTransactionId, false,
- fast_forward, xl_routine, prepare_write,
- do_write, update_progress);
+ fast_forward, page_read, cleanup_cb,
+ prepare_write, do_write, update_progress);
/* call output plugin initialization callback */
old_context = MemoryContextSwitchTo(ctx->context);
@@ -585,7 +589,13 @@ DecodingContextFindStartpoint(LogicalDecodingContext *ctx)
char *err = NULL;
/* the read_page callback waits for new WAL */
- record = XLogReadRecord(ctx->reader, &err);
+ while (XLogReadRecord(ctx->reader, &record, &err) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!ctx->page_read(ctx->reader))
+ break;
+ }
+
if (err)
elog(ERROR, "%s", err);
if (!record)
diff --git a/src/backend/replication/logical/logicalfuncs.c b/src/backend/replication/logical/logicalfuncs.c
index f7e055874e..b69f6911c5 100644
--- a/src/backend/replication/logical/logicalfuncs.c
+++ b/src/backend/replication/logical/logicalfuncs.c
@@ -233,9 +233,8 @@ pg_logical_slot_get_changes_guts(FunctionCallInfo fcinfo, bool confirm, bool bin
ctx = CreateDecodingContext(InvalidXLogRecPtr,
options,
false,
- XL_ROUTINE(.page_read = read_local_xlog_page,
- .segment_open = wal_segment_open,
- .segment_close = wal_segment_close),
+ read_local_xlog_page,
+ wal_segment_close,
LogicalOutputPrepareWrite,
LogicalOutputWrite, NULL);
@@ -284,7 +283,13 @@ pg_logical_slot_get_changes_guts(FunctionCallInfo fcinfo, bool confirm, bool bin
XLogRecord *record;
char *errm = NULL;
- record = XLogReadRecord(ctx->reader, &errm);
+ while (XLogReadRecord(ctx->reader, &record, &errm) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!ctx->page_read(ctx->reader))
+ break;
+ }
+
if (errm)
elog(ERROR, "%s", errm);
diff --git a/src/backend/replication/slotfuncs.c b/src/backend/replication/slotfuncs.c
index 9817b44113..deb7cf15eb 100644
--- a/src/backend/replication/slotfuncs.c
+++ b/src/backend/replication/slotfuncs.c
@@ -153,9 +153,8 @@ create_logical_replication_slot(char *name, char *plugin,
ctx = CreateInitDecodingContext(plugin, NIL,
false, /* just catalogs is OK */
restart_lsn,
- XL_ROUTINE(.page_read = read_local_xlog_page,
- .segment_open = wal_segment_open,
- .segment_close = wal_segment_close),
+ read_local_xlog_page,
+ wal_segment_close,
NULL, NULL, NULL);
/*
@@ -512,9 +511,8 @@ pg_logical_replication_slot_advance(XLogRecPtr moveto)
ctx = CreateDecodingContext(InvalidXLogRecPtr,
NIL,
true, /* fast_forward */
- XL_ROUTINE(.page_read = read_local_xlog_page,
- .segment_open = wal_segment_open,
- .segment_close = wal_segment_close),
+ read_local_xlog_page,
+ wal_segment_close,
NULL, NULL, NULL);
/*
@@ -536,7 +534,13 @@ pg_logical_replication_slot_advance(XLogRecPtr moveto)
* Read records. No changes are generated in fast_forward mode,
* but snapbuilder/slot statuses are updated properly.
*/
- record = XLogReadRecord(ctx->reader, &errm);
+ while (XLogReadRecord(ctx->reader, &record, &errm) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!ctx->page_read(ctx->reader))
+ break;
+ }
+
if (errm)
elog(ERROR, "%s", errm);
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index c36b91ae5e..062093727d 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -580,10 +580,7 @@ StartReplication(StartReplicationCmd *cmd)
/* create xlogreader for physical replication */
xlogreader =
- XLogReaderAllocate(wal_segment_size, NULL,
- XL_ROUTINE(.segment_open = WalSndSegmentOpen,
- .segment_close = wal_segment_close),
- NULL);
+ XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
if (!xlogreader)
ereport(ERROR,
@@ -807,9 +804,11 @@ StartReplication(StartReplicationCmd *cmd)
* set every time WAL is flushed.
*/
static bool
-logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetRecPtr, char *cur_page)
+logical_read_xlog_page(XLogReaderState *state)
{
+ XLogRecPtr targetPagePtr = state->readPagePtr;
+ int reqLen = state->readLen;
+ char *cur_page = state->readBuf;
XLogRecPtr flushptr;
int count;
WALReadError errinfo;
@@ -837,7 +836,7 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
count = flushptr - targetPagePtr; /* part of the page available */
/* now actually read the data, we know it's there */
- if (!WALRead(state,
+ if (!WALRead(state, WalSndSegmentOpen, wal_segment_close,
cur_page,
targetPagePtr,
XLOG_BLCKSZ,
@@ -1011,9 +1010,8 @@ CreateReplicationSlot(CreateReplicationSlotCmd *cmd)
ctx = CreateInitDecodingContext(cmd->plugin, NIL, need_full_snapshot,
InvalidXLogRecPtr,
- XL_ROUTINE(.page_read = logical_read_xlog_page,
- .segment_open = WalSndSegmentOpen,
- .segment_close = wal_segment_close),
+ logical_read_xlog_page,
+ wal_segment_close,
WalSndPrepareWrite, WalSndWriteData,
WalSndUpdateProgress);
@@ -1171,9 +1169,8 @@ StartLogicalReplication(StartReplicationCmd *cmd)
*/
logical_decoding_ctx =
CreateDecodingContext(cmd->startpoint, cmd->options, false,
- XL_ROUTINE(.page_read = logical_read_xlog_page,
- .segment_open = WalSndSegmentOpen,
- .segment_close = wal_segment_close),
+ logical_read_xlog_page,
+ wal_segment_close,
WalSndPrepareWrite, WalSndWriteData,
WalSndUpdateProgress);
xlogreader = logical_decoding_ctx->reader;
@@ -2749,7 +2746,7 @@ XLogSendPhysical(void)
enlargeStringInfo(&output_message, nbytes);
retry:
- if (!WALRead(xlogreader,
+ if (!WALRead(xlogreader, WalSndSegmentOpen, wal_segment_close,
&output_message.data[output_message.len],
startptr,
nbytes,
@@ -2847,7 +2844,12 @@ XLogSendLogical(void)
*/
WalSndCaughtUp = false;
- record = XLogReadRecord(logical_decoding_ctx->reader, &errm);
+ while (XLogReadRecord(logical_decoding_ctx->reader, &record, &errm) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!logical_decoding_ctx->page_read(logical_decoding_ctx->reader))
+ break;
+ }
/* xlog record was invalid */
if (errm != NULL)
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index cf119848b0..712c85281c 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -41,15 +41,9 @@ static int xlogreadfd = -1;
static XLogSegNo xlogreadsegno = -1;
static char xlogfpath[MAXPGPATH];
-typedef struct XLogPageReadPrivate
-{
- const char *restoreCommand;
- int tliIndex;
-} XLogPageReadPrivate;
-
-static bool SimpleXLogPageRead(XLogReaderState *xlogreader,
- XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
+static bool SimpleXLogPageRead(XLogReaderState *xlogreader,
+ const char *datadir, int *tliIndex,
+ const char *restoreCommand);
/*
* Read WAL from the datadir/pg_wal, starting from 'startpoint' on timeline
@@ -66,20 +60,22 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
XLogRecord *record;
XLogReaderState *xlogreader;
char *errormsg;
- XLogPageReadPrivate private;
- private.tliIndex = tliIndex;
- private.restoreCommand = restoreCommand;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir,
- XL_ROUTINE(.page_read = &SimpleXLogPageRead),
- &private);
+ xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+
if (xlogreader == NULL)
pg_fatal("out of memory");
XLogBeginRead(xlogreader, startpoint);
do
{
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!SimpleXLogPageRead(xlogreader, datadir,
+ &tliIndex, restoreCommand))
+ break;
+ }
if (record == NULL)
{
@@ -123,19 +119,19 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex,
XLogRecord *record;
XLogReaderState *xlogreader;
char *errormsg;
- XLogPageReadPrivate private;
XLogRecPtr endptr;
- private.tliIndex = tliIndex;
- private.restoreCommand = restoreCommand;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir,
- XL_ROUTINE(.page_read = &SimpleXLogPageRead),
- &private);
+ xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
XLogBeginRead(xlogreader, ptr);
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!SimpleXLogPageRead(xlogreader, datadir, &tliIndex, restoreCommand))
+ break;
+ }
if (record == NULL)
{
if (errormsg)
@@ -170,7 +166,6 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
XLogRecPtr searchptr;
XLogReaderState *xlogreader;
char *errormsg;
- XLogPageReadPrivate private;
/*
* The given fork pointer points to the end of the last common record,
@@ -186,11 +181,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
forkptr += SizeOfXLogShortPHD;
}
- private.tliIndex = tliIndex;
- private.restoreCommand = restoreCommand;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir,
- XL_ROUTINE(.page_read = &SimpleXLogPageRead),
- &private);
+ xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
@@ -200,7 +191,13 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
uint8 info;
XLogBeginRead(xlogreader, searchptr);
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!SimpleXLogPageRead(xlogreader, datadir,
+ &tliIndex, restoreCommand))
+ break;
+ }
if (record == NULL)
{
@@ -247,10 +244,11 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
/* XLogReader callback function, to read a WAL page */
static bool
-SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *readBuf)
+SimpleXLogPageRead(XLogReaderState *xlogreader, const char *datadir,
+ int *tliIndex, const char *restoreCommand)
{
- XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
+ XLogRecPtr targetPagePtr = xlogreader->readPagePtr;
+ char *readBuf = xlogreader->readBuf;
uint32 targetPageOff;
XLogRecPtr targetSegEnd;
XLogSegNo targetSegNo;
@@ -283,14 +281,14 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
* be done both forward and backward, consider also switching timeline
* accordingly.
*/
- while (private->tliIndex < targetNentries - 1 &&
- targetHistory[private->tliIndex].end < targetSegEnd)
- private->tliIndex++;
- while (private->tliIndex > 0 &&
- targetHistory[private->tliIndex].begin >= targetSegEnd)
- private->tliIndex--;
+ while (*tliIndex < targetNentries - 1 &&
+ targetHistory[*tliIndex].end < targetSegEnd)
+ (*tliIndex)++;
+ while (*tliIndex > 0 &&
+ targetHistory[*tliIndex].begin >= targetSegEnd)
+ (*tliIndex)--;
- XLogFileName(xlogfname, targetHistory[private->tliIndex].tli,
+ XLogFileName(xlogfname, targetHistory[*tliIndex].tli,
xlogreadsegno, WalSegSz);
snprintf(xlogfpath, MAXPGPATH, "%s/" XLOGDIR "/%s",
@@ -303,7 +301,7 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
/*
* If we have no restore_command to execute, then exit.
*/
- if (private->restoreCommand == NULL)
+ if (restoreCommand == NULL)
{
pg_log_error("could not open file \"%s\": %m", xlogfpath);
xlogreader->readLen = -1;
@@ -317,7 +315,7 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
xlogreadfd = RestoreArchivedFile(xlogreader->segcxt.ws_dir,
xlogfname,
WalSegSz,
- private->restoreCommand);
+ restoreCommand);
if (xlogreadfd < 0)
{
@@ -359,7 +357,7 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
Assert(targetSegNo == xlogreadsegno);
- xlogreader->seg.ws_tli = targetHistory[private->tliIndex].tli;
+ xlogreader->seg.ws_tli = targetHistory[*tliIndex].tli;
xlogreader->readLen = XLOG_BLCKSZ;
return true;
}
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index 0f0e0723a3..f514de6b13 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -330,12 +330,17 @@ WALDumpCloseSegment(XLogReaderState *state)
state->seg.ws_file = -1;
}
-/* pg_waldump's XLogReaderRoutine->page_read callback */
+/*
+ * pg_waldump's WAL page rader, also used as page_read callback for
+ * XLogFindNextRecord
+ */
static bool
-WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetPtr, char *readBuff)
+WALDumpReadPage(XLogReaderState *state, void *priv)
{
- XLogDumpPrivate *private = state->private_data;
+ XLogRecPtr targetPagePtr = state->readPagePtr;
+ int reqLen = state->readLen;
+ char *readBuff = state->readBuf;
+ XLogDumpPrivate *private = (XLogDumpPrivate *) priv;
int count = XLOG_BLCKSZ;
WALReadError errinfo;
@@ -353,8 +358,8 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
}
}
- if (!WALRead(state, readBuff, targetPagePtr, count, private->timeline,
- &errinfo))
+ if (!WALRead(state, WALDumpOpenSegment, WALDumpCloseSegment,
+ readBuff, targetPagePtr, count, private->timeline, &errinfo))
{
WALOpenSegment *seg = &errinfo.wre_seg;
char fname[MAXPGPATH];
@@ -374,6 +379,7 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
(Size) errinfo.wre_req);
}
+ Assert(count >= state->readLen);
state->readLen = count;
return true;
}
@@ -1035,16 +1041,14 @@ main(int argc, char **argv)
/* we have everything we need, start reading */
xlogreader_state =
- XLogReaderAllocate(WalSegSz, waldir,
- XL_ROUTINE(.page_read = WALDumpReadPage,
- .segment_open = WALDumpOpenSegment,
- .segment_close = WALDumpCloseSegment),
- &private);
+ XLogReaderAllocate(WalSegSz, waldir, WALDumpCloseSegment);
+
if (!xlogreader_state)
fatal_error("out of memory");
/* first find a valid recptr to start from */
- first_record = XLogFindNextRecord(xlogreader_state, private.startptr);
+ first_record = XLogFindNextRecord(xlogreader_state, private.startptr,
+ &WALDumpReadPage, (void*) &private);
if (first_record == InvalidXLogRecPtr)
fatal_error("could not find a valid record after %X/%X",
@@ -1067,7 +1071,13 @@ main(int argc, char **argv)
for (;;)
{
/* try to read the next record */
- record = XLogReadRecord(xlogreader_state, &errormsg);
+ while (XLogReadRecord(xlogreader_state, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!WALDumpReadPage(xlogreader_state, (void *) &private))
+ break;
+ }
+
if (!record)
{
if (!config.follow || private.endptr_reached)
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index 23d9e70a59..4e9268b553 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -57,64 +57,15 @@ typedef struct WALSegmentContext
typedef struct XLogReaderState XLogReaderState;
-/* Function type definition for the read_page callback */
-typedef bool (*XLogPageReadCB) (XLogReaderState *xlogreader,
- XLogRecPtr targetPagePtr,
- int reqLen,
- XLogRecPtr targetRecPtr,
- char *readBuf);
+/* Function type definition for the segment cleanup callback */
+typedef void (*WALSegmentCleanupCB) (XLogReaderState *xlogreader);
+
+/* Function type definition for the open/close callbacks for WALRead() */
typedef void (*WALSegmentOpenCB) (XLogReaderState *xlogreader,
XLogSegNo nextSegNo,
TimeLineID *tli_p);
typedef void (*WALSegmentCloseCB) (XLogReaderState *xlogreader);
-typedef struct XLogReaderRoutine
-{
- /*
- * Data input callback
- *
- * This callback shall read at least reqLen valid bytes of the xlog page
- * starting at targetPagePtr, and store them in readBuf. The callback
- * shall return the number of bytes read (never more than XLOG_BLCKSZ), or
- * -1 on failure. The callback shall sleep, if necessary, to wait for the
- * requested bytes to become available. The callback will not be invoked
- * again for the same page unless more than the returned number of bytes
- * are needed.
- *
- * targetRecPtr is the position of the WAL record we're reading. Usually
- * it is equal to targetPagePtr + reqLen, but sometimes xlogreader needs
- * to read and verify the page or segment header, before it reads the
- * actual WAL record it's interested in. In that case, targetRecPtr can
- * be used to determine which timeline to read the page from.
- *
- * The callback shall set ->seg.ws_tli to the TLI of the file the page was
- * read from.
- */
- XLogPageReadCB page_read;
-
- /*
- * Callback to open the specified WAL segment for reading. ->seg.ws_file
- * shall be set to the file descriptor of the opened segment. In case of
- * failure, an error shall be raised by the callback and it shall not
- * return.
- *
- * "nextSegNo" is the number of the segment to be opened.
- *
- * "tli_p" is an input/output argument. WALRead() uses it to pass the
- * timeline in which the new segment should be found, but the callback can
- * use it to return the TLI that it actually opened.
- */
- WALSegmentOpenCB segment_open;
-
- /*
- * WAL segment close callback. ->seg.ws_file shall be set to a negative
- * number.
- */
- WALSegmentCloseCB segment_close;
-} XLogReaderRoutine;
-
-#define XL_ROUTINE(...) &(XLogReaderRoutine){__VA_ARGS__}
-
typedef struct
{
/* Is this block ref in use? */
@@ -144,12 +95,35 @@ typedef struct
uint16 data_bufsz;
} DecodedBkpBlock;
+/* Return code from XLogReadRecord */
+typedef enum XLogReadRecordResult
+{
+ XLREAD_SUCCESS, /* record is successfully read */
+ XLREAD_NEED_DATA, /* need more data. see XLogReadRecord. */
+ XLREAD_FAIL /* failed during reading a record */
+} XLogReadRecordResult;
+
+/*
+ * internal state of XLogReadRecord
+ *
+ * XLogReadState runs a state machine while reading a record. Theses states
+ * are not seen outside the function. Each state may repeat several times
+ * exiting requesting caller for new data. See the comment of XLogReadRecrod
+ * for details.
+ */
+typedef enum XLogReadRecordState {
+ XLREAD_NEXT_RECORD,
+ XLREAD_TOT_LEN,
+ XLREAD_FIRST_FRAGMENT,
+ XLREAD_CONTINUATION
+} XLogReadRecordState;
+
struct XLogReaderState
{
/*
* Operational callbacks
*/
- XLogReaderRoutine routine;
+ WALSegmentCleanupCB cleanup_cb;
/* ----------------------------------------
* Public parameters
@@ -162,18 +136,14 @@ struct XLogReaderState
*/
uint64 system_identifier;
- /*
- * Opaque data for callbacks to use. Not used by XLogReader.
- */
- void *private_data;
-
/*
* Start and end point of last record read. EndRecPtr is also used as the
* position to read next. Calling XLogBeginRead() sets EndRecPtr to the
* starting position and ReadRecPtr to invalid.
*/
- XLogRecPtr ReadRecPtr; /* start of last record read */
+ XLogRecPtr ReadRecPtr; /* start of last record read or being read */
XLogRecPtr EndRecPtr; /* end+1 of last record read */
+ XLogRecPtr PrevRecPtr; /* start of previous record read */
/* ----------------------------------------
* Communication with page reader
@@ -187,7 +157,9 @@ struct XLogReaderState
* the request, or -1 on error */
TimeLineID readPageTLI; /* TLI for data currently in readBuf */
char *readBuf; /* buffer to store data */
- bool page_verified; /* is the page on the buffer verified? */
+ bool page_verified; /* is the page header on the buffer verified? */
+ bool record_verified;/* is the current record header verified? */
+
/* ----------------------------------------
@@ -229,8 +201,6 @@ struct XLogReaderState
XLogRecPtr latestPagePtr;
TimeLineID latestPageTLI;
- /* beginning of the WAL record being read. */
- XLogRecPtr currRecPtr;
/* timeline to read it from, 0 if a lookup is required */
TimeLineID currTLI;
@@ -257,6 +227,15 @@ struct XLogReaderState
char *readRecordBuf;
uint32 readRecordBufSize;
+ /*
+ * XLogReadRecord() state
+ */
+ XLogReadRecordState readRecordState;/* state machine state */
+ int recordGotLen; /* amount of current record that has
+ * already been read */
+ int recordRemainLen; /* length of current record that remains */
+ XLogRecPtr recordContRecPtr; /* where the current record continues */
+
/* Buffer to hold error message */
char *errormsg_buf;
};
@@ -264,9 +243,7 @@ struct XLogReaderState
/* Get a new XLogReader */
extern XLogReaderState *XLogReaderAllocate(int wal_segment_size,
const char *waldir,
- XLogReaderRoutine *routine,
- void *private_data);
-extern XLogReaderRoutine *LocalXLogReaderRoutine(void);
+ WALSegmentCleanupCB cleanup_cb);
/* Free an XLogReader */
extern void XLogReaderFree(XLogReaderState *state);
@@ -274,12 +251,17 @@ extern void XLogReaderFree(XLogReaderState *state);
/* Position the XLogReader to given record */
extern void XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr);
#ifdef FRONTEND
-extern XLogRecPtr XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr);
+/* Function type definition for the read_page callback */
+typedef bool (*XLogFindNextRecordCB) (XLogReaderState *xlogreader,
+ void *private);
+extern XLogRecPtr XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
+ XLogFindNextRecordCB read_page, void *private);
#endif /* FRONTEND */
/* Read the next XLog record. Returns NULL on end-of-WAL or failure */
-extern struct XLogRecord *XLogReadRecord(XLogReaderState *state,
- char **errormsg);
+extern XLogReadRecordResult XLogReadRecord(XLogReaderState *state,
+ XLogRecord **record,
+ char **errormsg);
/* Validate a page */
extern bool XLogReaderValidatePageHeader(XLogReaderState *state,
@@ -299,6 +281,7 @@ typedef struct WALReadError
} WALReadError;
extern bool WALRead(XLogReaderState *state,
+ WALSegmentOpenCB segopenfn, WALSegmentCloseCB sgclosefn,
char *buf, XLogRecPtr startptr, Size count,
TimeLineID tli, WALReadError *errinfo);
diff --git a/src/include/access/xlogutils.h b/src/include/access/xlogutils.h
index 364a21c4ea..397fb27fc2 100644
--- a/src/include/access/xlogutils.h
+++ b/src/include/access/xlogutils.h
@@ -47,9 +47,7 @@ extern Buffer XLogReadBufferExtended(RelFileNode rnode, ForkNumber forknum,
extern Relation CreateFakeRelcacheEntry(RelFileNode rnode);
extern void FreeFakeRelcacheEntry(Relation fakerel);
-extern bool read_local_xlog_page(XLogReaderState *state,
- XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetRecPtr, char *cur_page);
+extern bool read_local_xlog_page(XLogReaderState *state);
extern void wal_segment_open(XLogReaderState *state,
XLogSegNo nextSegNo,
TimeLineID *tli_p);
diff --git a/src/include/pg_config_manual.h b/src/include/pg_config_manual.h
index f10ad0acd6..94749609b4 100644
--- a/src/include/pg_config_manual.h
+++ b/src/include/pg_config_manual.h
@@ -377,7 +377,7 @@
* Enable debugging print statements for WAL-related operations; see
* also the wal_debug GUC var.
*/
-/* #define WAL_DEBUG */
+#define WAL_DEBUG
/*
* Enable tracing of resource consumption during sort operations;
diff --git a/src/include/replication/logical.h b/src/include/replication/logical.h
index c253403372..b936f31393 100644
--- a/src/include/replication/logical.h
+++ b/src/include/replication/logical.h
@@ -29,6 +29,10 @@ typedef void (*LogicalOutputPluginWriterUpdateProgress) (struct LogicalDecodingC
TransactionId xid
);
+typedef struct LogicalDecodingContext LogicalDecodingContext;
+
+typedef bool (*LogicalDecodingXLogPageReadCB)(XLogReaderState *ctx);
+
typedef struct LogicalDecodingContext
{
/* memory context this is all allocated in */
@@ -39,6 +43,7 @@ typedef struct LogicalDecodingContext
/* infrastructure pieces for decoding */
XLogReaderState *reader;
+ LogicalDecodingXLogPageReadCB page_read;
struct ReorderBuffer *reorder;
struct SnapBuild *snapshot_builder;
@@ -105,14 +110,16 @@ extern LogicalDecodingContext *CreateInitDecodingContext(const char *plugin,
List *output_plugin_options,
bool need_full_snapshot,
XLogRecPtr restart_lsn,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress);
extern LogicalDecodingContext *CreateDecodingContext(XLogRecPtr start_lsn,
List *output_plugin_options,
bool fast_forward,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress);
--
2.27.0
----Next_Part(Thu_Mar__4_11_28_53_2021_014)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
filename="v17-0003-Remove-globals-readOff-readLen-and-readSegNo.patch"
^ permalink raw reply [nested|flat] 70+ messages in thread
* [PATCH v8 2/4] Move page-reader out of XLogReadRecord
@ 2019-09-10 03:58 Kyotaro Horiguchi <[email protected]>
0 siblings, 0 replies; 70+ messages in thread
From: Kyotaro Horiguchi @ 2019-09-10 03:58 UTC (permalink / raw)
This is the second step of removing callbacks from WAL record reader.
Since it is essential to take in additional data while reading a
record, the function have to ask caller for new data while keeping
working state. Thus the function is turned into a state machine.
---
src/backend/access/transam/twophase.c | 11 +-
src/backend/access/transam/xlog.c | 54 +-
src/backend/access/transam/xlogreader.c | 681 +++++++++++-------
src/backend/access/transam/xlogutils.c | 12 +-
src/backend/replication/logical/logical.c | 17 +-
.../replication/logical/logicalfuncs.c | 8 +-
src/backend/replication/slotfuncs.c | 8 +-
src/backend/replication/walsender.c | 13 +-
src/bin/pg_rewind/parsexlog.c | 80 +-
src/bin/pg_waldump/pg_waldump.c | 24 +-
src/include/access/xlogreader.h | 90 +--
src/include/access/xlogutils.h | 4 +-
src/include/replication/logical.h | 9 +-
13 files changed, 601 insertions(+), 410 deletions(-)
diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index 2f7d4ed59a..24b08810ac 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -1330,8 +1330,7 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
XLogReaderState *xlogreader;
char *errormsg;
- xlogreader = XLogReaderAllocate(wal_segment_size, NULL,
- &read_local_xlog_page, NULL);
+ xlogreader = XLogReaderAllocate(wal_segment_size, NULL);
if (!xlogreader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
@@ -1339,7 +1338,13 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
errdetail("Failed while allocating a WAL reading processor.")));
XLogBeginRead(xlogreader, lsn);
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!read_local_xlog_page(xlogreader))
+ break;
+ }
+
if (record == NULL)
ereport(ERROR,
(errcode_for_file_access(),
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index b0ad9376d6..e380eaa186 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -827,13 +827,6 @@ static XLogSource currentSource = XLOG_FROM_ANY;
static bool lastSourceFailed = false;
static bool pendingWalRcvRestart = false;
-typedef struct XLogPageReadPrivate
-{
- int emode;
- bool fetching_ckpt; /* are we fetching a checkpoint record? */
- bool randAccess;
-} XLogPageReadPrivate;
-
/*
* These variables track when we last obtained some WAL data to process,
* and where we got it from. (XLogReceiptSource is initially the same as
@@ -908,8 +901,8 @@ static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath,
static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli,
XLogSource source, bool notfoundOk);
static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, XLogSource source);
-static bool XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
+static bool XLogPageRead(XLogReaderState *xlogreader,
+ bool fetching_ckpt, int emode, bool randAccess);
static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
bool fetching_ckpt, XLogRecPtr tliRecPtr);
static int emode_for_corrupt_record(int emode, XLogRecPtr RecPtr);
@@ -1221,8 +1214,7 @@ XLogInsertRecord(XLogRecData *rdata,
appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len);
if (!debug_reader)
- debug_reader = XLogReaderAllocate(wal_segment_size, NULL,
- NULL, NULL);
+ debug_reader = XLogReaderAllocate(wal_segment_size, NULL);
if (!debug_reader)
{
@@ -4322,15 +4314,10 @@ CleanupBackupHistory(void)
* record is available.
*/
static XLogRecord *
-ReadRecord(XLogReaderState *xlogreader, int emode,
- bool fetching_ckpt)
+ReadRecord(XLogReaderState *xlogreader, int emode, bool fetching_ckpt)
{
XLogRecord *record;
- XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
-
- private->fetching_ckpt = fetching_ckpt;
- private->emode = emode;
- private->randAccess = (xlogreader->ReadRecPtr == InvalidXLogRecPtr);
+ bool randAccess = (xlogreader->ReadRecPtr == InvalidXLogRecPtr);
/* This is the first attempt to read this page. */
lastSourceFailed = false;
@@ -4338,8 +4325,16 @@ ReadRecord(XLogReaderState *xlogreader, int emode,
for (;;)
{
char *errormsg;
+ XLogReadRecordResult result;
+
+ while ((result = XLogReadRecord(xlogreader, &record, &errormsg))
+ == XLREAD_NEED_DATA)
+ {
+ if (!XLogPageRead(xlogreader, fetching_ckpt, emode, randAccess))
+ break;
+
+ }
- record = XLogReadRecord(xlogreader, &errormsg);
ReadRecPtr = xlogreader->ReadRecPtr;
EndRecPtr = xlogreader->EndRecPtr;
if (record == NULL)
@@ -6309,7 +6304,6 @@ StartupXLOG(void)
bool backupFromStandby = false;
DBState dbstate_at_startup;
XLogReaderState *xlogreader;
- XLogPageReadPrivate private;
bool fast_promoted = false;
struct stat st;
@@ -6465,9 +6459,7 @@ StartupXLOG(void)
OwnLatch(&XLogCtl->recoveryWakeupLatch);
/* Set up XLOG reader facility */
- MemSet(&private, 0, sizeof(XLogPageReadPrivate));
- xlogreader = XLogReaderAllocate(wal_segment_size, NULL,
- &XLogPageRead, &private);
+ xlogreader = XLogReaderAllocate(wal_segment_size, NULL);
if (!xlogreader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
@@ -11810,12 +11802,13 @@ CancelBackup(void)
* sleep and retry.
*/
static bool
-XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetRecPtr, char *readBuf)
+XLogPageRead(XLogReaderState *xlogreader,
+ bool fetching_ckpt, int emode, bool randAccess)
{
- XLogPageReadPrivate *private =
- (XLogPageReadPrivate *) xlogreader->private_data;
- int emode = private->emode;
+ char *readBuf = xlogreader->readBuf;
+ XLogRecPtr targetPagePtr = xlogreader->readPagePtr;
+ int reqLen = xlogreader->readLen;
+ XLogRecPtr targetRecPtr = xlogreader->ReadRecPtr;
uint32 targetPageOff;
XLogSegNo targetSegNo PG_USED_FOR_ASSERTS_ONLY;
int r;
@@ -11858,8 +11851,8 @@ retry:
flushedUpto < targetPagePtr + reqLen))
{
if (!WaitForWALToBecomeAvailable(targetPagePtr + reqLen,
- private->randAccess,
- private->fetching_ckpt,
+ randAccess,
+ fetching_ckpt,
targetRecPtr))
{
if (readFile >= 0)
@@ -11964,6 +11957,7 @@ retry:
goto next_record_is_invalid;
}
+ Assert(xlogreader->readPagePtr == targetPagePtr);
xlogreader->readLen = readLen;
return true;
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index 6250093dd9..7c8c6fc314 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -40,7 +40,7 @@ static bool XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr,
int reqLen, bool header_inclusive);
static void XLogReaderInvalReadState(XLogReaderState *state);
static bool ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
- XLogRecPtr PrevRecPtr, XLogRecord *record, bool randAccess);
+ XLogRecPtr PrevRecPtr, XLogRecord *record);
static bool ValidXLogRecord(XLogReaderState *state, XLogRecord *record,
XLogRecPtr recptr);
static void ResetDecoder(XLogReaderState *state);
@@ -70,8 +70,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...)
* Returns NULL if the xlogreader couldn't be allocated.
*/
XLogReaderState *
-XLogReaderAllocate(int wal_segment_size, const char *waldir,
- XLogPageReadCB pagereadfunc, void *private_data)
+XLogReaderAllocate(int wal_segment_size, const char *waldir)
{
XLogReaderState *state;
@@ -102,9 +101,6 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
WALOpenSegmentInit(&state->seg, &state->segcxt, wal_segment_size,
waldir);
- state->read_page = pagereadfunc;
- /* system_identifier initialized to zeroes above */
- state->private_data = private_data;
/* ReadRecPtr, EndRecPtr and readLen initialized to zeroes above */
state->errormsg_buf = palloc_extended(MAX_ERRORMSG_LEN + 1,
MCXT_ALLOC_NO_OOM);
@@ -242,6 +238,7 @@ XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
/* Begin at the passed-in record pointer. */
state->EndRecPtr = RecPtr;
state->ReadRecPtr = InvalidXLogRecPtr;
+ state->readRecordState = XLREAD_NEXT_RECORD;
}
/*
@@ -250,314 +247,452 @@ XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
* XLogBeginRead() or XLogFindNextRecord() must be called before the first call
* to XLogReadRecord().
*
- * If the read_page callback fails to read the requested data, NULL is
- * returned. The callback is expected to have reported the error; errormsg
- * is set to NULL.
+ * This function may return XLREAD_NEED_DATA several times before returning a
+ * result record. The caller shall read in some new data then call this
+ * function again with the same parameters.
*
- * If the reading fails for some other reason, NULL is also returned, and
- * *errormsg is set to a string with details of the failure.
+ * When a record is successfully read, returns XLREAD_SUCCESS with result
+ * record being stored in *record. Otherwise *record is NULL.
*
- * The returned pointer (or *errormsg) points to an internal buffer that's
- * valid until the next call to XLogReadRecord.
+ * Returns XLREAD_NEED_DATA if more data is needed to finish reading the
+ * current record. In that case, state->readPagePtr and state->readLen inform
+ * the desired position and minimum length of data needed. The caller shall
+ * read in the requested data and set state->readBuf to point to a buffer
+ * containing it. The caller must also set state->readPageTLI and
+ * state->readLen to indicate the timeline that it was read from, and the
+ * length of data that is now available (which must be >= given readLen),
+ * respectively.
+ *
+ * If invalid data is encountered, returns XLREAD_FAIL with *record being set to
+ * NULL. *errormsg is set to a string with details of the failure.
+ * The returned pointer (or *errormsg) points to an internal buffer that's valid
+ * until the next call to XLogReadRecord.
+ *
+ *
+ * This function runs a state machine consists of the following states.
+ *
+ * XLREAD_NEXT_RECORD :
+ * The initial state, if called with valid RecPtr, try to read a record at
+ * that position. If invalid RecPtr is given try to read a record just after
+ * the last one previously read.
+ * This state ens after setting ReadRecPtr. Then goes to XLREAD_TOT_LEN.
+ *
+ * XLREAD_TOT_LEN:
+ * Examining record header. Ends after reading record total
+ * length. recordRemainLen and recordGotLen are initialized.
+ *
+ * XLREAD_FIRST_FRAGMENT:
+ * Reading the first fragment. Ends with finishing reading a single
+ * record. Goes to XLREAD_NEXT_RECORD if that's all or
+ * XLREAD_CONTINUATION if we have continuation.
+
+ * XLREAD_CONTINUATION:
+ * Reading continuation of record. Ends with finishing the whole record then
+ * goes to XLREAD_NEXT_RECORD. During this state, recordRemainLen indicates
+ * how much is left and readRecordBuf holds the partially assert
+ * record.recordContRecPtr points to the beginning of the next page where to
+ * continue.
+ *
+ * If wrong data found in any state, the state machine stays at the current
+ * state. This behavior allows to continue reading a reacord switching among
+ * different souces, while streaming replication.
*/
-XLogRecord *
-XLogReadRecord(XLogReaderState *state, char **errormsg)
+XLogReadRecordResult
+XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
{
- XLogRecPtr RecPtr;
- XLogRecord *record;
- XLogRecPtr targetPagePtr;
- bool randAccess;
- uint32 len,
- total_len;
- uint32 targetRecOff;
- uint32 pageHeaderSize;
- bool gotheader;
+ XLogRecord *prec;
- /*
- * randAccess indicates whether to verify the previous-record pointer of
- * the record we're reading. We only do this if we're reading
- * sequentially, which is what we initially assume.
- */
- randAccess = false;
+ *record = NULL;
/* reset error state */
*errormsg = NULL;
state->errormsg_buf[0] = '\0';
- ResetDecoder(state);
-
- RecPtr = state->EndRecPtr;
-
- if (state->ReadRecPtr != InvalidXLogRecPtr)
- {
- /* read the record after the one we just read */
-
- /*
- * EndRecPtr is pointing to end+1 of the previous WAL record. If
- * we're at a page boundary, no more records can fit on the current
- * page. We must skip over the page header, but we can't do that until
- * we've read in the page, since the header size is variable.
- */
- }
- else
- {
- /*
- * Caller supplied a position to start at.
- *
- * In this case, EndRecPtr should already be pointing to a valid
- * record starting position.
- */
- Assert(XRecOffIsValid(RecPtr));
- randAccess = true;
- }
-
- state->currRecPtr = RecPtr;
-
- targetPagePtr = RecPtr - (RecPtr % XLOG_BLCKSZ);
- targetRecOff = RecPtr % XLOG_BLCKSZ;
-
- /*
- * Read the page containing the record into state->readBuf. Request enough
- * byte to cover the whole record header, or at least the part of it that
- * fits on the same page.
- */
- while (XLogNeedData(state, targetPagePtr,
- Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
- targetRecOff != 0))
- {
- if (!state->read_page(state, state->readPagePtr, state->readLen,
- RecPtr, state->readBuf))
- break;
- }
-
- if (!state->page_verified)
- goto err;
-
- /*
- * We have at least the page header, so we can examine it now.
- */
- pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
- if (targetRecOff == 0)
- {
- /*
- * At page start, so skip over page header.
- */
- RecPtr += pageHeaderSize;
- targetRecOff = pageHeaderSize;
- }
- else if (targetRecOff < pageHeaderSize)
- {
- report_invalid_record(state, "invalid record offset at %X/%X",
- (uint32) (RecPtr >> 32), (uint32) RecPtr);
- goto err;
- }
-
- if ((((XLogPageHeader) state->readBuf)->xlp_info & XLP_FIRST_IS_CONTRECORD) &&
- targetRecOff == pageHeaderSize)
- {
- report_invalid_record(state, "contrecord is requested by %X/%X",
- (uint32) (RecPtr >> 32), (uint32) RecPtr);
- goto err;
- }
-
- /* XLogNeedData has verified the page header */
- Assert(pageHeaderSize <= state->readLen);
-
- /*
- * Read the record length.
- *
- * NB: Even though we use an XLogRecord pointer here, the whole record
- * header might not fit on this page. xl_tot_len is the first field of the
- * struct, so it must be on this page (the records are MAXALIGNed), but we
- * cannot access any other fields until we've verified that we got the
- * whole header.
- */
- record = (XLogRecord *) (state->readBuf + RecPtr % XLOG_BLCKSZ);
- total_len = record->xl_tot_len;
-
- /*
- * If the whole record header is on this page, validate it immediately.
- * Otherwise do just a basic sanity check on xl_tot_len, and validate the
- * rest of the header after reading it from the next page. The xl_tot_len
- * check is necessary here to ensure that we enter the "Need to reassemble
- * record" code path below; otherwise we might fail to apply
- * ValidXLogRecordHeader at all.
- */
- if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
- {
- if (!ValidXLogRecordHeader(state, RecPtr, state->ReadRecPtr, record,
- randAccess))
- goto err;
- gotheader = true;
- }
- else
- {
- /* XXX: more validation should be done here */
- if (total_len < SizeOfXLogRecord)
- {
- report_invalid_record(state,
- "invalid record length at %X/%X: wanted %u, got %u",
- (uint32) (RecPtr >> 32), (uint32) RecPtr,
- (uint32) SizeOfXLogRecord, total_len);
- goto err;
- }
- gotheader = false;
- }
-
- len = XLOG_BLCKSZ - RecPtr % XLOG_BLCKSZ;
- if (total_len > len)
+ switch (state->readRecordState)
{
- /* Need to reassemble record */
- char *contdata;
- XLogPageHeader pageHeader;
- char *buffer;
- uint32 gotlen;
-
- /*
- * Enlarge readRecordBuf as needed.
- */
- if (total_len > state->readRecordBufSize &&
- !allocate_recordbuf(state, total_len))
- {
- /* We treat this as a "bogus data" condition */
- report_invalid_record(state, "record length %u at %X/%X too long",
- total_len,
- (uint32) (RecPtr >> 32), (uint32) RecPtr);
- goto err;
- }
+ case XLREAD_NEXT_RECORD:
+ ResetDecoder(state);
- /* Copy the first fragment of the record from the first page. */
- memcpy(state->readRecordBuf,
- state->readBuf + RecPtr % XLOG_BLCKSZ, len);
- buffer = state->readRecordBuf + len;
- gotlen = len;
-
- do
- {
- int rest_len = total_len - gotlen;
-
- /* Calculate pointer to beginning of next page */
- targetPagePtr += XLOG_BLCKSZ;
-
- /* Wait for the next page to become available */
- while (XLogNeedData(state, targetPagePtr,
- Min(rest_len, XLOG_BLCKSZ),
- false))
+ if (state->ReadRecPtr != InvalidXLogRecPtr)
{
- if (!state->read_page(state, state->readPagePtr, state->readLen,
- state->ReadRecPtr, state->readBuf))
- break;
+ /* read the record after the one we just read */
+
+ /*
+ * EndRecPtr is pointing to end+1 of the previous WAL record.
+ * If we're at a page boundary, no more records can fit on the
+ * current page. We must skip over the page header, but we
+ * can't do that until we've read in the page, since the header
+ * size is variable.
+ */
+ state->PrevRecPtr = state->ReadRecPtr;
+ state->ReadRecPtr = state->EndRecPtr;
+ }
+ else
+ {
+ /*
+ * Caller supplied a position to start at.
+ *
+ * In this case, EndRecPtr should already be pointing to a
+ * valid record starting position.
+ */
+ Assert(XRecOffIsValid(state->EndRecPtr));
+ state->ReadRecPtr = state->EndRecPtr;
+
+ /*
+ * We cannot verify the previous-record pointer when we're
+ * seeking to a particular record. Reset PrevRecPtr so that we
+ * won't try doing that.
+ */
+ state->PrevRecPtr = InvalidXLogRecPtr;
+ state->EndRecPtr = InvalidXLogRecPtr; /* to be tidy */
}
+ state->record_verified = false;
+ state->readRecordState = XLREAD_TOT_LEN;
+ /* fall through */
+
+ case XLREAD_TOT_LEN:
+ {
+ uint32 total_len;
+ uint32 pageHeaderSize;
+ XLogRecPtr targetPagePtr;
+ uint32 targetRecOff;
+ XLogPageHeader pageHeader;
+
+ targetPagePtr =
+ state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
+ targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+
+ /*
+ * Check if we have enough data. For the first record in the page,
+ * the requesting length doesn't contain page header.
+ */
+ if (XLogNeedData(state, targetPagePtr,
+ Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+ targetRecOff != 0))
+ return XLREAD_NEED_DATA;
+
+ /* error out if caller supplied bogus page */
if (!state->page_verified)
goto err;
- Assert(SizeOfXLogShortPHD <= state->readLen);
+ /* examine page header now. */
+ pageHeaderSize =
+ XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+ if (targetRecOff == 0)
+ {
+ /* At page start, so skip over page header. */
+ state->ReadRecPtr += pageHeaderSize;
+ targetRecOff = pageHeaderSize;
+ }
+ else if (targetRecOff < pageHeaderSize)
+ {
+ report_invalid_record(state, "invalid record offset at %X/%X",
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr);
+ goto err;
+ }
- /* Check that the continuation on next page looks valid */
pageHeader = (XLogPageHeader) state->readBuf;
- if (!(pageHeader->xlp_info & XLP_FIRST_IS_CONTRECORD))
+ if ((pageHeader->xlp_info & XLP_FIRST_IS_CONTRECORD) &&
+ targetRecOff == pageHeaderSize)
{
- report_invalid_record(state,
- "there is no contrecord flag at %X/%X",
- (uint32) (RecPtr >> 32), (uint32) RecPtr);
+ report_invalid_record(state, "contrecord is requested by %X/%X",
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr);
goto err;
}
- /*
- * Cross-check that xlp_rem_len agrees with how much of the record
- * we expect there to be left.
- */
- if (pageHeader->xlp_rem_len == 0 ||
- total_len != (pageHeader->xlp_rem_len + gotlen))
- {
- report_invalid_record(state,
- "invalid contrecord length %u at %X/%X",
- pageHeader->xlp_rem_len,
- (uint32) (RecPtr >> 32), (uint32) RecPtr);
- goto err;
- }
-
- /* Append the continuation from this page to the buffer */
- pageHeaderSize = XLogPageHeaderSize(pageHeader);
-
+ /* XLogNeedData has verified the page header */
Assert(pageHeaderSize <= state->readLen);
- contdata = (char *) state->readBuf + pageHeaderSize;
- len = XLOG_BLCKSZ - pageHeaderSize;
- if (pageHeader->xlp_rem_len < len)
- len = pageHeader->xlp_rem_len;
+ /*
+ * Read the record length.
+ *
+ * NB: Even though we use an XLogRecord pointer here, the whole
+ * record header might not fit on this page. xl_tot_len is the first
+ * field of the struct, so it must be on this page (the records are
+ * MAXALIGNed), but we cannot access any other fields until we've
+ * verified that we got the whole header.
+ */
+ prec = (XLogRecord *) (state->readBuf +
+ state->ReadRecPtr % XLOG_BLCKSZ);
+ total_len = prec->xl_tot_len;
- Assert (pageHeaderSize + len <= state->readLen);
- memcpy(buffer, (char *) contdata, len);
- buffer += len;
- gotlen += len;
+ /*
+ * If the whole record header is on this page, validate it
+ * immediately. Otherwise do just a basic sanity check on
+ * xl_tot_len, and validate the rest of the header after reading it
+ * from the next page. The xl_tot_len check is necessary here to
+ * ensure that we enter the XLREAD_CONTINUATION state below;
+ * otherwise we might fail to apply ValidXLogRecordHeader at all.
+ */
+ if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
+ {
+ if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
+ state->PrevRecPtr, prec))
+ goto err;
- /* If we just reassembled the record header, validate it. */
- if (!gotheader)
+ state->record_verified = true;
+ }
+ else
{
- record = (XLogRecord *) state->readRecordBuf;
- if (!ValidXLogRecordHeader(state, RecPtr, state->ReadRecPtr,
- record, randAccess))
+ /* XXX: more validation should be done here */
+ if (total_len < SizeOfXLogRecord)
+ {
+ report_invalid_record(state,
+ "invalid record length at %X/%X: wanted %u, got %u",
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr,
+ (uint32) SizeOfXLogRecord, total_len);
goto err;
- gotheader = true;
+ }
}
- } while (gotlen < total_len);
-
- Assert(gotheader);
- record = (XLogRecord *) state->readRecordBuf;
- if (!ValidXLogRecord(state, record, RecPtr))
- goto err;
+ /*
+ * Wait for the rest of the record, or the part of the record that
+ * fit on the first page if crossed a page boundary, to become
+ * available.
+ */
+ state->recordGotLen = 0;
+ state->recordRemainLen = total_len;
+ state->readRecordState = XLREAD_FIRST_FRAGMENT;
+ }
+ /* fall through */
- pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
- state->ReadRecPtr = RecPtr;
- state->EndRecPtr = targetPagePtr + pageHeaderSize
- + MAXALIGN(pageHeader->xlp_rem_len);
- }
- else
- {
- /* Wait for the record data to become available */
- while (XLogNeedData(state, targetPagePtr,
- Min(targetRecOff + total_len, XLOG_BLCKSZ), true))
+ case XLREAD_FIRST_FRAGMENT:
{
- if (!state->read_page(state, state->readPagePtr, state->readLen,
- state->ReadRecPtr, state->readBuf))
+ uint32 total_len = state->recordRemainLen;
+ uint32 request_len;
+ uint32 record_len;
+ XLogRecPtr targetPagePtr;
+ uint32 targetRecOff;
+
+ /*
+ * Wait for the rest of the record on the first page to become
+ * available
+ */
+ targetPagePtr =
+ state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
+ targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+
+ request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ);
+ record_len = request_len - targetRecOff;
+
+ /* ReadRecPtr contains page header */
+ Assert (targetRecOff != 0);
+ if (XLogNeedData(state, targetPagePtr, request_len, true))
+ return XLREAD_NEED_DATA;
+
+ /* error out if caller supplied bogus page */
+ if (!state->page_verified)
+ goto err;
+
+ prec = (XLogRecord *) (state->readBuf + targetRecOff);
+
+ /* validate record header if not yet */
+ if (!state->record_verified && record_len >= SizeOfXLogRecord)
+ {
+ if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
+ state->PrevRecPtr, prec))
+ goto err;
+
+ state->record_verified = true;
+ }
+
+
+ if (total_len == record_len)
+ {
+ /* Record does not cross a page boundary */
+ Assert(state->record_verified);
+
+ if (!ValidXLogRecord(state, prec, state->ReadRecPtr))
+ goto err;
+
+ state->record_verified = true; /* to be tidy */
+
+ /* We already checked the header earlier */
+ state->EndRecPtr = state->ReadRecPtr + MAXALIGN(record_len);
+
+ *record = prec;
+ state->readRecordState = XLREAD_NEXT_RECORD;
break;
+ }
+
+ /*
+ * The record continues on the next page. Need to reassemble
+ * record
+ */
+ Assert(total_len > record_len);
+
+ /* Enlarge readRecordBuf as needed. */
+ if (total_len > state->readRecordBufSize &&
+ !allocate_recordbuf(state, total_len))
+ {
+ /* We treat this as a "bogus data" condition */
+ report_invalid_record(state,
+ "record length %u at %X/%X too long",
+ total_len,
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr);
+ goto err;
+ }
+
+ /* Copy the first fragment of the record from the first page. */
+ memcpy(state->readRecordBuf, state->readBuf + targetRecOff,
+ record_len);
+ state->recordGotLen += record_len;
+ state->recordRemainLen -= record_len;
+
+ /* Calculate pointer to beginning of next page */
+ state->recordContRecPtr = state->ReadRecPtr + record_len;
+ Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0);
+
+ state->readRecordState = XLREAD_CONTINUATION;
}
+ /* fall through */
+
+ case XLREAD_CONTINUATION:
+ {
+ XLogPageHeader pageHeader;
+ uint32 pageHeaderSize;
+ XLogRecPtr targetPagePtr;
+
+ /* we enter this state only if we haven't read the whole record. */
+ Assert (state->recordRemainLen > 0);
+
+ while(state->recordRemainLen > 0)
+ {
+ char *contdata;
+ uint32 request_len;
+ uint32 record_len;
+
+ /* Wait for the next page to become available */
+ targetPagePtr = state->recordContRecPtr;
+
+ /* this request contains page header */
+ Assert (targetPagePtr != 0);
+ if (XLogNeedData(state, targetPagePtr,
+ Min(state->recordRemainLen, XLOG_BLCKSZ),
+ false))
+ return XLREAD_NEED_DATA;
+
+ if (!state->page_verified)
+ goto err;
+
+ Assert(SizeOfXLogShortPHD <= state->readLen);
- if (!state->page_verified)
- goto err;
+ /* Check that the continuation on next page looks valid */
+ pageHeader = (XLogPageHeader) state->readBuf;
+ if (!(pageHeader->xlp_info & XLP_FIRST_IS_CONTRECORD))
+ {
+ report_invalid_record(
+ state,
+ "there is no contrecord flag at %X/%X reading %X/%X",
+ (uint32) (state->recordContRecPtr >> 32),
+ (uint32) state->recordContRecPtr,
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr);
+ goto err;
+ }
- /* Record does not cross a page boundary */
- if (!ValidXLogRecord(state, record, RecPtr))
- goto err;
+ /*
+ * Cross-check that xlp_rem_len agrees with how much of the
+ * record we expect there to be left.
+ */
+ if (pageHeader->xlp_rem_len == 0 ||
+ pageHeader->xlp_rem_len != state->recordRemainLen)
+ {
+ report_invalid_record(
+ state,
+ "invalid contrecord length %u at %X/%X reading %X/%X, expected %u",
+ pageHeader->xlp_rem_len,
+ (uint32) (state->recordContRecPtr >> 32),
+ (uint32) state->recordContRecPtr,
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr,
+ state->recordRemainLen);
+ goto err;
+ }
- state->EndRecPtr = RecPtr + MAXALIGN(total_len);
+ /* Append the continuation from this page to the buffer */
+ pageHeaderSize = XLogPageHeaderSize(pageHeader);
- state->ReadRecPtr = RecPtr;
+ /*
+ * XLogNeedData should have ensured that the whole page header
+ * was read
+ */
+ Assert(state->readLen >= pageHeaderSize);
+
+ contdata = (char *) state->readBuf + pageHeaderSize;
+ record_len = XLOG_BLCKSZ - pageHeaderSize;
+ if (pageHeader->xlp_rem_len < record_len)
+ record_len = pageHeader->xlp_rem_len;
+
+ request_len = record_len + pageHeaderSize;
+
+ /* XLogNeedData should have ensured all needed data was read */
+ Assert (state->readLen >= request_len);
+
+ memcpy(state->readRecordBuf + state->recordGotLen,
+ (char *) contdata, record_len);
+ state->recordGotLen += record_len;
+ state->recordRemainLen -= record_len;
+
+ /* If we just reassembled the record header, validate it. */
+ if (!state->record_verified)
+ {
+ Assert(state->recordGotLen >= SizeOfXLogRecord);
+ if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
+ state->PrevRecPtr,
+ (XLogRecord *) state->readRecordBuf))
+ goto err;
+
+ state->record_verified = true;
+ }
+
+ /* Calculate pointer to beginning of next page, and continue */
+ state->recordContRecPtr += XLOG_BLCKSZ;
+ }
+
+ /* targetPagePtr is pointing the last-read page here */
+ prec = (XLogRecord *) state->readRecordBuf;
+ if (!ValidXLogRecord(state, prec, state->ReadRecPtr))
+ goto err;
+
+ pageHeaderSize =
+ XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+ state->EndRecPtr = targetPagePtr + pageHeaderSize
+ + MAXALIGN(pageHeader->xlp_rem_len);
+
+ *record = prec;
+ state->readRecordState = XLREAD_NEXT_RECORD;
+ break;
+ }
}
/*
* Special processing if it's an XLOG SWITCH record
*/
- if (record->xl_rmid == RM_XLOG_ID &&
- (record->xl_info & ~XLR_INFO_MASK) == XLOG_SWITCH)
+ if ((*record)->xl_rmid == RM_XLOG_ID &&
+ ((*record)->xl_info & ~XLR_INFO_MASK) == XLOG_SWITCH)
{
/* Pretend it extends to end of segment */
state->EndRecPtr += state->segcxt.ws_segsize - 1;
state->EndRecPtr -= XLogSegmentOffset(state->EndRecPtr, state->segcxt.ws_segsize);
}
- if (DecodeXLogRecord(state, record, errormsg))
- return record;
- else
- return NULL;
+ Assert (!*record || state->readLen >= 0);
+ if (DecodeXLogRecord(state, *record, errormsg))
+ return XLREAD_SUCCESS;
+
+ *record = NULL;
+ return XLREAD_FAIL;
err:
/*
- * Invalidate the read state. We might read from a different source after
+ * Invalidate the read page. We might read from a different source after
* failure.
*/
XLogReaderInvalReadState(state);
@@ -565,7 +700,8 @@ err:
if (state->errormsg_buf[0] != '\0')
*errormsg = state->errormsg_buf;
- return NULL;
+ *record = NULL;
+ return XLREAD_FAIL;
}
/*
@@ -718,11 +854,12 @@ XLogReaderInvalReadState(XLogReaderState *state)
*
* This is just a convenience subroutine to avoid duplicated code in
* XLogReadRecord. It's not intended for use from anywhere else.
+ *
+ * If PrevRecPtr is valid, the xl_prev is is cross-checked with it.
*/
static bool
ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
- XLogRecPtr PrevRecPtr, XLogRecord *record,
- bool randAccess)
+ XLogRecPtr PrevRecPtr, XLogRecord *record)
{
if (record->xl_tot_len < SizeOfXLogRecord)
{
@@ -740,7 +877,7 @@ ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
(uint32) RecPtr);
return false;
}
- if (randAccess)
+ if (PrevRecPtr == InvalidXLogRecPtr)
{
/*
* We can't exactly verify the prev-link, but surely it should be less
@@ -972,11 +1109,14 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
* XLogReadRecord() will read the next valid record.
*/
XLogRecPtr
-XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
+XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
+ XLogFindNextRecordCB read_page, void *private)
{
XLogRecPtr tmpRecPtr;
XLogRecPtr found = InvalidXLogRecPtr;
XLogPageHeader header;
+ XLogRecord *record;
+ XLogReadRecordResult result;
char *errormsg;
Assert(!XLogRecPtrIsInvalid(RecPtr));
@@ -1009,8 +1149,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
while(XLogNeedData(state, targetPagePtr, targetRecOff,
targetRecOff != 0))
{
- if (!state->read_page(state, state->readPagePtr, state->readLen,
- state->ReadRecPtr, state->readBuf))
+ if (!read_page(state, private))
break;
}
@@ -1062,8 +1201,16 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
* or we just jumped over the remaining data of a continuation.
*/
XLogBeginRead(state, tmpRecPtr);
- while (XLogReadRecord(state, &errormsg) != NULL)
+ while ((result = XLogReadRecord(state, &record, &errormsg)) !=
+ XLREAD_FAIL)
{
+ if (result == XLREAD_NEED_DATA)
+ {
+ if (!read_page(state, private))
+ goto err;
+ continue;
+ }
+
/* past the record we've found, break out */
if (RecPtr <= state->ReadRecPtr)
{
diff --git a/src/backend/access/transam/xlogutils.c b/src/backend/access/transam/xlogutils.c
index 2464a0d092..82f1ed2d1a 100644
--- a/src/backend/access/transam/xlogutils.c
+++ b/src/backend/access/transam/xlogutils.c
@@ -689,8 +689,7 @@ XLogTruncateRelation(RelFileNode rnode, ForkNumber forkNum,
void
XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wantLength)
{
- const XLogRecPtr lastReadPage = state->seg.ws_segno *
- state->segcxt.ws_segsize + state->readLen;
+ const XLogRecPtr lastReadPage = state->readPagePtr;
Assert(wantPage != InvalidXLogRecPtr && wantPage % XLOG_BLCKSZ == 0);
Assert(wantLength <= XLOG_BLCKSZ);
@@ -705,7 +704,7 @@ XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wa
* current TLI has since become historical.
*/
if (lastReadPage == wantPage &&
- state->readLen != 0 &&
+ state->page_verified &&
lastReadPage + state->readLen >= wantPage + Min(wantLength, XLOG_BLCKSZ - 1))
return;
@@ -823,9 +822,11 @@ wal_segment_open(XLogSegNo nextSegNo, WALSegmentContext * segcxt,
* loop for now.
*/
bool
-read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *cur_page)
+read_local_xlog_page(XLogReaderState *state)
{
+ XLogRecPtr targetPagePtr = state->readPagePtr;
+ int reqLen = state->readLen;
+ char *cur_page = state->readBuf;
XLogRecPtr read_upto,
loc;
TimeLineID tli;
@@ -943,6 +944,7 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
WALReadRaiseError(&errinfo);
/* number of valid bytes in the buffer */
+ state->readPagePtr = targetPagePtr;
state->readLen = count;
return true;
}
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index 5adf253583..7782e3ad2f 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -120,7 +120,7 @@ StartupDecodingContext(List *output_plugin_options,
TransactionId xmin_horizon,
bool need_full_snapshot,
bool fast_forward,
- XLogPageReadCB read_page,
+ LogicalDecodingXLogReadPageCB read_page,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress)
@@ -169,11 +169,12 @@ StartupDecodingContext(List *output_plugin_options,
ctx->slot = slot;
- ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, read_page, ctx);
+ ctx->reader = XLogReaderAllocate(wal_segment_size, NULL);
if (!ctx->reader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
errmsg("out of memory")));
+ ctx->read_page = read_page;
ctx->reorder = ReorderBufferAllocate();
ctx->snapshot_builder =
@@ -230,7 +231,7 @@ CreateInitDecodingContext(char *plugin,
List *output_plugin_options,
bool need_full_snapshot,
XLogRecPtr restart_lsn,
- XLogPageReadCB read_page,
+ LogicalDecodingXLogReadPageCB read_page,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress)
@@ -372,7 +373,7 @@ LogicalDecodingContext *
CreateDecodingContext(XLogRecPtr start_lsn,
List *output_plugin_options,
bool fast_forward,
- XLogPageReadCB read_page,
+ LogicalDecodingXLogReadPageCB read_page,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress)
@@ -479,7 +480,13 @@ DecodingContextFindStartpoint(LogicalDecodingContext *ctx)
char *err = NULL;
/* the read_page callback waits for new WAL */
- record = XLogReadRecord(ctx->reader, &err);
+ while (XLogReadRecord(ctx->reader, &record, &err) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!ctx->read_page(ctx->reader))
+ break;
+ }
+
if (err)
elog(ERROR, "%s", err);
if (!record)
diff --git a/src/backend/replication/logical/logicalfuncs.c b/src/backend/replication/logical/logicalfuncs.c
index f5384f1df8..8f2b2dd1fe 100644
--- a/src/backend/replication/logical/logicalfuncs.c
+++ b/src/backend/replication/logical/logicalfuncs.c
@@ -269,7 +269,13 @@ pg_logical_slot_get_changes_guts(FunctionCallInfo fcinfo, bool confirm, bool bin
XLogRecord *record;
char *errm = NULL;
- record = XLogReadRecord(ctx->reader, &errm);
+ while (XLogReadRecord(ctx->reader, &record, &errm) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!ctx->read_page(ctx->reader))
+ break;
+ }
+
if (errm)
elog(ERROR, "%s", errm);
diff --git a/src/backend/replication/slotfuncs.c b/src/backend/replication/slotfuncs.c
index f776de3df7..bd34042458 100644
--- a/src/backend/replication/slotfuncs.c
+++ b/src/backend/replication/slotfuncs.c
@@ -489,7 +489,13 @@ pg_logical_replication_slot_advance(XLogRecPtr moveto)
* Read records. No changes are generated in fast_forward mode,
* but snapbuilder/slot statuses are updated properly.
*/
- record = XLogReadRecord(ctx->reader, &errm);
+ while (XLogReadRecord(ctx->reader, &record, &errm) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!ctx->read_page(ctx->reader))
+ break;
+ }
+
if (errm)
elog(ERROR, "%s", errm);
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index e1c1de22c5..0dab43545d 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -799,9 +799,11 @@ StartReplication(StartReplicationCmd *cmd)
* set every time WAL is flushed.
*/
static bool
-logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetRecPtr, char *cur_page)
+logical_read_xlog_page(XLogReaderState *state)
{
+ XLogRecPtr targetPagePtr = state->readPagePtr;
+ int reqLen = state->readLen;
+ char *cur_page = state->readBuf;
XLogRecPtr flushptr;
int count;
WALReadError errinfo;
@@ -2834,7 +2836,12 @@ XLogSendLogical(void)
*/
WalSndCaughtUp = false;
- record = XLogReadRecord(logical_decoding_ctx->reader, &errm);
+ while (XLogReadRecord(logical_decoding_ctx->reader, &record, &errm) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!logical_decoding_ctx->read_page(logical_decoding_ctx->reader))
+ break;
+ }
/* xlog record was invalid */
if (errm != NULL)
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index 1b0199f52f..5b16681a37 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -40,15 +40,9 @@ static int xlogreadfd = -1;
static XLogSegNo xlogreadsegno = -1;
static char xlogfpath[MAXPGPATH];
-typedef struct XLogPageReadPrivate
-{
- const char *restoreCommand;
- int tliIndex;
-} XLogPageReadPrivate;
-
-static bool SimpleXLogPageRead(XLogReaderState *xlogreader,
- XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
+static bool SimpleXLogPageRead(XLogReaderState *xlogreader,
+ const char *datadir, int *tliIndex,
+ const char *restoreCommand);
/*
* Read WAL from the datadir/pg_wal, starting from 'startpoint' on timeline
@@ -62,19 +56,22 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
XLogRecord *record;
XLogReaderState *xlogreader;
char *errormsg;
- XLogPageReadPrivate private;
- private.tliIndex = tliIndex;
- private.restoreCommand = restoreCommand;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir, &SimpleXLogPageRead,
- &private);
+ xlogreader = XLogReaderAllocate(WalSegSz, datadir);
+
if (xlogreader == NULL)
pg_fatal("out of memory");
XLogBeginRead(xlogreader, startpoint);
do
{
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!SimpleXLogPageRead(xlogreader, datadir,
+ &tliIndex, restoreCommand))
+ break;
+ }
if (record == NULL)
{
@@ -111,17 +108,19 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex)
XLogRecord *record;
XLogReaderState *xlogreader;
char *errormsg;
- XLogPageReadPrivate private;
XLogRecPtr endptr;
- private.tliIndex = tliIndex;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir, &SimpleXLogPageRead,
- &private);
+ xlogreader = XLogReaderAllocate(WalSegSz, datadir);
if (xlogreader == NULL)
pg_fatal("out of memory");
XLogBeginRead(xlogreader, ptr);
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!SimpleXLogPageRead(xlogreader, datadir, &tliIndex, NULL))
+ break;
+ }
if (record == NULL)
{
if (errormsg)
@@ -156,7 +155,6 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
XLogRecPtr searchptr;
XLogReaderState *xlogreader;
char *errormsg;
- XLogPageReadPrivate private;
/*
* The given fork pointer points to the end of the last common record,
@@ -172,10 +170,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
forkptr += SizeOfXLogShortPHD;
}
- private.tliIndex = tliIndex;
- private.restoreCommand = restoreCommand;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir, &SimpleXLogPageRead,
- &private);
+ xlogreader = XLogReaderAllocate(WalSegSz, datadir);
if (xlogreader == NULL)
pg_fatal("out of memory");
@@ -185,7 +180,13 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
uint8 info;
XLogBeginRead(xlogreader, searchptr);
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!SimpleXLogPageRead(xlogreader, datadir,
+ &tliIndex, restoreCommand))
+ break;
+ }
if (record == NULL)
{
@@ -232,10 +233,11 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
/* XLogReader callback function, to read a WAL page */
static bool
-SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *readBuf)
+SimpleXLogPageRead(XLogReaderState *xlogreader, const char *datadir,
+ int *tliIndex, const char *restoreCommand)
{
- XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
+ XLogRecPtr targetPagePtr = xlogreader->readPagePtr;
+ char *readBuf = xlogreader->readBuf;
uint32 targetPageOff;
XLogRecPtr targetSegEnd;
XLogSegNo targetSegNo;
@@ -268,14 +270,14 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
* be done both forward and backward, consider also switching timeline
* accordingly.
*/
- while (private->tliIndex < targetNentries - 1 &&
- targetHistory[private->tliIndex].end < targetSegEnd)
- private->tliIndex++;
- while (private->tliIndex > 0 &&
- targetHistory[private->tliIndex].begin >= targetSegEnd)
- private->tliIndex--;
+ while (*tliIndex < targetNentries - 1 &&
+ targetHistory[*tliIndex].end < targetSegEnd)
+ (*tliIndex)++;
+ while (*tliIndex > 0 &&
+ targetHistory[*tliIndex].begin >= targetSegEnd)
+ (*tliIndex)--;
- XLogFileName(xlogfname, targetHistory[private->tliIndex].tli,
+ XLogFileName(xlogfname, targetHistory[*tliIndex].tli,
xlogreadsegno, WalSegSz);
snprintf(xlogfpath, MAXPGPATH, "%s/" XLOGDIR "/%s",
@@ -288,7 +290,7 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
/*
* If we have no restore_command to execute, then exit.
*/
- if (private->restoreCommand == NULL)
+ if (restoreCommand == NULL)
{
pg_log_error("could not open file \"%s\": %m", xlogfpath);
xlogreader->readLen = -1;
@@ -302,7 +304,7 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
xlogreadfd = RestoreArchivedFile(xlogreader->segcxt.ws_dir,
xlogfname,
WalSegSz,
- private->restoreCommand);
+ restoreCommand);
if (xlogreadfd < 0)
{
@@ -344,7 +346,7 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
Assert(targetSegNo == xlogreadsegno);
- xlogreader->seg.ws_tli = targetHistory[private->tliIndex].tli;
+ xlogreader->seg.ws_tli = targetHistory[*tliIndex].tli;
xlogreader->readLen = XLOG_BLCKSZ;
return true;
}
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index 33fde23e7b..1c42050277 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -324,10 +324,12 @@ WALDumpOpenSegment(XLogSegNo nextSegNo, WALSegmentContext *segcxt,
* XLogReader read_page callback
*/
static bool
-WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetPtr, char *readBuff)
+WALDumpReadPage(XLogReaderState *state, void *priv)
{
- XLogDumpPrivate *private = state->private_data;
+ XLogRecPtr targetPagePtr = state->readPagePtr;
+ int reqLen = state->readLen;
+ char *readBuff = state->readBuf;
+ XLogDumpPrivate *private = (XLogDumpPrivate *) priv;
int count = XLOG_BLCKSZ;
WALReadError errinfo;
@@ -366,6 +368,7 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
(Size) errinfo.wre_req);
}
+ Assert(count >= state->readLen);
state->readLen = count;
return true;
}
@@ -1033,13 +1036,14 @@ main(int argc, char **argv)
/* done with argument parsing, do the actual work */
/* we have everything we need, start reading */
- xlogreader_state = XLogReaderAllocate(WalSegSz, waldir, WALDumpReadPage,
- &private);
+ xlogreader_state = XLogReaderAllocate(WalSegSz, waldir);
+
if (!xlogreader_state)
fatal_error("out of memory");
/* first find a valid recptr to start from */
- first_record = XLogFindNextRecord(xlogreader_state, private.startptr);
+ first_record = XLogFindNextRecord(xlogreader_state, private.startptr,
+ &WALDumpReadPage, (void*) &private);
if (first_record == InvalidXLogRecPtr)
fatal_error("could not find a valid record after %X/%X",
@@ -1063,7 +1067,13 @@ main(int argc, char **argv)
for (;;)
{
/* try to read the next record */
- record = XLogReadRecord(xlogreader_state, &errormsg);
+ while (XLogReadRecord(xlogreader_state, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!WALDumpReadPage(xlogreader_state, (void *) &private))
+ break;
+ }
+
if (!record)
{
if (!config.follow || private.endptr_reached)
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index 6ad953eea3..e59e42bee3 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -50,13 +50,6 @@ typedef struct WALSegmentContext
typedef struct XLogReaderState XLogReaderState;
-/* Function type definition for the read_page callback */
-typedef bool (*XLogPageReadCB) (XLogReaderState *xlogreader,
- XLogRecPtr targetPagePtr,
- int reqLen,
- XLogRecPtr targetRecPtr,
- char *readBuf);
-
typedef struct
{
/* Is this block ref in use? */
@@ -86,6 +79,29 @@ typedef struct
uint16 data_bufsz;
} DecodedBkpBlock;
+/* Return code from XLogReadRecord */
+typedef enum XLogReadRecordResult
+{
+ XLREAD_SUCCESS, /* record is successfully read */
+ XLREAD_NEED_DATA, /* need more data. see XLogReadRecord. */
+ XLREAD_FAIL /* failed during reading a record */
+} XLogReadRecordResult;
+
+/*
+ * internal state of XLogReadRecord
+ *
+ * XLogReadState runs a state machine while reading a record. Theses states
+ * are not seen outside the function. Each state may repeat several times
+ * exiting requesting caller for new data. See the comment of XLogReadRecrod
+ * for details.
+ */
+typedef enum XLogReadRecordState {
+ XLREAD_NEXT_RECORD,
+ XLREAD_TOT_LEN,
+ XLREAD_FIRST_FRAGMENT,
+ XLREAD_CONTINUATION
+} XLogReadRecordState;
+
struct XLogReaderState
{
/* ----------------------------------------
@@ -93,46 +109,20 @@ struct XLogReaderState
* ----------------------------------------
*/
- /*
- * Data input callback (mandatory).
- *
- * This callback shall read at least reqLen valid bytes of the xlog page
- * starting at targetPagePtr, and store them in readBuf. The callback
- * shall return the number of bytes read (never more than XLOG_BLCKSZ), or
- * -1 on failure. The callback shall sleep, if necessary, to wait for the
- * requested bytes to become available. The callback will not be invoked
- * again for the same page unless more than the returned number of bytes
- * are needed.
- *
- * targetRecPtr is the position of the WAL record we're reading. Usually
- * it is equal to targetPagePtr + reqLen, but sometimes xlogreader needs
- * to read and verify the page or segment header, before it reads the
- * actual WAL record it's interested in. In that case, targetRecPtr can
- * be used to determine which timeline to read the page from.
- *
- * The callback shall set ->seg.ws_tli to the TLI of the file the page was
- * read from.
- */
- XLogPageReadCB read_page;
-
/*
* System identifier of the xlog files we're about to read. Set to zero
* (the default value) if unknown or unimportant.
*/
uint64 system_identifier;
- /*
- * Opaque data for callbacks to use. Not used by XLogReader.
- */
- void *private_data;
-
/*
* Start and end point of last record read. EndRecPtr is also used as the
* position to read next. Calling XLogBeginRead() sets EndRecPtr to the
* starting position and ReadRecPtr to invalid.
*/
- XLogRecPtr ReadRecPtr; /* start of last record read */
+ XLogRecPtr ReadRecPtr; /* start of last record read or being read */
XLogRecPtr EndRecPtr; /* end+1 of last record read */
+ XLogRecPtr PrevRecPtr; /* start of previous record read */
/* ----------------------------------------
* Communication with page reader
@@ -146,7 +136,9 @@ struct XLogReaderState
* the request, or -1 on error */
TimeLineID readPageTLI; /* TLI for data currently in readBuf */
char *readBuf; /* buffer to store data */
- bool page_verified; /* is the page on the buffer verified? */
+ bool page_verified; /* is the page header on the buffer verified? */
+ bool record_verified;/* is the current record header verified? */
+
/* ----------------------------------------
@@ -186,8 +178,6 @@ struct XLogReaderState
XLogRecPtr latestPagePtr;
TimeLineID latestPageTLI;
- /* beginning of the WAL record being read. */
- XLogRecPtr currRecPtr;
/* timeline to read it from, 0 if a lookup is required */
TimeLineID currTLI;
@@ -214,15 +204,22 @@ struct XLogReaderState
char *readRecordBuf;
uint32 readRecordBufSize;
+ /*
+ * XLogReadRecord() state
+ */
+ XLogReadRecordState readRecordState;/* state machine state */
+ int recordGotLen; /* amount of current record that has
+ * already been read */
+ int recordRemainLen; /* length of current record that remains */
+ XLogRecPtr recordContRecPtr; /* where the current record continues */
+
/* Buffer to hold error message */
char *errormsg_buf;
};
/* Get a new XLogReader */
extern XLogReaderState *XLogReaderAllocate(int wal_segment_size,
- const char *waldir,
- XLogPageReadCB pagereadfunc,
- void *private_data);
+ const char *waldir);
/* Free an XLogReader */
extern void XLogReaderFree(XLogReaderState *state);
@@ -252,12 +249,17 @@ extern void WALOpenSegmentInit(WALOpenSegment *seg, WALSegmentContext *segcxt,
/* Position the XLogReader to given record */
extern void XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr);
#ifdef FRONTEND
-extern XLogRecPtr XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr);
+/* Function type definition for the read_page callback */
+typedef bool (*XLogFindNextRecordCB) (XLogReaderState *xlogreader,
+ void *private);
+extern XLogRecPtr XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
+ XLogFindNextRecordCB read_page, void *private);
#endif /* FRONTEND */
/* Read the next XLog record. Returns NULL on end-of-WAL or failure */
-extern struct XLogRecord *XLogReadRecord(XLogReaderState *state,
- char **errormsg);
+extern XLogReadRecordResult XLogReadRecord(XLogReaderState *state,
+ XLogRecord **record,
+ char **errormsg);
/* Validate a page */
extern bool XLogReaderValidatePageHeader(XLogReaderState *state,
diff --git a/src/include/access/xlogutils.h b/src/include/access/xlogutils.h
index dc7d894e5d..312acb4fa3 100644
--- a/src/include/access/xlogutils.h
+++ b/src/include/access/xlogutils.h
@@ -47,9 +47,7 @@ extern Buffer XLogReadBufferExtended(RelFileNode rnode, ForkNumber forknum,
extern Relation CreateFakeRelcacheEntry(RelFileNode rnode);
extern void FreeFakeRelcacheEntry(Relation fakerel);
-extern bool read_local_xlog_page(XLogReaderState *state,
- XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetRecPtr, char *cur_page);
+extern bool read_local_xlog_page(XLogReaderState *state);
extern void XLogReadDetermineTimeline(XLogReaderState *state,
XLogRecPtr wantPage, uint32 wantLength);
diff --git a/src/include/replication/logical.h b/src/include/replication/logical.h
index 3b7ca7f1da..cbe9ed751c 100644
--- a/src/include/replication/logical.h
+++ b/src/include/replication/logical.h
@@ -29,6 +29,10 @@ typedef void (*LogicalOutputPluginWriterUpdateProgress) (struct LogicalDecodingC
TransactionId xid
);
+typedef struct LogicalDecodingContext LogicalDecodingContext;
+
+typedef bool (*LogicalDecodingXLogReadPageCB)(XLogReaderState *ctx);
+
typedef struct LogicalDecodingContext
{
/* memory context this is all allocated in */
@@ -39,6 +43,7 @@ typedef struct LogicalDecodingContext
/* infrastructure pieces for decoding */
XLogReaderState *reader;
+ LogicalDecodingXLogReadPageCB read_page;
struct ReorderBuffer *reorder;
struct SnapBuild *snapshot_builder;
@@ -95,14 +100,14 @@ extern LogicalDecodingContext *CreateInitDecodingContext(char *plugin,
List *output_plugin_options,
bool need_full_snapshot,
XLogRecPtr restart_lsn,
- XLogPageReadCB read_page,
+ LogicalDecodingXLogReadPageCB read_page,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress);
extern LogicalDecodingContext *CreateDecodingContext(XLogRecPtr start_lsn,
List *output_plugin_options,
bool fast_forward,
- XLogPageReadCB read_page,
+ LogicalDecodingXLogReadPageCB read_page,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress);
--
2.18.2
----Next_Part(Tue_Apr_21_17_04_27_2020_275)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
filename="v8-0003-Remove-globals-readOff-readLen-and-readSegNo.patch"
^ permalink raw reply [nested|flat] 70+ messages in thread
* [PATCH v17 2/4] Move page-reader out of XLogReadRecord
@ 2019-09-10 03:58 Kyotaro Horiguchi <[email protected]>
0 siblings, 0 replies; 70+ messages in thread
From: Kyotaro Horiguchi @ 2019-09-10 03:58 UTC (permalink / raw)
This is the second step of removing callbacks from WAL record reader.
Since it is essential to take in additional data while reading a
record, the function have to ask caller for new data while keeping
working state. Thus the function is turned into a state machine.
---
src/backend/access/transam/twophase.c | 15 +-
src/backend/access/transam/xlog.c | 58 +-
src/backend/access/transam/xlogreader.c | 697 +++++++++++-------
src/backend/access/transam/xlogutils.c | 17 +-
src/backend/replication/logical/logical.c | 26 +-
.../replication/logical/logicalfuncs.c | 13 +-
src/backend/replication/slotfuncs.c | 18 +-
src/backend/replication/walsender.c | 32 +-
src/bin/pg_rewind/parsexlog.c | 84 ++-
src/bin/pg_waldump/pg_waldump.c | 36 +-
src/include/access/xlogreader.h | 121 ++-
src/include/access/xlogutils.h | 4 +-
src/include/pg_config_manual.h | 2 +-
src/include/replication/logical.h | 11 +-
14 files changed, 646 insertions(+), 488 deletions(-)
diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index 80d2d20d6c..7d5a96a5f5 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -1325,11 +1325,8 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
XLogReaderState *xlogreader;
char *errormsg;
- xlogreader = XLogReaderAllocate(wal_segment_size, NULL,
- XL_ROUTINE(.page_read = &read_local_xlog_page,
- .segment_open = &wal_segment_open,
- .segment_close = &wal_segment_close),
- NULL);
+ xlogreader = XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+
if (!xlogreader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
@@ -1337,7 +1334,13 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
errdetail("Failed while allocating a WAL reading processor.")));
XLogBeginRead(xlogreader, lsn);
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!read_local_xlog_page(xlogreader))
+ break;
+ }
+
if (record == NULL)
ereport(ERROR,
(errcode_for_file_access(),
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 606328a65a..670996ae67 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -836,13 +836,6 @@ static XLogSource currentSource = XLOG_FROM_ANY;
static bool lastSourceFailed = false;
static bool pendingWalRcvRestart = false;
-typedef struct XLogPageReadPrivate
-{
- int emode;
- bool fetching_ckpt; /* are we fetching a checkpoint record? */
- bool randAccess;
-} XLogPageReadPrivate;
-
/*
* These variables track when we last obtained some WAL data to process,
* and where we got it from. (XLogReceiptSource is initially the same as
@@ -917,8 +910,8 @@ static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath,
static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli,
XLogSource source, bool notfoundOk);
static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, XLogSource source);
-static bool XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
+static bool XLogPageRead(XLogReaderState *xlogreader,
+ bool fetching_ckpt, int emode, bool randAccess);
static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
bool fetching_ckpt, XLogRecPtr tliRecPtr);
static int emode_for_corrupt_record(int emode, XLogRecPtr RecPtr);
@@ -1231,8 +1224,7 @@ XLogInsertRecord(XLogRecData *rdata,
appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len);
if (!debug_reader)
- debug_reader = XLogReaderAllocate(wal_segment_size, NULL,
- XL_ROUTINE(), NULL);
+ debug_reader = XLogReaderAllocate(wal_segment_size, NULL, NULL);
if (!debug_reader)
{
@@ -4344,15 +4336,10 @@ CleanupBackupHistory(void)
* record is available.
*/
static XLogRecord *
-ReadRecord(XLogReaderState *xlogreader, int emode,
- bool fetching_ckpt)
+ReadRecord(XLogReaderState *xlogreader, int emode, bool fetching_ckpt)
{
XLogRecord *record;
- XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
-
- private->fetching_ckpt = fetching_ckpt;
- private->emode = emode;
- private->randAccess = (xlogreader->ReadRecPtr == InvalidXLogRecPtr);
+ bool randAccess = (xlogreader->ReadRecPtr == InvalidXLogRecPtr);
/* This is the first attempt to read this page. */
lastSourceFailed = false;
@@ -4360,8 +4347,16 @@ ReadRecord(XLogReaderState *xlogreader, int emode,
for (;;)
{
char *errormsg;
+ XLogReadRecordResult result;
+
+ while ((result = XLogReadRecord(xlogreader, &record, &errormsg))
+ == XLREAD_NEED_DATA)
+ {
+ if (!XLogPageRead(xlogreader, fetching_ckpt, emode, randAccess))
+ break;
+
+ }
- record = XLogReadRecord(xlogreader, &errormsg);
ReadRecPtr = xlogreader->ReadRecPtr;
EndRecPtr = xlogreader->EndRecPtr;
if (record == NULL)
@@ -6380,7 +6375,6 @@ StartupXLOG(void)
bool backupFromStandby = false;
DBState dbstate_at_startup;
XLogReaderState *xlogreader;
- XLogPageReadPrivate private;
bool promoted = false;
struct stat st;
@@ -6539,13 +6533,9 @@ StartupXLOG(void)
OwnLatch(&XLogCtl->recoveryWakeupLatch);
/* Set up XLOG reader facility */
- MemSet(&private, 0, sizeof(XLogPageReadPrivate));
xlogreader =
- XLogReaderAllocate(wal_segment_size, NULL,
- XL_ROUTINE(.page_read = &XLogPageRead,
- .segment_open = NULL,
- .segment_close = wal_segment_close),
- &private);
+ XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+
if (!xlogreader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
@@ -11987,12 +11977,13 @@ CancelBackup(void)
* sleep and retry.
*/
static bool
-XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetRecPtr, char *readBuf)
+XLogPageRead(XLogReaderState *xlogreader,
+ bool fetching_ckpt, int emode, bool randAccess)
{
- XLogPageReadPrivate *private =
- (XLogPageReadPrivate *) xlogreader->private_data;
- int emode = private->emode;
+ char *readBuf = xlogreader->readBuf;
+ XLogRecPtr targetPagePtr = xlogreader->readPagePtr;
+ int reqLen = xlogreader->readLen;
+ XLogRecPtr targetRecPtr = xlogreader->ReadRecPtr;
uint32 targetPageOff;
XLogSegNo targetSegNo PG_USED_FOR_ASSERTS_ONLY;
int r;
@@ -12035,8 +12026,8 @@ retry:
flushedUpto < targetPagePtr + reqLen))
{
if (!WaitForWALToBecomeAvailable(targetPagePtr + reqLen,
- private->randAccess,
- private->fetching_ckpt,
+ randAccess,
+ fetching_ckpt,
targetRecPtr))
{
if (readFile >= 0)
@@ -12141,6 +12132,7 @@ retry:
goto next_record_is_invalid;
}
+ Assert(xlogreader->readPagePtr == targetPagePtr);
xlogreader->readLen = readLen;
return true;
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index 7d04995b7b..ba3e4868f7 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -40,7 +40,7 @@ static bool XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr,
int reqLen, bool header_inclusive);
static void XLogReaderInvalReadState(XLogReaderState *state);
static bool ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
- XLogRecPtr PrevRecPtr, XLogRecord *record, bool randAccess);
+ XLogRecPtr PrevRecPtr, XLogRecord *record);
static bool ValidXLogRecord(XLogReaderState *state, XLogRecord *record,
XLogRecPtr recptr);
static void ResetDecoder(XLogReaderState *state);
@@ -73,7 +73,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...)
*/
XLogReaderState *
XLogReaderAllocate(int wal_segment_size, const char *waldir,
- XLogReaderRoutine *routine, void *private_data)
+ WALSegmentCleanupCB cleanup_cb)
{
XLogReaderState *state;
@@ -84,7 +84,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
return NULL;
/* initialize caller-provided support functions */
- state->routine = *routine;
+ state->cleanup_cb = cleanup_cb;
state->max_block_id = -1;
@@ -107,8 +107,6 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
WALOpenSegmentInit(&state->seg, &state->segcxt, wal_segment_size,
waldir);
- /* system_identifier initialized to zeroes above */
- state->private_data = private_data;
/* ReadRecPtr, EndRecPtr and readLen initialized to zeroes above */
state->errormsg_buf = palloc_extended(MAX_ERRORMSG_LEN + 1,
MCXT_ALLOC_NO_OOM);
@@ -140,8 +138,8 @@ XLogReaderFree(XLogReaderState *state)
{
int block_id;
- if (state->seg.ws_file != -1)
- state->routine.segment_close(state);
+ if (state->seg.ws_file >= 0)
+ state->cleanup_cb(state);
for (block_id = 0; block_id <= XLR_MAX_BLOCK_ID; block_id++)
{
@@ -246,6 +244,7 @@ XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
/* Begin at the passed-in record pointer. */
state->EndRecPtr = RecPtr;
state->ReadRecPtr = InvalidXLogRecPtr;
+ state->readRecordState = XLREAD_NEXT_RECORD;
}
/*
@@ -254,317 +253,449 @@ XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
* XLogBeginRead() or XLogFindNextRecord() must be called before the first call
* to XLogReadRecord().
*
- * If the page_read callback fails to read the requested data, NULL is
- * returned. The callback is expected to have reported the error; errormsg
- * is set to NULL.
+ * This function may return XLREAD_NEED_DATA several times before returning a
+ * result record. The caller shall read in some new data then call this
+ * function again with the same parameters.
*
- * If the reading fails for some other reason, NULL is also returned, and
- * *errormsg is set to a string with details of the failure.
+ * When a record is successfully read, returns XLREAD_SUCCESS with result
+ * record being stored in *record. Otherwise *record is NULL.
*
- * The returned pointer (or *errormsg) points to an internal buffer that's
- * valid until the next call to XLogReadRecord.
+ * Returns XLREAD_NEED_DATA if more data is needed to finish reading the
+ * current record. In that case, state->readPagePtr and state->readLen inform
+ * the desired position and minimum length of data needed. The caller shall
+ * read in the requested data and set state->readBuf to point to a buffer
+ * containing it. The caller must also set state->readPageTLI and
+ * state->readLen to indicate the timeline that it was read from, and the
+ * length of data that is now available (which must be >= given readLen),
+ * respectively.
+ *
+ * If invalid data is encountered, returns XLREAD_FAIL with *record being set to
+ * NULL. *errormsg is set to a string with details of the failure.
+ * The returned pointer (or *errormsg) points to an internal buffer that's valid
+ * until the next call to XLogReadRecord.
+ *
+ *
+ * This function runs a state machine consists of the following states.
+ *
+ * XLREAD_NEXT_RECORD :
+ * The initial state, if called with valid RecPtr, try to read a record at
+ * that position. If invalid RecPtr is given try to read a record just after
+ * the last one previously read.
+ * This state ens after setting ReadRecPtr. Then goes to XLREAD_TOT_LEN.
+ *
+ * XLREAD_TOT_LEN:
+ * Examining record header. Ends after reading record total
+ * length. recordRemainLen and recordGotLen are initialized.
+ *
+ * XLREAD_FIRST_FRAGMENT:
+ * Reading the first fragment. Ends with finishing reading a single
+ * record. Goes to XLREAD_NEXT_RECORD if that's all or
+ * XLREAD_CONTINUATION if we have continuation.
+
+ * XLREAD_CONTINUATION:
+ * Reading continuation of record. Ends with finishing the whole record then
+ * goes to XLREAD_NEXT_RECORD. During this state, recordRemainLen indicates
+ * how much is left and readRecordBuf holds the partially assert
+ * record.recordContRecPtr points to the beginning of the next page where to
+ * continue.
+ *
+ * If wrong data found in any state, the state machine stays at the current
+ * state. This behavior allows to continue reading a reacord switching among
+ * different souces, while streaming replication.
*/
-XLogRecord *
-XLogReadRecord(XLogReaderState *state, char **errormsg)
+XLogReadRecordResult
+XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
{
- XLogRecPtr RecPtr;
- XLogRecord *record;
- XLogRecPtr targetPagePtr;
- bool randAccess;
- uint32 len,
- total_len;
- uint32 targetRecOff;
- uint32 pageHeaderSize;
- bool gotheader;
+ XLogRecord *prec;
- /*
- * randAccess indicates whether to verify the previous-record pointer of
- * the record we're reading. We only do this if we're reading
- * sequentially, which is what we initially assume.
- */
- randAccess = false;
+ *record = NULL;
/* reset error state */
*errormsg = NULL;
state->errormsg_buf[0] = '\0';
- ResetDecoder(state);
-
- RecPtr = state->EndRecPtr;
-
- if (state->ReadRecPtr != InvalidXLogRecPtr)
- {
- /* read the record after the one we just read */
-
- /*
- * EndRecPtr is pointing to end+1 of the previous WAL record. If
- * we're at a page boundary, no more records can fit on the current
- * page. We must skip over the page header, but we can't do that until
- * we've read in the page, since the header size is variable.
- */
- }
- else
- {
- /*
- * Caller supplied a position to start at.
- *
- * In this case, EndRecPtr should already be pointing to a valid
- * record starting position.
- */
- Assert(XRecOffIsValid(RecPtr));
- randAccess = true;
- }
-
- state->currRecPtr = RecPtr;
-
- targetPagePtr = RecPtr - (RecPtr % XLOG_BLCKSZ);
- targetRecOff = RecPtr % XLOG_BLCKSZ;
-
- /*
- * Read the page containing the record into state->readBuf. Request enough
- * byte to cover the whole record header, or at least the part of it that
- * fits on the same page.
- */
- while (XLogNeedData(state, targetPagePtr,
- Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
- targetRecOff != 0))
- {
- if (!state->routine.page_read(state, state->readPagePtr, state->readLen,
- RecPtr, state->readBuf))
- break;
- }
-
- if (!state->page_verified)
- goto err;
-
- /*
- * We have at least the page header, so we can examine it now.
- */
- pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
- if (targetRecOff == 0)
- {
- /*
- * At page start, so skip over page header.
- */
- RecPtr += pageHeaderSize;
- targetRecOff = pageHeaderSize;
- }
- else if (targetRecOff < pageHeaderSize)
- {
- report_invalid_record(state, "invalid record offset at %X/%X",
- LSN_FORMAT_ARGS(RecPtr));
- goto err;
- }
-
- if ((((XLogPageHeader) state->readBuf)->xlp_info & XLP_FIRST_IS_CONTRECORD) &&
- targetRecOff == pageHeaderSize)
- {
- report_invalid_record(state, "contrecord is requested by %X/%X",
- LSN_FORMAT_ARGS(RecPtr));
- goto err;
- }
-
- /* XLogNeedData has verified the page header */
- Assert(pageHeaderSize <= state->readLen);
-
- /*
- * Read the record length.
- *
- * NB: Even though we use an XLogRecord pointer here, the whole record
- * header might not fit on this page. xl_tot_len is the first field of the
- * struct, so it must be on this page (the records are MAXALIGNed), but we
- * cannot access any other fields until we've verified that we got the
- * whole header.
- */
- record = (XLogRecord *) (state->readBuf + RecPtr % XLOG_BLCKSZ);
- total_len = record->xl_tot_len;
-
- /*
- * If the whole record header is on this page, validate it immediately.
- * Otherwise do just a basic sanity check on xl_tot_len, and validate the
- * rest of the header after reading it from the next page. The xl_tot_len
- * check is necessary here to ensure that we enter the "Need to reassemble
- * record" code path below; otherwise we might fail to apply
- * ValidXLogRecordHeader at all.
- */
- if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
- {
- if (!ValidXLogRecordHeader(state, RecPtr, state->ReadRecPtr, record,
- randAccess))
- goto err;
- gotheader = true;
- }
- else
- {
- /* XXX: more validation should be done here */
- if (total_len < SizeOfXLogRecord)
- {
- report_invalid_record(state,
- "invalid record length at %X/%X: wanted %u, got %u",
- LSN_FORMAT_ARGS(RecPtr),
- (uint32) SizeOfXLogRecord, total_len);
- goto err;
- }
- gotheader = false;
- }
-
- len = XLOG_BLCKSZ - RecPtr % XLOG_BLCKSZ;
- if (total_len > len)
+ switch (state->readRecordState)
{
- /* Need to reassemble record */
- char *contdata;
- XLogPageHeader pageHeader;
- char *buffer;
- uint32 gotlen;
-
- /*
- * Enlarge readRecordBuf as needed.
- */
- if (total_len > state->readRecordBufSize &&
- !allocate_recordbuf(state, total_len))
- {
- /* We treat this as a "bogus data" condition */
- report_invalid_record(state, "record length %u at %X/%X too long",
- total_len, LSN_FORMAT_ARGS(RecPtr));
- goto err;
- }
+ case XLREAD_NEXT_RECORD:
+ ResetDecoder(state);
- /* Copy the first fragment of the record from the first page. */
- memcpy(state->readRecordBuf,
- state->readBuf + RecPtr % XLOG_BLCKSZ, len);
- buffer = state->readRecordBuf + len;
- gotlen = len;
-
- do
- {
- int rest_len = total_len - gotlen;
-
- /* Calculate pointer to beginning of next page */
- targetPagePtr += XLOG_BLCKSZ;
-
- /* Wait for the next page to become available */
- while (XLogNeedData(state, targetPagePtr,
- Min(rest_len, XLOG_BLCKSZ),
- false))
+ if (state->ReadRecPtr != InvalidXLogRecPtr)
{
- if (!state->routine.page_read(state, state->readPagePtr,
- state->readLen,
- state->ReadRecPtr,
- state->readBuf))
- break;
+ /* read the record after the one we just read */
+
+ /*
+ * EndRecPtr is pointing to end+1 of the previous WAL record.
+ * If we're at a page boundary, no more records can fit on the
+ * current page. We must skip over the page header, but we
+ * can't do that until we've read in the page, since the header
+ * size is variable.
+ */
+ state->PrevRecPtr = state->ReadRecPtr;
+ state->ReadRecPtr = state->EndRecPtr;
+ }
+ else
+ {
+ /*
+ * Caller supplied a position to start at.
+ *
+ * In this case, EndRecPtr should already be pointing to a
+ * valid record starting position.
+ */
+ Assert(XRecOffIsValid(state->EndRecPtr));
+ state->ReadRecPtr = state->EndRecPtr;
+
+ /*
+ * We cannot verify the previous-record pointer when we're
+ * seeking to a particular record. Reset PrevRecPtr so that we
+ * won't try doing that.
+ */
+ state->PrevRecPtr = InvalidXLogRecPtr;
+ state->EndRecPtr = InvalidXLogRecPtr; /* to be tidy */
}
+ state->record_verified = false;
+ state->readRecordState = XLREAD_TOT_LEN;
+ /* fall through */
+
+ case XLREAD_TOT_LEN:
+ {
+ uint32 total_len;
+ uint32 pageHeaderSize;
+ XLogRecPtr targetPagePtr;
+ uint32 targetRecOff;
+ XLogPageHeader pageHeader;
+
+ targetPagePtr =
+ state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
+ targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+
+ /*
+ * Check if we have enough data. For the first record in the page,
+ * the requesting length doesn't contain page header.
+ */
+ if (XLogNeedData(state, targetPagePtr,
+ Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+ targetRecOff != 0))
+ return XLREAD_NEED_DATA;
+
+ /* error out if caller supplied bogus page */
if (!state->page_verified)
goto err;
- Assert(SizeOfXLogShortPHD <= state->readLen);
+ /* examine page header now. */
+ pageHeaderSize =
+ XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+ if (targetRecOff == 0)
+ {
+ /* At page start, so skip over page header. */
+ state->ReadRecPtr += pageHeaderSize;
+ targetRecOff = pageHeaderSize;
+ }
+ else if (targetRecOff < pageHeaderSize)
+ {
+ report_invalid_record(state, "invalid record offset at %X/%X",
+ LSN_FORMAT_ARGS(state->ReadRecPtr));
+ goto err;
+ }
- /* Check that the continuation on next page looks valid */
pageHeader = (XLogPageHeader) state->readBuf;
- if (!(pageHeader->xlp_info & XLP_FIRST_IS_CONTRECORD))
+ if ((pageHeader->xlp_info & XLP_FIRST_IS_CONTRECORD) &&
+ targetRecOff == pageHeaderSize)
{
- report_invalid_record(state,
- "there is no contrecord flag at %X/%X",
- LSN_FORMAT_ARGS(RecPtr));
+ report_invalid_record(state, "contrecord is requested by %X/%X",
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr);
goto err;
}
- /*
- * Cross-check that xlp_rem_len agrees with how much of the record
- * we expect there to be left.
- */
- if (pageHeader->xlp_rem_len == 0 ||
- total_len != (pageHeader->xlp_rem_len + gotlen))
- {
- report_invalid_record(state,
- "invalid contrecord length %u (expected %lld) at %X/%X",
- pageHeader->xlp_rem_len,
- ((long long) total_len) - gotlen,
- LSN_FORMAT_ARGS(RecPtr));
- goto err;
- }
-
- /* Append the continuation from this page to the buffer */
- pageHeaderSize = XLogPageHeaderSize(pageHeader);
-
+ /* XLogNeedData has verified the page header */
Assert(pageHeaderSize <= state->readLen);
- contdata = (char *) state->readBuf + pageHeaderSize;
- len = XLOG_BLCKSZ - pageHeaderSize;
- if (pageHeader->xlp_rem_len < len)
- len = pageHeader->xlp_rem_len;
+ /*
+ * Read the record length.
+ *
+ * NB: Even though we use an XLogRecord pointer here, the whole
+ * record header might not fit on this page. xl_tot_len is the first
+ * field of the struct, so it must be on this page (the records are
+ * MAXALIGNed), but we cannot access any other fields until we've
+ * verified that we got the whole header.
+ */
+ prec = (XLogRecord *) (state->readBuf +
+ state->ReadRecPtr % XLOG_BLCKSZ);
+ total_len = prec->xl_tot_len;
- Assert (pageHeaderSize + len <= state->readLen);
- memcpy(buffer, (char *) contdata, len);
- buffer += len;
- gotlen += len;
+ /*
+ * If the whole record header is on this page, validate it
+ * immediately. Otherwise do just a basic sanity check on
+ * xl_tot_len, and validate the rest of the header after reading it
+ * from the next page. The xl_tot_len check is necessary here to
+ * ensure that we enter the XLREAD_CONTINUATION state below;
+ * otherwise we might fail to apply ValidXLogRecordHeader at all.
+ */
+ if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
+ {
+ if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
+ state->PrevRecPtr, prec))
+ goto err;
- /* If we just reassembled the record header, validate it. */
- if (!gotheader)
+ state->record_verified = true;
+ }
+ else
{
- record = (XLogRecord *) state->readRecordBuf;
- if (!ValidXLogRecordHeader(state, RecPtr, state->ReadRecPtr,
- record, randAccess))
+ /* XXX: more validation should be done here */
+ if (total_len < SizeOfXLogRecord)
+ {
+ report_invalid_record(state,
+ "invalid record length at %X/%X: wanted %u, got %u",
+ LSN_FORMAT_ARGS(state->ReadRecPtr),
+ (uint32) SizeOfXLogRecord, total_len);
goto err;
- gotheader = true;
+ }
}
- } while (gotlen < total_len);
-
- Assert(gotheader);
- record = (XLogRecord *) state->readRecordBuf;
- if (!ValidXLogRecord(state, record, RecPtr))
- goto err;
+ /*
+ * Wait for the rest of the record, or the part of the record that
+ * fit on the first page if crossed a page boundary, to become
+ * available.
+ */
+ state->recordGotLen = 0;
+ state->recordRemainLen = total_len;
+ state->readRecordState = XLREAD_FIRST_FRAGMENT;
+ }
+ /* fall through */
- pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
- state->ReadRecPtr = RecPtr;
- state->EndRecPtr = targetPagePtr + pageHeaderSize
- + MAXALIGN(pageHeader->xlp_rem_len);
- }
- else
- {
- /* Wait for the record data to become available */
- while (XLogNeedData(state, targetPagePtr,
- Min(targetRecOff + total_len, XLOG_BLCKSZ), true))
+ case XLREAD_FIRST_FRAGMENT:
{
- if (!state->routine.page_read(state, state->readPagePtr,
- state->readLen,
- state->ReadRecPtr, state->readBuf))
+ uint32 total_len = state->recordRemainLen;
+ uint32 request_len;
+ uint32 record_len;
+ XLogRecPtr targetPagePtr;
+ uint32 targetRecOff;
+
+ /*
+ * Wait for the rest of the record on the first page to become
+ * available
+ */
+ targetPagePtr =
+ state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
+ targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+
+ request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ);
+ record_len = request_len - targetRecOff;
+
+ /* ReadRecPtr contains page header */
+ Assert (targetRecOff != 0);
+ if (XLogNeedData(state, targetPagePtr, request_len, true))
+ return XLREAD_NEED_DATA;
+
+ /* error out if caller supplied bogus page */
+ if (!state->page_verified)
+ goto err;
+
+ prec = (XLogRecord *) (state->readBuf + targetRecOff);
+
+ /* validate record header if not yet */
+ if (!state->record_verified && record_len >= SizeOfXLogRecord)
+ {
+ if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
+ state->PrevRecPtr, prec))
+ goto err;
+
+ state->record_verified = true;
+ }
+
+
+ if (total_len == record_len)
+ {
+ /* Record does not cross a page boundary */
+ Assert(state->record_verified);
+
+ if (!ValidXLogRecord(state, prec, state->ReadRecPtr))
+ goto err;
+
+ state->record_verified = true; /* to be tidy */
+
+ /* We already checked the header earlier */
+ state->EndRecPtr = state->ReadRecPtr + MAXALIGN(record_len);
+
+ *record = prec;
+ state->readRecordState = XLREAD_NEXT_RECORD;
break;
+ }
+
+ /*
+ * The record continues on the next page. Need to reassemble
+ * record
+ */
+ Assert(total_len > record_len);
+
+ /* Enlarge readRecordBuf as needed. */
+ if (total_len > state->readRecordBufSize &&
+ !allocate_recordbuf(state, total_len))
+ {
+ /* We treat this as a "bogus data" condition */
+ report_invalid_record(state,
+ "record length %u at %X/%X too long",
+ total_len,
+ LSN_FORMAT_ARGS(state->ReadRecPtr));
+ goto err;
+ }
+
+ /* Copy the first fragment of the record from the first page. */
+ memcpy(state->readRecordBuf, state->readBuf + targetRecOff,
+ record_len);
+ state->recordGotLen += record_len;
+ state->recordRemainLen -= record_len;
+
+ /* Calculate pointer to beginning of next page */
+ state->recordContRecPtr = state->ReadRecPtr + record_len;
+ Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0);
+
+ state->readRecordState = XLREAD_CONTINUATION;
}
+ /* fall through */
+
+ case XLREAD_CONTINUATION:
+ {
+ XLogPageHeader pageHeader;
+ uint32 pageHeaderSize;
+ XLogRecPtr targetPagePtr;
+
+ /* we enter this state only if we haven't read the whole record. */
+ Assert (state->recordRemainLen > 0);
+
+ while(state->recordRemainLen > 0)
+ {
+ char *contdata;
+ uint32 request_len;
+ uint32 record_len;
+
+ /* Wait for the next page to become available */
+ targetPagePtr = state->recordContRecPtr;
+
+ /* this request contains page header */
+ Assert (targetPagePtr != 0);
+ if (XLogNeedData(state, targetPagePtr,
+ Min(state->recordRemainLen, XLOG_BLCKSZ),
+ false))
+ return XLREAD_NEED_DATA;
+
+ if (!state->page_verified)
+ goto err;
+
+ Assert(SizeOfXLogShortPHD <= state->readLen);
- if (!state->page_verified)
- goto err;
+ /* Check that the continuation on next page looks valid */
+ pageHeader = (XLogPageHeader) state->readBuf;
+ if (!(pageHeader->xlp_info & XLP_FIRST_IS_CONTRECORD))
+ {
+ report_invalid_record(
+ state,
+ "there is no contrecord flag at %X/%X reading %X/%X",
+ (uint32) (state->recordContRecPtr >> 32),
+ (uint32) state->recordContRecPtr,
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr);
+ goto err;
+ }
- /* Record does not cross a page boundary */
- if (!ValidXLogRecord(state, record, RecPtr))
- goto err;
+ /*
+ * Cross-check that xlp_rem_len agrees with how much of the
+ * record we expect there to be left.
+ */
+ if (pageHeader->xlp_rem_len == 0 ||
+ pageHeader->xlp_rem_len != state->recordRemainLen)
+ {
+ report_invalid_record(
+ state,
+ "invalid contrecord length %u at %X/%X reading %X/%X, expected %u",
+ pageHeader->xlp_rem_len,
+ (uint32) (state->recordContRecPtr >> 32),
+ (uint32) state->recordContRecPtr,
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr,
+ state->recordRemainLen);
+ goto err;
+ }
- state->EndRecPtr = RecPtr + MAXALIGN(total_len);
+ /* Append the continuation from this page to the buffer */
+ pageHeaderSize = XLogPageHeaderSize(pageHeader);
- state->ReadRecPtr = RecPtr;
+ /*
+ * XLogNeedData should have ensured that the whole page header
+ * was read
+ */
+ Assert(state->readLen >= pageHeaderSize);
+
+ contdata = (char *) state->readBuf + pageHeaderSize;
+ record_len = XLOG_BLCKSZ - pageHeaderSize;
+ if (pageHeader->xlp_rem_len < record_len)
+ record_len = pageHeader->xlp_rem_len;
+
+ request_len = record_len + pageHeaderSize;
+
+ /* XLogNeedData should have ensured all needed data was read */
+ Assert (state->readLen >= request_len);
+
+ memcpy(state->readRecordBuf + state->recordGotLen,
+ (char *) contdata, record_len);
+ state->recordGotLen += record_len;
+ state->recordRemainLen -= record_len;
+
+ /* If we just reassembled the record header, validate it. */
+ if (!state->record_verified)
+ {
+ Assert(state->recordGotLen >= SizeOfXLogRecord);
+ if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
+ state->PrevRecPtr,
+ (XLogRecord *) state->readRecordBuf))
+ goto err;
+
+ state->record_verified = true;
+ }
+
+ /* Calculate pointer to beginning of next page, and continue */
+ state->recordContRecPtr += XLOG_BLCKSZ;
+ }
+
+ /* targetPagePtr is pointing the last-read page here */
+ prec = (XLogRecord *) state->readRecordBuf;
+ if (!ValidXLogRecord(state, prec, state->ReadRecPtr))
+ goto err;
+
+ pageHeaderSize =
+ XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+ state->EndRecPtr = targetPagePtr + pageHeaderSize
+ + MAXALIGN(pageHeader->xlp_rem_len);
+
+ *record = prec;
+ state->readRecordState = XLREAD_NEXT_RECORD;
+ break;
+ }
}
/*
* Special processing if it's an XLOG SWITCH record
*/
- if (record->xl_rmid == RM_XLOG_ID &&
- (record->xl_info & ~XLR_INFO_MASK) == XLOG_SWITCH)
+ if ((*record)->xl_rmid == RM_XLOG_ID &&
+ ((*record)->xl_info & ~XLR_INFO_MASK) == XLOG_SWITCH)
{
/* Pretend it extends to end of segment */
state->EndRecPtr += state->segcxt.ws_segsize - 1;
state->EndRecPtr -= XLogSegmentOffset(state->EndRecPtr, state->segcxt.ws_segsize);
}
- if (DecodeXLogRecord(state, record, errormsg))
- return record;
- else
- return NULL;
+ Assert (!*record || state->readLen >= 0);
+ if (DecodeXLogRecord(state, *record, errormsg))
+ return XLREAD_SUCCESS;
+
+ *record = NULL;
+ return XLREAD_FAIL;
err:
/*
- * Invalidate the read state. We might read from a different source after
+ * Invalidate the read page. We might read from a different source after
* failure.
*/
XLogReaderInvalReadState(state);
@@ -572,7 +703,8 @@ err:
if (state->errormsg_buf[0] != '\0')
*errormsg = state->errormsg_buf;
- return NULL;
+ *record = NULL;
+ return XLREAD_FAIL;
}
/*
@@ -724,11 +856,12 @@ XLogReaderInvalReadState(XLogReaderState *state)
*
* This is just a convenience subroutine to avoid duplicated code in
* XLogReadRecord. It's not intended for use from anywhere else.
+ *
+ * If PrevRecPtr is valid, the xl_prev is is cross-checked with it.
*/
static bool
ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
- XLogRecPtr PrevRecPtr, XLogRecord *record,
- bool randAccess)
+ XLogRecPtr PrevRecPtr, XLogRecord *record)
{
if (record->xl_tot_len < SizeOfXLogRecord)
{
@@ -745,7 +878,7 @@ ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
record->xl_rmid, LSN_FORMAT_ARGS(RecPtr));
return false;
}
- if (randAccess)
+ if (PrevRecPtr == InvalidXLogRecPtr)
{
/*
* We can't exactly verify the prev-link, but surely it should be less
@@ -975,11 +1108,14 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
* XLogReadRecord() will read the next valid record.
*/
XLogRecPtr
-XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
+XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
+ XLogFindNextRecordCB read_page, void *private)
{
XLogRecPtr tmpRecPtr;
XLogRecPtr found = InvalidXLogRecPtr;
XLogPageHeader header;
+ XLogRecord *record;
+ XLogReadRecordResult result;
char *errormsg;
Assert(!XLogRecPtrIsInvalid(RecPtr));
@@ -1012,9 +1148,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
while(XLogNeedData(state, targetPagePtr, targetRecOff,
targetRecOff != 0))
{
- if (!state->routine.page_read(state, state->readPagePtr,
- state->readLen,
- state->ReadRecPtr, state->readBuf))
+ if (!read_page(state, private))
break;
}
@@ -1066,8 +1200,16 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
* or we just jumped over the remaining data of a continuation.
*/
XLogBeginRead(state, tmpRecPtr);
- while (XLogReadRecord(state, &errormsg) != NULL)
+ while ((result = XLogReadRecord(state, &record, &errormsg)) !=
+ XLREAD_FAIL)
{
+ if (result == XLREAD_NEED_DATA)
+ {
+ if (!read_page(state, private))
+ goto err;
+ continue;
+ }
+
/* past the record we've found, break out */
if (RecPtr <= state->ReadRecPtr)
{
@@ -1087,9 +1229,9 @@ err:
#endif /* FRONTEND */
/*
- * Helper function to ease writing of XLogRoutine->page_read callbacks.
- * If this function is used, caller must supply a segment_open callback in
- * 'state', as that is used here.
+ * Helper function to ease writing of page_read callback.
+ * If this function is used, caller must supply a segment_open callback and
+ * segment_close callback as that is used here.
*
* Read 'count' bytes into 'buf', starting at location 'startptr', from WAL
* fetched from timeline 'tli'.
@@ -1102,6 +1244,7 @@ err:
*/
bool
WALRead(XLogReaderState *state,
+ WALSegmentOpenCB segopenfn, WALSegmentCloseCB segclosefn,
char *buf, XLogRecPtr startptr, Size count, TimeLineID tli,
WALReadError *errinfo)
{
@@ -1133,10 +1276,10 @@ WALRead(XLogReaderState *state,
XLogSegNo nextSegNo;
if (state->seg.ws_file >= 0)
- state->routine.segment_close(state);
+ segclosefn(state);
XLByteToSeg(recptr, nextSegNo, state->segcxt.ws_segsize);
- state->routine.segment_open(state, nextSegNo, &tli);
+ segopenfn(state, nextSegNo, &tli);
/* This shouldn't happen -- indicates a bug in segment_open */
Assert(state->seg.ws_file >= 0);
diff --git a/src/backend/access/transam/xlogutils.c b/src/backend/access/transam/xlogutils.c
index 46eda33f25..b003990745 100644
--- a/src/backend/access/transam/xlogutils.c
+++ b/src/backend/access/transam/xlogutils.c
@@ -686,8 +686,7 @@ XLogTruncateRelation(RelFileNode rnode, ForkNumber forkNum,
void
XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wantLength)
{
- const XLogRecPtr lastReadPage = state->seg.ws_segno *
- state->segcxt.ws_segsize + state->readLen;
+ const XLogRecPtr lastReadPage = state->readPagePtr;
Assert(wantPage != InvalidXLogRecPtr && wantPage % XLOG_BLCKSZ == 0);
Assert(wantLength <= XLOG_BLCKSZ);
@@ -702,7 +701,7 @@ XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wa
* current TLI has since become historical.
*/
if (lastReadPage == wantPage &&
- state->readLen != 0 &&
+ state->page_verified &&
lastReadPage + state->readLen >= wantPage + Min(wantLength, XLOG_BLCKSZ - 1))
return;
@@ -788,6 +787,7 @@ wal_segment_open(XLogReaderState *state, XLogSegNo nextSegNo,
char path[MAXPGPATH];
XLogFilePath(path, tli, nextSegNo, state->segcxt.ws_segsize);
+ elog(LOG, "HOGE: %lu, %d => %s", nextSegNo, tli, path);
state->seg.ws_file = BasicOpenFile(path, O_RDONLY | PG_BINARY);
if (state->seg.ws_file >= 0)
return;
@@ -825,9 +825,11 @@ wal_segment_close(XLogReaderState *state)
* loop for now.
*/
bool
-read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *cur_page)
+read_local_xlog_page(XLogReaderState *state)
{
+ XLogRecPtr targetPagePtr = state->readPagePtr;
+ int reqLen = state->readLen;
+ char *cur_page = state->readBuf;
XLogRecPtr read_upto,
loc;
TimeLineID tli;
@@ -940,11 +942,12 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
* as 'count', read the whole page anyway. It's guaranteed to be
* zero-padded up to the page boundary if it's incomplete.
*/
- if (!WALRead(state, cur_page, targetPagePtr, XLOG_BLCKSZ, tli,
- &errinfo))
+ if (!WALRead(state, wal_segment_open, wal_segment_close,
+ cur_page, targetPagePtr, XLOG_BLCKSZ, tli, &errinfo))
WALReadRaiseError(&errinfo);
/* number of valid bytes in the buffer */
+ state->readPagePtr = targetPagePtr;
state->readLen = count;
return true;
}
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index 37b75deb72..9cbd8afa49 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -148,7 +148,8 @@ StartupDecodingContext(List *output_plugin_options,
TransactionId xmin_horizon,
bool need_full_snapshot,
bool fast_forward,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress)
@@ -198,11 +199,12 @@ StartupDecodingContext(List *output_plugin_options,
ctx->slot = slot;
- ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, xl_routine, ctx);
+ ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, cleanup_cb);
if (!ctx->reader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
errmsg("out of memory")));
+ ctx->page_read = page_read;
ctx->reorder = ReorderBufferAllocate();
ctx->snapshot_builder =
@@ -319,7 +321,8 @@ CreateInitDecodingContext(const char *plugin,
List *output_plugin_options,
bool need_full_snapshot,
XLogRecPtr restart_lsn,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress)
@@ -422,7 +425,7 @@ CreateInitDecodingContext(const char *plugin,
ctx = StartupDecodingContext(NIL, restart_lsn, xmin_horizon,
need_full_snapshot, false,
- xl_routine, prepare_write, do_write,
+ page_read, cleanup_cb, prepare_write, do_write,
update_progress);
/* call output plugin initialization callback */
@@ -476,7 +479,8 @@ LogicalDecodingContext *
CreateDecodingContext(XLogRecPtr start_lsn,
List *output_plugin_options,
bool fast_forward,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress)
@@ -528,8 +532,8 @@ CreateDecodingContext(XLogRecPtr start_lsn,
ctx = StartupDecodingContext(output_plugin_options,
start_lsn, InvalidTransactionId, false,
- fast_forward, xl_routine, prepare_write,
- do_write, update_progress);
+ fast_forward, page_read, cleanup_cb,
+ prepare_write, do_write, update_progress);
/* call output plugin initialization callback */
old_context = MemoryContextSwitchTo(ctx->context);
@@ -585,7 +589,13 @@ DecodingContextFindStartpoint(LogicalDecodingContext *ctx)
char *err = NULL;
/* the read_page callback waits for new WAL */
- record = XLogReadRecord(ctx->reader, &err);
+ while (XLogReadRecord(ctx->reader, &record, &err) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!ctx->page_read(ctx->reader))
+ break;
+ }
+
if (err)
elog(ERROR, "%s", err);
if (!record)
diff --git a/src/backend/replication/logical/logicalfuncs.c b/src/backend/replication/logical/logicalfuncs.c
index f7e055874e..b69f6911c5 100644
--- a/src/backend/replication/logical/logicalfuncs.c
+++ b/src/backend/replication/logical/logicalfuncs.c
@@ -233,9 +233,8 @@ pg_logical_slot_get_changes_guts(FunctionCallInfo fcinfo, bool confirm, bool bin
ctx = CreateDecodingContext(InvalidXLogRecPtr,
options,
false,
- XL_ROUTINE(.page_read = read_local_xlog_page,
- .segment_open = wal_segment_open,
- .segment_close = wal_segment_close),
+ read_local_xlog_page,
+ wal_segment_close,
LogicalOutputPrepareWrite,
LogicalOutputWrite, NULL);
@@ -284,7 +283,13 @@ pg_logical_slot_get_changes_guts(FunctionCallInfo fcinfo, bool confirm, bool bin
XLogRecord *record;
char *errm = NULL;
- record = XLogReadRecord(ctx->reader, &errm);
+ while (XLogReadRecord(ctx->reader, &record, &errm) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!ctx->page_read(ctx->reader))
+ break;
+ }
+
if (errm)
elog(ERROR, "%s", errm);
diff --git a/src/backend/replication/slotfuncs.c b/src/backend/replication/slotfuncs.c
index 9817b44113..deb7cf15eb 100644
--- a/src/backend/replication/slotfuncs.c
+++ b/src/backend/replication/slotfuncs.c
@@ -153,9 +153,8 @@ create_logical_replication_slot(char *name, char *plugin,
ctx = CreateInitDecodingContext(plugin, NIL,
false, /* just catalogs is OK */
restart_lsn,
- XL_ROUTINE(.page_read = read_local_xlog_page,
- .segment_open = wal_segment_open,
- .segment_close = wal_segment_close),
+ read_local_xlog_page,
+ wal_segment_close,
NULL, NULL, NULL);
/*
@@ -512,9 +511,8 @@ pg_logical_replication_slot_advance(XLogRecPtr moveto)
ctx = CreateDecodingContext(InvalidXLogRecPtr,
NIL,
true, /* fast_forward */
- XL_ROUTINE(.page_read = read_local_xlog_page,
- .segment_open = wal_segment_open,
- .segment_close = wal_segment_close),
+ read_local_xlog_page,
+ wal_segment_close,
NULL, NULL, NULL);
/*
@@ -536,7 +534,13 @@ pg_logical_replication_slot_advance(XLogRecPtr moveto)
* Read records. No changes are generated in fast_forward mode,
* but snapbuilder/slot statuses are updated properly.
*/
- record = XLogReadRecord(ctx->reader, &errm);
+ while (XLogReadRecord(ctx->reader, &record, &errm) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!ctx->page_read(ctx->reader))
+ break;
+ }
+
if (errm)
elog(ERROR, "%s", errm);
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index c36b91ae5e..062093727d 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -580,10 +580,7 @@ StartReplication(StartReplicationCmd *cmd)
/* create xlogreader for physical replication */
xlogreader =
- XLogReaderAllocate(wal_segment_size, NULL,
- XL_ROUTINE(.segment_open = WalSndSegmentOpen,
- .segment_close = wal_segment_close),
- NULL);
+ XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
if (!xlogreader)
ereport(ERROR,
@@ -807,9 +804,11 @@ StartReplication(StartReplicationCmd *cmd)
* set every time WAL is flushed.
*/
static bool
-logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetRecPtr, char *cur_page)
+logical_read_xlog_page(XLogReaderState *state)
{
+ XLogRecPtr targetPagePtr = state->readPagePtr;
+ int reqLen = state->readLen;
+ char *cur_page = state->readBuf;
XLogRecPtr flushptr;
int count;
WALReadError errinfo;
@@ -837,7 +836,7 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
count = flushptr - targetPagePtr; /* part of the page available */
/* now actually read the data, we know it's there */
- if (!WALRead(state,
+ if (!WALRead(state, WalSndSegmentOpen, wal_segment_close,
cur_page,
targetPagePtr,
XLOG_BLCKSZ,
@@ -1011,9 +1010,8 @@ CreateReplicationSlot(CreateReplicationSlotCmd *cmd)
ctx = CreateInitDecodingContext(cmd->plugin, NIL, need_full_snapshot,
InvalidXLogRecPtr,
- XL_ROUTINE(.page_read = logical_read_xlog_page,
- .segment_open = WalSndSegmentOpen,
- .segment_close = wal_segment_close),
+ logical_read_xlog_page,
+ wal_segment_close,
WalSndPrepareWrite, WalSndWriteData,
WalSndUpdateProgress);
@@ -1171,9 +1169,8 @@ StartLogicalReplication(StartReplicationCmd *cmd)
*/
logical_decoding_ctx =
CreateDecodingContext(cmd->startpoint, cmd->options, false,
- XL_ROUTINE(.page_read = logical_read_xlog_page,
- .segment_open = WalSndSegmentOpen,
- .segment_close = wal_segment_close),
+ logical_read_xlog_page,
+ wal_segment_close,
WalSndPrepareWrite, WalSndWriteData,
WalSndUpdateProgress);
xlogreader = logical_decoding_ctx->reader;
@@ -2749,7 +2746,7 @@ XLogSendPhysical(void)
enlargeStringInfo(&output_message, nbytes);
retry:
- if (!WALRead(xlogreader,
+ if (!WALRead(xlogreader, WalSndSegmentOpen, wal_segment_close,
&output_message.data[output_message.len],
startptr,
nbytes,
@@ -2847,7 +2844,12 @@ XLogSendLogical(void)
*/
WalSndCaughtUp = false;
- record = XLogReadRecord(logical_decoding_ctx->reader, &errm);
+ while (XLogReadRecord(logical_decoding_ctx->reader, &record, &errm) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!logical_decoding_ctx->page_read(logical_decoding_ctx->reader))
+ break;
+ }
/* xlog record was invalid */
if (errm != NULL)
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index cf119848b0..712c85281c 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -41,15 +41,9 @@ static int xlogreadfd = -1;
static XLogSegNo xlogreadsegno = -1;
static char xlogfpath[MAXPGPATH];
-typedef struct XLogPageReadPrivate
-{
- const char *restoreCommand;
- int tliIndex;
-} XLogPageReadPrivate;
-
-static bool SimpleXLogPageRead(XLogReaderState *xlogreader,
- XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
+static bool SimpleXLogPageRead(XLogReaderState *xlogreader,
+ const char *datadir, int *tliIndex,
+ const char *restoreCommand);
/*
* Read WAL from the datadir/pg_wal, starting from 'startpoint' on timeline
@@ -66,20 +60,22 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
XLogRecord *record;
XLogReaderState *xlogreader;
char *errormsg;
- XLogPageReadPrivate private;
- private.tliIndex = tliIndex;
- private.restoreCommand = restoreCommand;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir,
- XL_ROUTINE(.page_read = &SimpleXLogPageRead),
- &private);
+ xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+
if (xlogreader == NULL)
pg_fatal("out of memory");
XLogBeginRead(xlogreader, startpoint);
do
{
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!SimpleXLogPageRead(xlogreader, datadir,
+ &tliIndex, restoreCommand))
+ break;
+ }
if (record == NULL)
{
@@ -123,19 +119,19 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex,
XLogRecord *record;
XLogReaderState *xlogreader;
char *errormsg;
- XLogPageReadPrivate private;
XLogRecPtr endptr;
- private.tliIndex = tliIndex;
- private.restoreCommand = restoreCommand;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir,
- XL_ROUTINE(.page_read = &SimpleXLogPageRead),
- &private);
+ xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
XLogBeginRead(xlogreader, ptr);
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!SimpleXLogPageRead(xlogreader, datadir, &tliIndex, restoreCommand))
+ break;
+ }
if (record == NULL)
{
if (errormsg)
@@ -170,7 +166,6 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
XLogRecPtr searchptr;
XLogReaderState *xlogreader;
char *errormsg;
- XLogPageReadPrivate private;
/*
* The given fork pointer points to the end of the last common record,
@@ -186,11 +181,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
forkptr += SizeOfXLogShortPHD;
}
- private.tliIndex = tliIndex;
- private.restoreCommand = restoreCommand;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir,
- XL_ROUTINE(.page_read = &SimpleXLogPageRead),
- &private);
+ xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
@@ -200,7 +191,13 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
uint8 info;
XLogBeginRead(xlogreader, searchptr);
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!SimpleXLogPageRead(xlogreader, datadir,
+ &tliIndex, restoreCommand))
+ break;
+ }
if (record == NULL)
{
@@ -247,10 +244,11 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
/* XLogReader callback function, to read a WAL page */
static bool
-SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *readBuf)
+SimpleXLogPageRead(XLogReaderState *xlogreader, const char *datadir,
+ int *tliIndex, const char *restoreCommand)
{
- XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
+ XLogRecPtr targetPagePtr = xlogreader->readPagePtr;
+ char *readBuf = xlogreader->readBuf;
uint32 targetPageOff;
XLogRecPtr targetSegEnd;
XLogSegNo targetSegNo;
@@ -283,14 +281,14 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
* be done both forward and backward, consider also switching timeline
* accordingly.
*/
- while (private->tliIndex < targetNentries - 1 &&
- targetHistory[private->tliIndex].end < targetSegEnd)
- private->tliIndex++;
- while (private->tliIndex > 0 &&
- targetHistory[private->tliIndex].begin >= targetSegEnd)
- private->tliIndex--;
+ while (*tliIndex < targetNentries - 1 &&
+ targetHistory[*tliIndex].end < targetSegEnd)
+ (*tliIndex)++;
+ while (*tliIndex > 0 &&
+ targetHistory[*tliIndex].begin >= targetSegEnd)
+ (*tliIndex)--;
- XLogFileName(xlogfname, targetHistory[private->tliIndex].tli,
+ XLogFileName(xlogfname, targetHistory[*tliIndex].tli,
xlogreadsegno, WalSegSz);
snprintf(xlogfpath, MAXPGPATH, "%s/" XLOGDIR "/%s",
@@ -303,7 +301,7 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
/*
* If we have no restore_command to execute, then exit.
*/
- if (private->restoreCommand == NULL)
+ if (restoreCommand == NULL)
{
pg_log_error("could not open file \"%s\": %m", xlogfpath);
xlogreader->readLen = -1;
@@ -317,7 +315,7 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
xlogreadfd = RestoreArchivedFile(xlogreader->segcxt.ws_dir,
xlogfname,
WalSegSz,
- private->restoreCommand);
+ restoreCommand);
if (xlogreadfd < 0)
{
@@ -359,7 +357,7 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
Assert(targetSegNo == xlogreadsegno);
- xlogreader->seg.ws_tli = targetHistory[private->tliIndex].tli;
+ xlogreader->seg.ws_tli = targetHistory[*tliIndex].tli;
xlogreader->readLen = XLOG_BLCKSZ;
return true;
}
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index 0f0e0723a3..f514de6b13 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -330,12 +330,17 @@ WALDumpCloseSegment(XLogReaderState *state)
state->seg.ws_file = -1;
}
-/* pg_waldump's XLogReaderRoutine->page_read callback */
+/*
+ * pg_waldump's WAL page rader, also used as page_read callback for
+ * XLogFindNextRecord
+ */
static bool
-WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetPtr, char *readBuff)
+WALDumpReadPage(XLogReaderState *state, void *priv)
{
- XLogDumpPrivate *private = state->private_data;
+ XLogRecPtr targetPagePtr = state->readPagePtr;
+ int reqLen = state->readLen;
+ char *readBuff = state->readBuf;
+ XLogDumpPrivate *private = (XLogDumpPrivate *) priv;
int count = XLOG_BLCKSZ;
WALReadError errinfo;
@@ -353,8 +358,8 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
}
}
- if (!WALRead(state, readBuff, targetPagePtr, count, private->timeline,
- &errinfo))
+ if (!WALRead(state, WALDumpOpenSegment, WALDumpCloseSegment,
+ readBuff, targetPagePtr, count, private->timeline, &errinfo))
{
WALOpenSegment *seg = &errinfo.wre_seg;
char fname[MAXPGPATH];
@@ -374,6 +379,7 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
(Size) errinfo.wre_req);
}
+ Assert(count >= state->readLen);
state->readLen = count;
return true;
}
@@ -1035,16 +1041,14 @@ main(int argc, char **argv)
/* we have everything we need, start reading */
xlogreader_state =
- XLogReaderAllocate(WalSegSz, waldir,
- XL_ROUTINE(.page_read = WALDumpReadPage,
- .segment_open = WALDumpOpenSegment,
- .segment_close = WALDumpCloseSegment),
- &private);
+ XLogReaderAllocate(WalSegSz, waldir, WALDumpCloseSegment);
+
if (!xlogreader_state)
fatal_error("out of memory");
/* first find a valid recptr to start from */
- first_record = XLogFindNextRecord(xlogreader_state, private.startptr);
+ first_record = XLogFindNextRecord(xlogreader_state, private.startptr,
+ &WALDumpReadPage, (void*) &private);
if (first_record == InvalidXLogRecPtr)
fatal_error("could not find a valid record after %X/%X",
@@ -1067,7 +1071,13 @@ main(int argc, char **argv)
for (;;)
{
/* try to read the next record */
- record = XLogReadRecord(xlogreader_state, &errormsg);
+ while (XLogReadRecord(xlogreader_state, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!WALDumpReadPage(xlogreader_state, (void *) &private))
+ break;
+ }
+
if (!record)
{
if (!config.follow || private.endptr_reached)
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index 23d9e70a59..4e9268b553 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -57,64 +57,15 @@ typedef struct WALSegmentContext
typedef struct XLogReaderState XLogReaderState;
-/* Function type definition for the read_page callback */
-typedef bool (*XLogPageReadCB) (XLogReaderState *xlogreader,
- XLogRecPtr targetPagePtr,
- int reqLen,
- XLogRecPtr targetRecPtr,
- char *readBuf);
+/* Function type definition for the segment cleanup callback */
+typedef void (*WALSegmentCleanupCB) (XLogReaderState *xlogreader);
+
+/* Function type definition for the open/close callbacks for WALRead() */
typedef void (*WALSegmentOpenCB) (XLogReaderState *xlogreader,
XLogSegNo nextSegNo,
TimeLineID *tli_p);
typedef void (*WALSegmentCloseCB) (XLogReaderState *xlogreader);
-typedef struct XLogReaderRoutine
-{
- /*
- * Data input callback
- *
- * This callback shall read at least reqLen valid bytes of the xlog page
- * starting at targetPagePtr, and store them in readBuf. The callback
- * shall return the number of bytes read (never more than XLOG_BLCKSZ), or
- * -1 on failure. The callback shall sleep, if necessary, to wait for the
- * requested bytes to become available. The callback will not be invoked
- * again for the same page unless more than the returned number of bytes
- * are needed.
- *
- * targetRecPtr is the position of the WAL record we're reading. Usually
- * it is equal to targetPagePtr + reqLen, but sometimes xlogreader needs
- * to read and verify the page or segment header, before it reads the
- * actual WAL record it's interested in. In that case, targetRecPtr can
- * be used to determine which timeline to read the page from.
- *
- * The callback shall set ->seg.ws_tli to the TLI of the file the page was
- * read from.
- */
- XLogPageReadCB page_read;
-
- /*
- * Callback to open the specified WAL segment for reading. ->seg.ws_file
- * shall be set to the file descriptor of the opened segment. In case of
- * failure, an error shall be raised by the callback and it shall not
- * return.
- *
- * "nextSegNo" is the number of the segment to be opened.
- *
- * "tli_p" is an input/output argument. WALRead() uses it to pass the
- * timeline in which the new segment should be found, but the callback can
- * use it to return the TLI that it actually opened.
- */
- WALSegmentOpenCB segment_open;
-
- /*
- * WAL segment close callback. ->seg.ws_file shall be set to a negative
- * number.
- */
- WALSegmentCloseCB segment_close;
-} XLogReaderRoutine;
-
-#define XL_ROUTINE(...) &(XLogReaderRoutine){__VA_ARGS__}
-
typedef struct
{
/* Is this block ref in use? */
@@ -144,12 +95,35 @@ typedef struct
uint16 data_bufsz;
} DecodedBkpBlock;
+/* Return code from XLogReadRecord */
+typedef enum XLogReadRecordResult
+{
+ XLREAD_SUCCESS, /* record is successfully read */
+ XLREAD_NEED_DATA, /* need more data. see XLogReadRecord. */
+ XLREAD_FAIL /* failed during reading a record */
+} XLogReadRecordResult;
+
+/*
+ * internal state of XLogReadRecord
+ *
+ * XLogReadState runs a state machine while reading a record. Theses states
+ * are not seen outside the function. Each state may repeat several times
+ * exiting requesting caller for new data. See the comment of XLogReadRecrod
+ * for details.
+ */
+typedef enum XLogReadRecordState {
+ XLREAD_NEXT_RECORD,
+ XLREAD_TOT_LEN,
+ XLREAD_FIRST_FRAGMENT,
+ XLREAD_CONTINUATION
+} XLogReadRecordState;
+
struct XLogReaderState
{
/*
* Operational callbacks
*/
- XLogReaderRoutine routine;
+ WALSegmentCleanupCB cleanup_cb;
/* ----------------------------------------
* Public parameters
@@ -162,18 +136,14 @@ struct XLogReaderState
*/
uint64 system_identifier;
- /*
- * Opaque data for callbacks to use. Not used by XLogReader.
- */
- void *private_data;
-
/*
* Start and end point of last record read. EndRecPtr is also used as the
* position to read next. Calling XLogBeginRead() sets EndRecPtr to the
* starting position and ReadRecPtr to invalid.
*/
- XLogRecPtr ReadRecPtr; /* start of last record read */
+ XLogRecPtr ReadRecPtr; /* start of last record read or being read */
XLogRecPtr EndRecPtr; /* end+1 of last record read */
+ XLogRecPtr PrevRecPtr; /* start of previous record read */
/* ----------------------------------------
* Communication with page reader
@@ -187,7 +157,9 @@ struct XLogReaderState
* the request, or -1 on error */
TimeLineID readPageTLI; /* TLI for data currently in readBuf */
char *readBuf; /* buffer to store data */
- bool page_verified; /* is the page on the buffer verified? */
+ bool page_verified; /* is the page header on the buffer verified? */
+ bool record_verified;/* is the current record header verified? */
+
/* ----------------------------------------
@@ -229,8 +201,6 @@ struct XLogReaderState
XLogRecPtr latestPagePtr;
TimeLineID latestPageTLI;
- /* beginning of the WAL record being read. */
- XLogRecPtr currRecPtr;
/* timeline to read it from, 0 if a lookup is required */
TimeLineID currTLI;
@@ -257,6 +227,15 @@ struct XLogReaderState
char *readRecordBuf;
uint32 readRecordBufSize;
+ /*
+ * XLogReadRecord() state
+ */
+ XLogReadRecordState readRecordState;/* state machine state */
+ int recordGotLen; /* amount of current record that has
+ * already been read */
+ int recordRemainLen; /* length of current record that remains */
+ XLogRecPtr recordContRecPtr; /* where the current record continues */
+
/* Buffer to hold error message */
char *errormsg_buf;
};
@@ -264,9 +243,7 @@ struct XLogReaderState
/* Get a new XLogReader */
extern XLogReaderState *XLogReaderAllocate(int wal_segment_size,
const char *waldir,
- XLogReaderRoutine *routine,
- void *private_data);
-extern XLogReaderRoutine *LocalXLogReaderRoutine(void);
+ WALSegmentCleanupCB cleanup_cb);
/* Free an XLogReader */
extern void XLogReaderFree(XLogReaderState *state);
@@ -274,12 +251,17 @@ extern void XLogReaderFree(XLogReaderState *state);
/* Position the XLogReader to given record */
extern void XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr);
#ifdef FRONTEND
-extern XLogRecPtr XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr);
+/* Function type definition for the read_page callback */
+typedef bool (*XLogFindNextRecordCB) (XLogReaderState *xlogreader,
+ void *private);
+extern XLogRecPtr XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
+ XLogFindNextRecordCB read_page, void *private);
#endif /* FRONTEND */
/* Read the next XLog record. Returns NULL on end-of-WAL or failure */
-extern struct XLogRecord *XLogReadRecord(XLogReaderState *state,
- char **errormsg);
+extern XLogReadRecordResult XLogReadRecord(XLogReaderState *state,
+ XLogRecord **record,
+ char **errormsg);
/* Validate a page */
extern bool XLogReaderValidatePageHeader(XLogReaderState *state,
@@ -299,6 +281,7 @@ typedef struct WALReadError
} WALReadError;
extern bool WALRead(XLogReaderState *state,
+ WALSegmentOpenCB segopenfn, WALSegmentCloseCB sgclosefn,
char *buf, XLogRecPtr startptr, Size count,
TimeLineID tli, WALReadError *errinfo);
diff --git a/src/include/access/xlogutils.h b/src/include/access/xlogutils.h
index 364a21c4ea..397fb27fc2 100644
--- a/src/include/access/xlogutils.h
+++ b/src/include/access/xlogutils.h
@@ -47,9 +47,7 @@ extern Buffer XLogReadBufferExtended(RelFileNode rnode, ForkNumber forknum,
extern Relation CreateFakeRelcacheEntry(RelFileNode rnode);
extern void FreeFakeRelcacheEntry(Relation fakerel);
-extern bool read_local_xlog_page(XLogReaderState *state,
- XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetRecPtr, char *cur_page);
+extern bool read_local_xlog_page(XLogReaderState *state);
extern void wal_segment_open(XLogReaderState *state,
XLogSegNo nextSegNo,
TimeLineID *tli_p);
diff --git a/src/include/pg_config_manual.h b/src/include/pg_config_manual.h
index f10ad0acd6..94749609b4 100644
--- a/src/include/pg_config_manual.h
+++ b/src/include/pg_config_manual.h
@@ -377,7 +377,7 @@
* Enable debugging print statements for WAL-related operations; see
* also the wal_debug GUC var.
*/
-/* #define WAL_DEBUG */
+#define WAL_DEBUG
/*
* Enable tracing of resource consumption during sort operations;
diff --git a/src/include/replication/logical.h b/src/include/replication/logical.h
index c253403372..b936f31393 100644
--- a/src/include/replication/logical.h
+++ b/src/include/replication/logical.h
@@ -29,6 +29,10 @@ typedef void (*LogicalOutputPluginWriterUpdateProgress) (struct LogicalDecodingC
TransactionId xid
);
+typedef struct LogicalDecodingContext LogicalDecodingContext;
+
+typedef bool (*LogicalDecodingXLogPageReadCB)(XLogReaderState *ctx);
+
typedef struct LogicalDecodingContext
{
/* memory context this is all allocated in */
@@ -39,6 +43,7 @@ typedef struct LogicalDecodingContext
/* infrastructure pieces for decoding */
XLogReaderState *reader;
+ LogicalDecodingXLogPageReadCB page_read;
struct ReorderBuffer *reorder;
struct SnapBuild *snapshot_builder;
@@ -105,14 +110,16 @@ extern LogicalDecodingContext *CreateInitDecodingContext(const char *plugin,
List *output_plugin_options,
bool need_full_snapshot,
XLogRecPtr restart_lsn,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress);
extern LogicalDecodingContext *CreateDecodingContext(XLogRecPtr start_lsn,
List *output_plugin_options,
bool fast_forward,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress);
--
2.27.0
----Next_Part(Thu_Mar__4_11_28_53_2021_014)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
filename="v17-0003-Remove-globals-readOff-readLen-and-readSegNo.patch"
^ permalink raw reply [nested|flat] 70+ messages in thread
* [PATCH v15 2/4] Move page-reader out of XLogReadRecord
@ 2019-09-10 03:58 Kyotaro Horiguchi <[email protected]>
0 siblings, 0 replies; 70+ messages in thread
From: Kyotaro Horiguchi @ 2019-09-10 03:58 UTC (permalink / raw)
This is the second step of removing callbacks from WAL record reader.
Since it is essential to take in additional data while reading a
record, the function have to ask caller for new data while keeping
working state. Thus the function is turned into a state machine.
---
src/backend/access/transam/twophase.c | 15 +-
src/backend/access/transam/xlog.c | 58 +-
src/backend/access/transam/xlogreader.c | 700 +++++++++++-------
src/backend/access/transam/xlogutils.c | 17 +-
src/backend/replication/logical/logical.c | 26 +-
.../replication/logical/logicalfuncs.c | 13 +-
src/backend/replication/slotfuncs.c | 18 +-
src/backend/replication/walsender.c | 32 +-
src/bin/pg_rewind/parsexlog.c | 84 +--
src/bin/pg_waldump/pg_waldump.c | 36 +-
src/include/access/xlogreader.h | 121 ++-
src/include/access/xlogutils.h | 4 +-
src/include/replication/logical.h | 11 +-
13 files changed, 648 insertions(+), 487 deletions(-)
diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index 9b2e59bf0e..b0d60a0d0f 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -1330,11 +1330,8 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
XLogReaderState *xlogreader;
char *errormsg;
- xlogreader = XLogReaderAllocate(wal_segment_size, NULL,
- XL_ROUTINE(.page_read = &read_local_xlog_page,
- .segment_open = &wal_segment_open,
- .segment_close = &wal_segment_close),
- NULL);
+ xlogreader = XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+
if (!xlogreader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
@@ -1342,7 +1339,13 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
errdetail("Failed while allocating a WAL reading processor.")));
XLogBeginRead(xlogreader, lsn);
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!read_local_xlog_page(xlogreader))
+ break;
+ }
+
if (record == NULL)
ereport(ERROR,
(errcode_for_file_access(),
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index e570e56a24..f9b0108602 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -831,13 +831,6 @@ static XLogSource currentSource = XLOG_FROM_ANY;
static bool lastSourceFailed = false;
static bool pendingWalRcvRestart = false;
-typedef struct XLogPageReadPrivate
-{
- int emode;
- bool fetching_ckpt; /* are we fetching a checkpoint record? */
- bool randAccess;
-} XLogPageReadPrivate;
-
/*
* These variables track when we last obtained some WAL data to process,
* and where we got it from. (XLogReceiptSource is initially the same as
@@ -912,8 +905,8 @@ static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath,
static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli,
XLogSource source, bool notfoundOk);
static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, XLogSource source);
-static bool XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
+static bool XLogPageRead(XLogReaderState *xlogreader,
+ bool fetching_ckpt, int emode, bool randAccess);
static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
bool fetching_ckpt, XLogRecPtr tliRecPtr);
static int emode_for_corrupt_record(int emode, XLogRecPtr RecPtr);
@@ -1225,8 +1218,7 @@ XLogInsertRecord(XLogRecData *rdata,
appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len);
if (!debug_reader)
- debug_reader = XLogReaderAllocate(wal_segment_size, NULL,
- XL_ROUTINE(), NULL);
+ debug_reader = XLogReaderAllocate(wal_segment_size, NULL NULL);
if (!debug_reader)
{
@@ -4326,15 +4318,10 @@ CleanupBackupHistory(void)
* record is available.
*/
static XLogRecord *
-ReadRecord(XLogReaderState *xlogreader, int emode,
- bool fetching_ckpt)
+ReadRecord(XLogReaderState *xlogreader, int emode, bool fetching_ckpt)
{
XLogRecord *record;
- XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
-
- private->fetching_ckpt = fetching_ckpt;
- private->emode = emode;
- private->randAccess = (xlogreader->ReadRecPtr == InvalidXLogRecPtr);
+ bool randAccess = (xlogreader->ReadRecPtr == InvalidXLogRecPtr);
/* This is the first attempt to read this page. */
lastSourceFailed = false;
@@ -4342,8 +4329,16 @@ ReadRecord(XLogReaderState *xlogreader, int emode,
for (;;)
{
char *errormsg;
+ XLogReadRecordResult result;
+
+ while ((result = XLogReadRecord(xlogreader, &record, &errormsg))
+ == XLREAD_NEED_DATA)
+ {
+ if (!XLogPageRead(xlogreader, fetching_ckpt, emode, randAccess))
+ break;
+
+ }
- record = XLogReadRecord(xlogreader, &errormsg);
ReadRecPtr = xlogreader->ReadRecPtr;
EndRecPtr = xlogreader->EndRecPtr;
if (record == NULL)
@@ -6320,7 +6315,6 @@ StartupXLOG(void)
bool backupFromStandby = false;
DBState dbstate_at_startup;
XLogReaderState *xlogreader;
- XLogPageReadPrivate private;
bool fast_promoted = false;
struct stat st;
@@ -6480,13 +6474,9 @@ StartupXLOG(void)
OwnLatch(&XLogCtl->recoveryWakeupLatch);
/* Set up XLOG reader facility */
- MemSet(&private, 0, sizeof(XLogPageReadPrivate));
xlogreader =
- XLogReaderAllocate(wal_segment_size, NULL,
- XL_ROUTINE(.page_read = &XLogPageRead,
- .segment_open = NULL,
- .segment_close = wal_segment_close),
- &private);
+ XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+
if (!xlogreader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
@@ -11860,12 +11850,13 @@ CancelBackup(void)
* sleep and retry.
*/
static bool
-XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetRecPtr, char *readBuf)
+XLogPageRead(XLogReaderState *xlogreader,
+ bool fetching_ckpt, int emode, bool randAccess)
{
- XLogPageReadPrivate *private =
- (XLogPageReadPrivate *) xlogreader->private_data;
- int emode = private->emode;
+ char *readBuf = xlogreader->readBuf;
+ XLogRecPtr targetPagePtr = xlogreader->readPagePtr;
+ int reqLen = xlogreader->readLen;
+ XLogRecPtr targetRecPtr = xlogreader->ReadRecPtr;
uint32 targetPageOff;
XLogSegNo targetSegNo PG_USED_FOR_ASSERTS_ONLY;
int r;
@@ -11908,8 +11899,8 @@ retry:
flushedUpto < targetPagePtr + reqLen))
{
if (!WaitForWALToBecomeAvailable(targetPagePtr + reqLen,
- private->randAccess,
- private->fetching_ckpt,
+ randAccess,
+ fetching_ckpt,
targetRecPtr))
{
if (readFile >= 0)
@@ -12014,6 +12005,7 @@ retry:
goto next_record_is_invalid;
}
+ Assert(xlogreader->readPagePtr == targetPagePtr);
xlogreader->readLen = readLen;
return true;
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index 3d599325ee..9281b57379 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -40,7 +40,7 @@ static bool XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr,
int reqLen, bool header_inclusive);
static void XLogReaderInvalReadState(XLogReaderState *state);
static bool ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
- XLogRecPtr PrevRecPtr, XLogRecord *record, bool randAccess);
+ XLogRecPtr PrevRecPtr, XLogRecord *record);
static bool ValidXLogRecord(XLogReaderState *state, XLogRecord *record,
XLogRecPtr recptr);
static void ResetDecoder(XLogReaderState *state);
@@ -73,7 +73,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...)
*/
XLogReaderState *
XLogReaderAllocate(int wal_segment_size, const char *waldir,
- XLogReaderRoutine *routine, void *private_data)
+ WALSegmentCleanupCB cleanup_cb)
{
XLogReaderState *state;
@@ -84,7 +84,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
return NULL;
/* initialize caller-provided support functions */
- state->routine = *routine;
+ state->cleanup_cb = cleanup_cb;
state->max_block_id = -1;
@@ -107,8 +107,6 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
WALOpenSegmentInit(&state->seg, &state->segcxt, wal_segment_size,
waldir);
- /* system_identifier initialized to zeroes above */
- state->private_data = private_data;
/* ReadRecPtr, EndRecPtr and readLen initialized to zeroes above */
state->errormsg_buf = palloc_extended(MAX_ERRORMSG_LEN + 1,
MCXT_ALLOC_NO_OOM);
@@ -140,8 +138,8 @@ XLogReaderFree(XLogReaderState *state)
{
int block_id;
- if (state->seg.ws_file != -1)
- state->routine.segment_close(state);
+ if (state->seg.ws_file >= 0)
+ state->cleanup_cb(state);
for (block_id = 0; block_id <= XLR_MAX_BLOCK_ID; block_id++)
{
@@ -246,6 +244,7 @@ XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
/* Begin at the passed-in record pointer. */
state->EndRecPtr = RecPtr;
state->ReadRecPtr = InvalidXLogRecPtr;
+ state->readRecordState = XLREAD_NEXT_RECORD;
}
/*
@@ -254,317 +253,452 @@ XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
* XLogBeginRead() or XLogFindNextRecord() must be called before the first call
* to XLogReadRecord().
*
- * If the page_read callback fails to read the requested data, NULL is
- * returned. The callback is expected to have reported the error; errormsg
- * is set to NULL.
+ * This function may return XLREAD_NEED_DATA several times before returning a
+ * result record. The caller shall read in some new data then call this
+ * function again with the same parameters.
*
- * If the reading fails for some other reason, NULL is also returned, and
- * *errormsg is set to a string with details of the failure.
+ * When a record is successfully read, returns XLREAD_SUCCESS with result
+ * record being stored in *record. Otherwise *record is NULL.
*
- * The returned pointer (or *errormsg) points to an internal buffer that's
- * valid until the next call to XLogReadRecord.
+ * Returns XLREAD_NEED_DATA if more data is needed to finish reading the
+ * current record. In that case, state->readPagePtr and state->readLen inform
+ * the desired position and minimum length of data needed. The caller shall
+ * read in the requested data and set state->readBuf to point to a buffer
+ * containing it. The caller must also set state->readPageTLI and
+ * state->readLen to indicate the timeline that it was read from, and the
+ * length of data that is now available (which must be >= given readLen),
+ * respectively.
+ *
+ * If invalid data is encountered, returns XLREAD_FAIL with *record being set to
+ * NULL. *errormsg is set to a string with details of the failure.
+ * The returned pointer (or *errormsg) points to an internal buffer that's valid
+ * until the next call to XLogReadRecord.
+ *
+ *
+ * This function runs a state machine consists of the following states.
+ *
+ * XLREAD_NEXT_RECORD :
+ * The initial state, if called with valid RecPtr, try to read a record at
+ * that position. If invalid RecPtr is given try to read a record just after
+ * the last one previously read.
+ * This state ens after setting ReadRecPtr. Then goes to XLREAD_TOT_LEN.
+ *
+ * XLREAD_TOT_LEN:
+ * Examining record header. Ends after reading record total
+ * length. recordRemainLen and recordGotLen are initialized.
+ *
+ * XLREAD_FIRST_FRAGMENT:
+ * Reading the first fragment. Ends with finishing reading a single
+ * record. Goes to XLREAD_NEXT_RECORD if that's all or
+ * XLREAD_CONTINUATION if we have continuation.
+
+ * XLREAD_CONTINUATION:
+ * Reading continuation of record. Ends with finishing the whole record then
+ * goes to XLREAD_NEXT_RECORD. During this state, recordRemainLen indicates
+ * how much is left and readRecordBuf holds the partially assert
+ * record.recordContRecPtr points to the beginning of the next page where to
+ * continue.
+ *
+ * If wrong data found in any state, the state machine stays at the current
+ * state. This behavior allows to continue reading a reacord switching among
+ * different souces, while streaming replication.
*/
-XLogRecord *
-XLogReadRecord(XLogReaderState *state, char **errormsg)
+XLogReadRecordResult
+XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
{
- XLogRecPtr RecPtr;
- XLogRecord *record;
- XLogRecPtr targetPagePtr;
- bool randAccess;
- uint32 len,
- total_len;
- uint32 targetRecOff;
- uint32 pageHeaderSize;
- bool gotheader;
+ XLogRecord *prec;
- /*
- * randAccess indicates whether to verify the previous-record pointer of
- * the record we're reading. We only do this if we're reading
- * sequentially, which is what we initially assume.
- */
- randAccess = false;
+ *record = NULL;
/* reset error state */
*errormsg = NULL;
state->errormsg_buf[0] = '\0';
- ResetDecoder(state);
-
- RecPtr = state->EndRecPtr;
-
- if (state->ReadRecPtr != InvalidXLogRecPtr)
- {
- /* read the record after the one we just read */
-
- /*
- * EndRecPtr is pointing to end+1 of the previous WAL record. If
- * we're at a page boundary, no more records can fit on the current
- * page. We must skip over the page header, but we can't do that until
- * we've read in the page, since the header size is variable.
- */
- }
- else
- {
- /*
- * Caller supplied a position to start at.
- *
- * In this case, EndRecPtr should already be pointing to a valid
- * record starting position.
- */
- Assert(XRecOffIsValid(RecPtr));
- randAccess = true;
- }
-
- state->currRecPtr = RecPtr;
-
- targetPagePtr = RecPtr - (RecPtr % XLOG_BLCKSZ);
- targetRecOff = RecPtr % XLOG_BLCKSZ;
-
- /*
- * Read the page containing the record into state->readBuf. Request enough
- * byte to cover the whole record header, or at least the part of it that
- * fits on the same page.
- */
- while (XLogNeedData(state, targetPagePtr,
- Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
- targetRecOff != 0))
- {
- if (!state->routine.page_read(state, state->readPagePtr, state->readLen,
- RecPtr, state->readBuf))
- break;
- }
-
- if (!state->page_verified)
- goto err;
-
- /*
- * We have at least the page header, so we can examine it now.
- */
- pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
- if (targetRecOff == 0)
- {
- /*
- * At page start, so skip over page header.
- */
- RecPtr += pageHeaderSize;
- targetRecOff = pageHeaderSize;
- }
- else if (targetRecOff < pageHeaderSize)
- {
- report_invalid_record(state, "invalid record offset at %X/%X",
- (uint32) (RecPtr >> 32), (uint32) RecPtr);
- goto err;
- }
-
- if ((((XLogPageHeader) state->readBuf)->xlp_info & XLP_FIRST_IS_CONTRECORD) &&
- targetRecOff == pageHeaderSize)
- {
- report_invalid_record(state, "contrecord is requested by %X/%X",
- (uint32) (RecPtr >> 32), (uint32) RecPtr);
- goto err;
- }
-
- /* XLogNeedData has verified the page header */
- Assert(pageHeaderSize <= state->readLen);
-
- /*
- * Read the record length.
- *
- * NB: Even though we use an XLogRecord pointer here, the whole record
- * header might not fit on this page. xl_tot_len is the first field of the
- * struct, so it must be on this page (the records are MAXALIGNed), but we
- * cannot access any other fields until we've verified that we got the
- * whole header.
- */
- record = (XLogRecord *) (state->readBuf + RecPtr % XLOG_BLCKSZ);
- total_len = record->xl_tot_len;
-
- /*
- * If the whole record header is on this page, validate it immediately.
- * Otherwise do just a basic sanity check on xl_tot_len, and validate the
- * rest of the header after reading it from the next page. The xl_tot_len
- * check is necessary here to ensure that we enter the "Need to reassemble
- * record" code path below; otherwise we might fail to apply
- * ValidXLogRecordHeader at all.
- */
- if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
- {
- if (!ValidXLogRecordHeader(state, RecPtr, state->ReadRecPtr, record,
- randAccess))
- goto err;
- gotheader = true;
- }
- else
- {
- /* XXX: more validation should be done here */
- if (total_len < SizeOfXLogRecord)
- {
- report_invalid_record(state,
- "invalid record length at %X/%X: wanted %u, got %u",
- (uint32) (RecPtr >> 32), (uint32) RecPtr,
- (uint32) SizeOfXLogRecord, total_len);
- goto err;
- }
- gotheader = false;
- }
-
- len = XLOG_BLCKSZ - RecPtr % XLOG_BLCKSZ;
- if (total_len > len)
+ switch (state->readRecordState)
{
- /* Need to reassemble record */
- char *contdata;
- XLogPageHeader pageHeader;
- char *buffer;
- uint32 gotlen;
-
- /*
- * Enlarge readRecordBuf as needed.
- */
- if (total_len > state->readRecordBufSize &&
- !allocate_recordbuf(state, total_len))
- {
- /* We treat this as a "bogus data" condition */
- report_invalid_record(state, "record length %u at %X/%X too long",
- total_len,
- (uint32) (RecPtr >> 32), (uint32) RecPtr);
- goto err;
- }
+ case XLREAD_NEXT_RECORD:
+ ResetDecoder(state);
- /* Copy the first fragment of the record from the first page. */
- memcpy(state->readRecordBuf,
- state->readBuf + RecPtr % XLOG_BLCKSZ, len);
- buffer = state->readRecordBuf + len;
- gotlen = len;
-
- do
- {
- int rest_len = total_len - gotlen;
-
- /* Calculate pointer to beginning of next page */
- targetPagePtr += XLOG_BLCKSZ;
-
- /* Wait for the next page to become available */
- while (XLogNeedData(state, targetPagePtr,
- Min(rest_len, XLOG_BLCKSZ),
- false))
+ if (state->ReadRecPtr != InvalidXLogRecPtr)
{
- if (!state->routine.page_read(state, state->readPagePtr,
- state->readLen,
- state->ReadRecPtr,
- state->readBuf))
- break;
+ /* read the record after the one we just read */
+
+ /*
+ * EndRecPtr is pointing to end+1 of the previous WAL record.
+ * If we're at a page boundary, no more records can fit on the
+ * current page. We must skip over the page header, but we
+ * can't do that until we've read in the page, since the header
+ * size is variable.
+ */
+ state->PrevRecPtr = state->ReadRecPtr;
+ state->ReadRecPtr = state->EndRecPtr;
+ }
+ else
+ {
+ /*
+ * Caller supplied a position to start at.
+ *
+ * In this case, EndRecPtr should already be pointing to a
+ * valid record starting position.
+ */
+ Assert(XRecOffIsValid(state->EndRecPtr));
+ state->ReadRecPtr = state->EndRecPtr;
+
+ /*
+ * We cannot verify the previous-record pointer when we're
+ * seeking to a particular record. Reset PrevRecPtr so that we
+ * won't try doing that.
+ */
+ state->PrevRecPtr = InvalidXLogRecPtr;
+ state->EndRecPtr = InvalidXLogRecPtr; /* to be tidy */
}
+ state->record_verified = false;
+ state->readRecordState = XLREAD_TOT_LEN;
+ /* fall through */
+
+ case XLREAD_TOT_LEN:
+ {
+ uint32 total_len;
+ uint32 pageHeaderSize;
+ XLogRecPtr targetPagePtr;
+ uint32 targetRecOff;
+ XLogPageHeader pageHeader;
+
+ targetPagePtr =
+ state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
+ targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+
+ /*
+ * Check if we have enough data. For the first record in the page,
+ * the requesting length doesn't contain page header.
+ */
+ if (XLogNeedData(state, targetPagePtr,
+ Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+ targetRecOff != 0))
+ return XLREAD_NEED_DATA;
+
+ /* error out if caller supplied bogus page */
if (!state->page_verified)
goto err;
- Assert(SizeOfXLogShortPHD <= state->readLen);
+ /* examine page header now. */
+ pageHeaderSize =
+ XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+ if (targetRecOff == 0)
+ {
+ /* At page start, so skip over page header. */
+ state->ReadRecPtr += pageHeaderSize;
+ targetRecOff = pageHeaderSize;
+ }
+ else if (targetRecOff < pageHeaderSize)
+ {
+ report_invalid_record(state, "invalid record offset at %X/%X",
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr);
+ goto err;
+ }
- /* Check that the continuation on next page looks valid */
pageHeader = (XLogPageHeader) state->readBuf;
- if (!(pageHeader->xlp_info & XLP_FIRST_IS_CONTRECORD))
+ if ((pageHeader->xlp_info & XLP_FIRST_IS_CONTRECORD) &&
+ targetRecOff == pageHeaderSize)
{
- report_invalid_record(state,
- "there is no contrecord flag at %X/%X",
- (uint32) (RecPtr >> 32), (uint32) RecPtr);
+ report_invalid_record(state, "contrecord is requested by %X/%X",
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr);
goto err;
}
- /*
- * Cross-check that xlp_rem_len agrees with how much of the record
- * we expect there to be left.
- */
- if (pageHeader->xlp_rem_len == 0 ||
- total_len != (pageHeader->xlp_rem_len + gotlen))
- {
- report_invalid_record(state,
- "invalid contrecord length %u at %X/%X",
- pageHeader->xlp_rem_len,
- (uint32) (RecPtr >> 32), (uint32) RecPtr);
- goto err;
- }
-
- /* Append the continuation from this page to the buffer */
- pageHeaderSize = XLogPageHeaderSize(pageHeader);
-
+ /* XLogNeedData has verified the page header */
Assert(pageHeaderSize <= state->readLen);
- contdata = (char *) state->readBuf + pageHeaderSize;
- len = XLOG_BLCKSZ - pageHeaderSize;
- if (pageHeader->xlp_rem_len < len)
- len = pageHeader->xlp_rem_len;
+ /*
+ * Read the record length.
+ *
+ * NB: Even though we use an XLogRecord pointer here, the whole
+ * record header might not fit on this page. xl_tot_len is the first
+ * field of the struct, so it must be on this page (the records are
+ * MAXALIGNed), but we cannot access any other fields until we've
+ * verified that we got the whole header.
+ */
+ prec = (XLogRecord *) (state->readBuf +
+ state->ReadRecPtr % XLOG_BLCKSZ);
+ total_len = prec->xl_tot_len;
- Assert (pageHeaderSize + len <= state->readLen);
- memcpy(buffer, (char *) contdata, len);
- buffer += len;
- gotlen += len;
+ /*
+ * If the whole record header is on this page, validate it
+ * immediately. Otherwise do just a basic sanity check on
+ * xl_tot_len, and validate the rest of the header after reading it
+ * from the next page. The xl_tot_len check is necessary here to
+ * ensure that we enter the XLREAD_CONTINUATION state below;
+ * otherwise we might fail to apply ValidXLogRecordHeader at all.
+ */
+ if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
+ {
+ if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
+ state->PrevRecPtr, prec))
+ goto err;
- /* If we just reassembled the record header, validate it. */
- if (!gotheader)
+ state->record_verified = true;
+ }
+ else
{
- record = (XLogRecord *) state->readRecordBuf;
- if (!ValidXLogRecordHeader(state, RecPtr, state->ReadRecPtr,
- record, randAccess))
+ /* XXX: more validation should be done here */
+ if (total_len < SizeOfXLogRecord)
+ {
+ report_invalid_record(state,
+ "invalid record length at %X/%X: wanted %u, got %u",
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr,
+ (uint32) SizeOfXLogRecord, total_len);
goto err;
- gotheader = true;
+ }
}
- } while (gotlen < total_len);
-
- Assert(gotheader);
- record = (XLogRecord *) state->readRecordBuf;
- if (!ValidXLogRecord(state, record, RecPtr))
- goto err;
+ /*
+ * Wait for the rest of the record, or the part of the record that
+ * fit on the first page if crossed a page boundary, to become
+ * available.
+ */
+ state->recordGotLen = 0;
+ state->recordRemainLen = total_len;
+ state->readRecordState = XLREAD_FIRST_FRAGMENT;
+ }
+ /* fall through */
- pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
- state->ReadRecPtr = RecPtr;
- state->EndRecPtr = targetPagePtr + pageHeaderSize
- + MAXALIGN(pageHeader->xlp_rem_len);
- }
- else
- {
- /* Wait for the record data to become available */
- while (XLogNeedData(state, targetPagePtr,
- Min(targetRecOff + total_len, XLOG_BLCKSZ), true))
+ case XLREAD_FIRST_FRAGMENT:
{
- if (!state->routine.page_read(state, state->readPagePtr,
- state->readLen,
- state->ReadRecPtr, state->readBuf))
+ uint32 total_len = state->recordRemainLen;
+ uint32 request_len;
+ uint32 record_len;
+ XLogRecPtr targetPagePtr;
+ uint32 targetRecOff;
+
+ /*
+ * Wait for the rest of the record on the first page to become
+ * available
+ */
+ targetPagePtr =
+ state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
+ targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+
+ request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ);
+ record_len = request_len - targetRecOff;
+
+ /* ReadRecPtr contains page header */
+ Assert (targetRecOff != 0);
+ if (XLogNeedData(state, targetPagePtr, request_len, true))
+ return XLREAD_NEED_DATA;
+
+ /* error out if caller supplied bogus page */
+ if (!state->page_verified)
+ goto err;
+
+ prec = (XLogRecord *) (state->readBuf + targetRecOff);
+
+ /* validate record header if not yet */
+ if (!state->record_verified && record_len >= SizeOfXLogRecord)
+ {
+ if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
+ state->PrevRecPtr, prec))
+ goto err;
+
+ state->record_verified = true;
+ }
+
+
+ if (total_len == record_len)
+ {
+ /* Record does not cross a page boundary */
+ Assert(state->record_verified);
+
+ if (!ValidXLogRecord(state, prec, state->ReadRecPtr))
+ goto err;
+
+ state->record_verified = true; /* to be tidy */
+
+ /* We already checked the header earlier */
+ state->EndRecPtr = state->ReadRecPtr + MAXALIGN(record_len);
+
+ *record = prec;
+ state->readRecordState = XLREAD_NEXT_RECORD;
break;
+ }
+
+ /*
+ * The record continues on the next page. Need to reassemble
+ * record
+ */
+ Assert(total_len > record_len);
+
+ /* Enlarge readRecordBuf as needed. */
+ if (total_len > state->readRecordBufSize &&
+ !allocate_recordbuf(state, total_len))
+ {
+ /* We treat this as a "bogus data" condition */
+ report_invalid_record(state,
+ "record length %u at %X/%X too long",
+ total_len,
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr);
+ goto err;
+ }
+
+ /* Copy the first fragment of the record from the first page. */
+ memcpy(state->readRecordBuf, state->readBuf + targetRecOff,
+ record_len);
+ state->recordGotLen += record_len;
+ state->recordRemainLen -= record_len;
+
+ /* Calculate pointer to beginning of next page */
+ state->recordContRecPtr = state->ReadRecPtr + record_len;
+ Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0);
+
+ state->readRecordState = XLREAD_CONTINUATION;
}
+ /* fall through */
+
+ case XLREAD_CONTINUATION:
+ {
+ XLogPageHeader pageHeader;
+ uint32 pageHeaderSize;
+ XLogRecPtr targetPagePtr;
+
+ /* we enter this state only if we haven't read the whole record. */
+ Assert (state->recordRemainLen > 0);
+
+ while(state->recordRemainLen > 0)
+ {
+ char *contdata;
+ uint32 request_len;
+ uint32 record_len;
+
+ /* Wait for the next page to become available */
+ targetPagePtr = state->recordContRecPtr;
+
+ /* this request contains page header */
+ Assert (targetPagePtr != 0);
+ if (XLogNeedData(state, targetPagePtr,
+ Min(state->recordRemainLen, XLOG_BLCKSZ),
+ false))
+ return XLREAD_NEED_DATA;
+
+ if (!state->page_verified)
+ goto err;
+
+ Assert(SizeOfXLogShortPHD <= state->readLen);
- if (!state->page_verified)
- goto err;
+ /* Check that the continuation on next page looks valid */
+ pageHeader = (XLogPageHeader) state->readBuf;
+ if (!(pageHeader->xlp_info & XLP_FIRST_IS_CONTRECORD))
+ {
+ report_invalid_record(
+ state,
+ "there is no contrecord flag at %X/%X reading %X/%X",
+ (uint32) (state->recordContRecPtr >> 32),
+ (uint32) state->recordContRecPtr,
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr);
+ goto err;
+ }
- /* Record does not cross a page boundary */
- if (!ValidXLogRecord(state, record, RecPtr))
- goto err;
+ /*
+ * Cross-check that xlp_rem_len agrees with how much of the
+ * record we expect there to be left.
+ */
+ if (pageHeader->xlp_rem_len == 0 ||
+ pageHeader->xlp_rem_len != state->recordRemainLen)
+ {
+ report_invalid_record(
+ state,
+ "invalid contrecord length %u at %X/%X reading %X/%X, expected %u",
+ pageHeader->xlp_rem_len,
+ (uint32) (state->recordContRecPtr >> 32),
+ (uint32) state->recordContRecPtr,
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr,
+ state->recordRemainLen);
+ goto err;
+ }
- state->EndRecPtr = RecPtr + MAXALIGN(total_len);
+ /* Append the continuation from this page to the buffer */
+ pageHeaderSize = XLogPageHeaderSize(pageHeader);
- state->ReadRecPtr = RecPtr;
+ /*
+ * XLogNeedData should have ensured that the whole page header
+ * was read
+ */
+ Assert(state->readLen >= pageHeaderSize);
+
+ contdata = (char *) state->readBuf + pageHeaderSize;
+ record_len = XLOG_BLCKSZ - pageHeaderSize;
+ if (pageHeader->xlp_rem_len < record_len)
+ record_len = pageHeader->xlp_rem_len;
+
+ request_len = record_len + pageHeaderSize;
+
+ /* XLogNeedData should have ensured all needed data was read */
+ Assert (state->readLen >= request_len);
+
+ memcpy(state->readRecordBuf + state->recordGotLen,
+ (char *) contdata, record_len);
+ state->recordGotLen += record_len;
+ state->recordRemainLen -= record_len;
+
+ /* If we just reassembled the record header, validate it. */
+ if (!state->record_verified)
+ {
+ Assert(state->recordGotLen >= SizeOfXLogRecord);
+ if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
+ state->PrevRecPtr,
+ (XLogRecord *) state->readRecordBuf))
+ goto err;
+
+ state->record_verified = true;
+ }
+
+ /* Calculate pointer to beginning of next page, and continue */
+ state->recordContRecPtr += XLOG_BLCKSZ;
+ }
+
+ /* targetPagePtr is pointing the last-read page here */
+ prec = (XLogRecord *) state->readRecordBuf;
+ if (!ValidXLogRecord(state, prec, state->ReadRecPtr))
+ goto err;
+
+ pageHeaderSize =
+ XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+ state->EndRecPtr = targetPagePtr + pageHeaderSize
+ + MAXALIGN(pageHeader->xlp_rem_len);
+
+ *record = prec;
+ state->readRecordState = XLREAD_NEXT_RECORD;
+ break;
+ }
}
/*
* Special processing if it's an XLOG SWITCH record
*/
- if (record->xl_rmid == RM_XLOG_ID &&
- (record->xl_info & ~XLR_INFO_MASK) == XLOG_SWITCH)
+ if ((*record)->xl_rmid == RM_XLOG_ID &&
+ ((*record)->xl_info & ~XLR_INFO_MASK) == XLOG_SWITCH)
{
/* Pretend it extends to end of segment */
state->EndRecPtr += state->segcxt.ws_segsize - 1;
state->EndRecPtr -= XLogSegmentOffset(state->EndRecPtr, state->segcxt.ws_segsize);
}
- if (DecodeXLogRecord(state, record, errormsg))
- return record;
- else
- return NULL;
+ Assert (!*record || state->readLen >= 0);
+ if (DecodeXLogRecord(state, *record, errormsg))
+ return XLREAD_SUCCESS;
+
+ *record = NULL;
+ return XLREAD_FAIL;
err:
/*
- * Invalidate the read state. We might read from a different source after
+ * Invalidate the read page. We might read from a different source after
* failure.
*/
XLogReaderInvalReadState(state);
@@ -572,7 +706,8 @@ err:
if (state->errormsg_buf[0] != '\0')
*errormsg = state->errormsg_buf;
- return NULL;
+ *record = NULL;
+ return XLREAD_FAIL;
}
/*
@@ -724,11 +859,12 @@ XLogReaderInvalReadState(XLogReaderState *state)
*
* This is just a convenience subroutine to avoid duplicated code in
* XLogReadRecord. It's not intended for use from anywhere else.
+ *
+ * If PrevRecPtr is valid, the xl_prev is is cross-checked with it.
*/
static bool
ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
- XLogRecPtr PrevRecPtr, XLogRecord *record,
- bool randAccess)
+ XLogRecPtr PrevRecPtr, XLogRecord *record)
{
if (record->xl_tot_len < SizeOfXLogRecord)
{
@@ -746,7 +882,7 @@ ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
(uint32) RecPtr);
return false;
}
- if (randAccess)
+ if (PrevRecPtr == InvalidXLogRecPtr)
{
/*
* We can't exactly verify the prev-link, but surely it should be less
@@ -978,11 +1114,14 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
* XLogReadRecord() will read the next valid record.
*/
XLogRecPtr
-XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
+XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
+ XLogFindNextRecordCB read_page, void *private)
{
XLogRecPtr tmpRecPtr;
XLogRecPtr found = InvalidXLogRecPtr;
XLogPageHeader header;
+ XLogRecord *record;
+ XLogReadRecordResult result;
char *errormsg;
Assert(!XLogRecPtrIsInvalid(RecPtr));
@@ -1015,9 +1154,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
while(XLogNeedData(state, targetPagePtr, targetRecOff,
targetRecOff != 0))
{
- if (!state->routine.page_read(state, state->readPagePtr,
- state->readLen,
- state->ReadRecPtr, state->readBuf))
+ if (!read_page(state, private))
break;
}
@@ -1069,8 +1206,16 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
* or we just jumped over the remaining data of a continuation.
*/
XLogBeginRead(state, tmpRecPtr);
- while (XLogReadRecord(state, &errormsg) != NULL)
+ while ((result = XLogReadRecord(state, &record, &errormsg)) !=
+ XLREAD_FAIL)
{
+ if (result == XLREAD_NEED_DATA)
+ {
+ if (!read_page(state, private))
+ goto err;
+ continue;
+ }
+
/* past the record we've found, break out */
if (RecPtr <= state->ReadRecPtr)
{
@@ -1090,9 +1235,9 @@ err:
#endif /* FRONTEND */
/*
- * Helper function to ease writing of XLogRoutine->page_read callbacks.
- * If this function is used, caller must supply a segment_open callback in
- * 'state', as that is used here.
+ * Helper function to ease writing of page_read callback.
+ * If this function is used, caller must supply a segment_open callback and
+ * segment_close callback as that is used here.
*
* Read 'count' bytes into 'buf', starting at location 'startptr', from WAL
* fetched from timeline 'tli'.
@@ -1105,6 +1250,7 @@ err:
*/
bool
WALRead(XLogReaderState *state,
+ WALSegmentOpenCB segopenfn, WALSegmentCloseCB segclosefn,
char *buf, XLogRecPtr startptr, Size count, TimeLineID tli,
WALReadError *errinfo)
{
@@ -1136,10 +1282,10 @@ WALRead(XLogReaderState *state,
XLogSegNo nextSegNo;
if (state->seg.ws_file >= 0)
- state->routine.segment_close(state);
+ segclosefn(state);
XLByteToSeg(recptr, nextSegNo, state->segcxt.ws_segsize);
- state->routine.segment_open(state, nextSegNo, &tli);
+ segopenfn(state, nextSegNo, &tli);
/* This shouldn't happen -- indicates a bug in segment_open */
Assert(state->seg.ws_file >= 0);
diff --git a/src/backend/access/transam/xlogutils.c b/src/backend/access/transam/xlogutils.c
index 1b10b6df20..303333571f 100644
--- a/src/backend/access/transam/xlogutils.c
+++ b/src/backend/access/transam/xlogutils.c
@@ -689,8 +689,7 @@ XLogTruncateRelation(RelFileNode rnode, ForkNumber forkNum,
void
XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wantLength)
{
- const XLogRecPtr lastReadPage = state->seg.ws_segno *
- state->segcxt.ws_segsize + state->readLen;
+ const XLogRecPtr lastReadPage = state->readPagePtr;
Assert(wantPage != InvalidXLogRecPtr && wantPage % XLOG_BLCKSZ == 0);
Assert(wantLength <= XLOG_BLCKSZ);
@@ -705,7 +704,7 @@ XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wa
* current TLI has since become historical.
*/
if (lastReadPage == wantPage &&
- state->readLen != 0 &&
+ state->page_verified &&
lastReadPage + state->readLen >= wantPage + Min(wantLength, XLOG_BLCKSZ - 1))
return;
@@ -792,6 +791,7 @@ wal_segment_open(XLogReaderState *state, XLogSegNo nextSegNo,
char path[MAXPGPATH];
XLogFilePath(path, tli, nextSegNo, state->segcxt.ws_segsize);
+ elog(LOG, "HOGE: %lu, %d => %s", nextSegNo, tli, path);
state->seg.ws_file = BasicOpenFile(path, O_RDONLY | PG_BINARY);
if (state->seg.ws_file >= 0)
return;
@@ -829,9 +829,11 @@ wal_segment_close(XLogReaderState *state)
* loop for now.
*/
bool
-read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *cur_page)
+read_local_xlog_page(XLogReaderState *state)
{
+ XLogRecPtr targetPagePtr = state->readPagePtr;
+ int reqLen = state->readLen;
+ char *cur_page = state->readBuf;
XLogRecPtr read_upto,
loc;
TimeLineID tli;
@@ -944,11 +946,12 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
* as 'count', read the whole page anyway. It's guaranteed to be
* zero-padded up to the page boundary if it's incomplete.
*/
- if (!WALRead(state, cur_page, targetPagePtr, XLOG_BLCKSZ, tli,
- &errinfo))
+ if (!WALRead(state, wal_segment_open, wal_segment_close,
+ cur_page, targetPagePtr, XLOG_BLCKSZ, tli, &errinfo))
WALReadRaiseError(&errinfo);
/* number of valid bytes in the buffer */
+ state->readPagePtr = targetPagePtr;
state->readLen = count;
return true;
}
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index 61902be3b0..a15b0b3355 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -120,7 +120,8 @@ StartupDecodingContext(List *output_plugin_options,
TransactionId xmin_horizon,
bool need_full_snapshot,
bool fast_forward,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress)
@@ -169,11 +170,12 @@ StartupDecodingContext(List *output_plugin_options,
ctx->slot = slot;
- ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, xl_routine, ctx);
+ ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, cleanup_cb);
if (!ctx->reader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
errmsg("out of memory")));
+ ctx->page_read = page_read;
ctx->reorder = ReorderBufferAllocate();
ctx->snapshot_builder =
@@ -231,7 +233,8 @@ CreateInitDecodingContext(char *plugin,
List *output_plugin_options,
bool need_full_snapshot,
XLogRecPtr restart_lsn,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress)
@@ -328,7 +331,7 @@ CreateInitDecodingContext(char *plugin,
ctx = StartupDecodingContext(NIL, restart_lsn, xmin_horizon,
need_full_snapshot, false,
- xl_routine, prepare_write, do_write,
+ page_read, cleanup_cb, prepare_write, do_write,
update_progress);
/* call output plugin initialization callback */
@@ -376,7 +379,8 @@ LogicalDecodingContext *
CreateDecodingContext(XLogRecPtr start_lsn,
List *output_plugin_options,
bool fast_forward,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress)
@@ -429,8 +433,8 @@ CreateDecodingContext(XLogRecPtr start_lsn,
ctx = StartupDecodingContext(output_plugin_options,
start_lsn, InvalidTransactionId, false,
- fast_forward, xl_routine, prepare_write,
- do_write, update_progress);
+ fast_forward, page_read, cleanup_cb,
+ prepare_write, do_write, update_progress);
/* call output plugin initialization callback */
old_context = MemoryContextSwitchTo(ctx->context);
@@ -483,7 +487,13 @@ DecodingContextFindStartpoint(LogicalDecodingContext *ctx)
char *err = NULL;
/* the read_page callback waits for new WAL */
- record = XLogReadRecord(ctx->reader, &err);
+ while (XLogReadRecord(ctx->reader, &record, &err) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!ctx->page_read(ctx->reader))
+ break;
+ }
+
if (err)
elog(ERROR, "%s", err);
if (!record)
diff --git a/src/backend/replication/logical/logicalfuncs.c b/src/backend/replication/logical/logicalfuncs.c
index b99c94e848..fcc81b358e 100644
--- a/src/backend/replication/logical/logicalfuncs.c
+++ b/src/backend/replication/logical/logicalfuncs.c
@@ -233,9 +233,8 @@ pg_logical_slot_get_changes_guts(FunctionCallInfo fcinfo, bool confirm, bool bin
ctx = CreateDecodingContext(InvalidXLogRecPtr,
options,
false,
- XL_ROUTINE(.page_read = read_local_xlog_page,
- .segment_open = wal_segment_open,
- .segment_close = wal_segment_close),
+ read_local_xlog_page,
+ wal_segment_close,
LogicalOutputPrepareWrite,
LogicalOutputWrite, NULL);
@@ -284,7 +283,13 @@ pg_logical_slot_get_changes_guts(FunctionCallInfo fcinfo, bool confirm, bool bin
XLogRecord *record;
char *errm = NULL;
- record = XLogReadRecord(ctx->reader, &errm);
+ while (XLogReadRecord(ctx->reader, &record, &errm) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!ctx->page_read(ctx->reader))
+ break;
+ }
+
if (errm)
elog(ERROR, "%s", errm);
diff --git a/src/backend/replication/slotfuncs.c b/src/backend/replication/slotfuncs.c
index 88033a79b2..8a678074d5 100644
--- a/src/backend/replication/slotfuncs.c
+++ b/src/backend/replication/slotfuncs.c
@@ -152,9 +152,8 @@ create_logical_replication_slot(char *name, char *plugin,
ctx = CreateInitDecodingContext(plugin, NIL,
false, /* just catalogs is OK */
restart_lsn,
- XL_ROUTINE(.page_read = read_local_xlog_page,
- .segment_open = wal_segment_open,
- .segment_close = wal_segment_close),
+ read_local_xlog_page,
+ wal_segment_close,
NULL, NULL, NULL);
/*
@@ -486,9 +485,8 @@ pg_logical_replication_slot_advance(XLogRecPtr moveto)
ctx = CreateDecodingContext(InvalidXLogRecPtr,
NIL,
true, /* fast_forward */
- XL_ROUTINE(.page_read = read_local_xlog_page,
- .segment_open = wal_segment_open,
- .segment_close = wal_segment_close),
+ read_local_xlog_page,
+ wal_segment_close,
NULL, NULL, NULL);
/*
@@ -513,7 +511,13 @@ pg_logical_replication_slot_advance(XLogRecPtr moveto)
* Read records. No changes are generated in fast_forward mode,
* but snapbuilder/slot statuses are updated properly.
*/
- record = XLogReadRecord(ctx->reader, &errm);
+ while (XLogReadRecord(ctx->reader, &record, &errm) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!ctx->page_read(ctx->reader))
+ break;
+ }
+
if (errm)
elog(ERROR, "%s", errm);
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index bf2711bddd..2fd24a9b14 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -580,10 +580,7 @@ StartReplication(StartReplicationCmd *cmd)
/* create xlogreader for physical replication */
xlogreader =
- XLogReaderAllocate(wal_segment_size, NULL,
- XL_ROUTINE(.segment_open = WalSndSegmentOpen,
- .segment_close = wal_segment_close),
- NULL);
+ XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
if (!xlogreader)
ereport(ERROR,
@@ -812,9 +809,11 @@ StartReplication(StartReplicationCmd *cmd)
* set every time WAL is flushed.
*/
static bool
-logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetRecPtr, char *cur_page)
+logical_read_xlog_page(XLogReaderState *state)
{
+ XLogRecPtr targetPagePtr = state->readPagePtr;
+ int reqLen = state->readLen;
+ char *cur_page = state->readBuf;
XLogRecPtr flushptr;
int count;
WALReadError errinfo;
@@ -842,7 +841,7 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
count = flushptr - targetPagePtr; /* part of the page available */
/* now actually read the data, we know it's there */
- if (!WALRead(state,
+ if (!WALRead(state, WalSndSegmentOpen, wal_segment_close,
cur_page,
targetPagePtr,
XLOG_BLCKSZ,
@@ -1014,9 +1013,8 @@ CreateReplicationSlot(CreateReplicationSlotCmd *cmd)
ctx = CreateInitDecodingContext(cmd->plugin, NIL, need_full_snapshot,
InvalidXLogRecPtr,
- XL_ROUTINE(.page_read = logical_read_xlog_page,
- .segment_open = WalSndSegmentOpen,
- .segment_close = wal_segment_close),
+ logical_read_xlog_page,
+ wal_segment_close,
WalSndPrepareWrite, WalSndWriteData,
WalSndUpdateProgress);
@@ -1179,9 +1177,8 @@ StartLogicalReplication(StartReplicationCmd *cmd)
*/
logical_decoding_ctx =
CreateDecodingContext(cmd->startpoint, cmd->options, false,
- XL_ROUTINE(.page_read = logical_read_xlog_page,
- .segment_open = WalSndSegmentOpen,
- .segment_close = wal_segment_close),
+ logical_read_xlog_page,
+ wal_segment_close,
WalSndPrepareWrite, WalSndWriteData,
WalSndUpdateProgress);
xlogreader = logical_decoding_ctx->reader;
@@ -2761,7 +2758,7 @@ XLogSendPhysical(void)
enlargeStringInfo(&output_message, nbytes);
retry:
- if (!WALRead(xlogreader,
+ if (!WALRead(xlogreader, WalSndSegmentOpen, wal_segment_close,
&output_message.data[output_message.len],
startptr,
nbytes,
@@ -2859,7 +2856,12 @@ XLogSendLogical(void)
*/
WalSndCaughtUp = false;
- record = XLogReadRecord(logical_decoding_ctx->reader, &errm);
+ while (XLogReadRecord(logical_decoding_ctx->reader, &record, &errm) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!logical_decoding_ctx->page_read(logical_decoding_ctx->reader))
+ break;
+ }
/* xlog record was invalid */
if (errm != NULL)
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index 0b7e73ae79..3dd6df4be6 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -40,15 +40,9 @@ static int xlogreadfd = -1;
static XLogSegNo xlogreadsegno = -1;
static char xlogfpath[MAXPGPATH];
-typedef struct XLogPageReadPrivate
-{
- const char *restoreCommand;
- int tliIndex;
-} XLogPageReadPrivate;
-
-static bool SimpleXLogPageRead(XLogReaderState *xlogreader,
- XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
+static bool SimpleXLogPageRead(XLogReaderState *xlogreader,
+ const char *datadir, int *tliIndex,
+ const char *restoreCommand);
/*
* Read WAL from the datadir/pg_wal, starting from 'startpoint' on timeline
@@ -62,20 +56,22 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
XLogRecord *record;
XLogReaderState *xlogreader;
char *errormsg;
- XLogPageReadPrivate private;
- private.tliIndex = tliIndex;
- private.restoreCommand = restoreCommand;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir,
- XL_ROUTINE(.page_read = &SimpleXLogPageRead),
- &private);
+ xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+
if (xlogreader == NULL)
pg_fatal("out of memory");
XLogBeginRead(xlogreader, startpoint);
do
{
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!SimpleXLogPageRead(xlogreader, datadir,
+ &tliIndex, restoreCommand))
+ break;
+ }
if (record == NULL)
{
@@ -113,19 +109,19 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex,
XLogRecord *record;
XLogReaderState *xlogreader;
char *errormsg;
- XLogPageReadPrivate private;
XLogRecPtr endptr;
- private.tliIndex = tliIndex;
- private.restoreCommand = restoreCommand;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir,
- XL_ROUTINE(.page_read = &SimpleXLogPageRead),
- &private);
+ xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
XLogBeginRead(xlogreader, ptr);
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!SimpleXLogPageRead(xlogreader, datadir, &tliIndex, restoreCommand))
+ break;
+ }
if (record == NULL)
{
if (errormsg)
@@ -160,7 +156,6 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
XLogRecPtr searchptr;
XLogReaderState *xlogreader;
char *errormsg;
- XLogPageReadPrivate private;
/*
* The given fork pointer points to the end of the last common record,
@@ -176,11 +171,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
forkptr += SizeOfXLogShortPHD;
}
- private.tliIndex = tliIndex;
- private.restoreCommand = restoreCommand;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir,
- XL_ROUTINE(.page_read = &SimpleXLogPageRead),
- &private);
+ xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
@@ -190,7 +181,13 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
uint8 info;
XLogBeginRead(xlogreader, searchptr);
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!SimpleXLogPageRead(xlogreader, datadir,
+ &tliIndex, restoreCommand))
+ break;
+ }
if (record == NULL)
{
@@ -237,10 +234,11 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
/* XLogReader callback function, to read a WAL page */
static bool
-SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *readBuf)
+SimpleXLogPageRead(XLogReaderState *xlogreader, const char *datadir,
+ int *tliIndex, const char *restoreCommand)
{
- XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
+ XLogRecPtr targetPagePtr = xlogreader->readPagePtr;
+ char *readBuf = xlogreader->readBuf;
uint32 targetPageOff;
XLogRecPtr targetSegEnd;
XLogSegNo targetSegNo;
@@ -273,14 +271,14 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
* be done both forward and backward, consider also switching timeline
* accordingly.
*/
- while (private->tliIndex < targetNentries - 1 &&
- targetHistory[private->tliIndex].end < targetSegEnd)
- private->tliIndex++;
- while (private->tliIndex > 0 &&
- targetHistory[private->tliIndex].begin >= targetSegEnd)
- private->tliIndex--;
+ while (*tliIndex < targetNentries - 1 &&
+ targetHistory[*tliIndex].end < targetSegEnd)
+ (*tliIndex)++;
+ while (*tliIndex > 0 &&
+ targetHistory[*tliIndex].begin >= targetSegEnd)
+ (*tliIndex)--;
- XLogFileName(xlogfname, targetHistory[private->tliIndex].tli,
+ XLogFileName(xlogfname, targetHistory[*tliIndex].tli,
xlogreadsegno, WalSegSz);
snprintf(xlogfpath, MAXPGPATH, "%s/" XLOGDIR "/%s",
@@ -293,7 +291,7 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
/*
* If we have no restore_command to execute, then exit.
*/
- if (private->restoreCommand == NULL)
+ if (restoreCommand == NULL)
{
pg_log_error("could not open file \"%s\": %m", xlogfpath);
xlogreader->readLen = -1;
@@ -307,7 +305,7 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
xlogreadfd = RestoreArchivedFile(xlogreader->segcxt.ws_dir,
xlogfname,
WalSegSz,
- private->restoreCommand);
+ restoreCommand);
if (xlogreadfd < 0)
{
@@ -349,7 +347,7 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
Assert(targetSegNo == xlogreadsegno);
- xlogreader->seg.ws_tli = targetHistory[private->tliIndex].tli;
+ xlogreader->seg.ws_tli = targetHistory[*tliIndex].tli;
xlogreader->readLen = XLOG_BLCKSZ;
return true;
}
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index ab0c28d9b7..02e71138cc 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -330,12 +330,17 @@ WALDumpCloseSegment(XLogReaderState *state)
state->seg.ws_file = -1;
}
-/* pg_waldump's XLogReaderRoutine->page_read callback */
+/*
+ * pg_waldump's WAL page rader, also used as page_read callback for
+ * XLogFindNextRecord
+ */
static bool
-WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetPtr, char *readBuff)
+WALDumpReadPage(XLogReaderState *state, void *priv)
{
- XLogDumpPrivate *private = state->private_data;
+ XLogRecPtr targetPagePtr = state->readPagePtr;
+ int reqLen = state->readLen;
+ char *readBuff = state->readBuf;
+ XLogDumpPrivate *private = (XLogDumpPrivate *) priv;
int count = XLOG_BLCKSZ;
WALReadError errinfo;
@@ -353,8 +358,8 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
}
}
- if (!WALRead(state, readBuff, targetPagePtr, count, private->timeline,
- &errinfo))
+ if (!WALRead(state, WALDumpOpenSegment, WALDumpCloseSegment,
+ readBuff, targetPagePtr, count, private->timeline, &errinfo))
{
WALOpenSegment *seg = &errinfo.wre_seg;
char fname[MAXPGPATH];
@@ -374,6 +379,7 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
(Size) errinfo.wre_req);
}
+ Assert(count >= state->readLen);
state->readLen = count;
return true;
}
@@ -1042,16 +1048,14 @@ main(int argc, char **argv)
/* we have everything we need, start reading */
xlogreader_state =
- XLogReaderAllocate(WalSegSz, waldir,
- XL_ROUTINE(.page_read = WALDumpReadPage,
- .segment_open = WALDumpOpenSegment,
- .segment_close = WALDumpCloseSegment),
- &private);
+ XLogReaderAllocate(WalSegSz, waldir, WALDumpCloseSegment);
+
if (!xlogreader_state)
fatal_error("out of memory");
/* first find a valid recptr to start from */
- first_record = XLogFindNextRecord(xlogreader_state, private.startptr);
+ first_record = XLogFindNextRecord(xlogreader_state, private.startptr,
+ &WALDumpReadPage, (void*) &private);
if (first_record == InvalidXLogRecPtr)
fatal_error("could not find a valid record after %X/%X",
@@ -1075,7 +1079,13 @@ main(int argc, char **argv)
for (;;)
{
/* try to read the next record */
- record = XLogReadRecord(xlogreader_state, &errormsg);
+ while (XLogReadRecord(xlogreader_state, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!WALDumpReadPage(xlogreader_state, (void *) &private))
+ break;
+ }
+
if (!record)
{
if (!config.follow || private.endptr_reached)
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index 6c8848e14f..6a7fe140cb 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -57,64 +57,15 @@ typedef struct WALSegmentContext
typedef struct XLogReaderState XLogReaderState;
-/* Function type definition for the read_page callback */
-typedef bool (*XLogPageReadCB) (XLogReaderState *xlogreader,
- XLogRecPtr targetPagePtr,
- int reqLen,
- XLogRecPtr targetRecPtr,
- char *readBuf);
+/* Function type definition for the segment cleanup callback */
+typedef void (*WALSegmentCleanupCB) (XLogReaderState *xlogreader);
+
+/* Function type definition for the open/close callbacks for WALRead() */
typedef void (*WALSegmentOpenCB) (XLogReaderState *xlogreader,
XLogSegNo nextSegNo,
TimeLineID *tli_p);
typedef void (*WALSegmentCloseCB) (XLogReaderState *xlogreader);
-typedef struct XLogReaderRoutine
-{
- /*
- * Data input callback
- *
- * This callback shall read at least reqLen valid bytes of the xlog page
- * starting at targetPagePtr, and store them in readBuf. The callback
- * shall return the number of bytes read (never more than XLOG_BLCKSZ), or
- * -1 on failure. The callback shall sleep, if necessary, to wait for the
- * requested bytes to become available. The callback will not be invoked
- * again for the same page unless more than the returned number of bytes
- * are needed.
- *
- * targetRecPtr is the position of the WAL record we're reading. Usually
- * it is equal to targetPagePtr + reqLen, but sometimes xlogreader needs
- * to read and verify the page or segment header, before it reads the
- * actual WAL record it's interested in. In that case, targetRecPtr can
- * be used to determine which timeline to read the page from.
- *
- * The callback shall set ->seg.ws_tli to the TLI of the file the page was
- * read from.
- */
- XLogPageReadCB page_read;
-
- /*
- * Callback to open the specified WAL segment for reading. ->seg.ws_file
- * shall be set to the file descriptor of the opened segment. In case of
- * failure, an error shall be raised by the callback and it shall not
- * return.
- *
- * "nextSegNo" is the number of the segment to be opened.
- *
- * "tli_p" is an input/output argument. WALRead() uses it to pass the
- * timeline in which the new segment should be found, but the callback can
- * use it to return the TLI that it actually opened.
- */
- WALSegmentOpenCB segment_open;
-
- /*
- * WAL segment close callback. ->seg.ws_file shall be set to a negative
- * number.
- */
- WALSegmentCloseCB segment_close;
-} XLogReaderRoutine;
-
-#define XL_ROUTINE(...) &(XLogReaderRoutine){__VA_ARGS__}
-
typedef struct
{
/* Is this block ref in use? */
@@ -144,12 +95,35 @@ typedef struct
uint16 data_bufsz;
} DecodedBkpBlock;
+/* Return code from XLogReadRecord */
+typedef enum XLogReadRecordResult
+{
+ XLREAD_SUCCESS, /* record is successfully read */
+ XLREAD_NEED_DATA, /* need more data. see XLogReadRecord. */
+ XLREAD_FAIL /* failed during reading a record */
+} XLogReadRecordResult;
+
+/*
+ * internal state of XLogReadRecord
+ *
+ * XLogReadState runs a state machine while reading a record. Theses states
+ * are not seen outside the function. Each state may repeat several times
+ * exiting requesting caller for new data. See the comment of XLogReadRecrod
+ * for details.
+ */
+typedef enum XLogReadRecordState {
+ XLREAD_NEXT_RECORD,
+ XLREAD_TOT_LEN,
+ XLREAD_FIRST_FRAGMENT,
+ XLREAD_CONTINUATION
+} XLogReadRecordState;
+
struct XLogReaderState
{
/*
* Operational callbacks
*/
- XLogReaderRoutine routine;
+ WALSegmentCleanupCB cleanup_cb;
/* ----------------------------------------
* Public parameters
@@ -162,18 +136,14 @@ struct XLogReaderState
*/
uint64 system_identifier;
- /*
- * Opaque data for callbacks to use. Not used by XLogReader.
- */
- void *private_data;
-
/*
* Start and end point of last record read. EndRecPtr is also used as the
* position to read next. Calling XLogBeginRead() sets EndRecPtr to the
* starting position and ReadRecPtr to invalid.
*/
- XLogRecPtr ReadRecPtr; /* start of last record read */
+ XLogRecPtr ReadRecPtr; /* start of last record read or being read */
XLogRecPtr EndRecPtr; /* end+1 of last record read */
+ XLogRecPtr PrevRecPtr; /* start of previous record read */
/* ----------------------------------------
* Communication with page reader
@@ -187,7 +157,9 @@ struct XLogReaderState
* the request, or -1 on error */
TimeLineID readPageTLI; /* TLI for data currently in readBuf */
char *readBuf; /* buffer to store data */
- bool page_verified; /* is the page on the buffer verified? */
+ bool page_verified; /* is the page header on the buffer verified? */
+ bool record_verified;/* is the current record header verified? */
+
/* ----------------------------------------
@@ -227,8 +199,6 @@ struct XLogReaderState
XLogRecPtr latestPagePtr;
TimeLineID latestPageTLI;
- /* beginning of the WAL record being read. */
- XLogRecPtr currRecPtr;
/* timeline to read it from, 0 if a lookup is required */
TimeLineID currTLI;
@@ -255,6 +225,15 @@ struct XLogReaderState
char *readRecordBuf;
uint32 readRecordBufSize;
+ /*
+ * XLogReadRecord() state
+ */
+ XLogReadRecordState readRecordState;/* state machine state */
+ int recordGotLen; /* amount of current record that has
+ * already been read */
+ int recordRemainLen; /* length of current record that remains */
+ XLogRecPtr recordContRecPtr; /* where the current record continues */
+
/* Buffer to hold error message */
char *errormsg_buf;
};
@@ -262,9 +241,7 @@ struct XLogReaderState
/* Get a new XLogReader */
extern XLogReaderState *XLogReaderAllocate(int wal_segment_size,
const char *waldir,
- XLogReaderRoutine *routine,
- void *private_data);
-extern XLogReaderRoutine *LocalXLogReaderRoutine(void);
+ WALSegmentCleanupCB cleanup_cb);
/* Free an XLogReader */
extern void XLogReaderFree(XLogReaderState *state);
@@ -272,12 +249,17 @@ extern void XLogReaderFree(XLogReaderState *state);
/* Position the XLogReader to given record */
extern void XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr);
#ifdef FRONTEND
-extern XLogRecPtr XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr);
+/* Function type definition for the read_page callback */
+typedef bool (*XLogFindNextRecordCB) (XLogReaderState *xlogreader,
+ void *private);
+extern XLogRecPtr XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
+ XLogFindNextRecordCB read_page, void *private);
#endif /* FRONTEND */
/* Read the next XLog record. Returns NULL on end-of-WAL or failure */
-extern struct XLogRecord *XLogReadRecord(XLogReaderState *state,
- char **errormsg);
+extern XLogReadRecordResult XLogReadRecord(XLogReaderState *state,
+ XLogRecord **record,
+ char **errormsg);
/* Validate a page */
extern bool XLogReaderValidatePageHeader(XLogReaderState *state,
@@ -297,6 +279,7 @@ typedef struct WALReadError
} WALReadError;
extern bool WALRead(XLogReaderState *state,
+ WALSegmentOpenCB segopenfn, WALSegmentCloseCB sgclosefn,
char *buf, XLogRecPtr startptr, Size count,
TimeLineID tli, WALReadError *errinfo);
diff --git a/src/include/access/xlogutils.h b/src/include/access/xlogutils.h
index c287edf206..482e4ced69 100644
--- a/src/include/access/xlogutils.h
+++ b/src/include/access/xlogutils.h
@@ -47,9 +47,7 @@ extern Buffer XLogReadBufferExtended(RelFileNode rnode, ForkNumber forknum,
extern Relation CreateFakeRelcacheEntry(RelFileNode rnode);
extern void FreeFakeRelcacheEntry(Relation fakerel);
-extern bool read_local_xlog_page(XLogReaderState *state,
- XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetRecPtr, char *cur_page);
+extern bool read_local_xlog_page(XLogReaderState *state);
extern void wal_segment_open(XLogReaderState *state,
XLogSegNo nextSegNo,
TimeLineID *tli_p);
diff --git a/src/include/replication/logical.h b/src/include/replication/logical.h
index c2f2475e5d..a743db49cf 100644
--- a/src/include/replication/logical.h
+++ b/src/include/replication/logical.h
@@ -29,6 +29,10 @@ typedef void (*LogicalOutputPluginWriterUpdateProgress) (struct LogicalDecodingC
TransactionId xid
);
+typedef struct LogicalDecodingContext LogicalDecodingContext;
+
+typedef bool (*LogicalDecodingXLogPageReadCB)(XLogReaderState *ctx);
+
typedef struct LogicalDecodingContext
{
/* memory context this is all allocated in */
@@ -39,6 +43,7 @@ typedef struct LogicalDecodingContext
/* infrastructure pieces for decoding */
XLogReaderState *reader;
+ LogicalDecodingXLogPageReadCB page_read;
struct ReorderBuffer *reorder;
struct SnapBuild *snapshot_builder;
@@ -95,14 +100,16 @@ extern LogicalDecodingContext *CreateInitDecodingContext(char *plugin,
List *output_plugin_options,
bool need_full_snapshot,
XLogRecPtr restart_lsn,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress);
extern LogicalDecodingContext *CreateDecodingContext(XLogRecPtr start_lsn,
List *output_plugin_options,
bool fast_forward,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress);
--
2.18.4
----Next_Part(Thu_Jul__2_13_53_30_2020_072)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
filename="v15-0003-Remove-globals-readOff-readLen-and-readSegNo.patch"
^ permalink raw reply [nested|flat] 70+ messages in thread
* [PATCH v13 2/4] Move page-reader out of XLogReadRecord
@ 2019-09-10 03:58 Kyotaro Horiguchi <[email protected]>
0 siblings, 0 replies; 70+ messages in thread
From: Kyotaro Horiguchi @ 2019-09-10 03:58 UTC (permalink / raw)
This is the second step of removing callbacks from WAL record reader.
Since it is essential to take in additional data while reading a
record, the function have to ask caller for new data while keeping
working state. Thus the function is turned into a state machine.
---
src/backend/access/transam/twophase.c | 11 +-
src/backend/access/transam/xlog.c | 54 +-
src/backend/access/transam/xlogreader.c | 693 +++++++++++-------
src/backend/access/transam/xlogutils.c | 12 +-
src/backend/replication/logical/logical.c | 17 +-
.../replication/logical/logicalfuncs.c | 14 +-
src/backend/replication/slotfuncs.c | 8 +-
src/backend/replication/walsender.c | 14 +-
src/bin/pg_rewind/parsexlog.c | 69 +-
src/bin/pg_waldump/pg_waldump.c | 24 +-
src/include/access/xlogreader.h | 90 +--
src/include/access/xlogutils.h | 4 +-
src/include/replication/logical.h | 9 +-
src/include/replication/logicalfuncs.h | 5 +-
14 files changed, 605 insertions(+), 419 deletions(-)
diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index 5adf956f41..f5f6278880 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -1330,8 +1330,7 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
XLogReaderState *xlogreader;
char *errormsg;
- xlogreader = XLogReaderAllocate(wal_segment_size, NULL,
- &read_local_xlog_page, NULL);
+ xlogreader = XLogReaderAllocate(wal_segment_size, NULL);
if (!xlogreader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
@@ -1339,7 +1338,13 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
errdetail("Failed while allocating a WAL reading processor.")));
XLogBeginRead(xlogreader, lsn);
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!read_local_xlog_page(xlogreader))
+ break;
+ }
+
if (record == NULL)
ereport(ERROR,
(errcode_for_file_access(),
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 357432ffa2..9d5d71ac8c 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -802,13 +802,6 @@ static XLogSource readSource = 0; /* XLOG_FROM_* code */
static XLogSource currentSource = 0; /* XLOG_FROM_* code */
static bool lastSourceFailed = false;
-typedef struct XLogPageReadPrivate
-{
- int emode;
- bool fetching_ckpt; /* are we fetching a checkpoint record? */
- bool randAccess;
-} XLogPageReadPrivate;
-
/*
* These variables track when we last obtained some WAL data to process,
* and where we got it from. (XLogReceiptSource is initially the same as
@@ -883,8 +876,8 @@ static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath,
static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli,
int source, bool notfoundOk);
static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, int source);
-static bool XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
+static bool XLogPageRead(XLogReaderState *xlogreader,
+ bool fetching_ckpt, int emode, bool randAccess);
static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
bool fetching_ckpt, XLogRecPtr tliRecPtr);
static int emode_for_corrupt_record(int emode, XLogRecPtr RecPtr);
@@ -1193,8 +1186,7 @@ XLogInsertRecord(XLogRecData *rdata,
appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len);
if (!debug_reader)
- debug_reader = XLogReaderAllocate(wal_segment_size, NULL,
- NULL, NULL);
+ debug_reader = XLogReaderAllocate(wal_segment_size, NULL);
if (!debug_reader)
{
@@ -4256,15 +4248,10 @@ CleanupBackupHistory(void)
* record is available.
*/
static XLogRecord *
-ReadRecord(XLogReaderState *xlogreader, int emode,
- bool fetching_ckpt)
+ReadRecord(XLogReaderState *xlogreader, int emode, bool fetching_ckpt)
{
XLogRecord *record;
- XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
-
- private->fetching_ckpt = fetching_ckpt;
- private->emode = emode;
- private->randAccess = (xlogreader->ReadRecPtr == InvalidXLogRecPtr);
+ bool randAccess = (xlogreader->ReadRecPtr == InvalidXLogRecPtr);
/* This is the first attempt to read this page. */
lastSourceFailed = false;
@@ -4272,8 +4259,16 @@ ReadRecord(XLogReaderState *xlogreader, int emode,
for (;;)
{
char *errormsg;
+ XLogReadRecordResult result;
+
+ while ((result = XLogReadRecord(xlogreader, &record, &errormsg))
+ == XLREAD_NEED_DATA)
+ {
+ if (!XLogPageRead(xlogreader, fetching_ckpt, emode, randAccess))
+ break;
+
+ }
- record = XLogReadRecord(xlogreader, &errormsg);
ReadRecPtr = xlogreader->ReadRecPtr;
EndRecPtr = xlogreader->EndRecPtr;
if (record == NULL)
@@ -6213,7 +6208,6 @@ StartupXLOG(void)
bool backupFromStandby = false;
DBState dbstate_at_startup;
XLogReaderState *xlogreader;
- XLogPageReadPrivate private;
bool fast_promoted = false;
struct stat st;
@@ -6368,9 +6362,7 @@ StartupXLOG(void)
OwnLatch(&XLogCtl->recoveryWakeupLatch);
/* Set up XLOG reader facility */
- MemSet(&private, 0, sizeof(XLogPageReadPrivate));
- xlogreader = XLogReaderAllocate(wal_segment_size, NULL,
- &XLogPageRead, &private);
+ xlogreader = XLogReaderAllocate(wal_segment_size, NULL);
if (!xlogreader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
@@ -11586,12 +11578,13 @@ CancelBackup(void)
* sleep and retry.
*/
static bool
-XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetRecPtr, char *readBuf)
+XLogPageRead(XLogReaderState *xlogreader,
+ bool fetching_ckpt, int emode, bool randAccess)
{
- XLogPageReadPrivate *private =
- (XLogPageReadPrivate *) xlogreader->private_data;
- int emode = private->emode;
+ char *readBuf = xlogreader->readBuf;
+ XLogRecPtr targetPagePtr = xlogreader->readPagePtr;
+ int reqLen = xlogreader->readLen;
+ XLogRecPtr targetRecPtr = xlogreader->ReadRecPtr;
uint32 targetPageOff;
XLogSegNo targetSegNo PG_USED_FOR_ASSERTS_ONLY;
int r;
@@ -11634,8 +11627,8 @@ retry:
receivedUpto < targetPagePtr + reqLen))
{
if (!WaitForWALToBecomeAvailable(targetPagePtr + reqLen,
- private->randAccess,
- private->fetching_ckpt,
+ randAccess,
+ fetching_ckpt,
targetRecPtr))
{
if (readFile >= 0)
@@ -11740,6 +11733,7 @@ retry:
goto next_record_is_invalid;
}
+ Assert(xlogreader->readPagePtr == targetPagePtr);
xlogreader->readLen = readLen;
return true;
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index 2c1500443e..ad85f50c77 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -40,7 +40,7 @@ static bool XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr,
int reqLen, bool header_inclusive);
static void XLogReaderInvalReadState(XLogReaderState *state);
static bool ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
- XLogRecPtr PrevRecPtr, XLogRecord *record, bool randAccess);
+ XLogRecPtr PrevRecPtr, XLogRecord *record);
static bool ValidXLogRecord(XLogReaderState *state, XLogRecord *record,
XLogRecPtr recptr);
static void ResetDecoder(XLogReaderState *state);
@@ -70,8 +70,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...)
* Returns NULL if the xlogreader couldn't be allocated.
*/
XLogReaderState *
-XLogReaderAllocate(int wal_segment_size, const char *waldir,
- XLogPageReadCB pagereadfunc, void *private_data)
+XLogReaderAllocate(int wal_segment_size, const char *waldir)
{
XLogReaderState *state;
@@ -102,9 +101,6 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
WALOpenSegmentInit(&state->seg, &state->segcxt, wal_segment_size,
waldir);
- state->read_page = pagereadfunc;
- /* system_identifier initialized to zeroes above */
- state->private_data = private_data;
/* ReadRecPtr, EndRecPtr and readLen initialized to zeroes above */
state->errormsg_buf = palloc_extended(MAX_ERRORMSG_LEN + 1,
MCXT_ALLOC_NO_OOM);
@@ -239,6 +235,7 @@ XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
/* Begin at the passed-in record pointer. */
state->EndRecPtr = RecPtr;
state->ReadRecPtr = InvalidXLogRecPtr;
+ state->readRecordState = XLREAD_NEXT_RECORD;
}
/*
@@ -247,314 +244,452 @@ XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
* XLogBeginRead() or XLogFindNextRecord() must be called before the first call
* to XLogReadRecord().
*
- * If the read_page callback fails to read the requested data, NULL is
- * returned. The callback is expected to have reported the error; errormsg
- * is set to NULL.
+ * This function may return XLREAD_NEED_DATA several times before returning a
+ * result record. The caller shall read in some new data then call this
+ * function again with the same parameters.
*
- * If the reading fails for some other reason, NULL is also returned, and
- * *errormsg is set to a string with details of the failure.
+ * When a record is successfully read, returns XLREAD_SUCCESS with result
+ * record being stored in *record. Otherwise *record is NULL.
*
- * The returned pointer (or *errormsg) points to an internal buffer that's
- * valid until the next call to XLogReadRecord.
- */
-XLogRecord *
-XLogReadRecord(XLogReaderState *state, char **errormsg)
-{
- XLogRecPtr RecPtr;
- XLogRecord *record;
- XLogRecPtr targetPagePtr;
- bool randAccess;
- uint32 len,
- total_len;
- uint32 targetRecOff;
- uint32 pageHeaderSize;
- bool gotheader;
+ * Returns XLREAD_NEED_DATA if more data is needed to finish reading the
+ * current record. In that case, state->readPagePtr and state->readLen inform
+ * the desired position and minimum length of data needed. The caller shall
+ * read in the requested data and set state->readBuf to point to a buffer
+ * containing it. The caller must also set state->readPageTLI and
+ * state->readLen to indicate the timeline that it was read from, and the
+ * length of data that is now available (which must be >= given readLen),
+ * respectively.
+ *
+ * If invalid data is encountered, returns XLREAD_FAIL with *record being set to
+ * NULL. *errormsg is set to a string with details of the failure.
+ * The returned pointer (or *errormsg) points to an internal buffer that's valid
+ * until the next call to XLogReadRecord.
+ *
+ *
+ * This function runs a state machine consists of the following states.
+ *
+ * XLREAD_NEXT_RECORD :
+ * The initial state, if called with valid RecPtr, try to read a record at
+ * that position. If invalid RecPtr is given try to read a record just after
+ * the last one previously read.
+ * This state ens after setting ReadRecPtr. Then goes to XLREAD_TOT_LEN.
+ *
+ * XLREAD_TOT_LEN:
+ * Examining record header. Ends after reading record total
+ * length. recordRemainLen and recordGotLen are initialized.
+ *
+ * XLREAD_FIRST_FRAGMENT:
+ * Reading the first fragment. Ends with finishing reading a single
+ * record. Goes to XLREAD_NEXT_RECORD if that's all or
+ * XLREAD_CONTINUATION if we have continuation.
- /*
- * randAccess indicates whether to verify the previous-record pointer of
- * the record we're reading. We only do this if we're reading
- * sequentially, which is what we initially assume.
- */
- randAccess = false;
+ * XLREAD_CONTINUATION:
+ * Reading continuation of record. Ends with finishing the whole record then
+ * goes to XLREAD_NEXT_RECORD. During this state, recordRemainLen indicates
+ * how much is left and readRecordBuf holds the partially assert
+ * record.recordContRecPtr points to the beginning of the next page where to
+ * continue.
+ *
+ * If wrong data found in any state, the state machine stays at the current
+ * state. This behavior allows to continue reading a reacord switching among
+ * different souces, while streaming replication.
+ */
+XLogReadRecordResult
+XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
+{
+ XLogRecord *prec;
+
+ *record = NULL;
/* reset error state */
*errormsg = NULL;
state->errormsg_buf[0] = '\0';
- ResetDecoder(state);
-
- RecPtr = state->EndRecPtr;
-
- if (state->ReadRecPtr != InvalidXLogRecPtr)
+ switch (state->readRecordState)
{
- /* read the record after the one we just read */
+ case XLREAD_NEXT_RECORD:
+ ResetDecoder(state);
- /*
- * EndRecPtr is pointing to end+1 of the previous WAL record. If
- * we're at a page boundary, no more records can fit on the current
- * page. We must skip over the page header, but we can't do that until
- * we've read in the page, since the header size is variable.
- */
- }
- else
- {
- /*
- * Caller supplied a position to start at.
- *
- * In this case, EndRecPtr should already be pointing to a valid
- * record starting position.
- */
- Assert(XRecOffIsValid(RecPtr));
- randAccess = true;
- }
-
- state->currRecPtr = RecPtr;
-
- targetPagePtr = RecPtr - (RecPtr % XLOG_BLCKSZ);
- targetRecOff = RecPtr % XLOG_BLCKSZ;
-
- /*
- * Read the page containing the record into state->readBuf. Request enough
- * byte to cover the whole record header, or at least the part of it that
- * fits on the same page.
- */
- while (XLogNeedData(state, targetPagePtr,
- Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
- targetRecOff != 0))
- {
- if (!state->read_page(state, state->readPagePtr, state->readLen,
- RecPtr, state->readBuf))
- break;
- }
-
- if (!state->page_verified)
- goto err;
-
- /*
- * We have at least the page header, so we can examine it now.
- */
- pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
- if (targetRecOff == 0)
- {
- /*
- * At page start, so skip over page header.
- */
- RecPtr += pageHeaderSize;
- targetRecOff = pageHeaderSize;
- }
- else if (targetRecOff < pageHeaderSize)
- {
- report_invalid_record(state, "invalid record offset at %X/%X",
- (uint32) (RecPtr >> 32), (uint32) RecPtr);
- goto err;
- }
-
- if ((((XLogPageHeader) state->readBuf)->xlp_info & XLP_FIRST_IS_CONTRECORD) &&
- targetRecOff == pageHeaderSize)
- {
- report_invalid_record(state, "contrecord is requested by %X/%X",
- (uint32) (RecPtr >> 32), (uint32) RecPtr);
- goto err;
- }
-
- /* XLogNeedData has verified the page header */
- Assert(pageHeaderSize <= state->readLen);
-
- /*
- * Read the record length.
- *
- * NB: Even though we use an XLogRecord pointer here, the whole record
- * header might not fit on this page. xl_tot_len is the first field of the
- * struct, so it must be on this page (the records are MAXALIGNed), but we
- * cannot access any other fields until we've verified that we got the
- * whole header.
- */
- record = (XLogRecord *) (state->readBuf + RecPtr % XLOG_BLCKSZ);
- total_len = record->xl_tot_len;
-
- /*
- * If the whole record header is on this page, validate it immediately.
- * Otherwise do just a basic sanity check on xl_tot_len, and validate the
- * rest of the header after reading it from the next page. The xl_tot_len
- * check is necessary here to ensure that we enter the "Need to reassemble
- * record" code path below; otherwise we might fail to apply
- * ValidXLogRecordHeader at all.
- */
- if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
- {
- if (!ValidXLogRecordHeader(state, RecPtr, state->ReadRecPtr, record,
- randAccess))
- goto err;
- gotheader = true;
- }
- else
- {
- /* XXX: more validation should be done here */
- if (total_len < SizeOfXLogRecord)
- {
- report_invalid_record(state,
- "invalid record length at %X/%X: wanted %u, got %u",
- (uint32) (RecPtr >> 32), (uint32) RecPtr,
- (uint32) SizeOfXLogRecord, total_len);
- goto err;
- }
- gotheader = false;
- }
-
- len = XLOG_BLCKSZ - RecPtr % XLOG_BLCKSZ;
- if (total_len > len)
- {
- /* Need to reassemble record */
- char *contdata;
- XLogPageHeader pageHeader;
- char *buffer;
- uint32 gotlen;
-
- /*
- * Enlarge readRecordBuf as needed.
- */
- if (total_len > state->readRecordBufSize &&
- !allocate_recordbuf(state, total_len))
- {
- /* We treat this as a "bogus data" condition */
- report_invalid_record(state, "record length %u at %X/%X too long",
- total_len,
- (uint32) (RecPtr >> 32), (uint32) RecPtr);
- goto err;
- }
-
- /* Copy the first fragment of the record from the first page. */
- memcpy(state->readRecordBuf,
- state->readBuf + RecPtr % XLOG_BLCKSZ, len);
- buffer = state->readRecordBuf + len;
- gotlen = len;
-
- do
- {
- int rest_len = total_len - gotlen;
-
- /* Calculate pointer to beginning of next page */
- targetPagePtr += XLOG_BLCKSZ;
-
- /* Wait for the next page to become available */
- while (XLogNeedData(state, targetPagePtr,
- Min(rest_len, XLOG_BLCKSZ),
- false))
+ if (state->ReadRecPtr != InvalidXLogRecPtr)
{
- if (!state->read_page(state, state->readPagePtr, state->readLen,
- state->ReadRecPtr, state->readBuf))
- break;
+ /* read the record after the one we just read */
+
+ /*
+ * EndRecPtr is pointing to end+1 of the previous WAL record.
+ * If we're at a page boundary, no more records can fit on the
+ * current page. We must skip over the page header, but we
+ * can't do that until we've read in the page, since the header
+ * size is variable.
+ */
+ state->PrevRecPtr = state->ReadRecPtr;
+ state->ReadRecPtr = state->EndRecPtr;
+ }
+ else
+ {
+ /*
+ * Caller supplied a position to start at.
+ *
+ * In this case, EndRecPtr should already be pointing to a
+ * valid record starting position.
+ */
+ Assert(XRecOffIsValid(state->EndRecPtr));
+ state->ReadRecPtr = state->EndRecPtr;
+
+ /*
+ * We cannot verify the previous-record pointer when we're
+ * seeking to a particular record. Reset PrevRecPtr so that we
+ * won't try doing that.
+ */
+ state->PrevRecPtr = InvalidXLogRecPtr;
+ state->EndRecPtr = InvalidXLogRecPtr; /* to be tidy */
}
+ state->record_verified = false;
+ state->readRecordState = XLREAD_TOT_LEN;
+ /* fall through */
+
+ case XLREAD_TOT_LEN:
+ {
+ uint32 total_len;
+ uint32 pageHeaderSize;
+ XLogRecPtr targetPagePtr;
+ uint32 targetRecOff;
+ XLogPageHeader pageHeader;
+
+ targetPagePtr =
+ state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
+ targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+
+ /*
+ * Check if we have enough data. For the first record in the page,
+ * the requesting length doesn't contain page header.
+ */
+ if (XLogNeedData(state, targetPagePtr,
+ Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+ targetRecOff != 0))
+ return XLREAD_NEED_DATA;
+
+ /* error out if caller supplied bogus page */
if (!state->page_verified)
goto err;
- Assert(SizeOfXLogShortPHD <= state->readLen);
-
- /* Check that the continuation on next page looks valid */
- pageHeader = (XLogPageHeader) state->readBuf;
- if (!(pageHeader->xlp_info & XLP_FIRST_IS_CONTRECORD))
+ /* examine page header now. */
+ pageHeaderSize =
+ XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+ if (targetRecOff == 0)
{
- report_invalid_record(state,
- "there is no contrecord flag at %X/%X",
- (uint32) (RecPtr >> 32), (uint32) RecPtr);
+ /* At page start, so skip over page header. */
+ state->ReadRecPtr += pageHeaderSize;
+ targetRecOff = pageHeaderSize;
+ }
+ else if (targetRecOff < pageHeaderSize)
+ {
+ report_invalid_record(state, "invalid record offset at %X/%X",
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr);
goto err;
}
+ pageHeader = (XLogPageHeader) state->readBuf;
+ if ((pageHeader->xlp_info & XLP_FIRST_IS_CONTRECORD) &&
+ targetRecOff == pageHeaderSize)
+ {
+ report_invalid_record(state, "contrecord is requested by %X/%X",
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr);
+ goto err;
+ }
+
+ /* XLogNeedData has verified the page header */
+ Assert(pageHeaderSize <= state->readLen);
+
+ /*
+ * Read the record length.
+ *
+ * NB: Even though we use an XLogRecord pointer here, the whole
+ * record header might not fit on this page. xl_tot_len is the first
+ * field of the struct, so it must be on this page (the records are
+ * MAXALIGNed), but we cannot access any other fields until we've
+ * verified that we got the whole header.
+ */
+ prec = (XLogRecord *) (state->readBuf +
+ state->ReadRecPtr % XLOG_BLCKSZ);
+ total_len = prec->xl_tot_len;
+
+ /*
+ * If the whole record header is on this page, validate it
+ * immediately. Otherwise do just a basic sanity check on
+ * xl_tot_len, and validate the rest of the header after reading it
+ * from the next page. The xl_tot_len check is necessary here to
+ * ensure that we enter the XLREAD_CONTINUATION state below;
+ * otherwise we might fail to apply ValidXLogRecordHeader at all.
+ */
+ if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
+ {
+ if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
+ state->PrevRecPtr, prec))
+ goto err;
+
+ state->record_verified = true;
+ }
+ else
+ {
+ /* XXX: more validation should be done here */
+ if (total_len < SizeOfXLogRecord)
+ {
+ report_invalid_record(state,
+ "invalid record length at %X/%X: wanted %u, got %u",
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr,
+ (uint32) SizeOfXLogRecord, total_len);
+ goto err;
+ }
+ }
+
/*
- * Cross-check that xlp_rem_len agrees with how much of the record
- * we expect there to be left.
+ * Wait for the rest of the record, or the part of the record that
+ * fit on the first page if crossed a page boundary, to become
+ * available.
*/
- if (pageHeader->xlp_rem_len == 0 ||
- total_len != (pageHeader->xlp_rem_len + gotlen))
+ state->recordGotLen = 0;
+ state->recordRemainLen = total_len;
+ state->readRecordState = XLREAD_FIRST_FRAGMENT;
+ }
+ /* fall through */
+
+ case XLREAD_FIRST_FRAGMENT:
+ {
+ uint32 total_len = state->recordRemainLen;
+ uint32 request_len;
+ uint32 record_len;
+ XLogRecPtr targetPagePtr;
+ uint32 targetRecOff;
+
+ /*
+ * Wait for the rest of the record on the first page to become
+ * available
+ */
+ targetPagePtr =
+ state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
+ targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+
+ request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ);
+ record_len = request_len - targetRecOff;
+
+ /* ReadRecPtr contains page header */
+ Assert (targetRecOff != 0);
+ if (XLogNeedData(state, targetPagePtr, request_len, true))
+ return XLREAD_NEED_DATA;
+
+ /* error out if caller supplied bogus page */
+ if (!state->page_verified)
+ goto err;
+
+ prec = (XLogRecord *) (state->readBuf + targetRecOff);
+
+ /* validate record header if not yet */
+ if (!state->record_verified && record_len >= SizeOfXLogRecord)
{
+ if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
+ state->PrevRecPtr, prec))
+ goto err;
+
+ state->record_verified = true;
+ }
+
+
+ if (total_len == record_len)
+ {
+ /* Record does not cross a page boundary */
+ Assert(state->record_verified);
+
+ if (!ValidXLogRecord(state, prec, state->ReadRecPtr))
+ goto err;
+
+ state->record_verified = true; /* to be tidy */
+
+ /* We already checked the header earlier */
+ state->EndRecPtr = state->ReadRecPtr + MAXALIGN(record_len);
+
+ *record = prec;
+ state->readRecordState = XLREAD_NEXT_RECORD;
+ break;
+ }
+
+ /*
+ * The record continues on the next page. Need to reassemble
+ * record
+ */
+ Assert(total_len > record_len);
+
+ /* Enlarge readRecordBuf as needed. */
+ if (total_len > state->readRecordBufSize &&
+ !allocate_recordbuf(state, total_len))
+ {
+ /* We treat this as a "bogus data" condition */
report_invalid_record(state,
- "invalid contrecord length %u at %X/%X",
- pageHeader->xlp_rem_len,
- (uint32) (RecPtr >> 32), (uint32) RecPtr);
+ "record length %u at %X/%X too long",
+ total_len,
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr);
goto err;
}
- /* Append the continuation from this page to the buffer */
- pageHeaderSize = XLogPageHeaderSize(pageHeader);
+ /* Copy the first fragment of the record from the first page. */
+ memcpy(state->readRecordBuf, state->readBuf + targetRecOff,
+ record_len);
+ state->recordGotLen += record_len;
+ state->recordRemainLen -= record_len;
- Assert(pageHeaderSize <= state->readLen);
+ /* Calculate pointer to beginning of next page */
+ state->recordContRecPtr = state->ReadRecPtr + record_len;
+ Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0);
- contdata = (char *) state->readBuf + pageHeaderSize;
- len = XLOG_BLCKSZ - pageHeaderSize;
- if (pageHeader->xlp_rem_len < len)
- len = pageHeader->xlp_rem_len;
-
- Assert (pageHeaderSize + len <= state->readLen);
- memcpy(buffer, (char *) contdata, len);
- buffer += len;
- gotlen += len;
-
- /* If we just reassembled the record header, validate it. */
- if (!gotheader)
- {
- record = (XLogRecord *) state->readRecordBuf;
- if (!ValidXLogRecordHeader(state, RecPtr, state->ReadRecPtr,
- record, randAccess))
- goto err;
- gotheader = true;
- }
- } while (gotlen < total_len);
-
- Assert(gotheader);
-
- record = (XLogRecord *) state->readRecordBuf;
- if (!ValidXLogRecord(state, record, RecPtr))
- goto err;
-
- pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
- state->ReadRecPtr = RecPtr;
- state->EndRecPtr = targetPagePtr + pageHeaderSize
- + MAXALIGN(pageHeader->xlp_rem_len);
- }
- else
- {
- /* Wait for the record data to become available */
- while (XLogNeedData(state, targetPagePtr,
- Min(targetRecOff + total_len, XLOG_BLCKSZ), true))
- {
- if (!state->read_page(state, state->readPagePtr, state->readLen,
- state->ReadRecPtr, state->readBuf))
- break;
+ state->readRecordState = XLREAD_CONTINUATION;
}
+ /* fall through */
- if (!state->page_verified)
- goto err;
+ case XLREAD_CONTINUATION:
+ {
+ XLogPageHeader pageHeader;
+ uint32 pageHeaderSize;
+ XLogRecPtr targetPagePtr;
- /* Record does not cross a page boundary */
- if (!ValidXLogRecord(state, record, RecPtr))
- goto err;
+ /* we enter this state only if we haven't read the whole record. */
+ Assert (state->recordRemainLen > 0);
- state->EndRecPtr = RecPtr + MAXALIGN(total_len);
+ while(state->recordRemainLen > 0)
+ {
+ char *contdata;
+ uint32 request_len;
+ uint32 record_len;
- state->ReadRecPtr = RecPtr;
+ /* Wait for the next page to become available */
+ targetPagePtr = state->recordContRecPtr;
+
+ /* this request contains page header */
+ Assert (targetPagePtr != 0);
+ if (XLogNeedData(state, targetPagePtr,
+ Min(state->recordRemainLen, XLOG_BLCKSZ),
+ false))
+ return XLREAD_NEED_DATA;
+
+ if (!state->page_verified)
+ goto err;
+
+ Assert(SizeOfXLogShortPHD <= state->readLen);
+
+ /* Check that the continuation on next page looks valid */
+ pageHeader = (XLogPageHeader) state->readBuf;
+ if (!(pageHeader->xlp_info & XLP_FIRST_IS_CONTRECORD))
+ {
+ report_invalid_record(
+ state,
+ "there is no contrecord flag at %X/%X reading %X/%X",
+ (uint32) (state->recordContRecPtr >> 32),
+ (uint32) state->recordContRecPtr,
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr);
+ goto err;
+ }
+
+ /*
+ * Cross-check that xlp_rem_len agrees with how much of the
+ * record we expect there to be left.
+ */
+ if (pageHeader->xlp_rem_len == 0 ||
+ pageHeader->xlp_rem_len != state->recordRemainLen)
+ {
+ report_invalid_record(
+ state,
+ "invalid contrecord length %u at %X/%X reading %X/%X, expected %u",
+ pageHeader->xlp_rem_len,
+ (uint32) (state->recordContRecPtr >> 32),
+ (uint32) state->recordContRecPtr,
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr,
+ state->recordRemainLen);
+ goto err;
+ }
+
+ /* Append the continuation from this page to the buffer */
+ pageHeaderSize = XLogPageHeaderSize(pageHeader);
+
+ /*
+ * XLogNeedData should have ensured that the whole page header
+ * was read
+ */
+ Assert(state->readLen >= pageHeaderSize);
+
+ contdata = (char *) state->readBuf + pageHeaderSize;
+ record_len = XLOG_BLCKSZ - pageHeaderSize;
+ if (pageHeader->xlp_rem_len < record_len)
+ record_len = pageHeader->xlp_rem_len;
+
+ request_len = record_len + pageHeaderSize;
+
+ /* XLogNeedData should have ensured all needed data was read */
+ Assert (state->readLen >= request_len);
+
+ memcpy(state->readRecordBuf + state->recordGotLen,
+ (char *) contdata, record_len);
+ state->recordGotLen += record_len;
+ state->recordRemainLen -= record_len;
+
+ /* If we just reassembled the record header, validate it. */
+ if (!state->record_verified)
+ {
+ Assert(state->recordGotLen >= SizeOfXLogRecord);
+ if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
+ state->PrevRecPtr,
+ (XLogRecord *) state->readRecordBuf))
+ goto err;
+
+ state->record_verified = true;
+ }
+
+ /* Calculate pointer to beginning of next page, and continue */
+ state->recordContRecPtr += XLOG_BLCKSZ;
+ }
+
+ /* targetPagePtr is pointing the last-read page here */
+ prec = (XLogRecord *) state->readRecordBuf;
+ if (!ValidXLogRecord(state, prec, state->ReadRecPtr))
+ goto err;
+
+ pageHeaderSize =
+ XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+ state->EndRecPtr = targetPagePtr + pageHeaderSize
+ + MAXALIGN(pageHeader->xlp_rem_len);
+
+ *record = prec;
+ state->readRecordState = XLREAD_NEXT_RECORD;
+ break;
+ }
}
/*
* Special processing if it's an XLOG SWITCH record
*/
- if (record->xl_rmid == RM_XLOG_ID &&
- (record->xl_info & ~XLR_INFO_MASK) == XLOG_SWITCH)
+ if ((*record)->xl_rmid == RM_XLOG_ID &&
+ ((*record)->xl_info & ~XLR_INFO_MASK) == XLOG_SWITCH)
{
/* Pretend it extends to end of segment */
state->EndRecPtr += state->segcxt.ws_segsize - 1;
state->EndRecPtr -= XLogSegmentOffset(state->EndRecPtr, state->segcxt.ws_segsize);
}
- if (DecodeXLogRecord(state, record, errormsg))
- return record;
- else
- return NULL;
+ Assert (!*record || state->readLen >= 0);
+ if (DecodeXLogRecord(state, *record, errormsg))
+ return XLREAD_SUCCESS;
+
+ *record = NULL;
+ return XLREAD_FAIL;
err:
/*
- * Invalidate the read state. We might read from a different source after
+ * Invalidate the read page. We might read from a different source after
* failure.
*/
XLogReaderInvalReadState(state);
@@ -562,7 +697,8 @@ err:
if (state->errormsg_buf[0] != '\0')
*errormsg = state->errormsg_buf;
- return NULL;
+ *record = NULL;
+ return XLREAD_FAIL;
}
/*
@@ -715,11 +851,12 @@ XLogReaderInvalReadState(XLogReaderState *state)
*
* This is just a convenience subroutine to avoid duplicated code in
* XLogReadRecord. It's not intended for use from anywhere else.
+ *
+ * If PrevRecPtr is valid, the xl_prev is is cross-checked with it.
*/
static bool
ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
- XLogRecPtr PrevRecPtr, XLogRecord *record,
- bool randAccess)
+ XLogRecPtr PrevRecPtr, XLogRecord *record)
{
if (record->xl_tot_len < SizeOfXLogRecord)
{
@@ -737,7 +874,7 @@ ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
(uint32) RecPtr);
return false;
}
- if (randAccess)
+ if (PrevRecPtr == InvalidXLogRecPtr)
{
/*
* We can't exactly verify the prev-link, but surely it should be less
@@ -969,11 +1106,14 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
* XLogReadRecord() will read the next valid record.
*/
XLogRecPtr
-XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
+XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
+ XLogFindNextRecordCB read_page, void *private)
{
XLogRecPtr tmpRecPtr;
XLogRecPtr found = InvalidXLogRecPtr;
XLogPageHeader header;
+ XLogRecord *record;
+ XLogReadRecordResult result;
char *errormsg;
Assert(!XLogRecPtrIsInvalid(RecPtr));
@@ -1006,8 +1146,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
while(XLogNeedData(state, targetPagePtr, targetRecOff,
targetRecOff != 0))
{
- if (!state->read_page(state, state->readPagePtr, state->readLen,
- state->ReadRecPtr, state->readBuf))
+ if (!read_page(state, private))
break;
}
@@ -1059,8 +1198,16 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
* or we just jumped over the remaining data of a continuation.
*/
XLogBeginRead(state, tmpRecPtr);
- while (XLogReadRecord(state, &errormsg) != NULL)
+ while ((result = XLogReadRecord(state, &record, &errormsg)) !=
+ XLREAD_FAIL)
{
+ if (result == XLREAD_NEED_DATA)
+ {
+ if (!read_page(state, private))
+ goto err;
+ continue;
+ }
+
/* past the record we've found, break out */
if (RecPtr <= state->ReadRecPtr)
{
diff --git a/src/backend/access/transam/xlogutils.c b/src/backend/access/transam/xlogutils.c
index 47676bf800..b2c2abc826 100644
--- a/src/backend/access/transam/xlogutils.c
+++ b/src/backend/access/transam/xlogutils.c
@@ -685,8 +685,7 @@ XLogTruncateRelation(RelFileNode rnode, ForkNumber forkNum,
void
XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wantLength)
{
- const XLogRecPtr lastReadPage = state->seg.ws_segno *
- state->segcxt.ws_segsize + state->readLen;
+ const XLogRecPtr lastReadPage = state->readPagePtr;
Assert(wantPage != InvalidXLogRecPtr && wantPage % XLOG_BLCKSZ == 0);
Assert(wantLength <= XLOG_BLCKSZ);
@@ -701,7 +700,7 @@ XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wa
* current TLI has since become historical.
*/
if (lastReadPage == wantPage &&
- state->readLen != 0 &&
+ state->page_verified &&
lastReadPage + state->readLen >= wantPage + Min(wantLength, XLOG_BLCKSZ - 1))
return;
@@ -819,9 +818,11 @@ wal_segment_open(XLogSegNo nextSegNo, WALSegmentContext * segcxt,
* loop for now.
*/
bool
-read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *cur_page)
+read_local_xlog_page(XLogReaderState *state)
{
+ XLogRecPtr targetPagePtr = state->readPagePtr;
+ int reqLen = state->readLen;
+ char *cur_page = state->readBuf;
XLogRecPtr read_upto,
loc;
TimeLineID tli;
@@ -939,6 +940,7 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
WALReadRaiseError(&errinfo);
/* number of valid bytes in the buffer */
+ state->readPagePtr = targetPagePtr;
state->readLen = count;
return true;
}
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index cf93200618..d0a002fae7 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -120,7 +120,7 @@ StartupDecodingContext(List *output_plugin_options,
TransactionId xmin_horizon,
bool need_full_snapshot,
bool fast_forward,
- XLogPageReadCB read_page,
+ LogicalDecodingXLogReadPageCB read_page,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress)
@@ -169,11 +169,12 @@ StartupDecodingContext(List *output_plugin_options,
ctx->slot = slot;
- ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, read_page, ctx);
+ ctx->reader = XLogReaderAllocate(wal_segment_size, NULL);
if (!ctx->reader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
errmsg("out of memory")));
+ ctx->read_page = read_page;
ctx->reorder = ReorderBufferAllocate();
ctx->snapshot_builder =
@@ -228,7 +229,7 @@ CreateInitDecodingContext(char *plugin,
List *output_plugin_options,
bool need_full_snapshot,
XLogRecPtr restart_lsn,
- XLogPageReadCB read_page,
+ LogicalDecodingXLogReadPageCB read_page,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress)
@@ -370,7 +371,7 @@ LogicalDecodingContext *
CreateDecodingContext(XLogRecPtr start_lsn,
List *output_plugin_options,
bool fast_forward,
- XLogPageReadCB read_page,
+ LogicalDecodingXLogReadPageCB read_page,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress)
@@ -477,7 +478,13 @@ DecodingContextFindStartpoint(LogicalDecodingContext *ctx)
char *err = NULL;
/* the read_page callback waits for new WAL */
- record = XLogReadRecord(ctx->reader, &err);
+ while (XLogReadRecord(ctx->reader, &record, &err) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!ctx->read_page(ctx))
+ break;
+ }
+
if (err)
elog(ERROR, "%s", err);
if (!record)
diff --git a/src/backend/replication/logical/logicalfuncs.c b/src/backend/replication/logical/logicalfuncs.c
index acc8ef73be..8158842432 100644
--- a/src/backend/replication/logical/logicalfuncs.c
+++ b/src/backend/replication/logical/logicalfuncs.c
@@ -107,11 +107,9 @@ check_permissions(void)
}
bool
-logical_read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *cur_page)
+logical_read_local_xlog_page(LogicalDecodingContext *ctx)
{
- return read_local_xlog_page(state, targetPagePtr, reqLen,
- targetRecPtr, cur_page);
+ return read_local_xlog_page(ctx->reader);
}
/*
@@ -279,7 +277,13 @@ pg_logical_slot_get_changes_guts(FunctionCallInfo fcinfo, bool confirm, bool bin
XLogRecord *record;
char *errm = NULL;
- record = XLogReadRecord(ctx->reader, &errm);
+ while (XLogReadRecord(ctx->reader, &record, &errm) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!ctx->read_page(ctx))
+ break;
+ }
+
if (errm)
elog(ERROR, "%s", errm);
diff --git a/src/backend/replication/slotfuncs.c b/src/backend/replication/slotfuncs.c
index 7c89694611..ccf96134a7 100644
--- a/src/backend/replication/slotfuncs.c
+++ b/src/backend/replication/slotfuncs.c
@@ -428,7 +428,13 @@ pg_logical_replication_slot_advance(XLogRecPtr moveto)
* Read records. No changes are generated in fast_forward mode,
* but snapbuilder/slot statuses are updated properly.
*/
- record = XLogReadRecord(ctx->reader, &errm);
+ while (XLogReadRecord(ctx->reader, &record, &errm) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!ctx->read_page(ctx))
+ break;
+ }
+
if (errm)
elog(ERROR, "%s", errm);
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index 9f8e5a592d..b9b3f07cad 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -763,9 +763,12 @@ StartReplication(StartReplicationCmd *cmd)
* set every time WAL is flushed.
*/
static bool
-logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetRecPtr, char *cur_page)
+logical_read_xlog_page(LogicalDecodingContext *ctx)
{
+ XLogReaderState *state = ctx->reader;
+ XLogRecPtr targetPagePtr = state->readPagePtr;
+ int reqLen = state->readLen;
+ char *cur_page = state->readBuf;
XLogRecPtr flushptr;
int count;
WALReadError errinfo;
@@ -2794,7 +2797,12 @@ XLogSendLogical(void)
*/
WalSndCaughtUp = false;
- record = XLogReadRecord(logical_decoding_ctx->reader, &errm);
+ while (XLogReadRecord(logical_decoding_ctx->reader, &record, &errm) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!logical_decoding_ctx->read_page(logical_decoding_ctx))
+ break;
+ }
/* xlog record was invalid */
if (errm != NULL)
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index 78ee9f3faa..55337b65f1 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -39,14 +39,8 @@ static int xlogreadfd = -1;
static XLogSegNo xlogreadsegno = -1;
static char xlogfpath[MAXPGPATH];
-typedef struct XLogPageReadPrivate
-{
- int tliIndex;
-} XLogPageReadPrivate;
-
-static bool SimpleXLogPageRead(XLogReaderState *xlogreader,
- XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
+static bool SimpleXLogPageRead(XLogReaderState *xlogreader,
+ const char *datadir, int *tliIndex);
/*
* Read WAL from the datadir/pg_wal, starting from 'startpoint' on timeline
@@ -60,18 +54,20 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
XLogRecord *record;
XLogReaderState *xlogreader;
char *errormsg;
- XLogPageReadPrivate private;
- private.tliIndex = tliIndex;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir, &SimpleXLogPageRead,
- &private);
+ xlogreader = XLogReaderAllocate(WalSegSz, datadir);
if (xlogreader == NULL)
pg_fatal("out of memory");
XLogBeginRead(xlogreader, startpoint);
do
{
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!SimpleXLogPageRead(xlogreader, datadir, &tliIndex))
+ break;
+ }
if (record == NULL)
{
@@ -108,17 +104,19 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex)
XLogRecord *record;
XLogReaderState *xlogreader;
char *errormsg;
- XLogPageReadPrivate private;
XLogRecPtr endptr;
- private.tliIndex = tliIndex;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir, &SimpleXLogPageRead,
- &private);
+ xlogreader = XLogReaderAllocate(WalSegSz, datadir);
if (xlogreader == NULL)
pg_fatal("out of memory");
XLogBeginRead(xlogreader, ptr);
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!SimpleXLogPageRead(xlogreader, datadir, &tliIndex))
+ break;
+ }
if (record == NULL)
{
if (errormsg)
@@ -153,7 +151,6 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
XLogRecPtr searchptr;
XLogReaderState *xlogreader;
char *errormsg;
- XLogPageReadPrivate private;
/*
* The given fork pointer points to the end of the last common record,
@@ -169,9 +166,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
forkptr += SizeOfXLogShortPHD;
}
- private.tliIndex = tliIndex;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir, &SimpleXLogPageRead,
- &private);
+ xlogreader = XLogReaderAllocate(WalSegSz, datadir);
if (xlogreader == NULL)
pg_fatal("out of memory");
@@ -181,7 +176,12 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
uint8 info;
XLogBeginRead(xlogreader, searchptr);
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!SimpleXLogPageRead(xlogreader, datadir, &tliIndex))
+ break;
+ }
if (record == NULL)
{
@@ -228,10 +228,11 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
/* XLogReader callback function, to read a WAL page */
static bool
-SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *readBuf)
+SimpleXLogPageRead(XLogReaderState *xlogreader, const char *datadir,
+ int *tliIndex)
{
- XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
+ XLogRecPtr targetPagePtr = xlogreader->readPagePtr;
+ char *readBuf = xlogreader->readBuf;
uint32 targetPageOff;
XLogRecPtr targetSegEnd;
XLogSegNo targetSegNo;
@@ -264,14 +265,14 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
* be done both forward and backward, consider also switching timeline
* accordingly.
*/
- while (private->tliIndex < targetNentries - 1 &&
- targetHistory[private->tliIndex].end < targetSegEnd)
- private->tliIndex++;
- while (private->tliIndex > 0 &&
- targetHistory[private->tliIndex].begin >= targetSegEnd)
- private->tliIndex--;
+ while (*tliIndex < targetNentries - 1 &&
+ targetHistory[*tliIndex].end < targetSegEnd)
+ (*tliIndex)++;
+ while (*tliIndex > 0 &&
+ targetHistory[*tliIndex].begin >= targetSegEnd)
+ (*tliIndex)--;
- XLogFileName(xlogfname, targetHistory[private->tliIndex].tli,
+ XLogFileName(xlogfname, targetHistory[*tliIndex].tli,
xlogreadsegno, WalSegSz);
snprintf(xlogfpath, MAXPGPATH, "%s/" XLOGDIR "/%s",
@@ -316,7 +317,7 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
Assert(targetSegNo == xlogreadsegno);
- xlogreader->seg.ws_tli = targetHistory[private->tliIndex].tli;
+ xlogreader->seg.ws_tli = targetHistory[*tliIndex].tli;
xlogreader->readLen = XLOG_BLCKSZ;
return true;
}
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index fcfe80730c..e82f184de2 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -323,10 +323,12 @@ WALDumpOpenSegment(XLogSegNo nextSegNo, WALSegmentContext *segcxt,
* XLogReader read_page callback
*/
static bool
-WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetPtr, char *readBuff)
+WALDumpReadPage(XLogReaderState *state, void *priv)
{
- XLogDumpPrivate *private = state->private_data;
+ XLogRecPtr targetPagePtr = state->readPagePtr;
+ int reqLen = state->readLen;
+ char *readBuff = state->readBuf;
+ XLogDumpPrivate *private = (XLogDumpPrivate *) priv;
int count = XLOG_BLCKSZ;
WALReadError errinfo;
@@ -365,6 +367,7 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
(Size) errinfo.wre_req);
}
+ Assert(count >= state->readLen);
state->readLen = count;
return true;
}
@@ -1025,13 +1028,14 @@ main(int argc, char **argv)
/* done with argument parsing, do the actual work */
/* we have everything we need, start reading */
- xlogreader_state = XLogReaderAllocate(WalSegSz, waldir, WALDumpReadPage,
- &private);
+ xlogreader_state = XLogReaderAllocate(WalSegSz, waldir);
+
if (!xlogreader_state)
fatal_error("out of memory");
/* first find a valid recptr to start from */
- first_record = XLogFindNextRecord(xlogreader_state, private.startptr);
+ first_record = XLogFindNextRecord(xlogreader_state, private.startptr,
+ &WALDumpReadPage, (void*) &private);
if (first_record == InvalidXLogRecPtr)
fatal_error("could not find a valid record after %X/%X",
@@ -1055,7 +1059,13 @@ main(int argc, char **argv)
for (;;)
{
/* try to read the next record */
- record = XLogReadRecord(xlogreader_state, &errormsg);
+ while (XLogReadRecord(xlogreader_state, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!WALDumpReadPage(xlogreader_state, (void *) &private))
+ break;
+ }
+
if (!record)
{
if (!config.follow || private.endptr_reached)
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index 6ad953eea3..e59e42bee3 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -50,13 +50,6 @@ typedef struct WALSegmentContext
typedef struct XLogReaderState XLogReaderState;
-/* Function type definition for the read_page callback */
-typedef bool (*XLogPageReadCB) (XLogReaderState *xlogreader,
- XLogRecPtr targetPagePtr,
- int reqLen,
- XLogRecPtr targetRecPtr,
- char *readBuf);
-
typedef struct
{
/* Is this block ref in use? */
@@ -86,6 +79,29 @@ typedef struct
uint16 data_bufsz;
} DecodedBkpBlock;
+/* Return code from XLogReadRecord */
+typedef enum XLogReadRecordResult
+{
+ XLREAD_SUCCESS, /* record is successfully read */
+ XLREAD_NEED_DATA, /* need more data. see XLogReadRecord. */
+ XLREAD_FAIL /* failed during reading a record */
+} XLogReadRecordResult;
+
+/*
+ * internal state of XLogReadRecord
+ *
+ * XLogReadState runs a state machine while reading a record. Theses states
+ * are not seen outside the function. Each state may repeat several times
+ * exiting requesting caller for new data. See the comment of XLogReadRecrod
+ * for details.
+ */
+typedef enum XLogReadRecordState {
+ XLREAD_NEXT_RECORD,
+ XLREAD_TOT_LEN,
+ XLREAD_FIRST_FRAGMENT,
+ XLREAD_CONTINUATION
+} XLogReadRecordState;
+
struct XLogReaderState
{
/* ----------------------------------------
@@ -93,46 +109,20 @@ struct XLogReaderState
* ----------------------------------------
*/
- /*
- * Data input callback (mandatory).
- *
- * This callback shall read at least reqLen valid bytes of the xlog page
- * starting at targetPagePtr, and store them in readBuf. The callback
- * shall return the number of bytes read (never more than XLOG_BLCKSZ), or
- * -1 on failure. The callback shall sleep, if necessary, to wait for the
- * requested bytes to become available. The callback will not be invoked
- * again for the same page unless more than the returned number of bytes
- * are needed.
- *
- * targetRecPtr is the position of the WAL record we're reading. Usually
- * it is equal to targetPagePtr + reqLen, but sometimes xlogreader needs
- * to read and verify the page or segment header, before it reads the
- * actual WAL record it's interested in. In that case, targetRecPtr can
- * be used to determine which timeline to read the page from.
- *
- * The callback shall set ->seg.ws_tli to the TLI of the file the page was
- * read from.
- */
- XLogPageReadCB read_page;
-
/*
* System identifier of the xlog files we're about to read. Set to zero
* (the default value) if unknown or unimportant.
*/
uint64 system_identifier;
- /*
- * Opaque data for callbacks to use. Not used by XLogReader.
- */
- void *private_data;
-
/*
* Start and end point of last record read. EndRecPtr is also used as the
* position to read next. Calling XLogBeginRead() sets EndRecPtr to the
* starting position and ReadRecPtr to invalid.
*/
- XLogRecPtr ReadRecPtr; /* start of last record read */
+ XLogRecPtr ReadRecPtr; /* start of last record read or being read */
XLogRecPtr EndRecPtr; /* end+1 of last record read */
+ XLogRecPtr PrevRecPtr; /* start of previous record read */
/* ----------------------------------------
* Communication with page reader
@@ -146,7 +136,9 @@ struct XLogReaderState
* the request, or -1 on error */
TimeLineID readPageTLI; /* TLI for data currently in readBuf */
char *readBuf; /* buffer to store data */
- bool page_verified; /* is the page on the buffer verified? */
+ bool page_verified; /* is the page header on the buffer verified? */
+ bool record_verified;/* is the current record header verified? */
+
/* ----------------------------------------
@@ -186,8 +178,6 @@ struct XLogReaderState
XLogRecPtr latestPagePtr;
TimeLineID latestPageTLI;
- /* beginning of the WAL record being read. */
- XLogRecPtr currRecPtr;
/* timeline to read it from, 0 if a lookup is required */
TimeLineID currTLI;
@@ -214,15 +204,22 @@ struct XLogReaderState
char *readRecordBuf;
uint32 readRecordBufSize;
+ /*
+ * XLogReadRecord() state
+ */
+ XLogReadRecordState readRecordState;/* state machine state */
+ int recordGotLen; /* amount of current record that has
+ * already been read */
+ int recordRemainLen; /* length of current record that remains */
+ XLogRecPtr recordContRecPtr; /* where the current record continues */
+
/* Buffer to hold error message */
char *errormsg_buf;
};
/* Get a new XLogReader */
extern XLogReaderState *XLogReaderAllocate(int wal_segment_size,
- const char *waldir,
- XLogPageReadCB pagereadfunc,
- void *private_data);
+ const char *waldir);
/* Free an XLogReader */
extern void XLogReaderFree(XLogReaderState *state);
@@ -252,12 +249,17 @@ extern void WALOpenSegmentInit(WALOpenSegment *seg, WALSegmentContext *segcxt,
/* Position the XLogReader to given record */
extern void XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr);
#ifdef FRONTEND
-extern XLogRecPtr XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr);
+/* Function type definition for the read_page callback */
+typedef bool (*XLogFindNextRecordCB) (XLogReaderState *xlogreader,
+ void *private);
+extern XLogRecPtr XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
+ XLogFindNextRecordCB read_page, void *private);
#endif /* FRONTEND */
/* Read the next XLog record. Returns NULL on end-of-WAL or failure */
-extern struct XLogRecord *XLogReadRecord(XLogReaderState *state,
- char **errormsg);
+extern XLogReadRecordResult XLogReadRecord(XLogReaderState *state,
+ XLogRecord **record,
+ char **errormsg);
/* Validate a page */
extern bool XLogReaderValidatePageHeader(XLogReaderState *state,
diff --git a/src/include/access/xlogutils.h b/src/include/access/xlogutils.h
index dc7d894e5d..312acb4fa3 100644
--- a/src/include/access/xlogutils.h
+++ b/src/include/access/xlogutils.h
@@ -47,9 +47,7 @@ extern Buffer XLogReadBufferExtended(RelFileNode rnode, ForkNumber forknum,
extern Relation CreateFakeRelcacheEntry(RelFileNode rnode);
extern void FreeFakeRelcacheEntry(Relation fakerel);
-extern bool read_local_xlog_page(XLogReaderState *state,
- XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetRecPtr, char *cur_page);
+extern bool read_local_xlog_page(XLogReaderState *state);
extern void XLogReadDetermineTimeline(XLogReaderState *state,
XLogRecPtr wantPage, uint32 wantLength);
diff --git a/src/include/replication/logical.h b/src/include/replication/logical.h
index 3b7ca7f1da..f0ccfc6895 100644
--- a/src/include/replication/logical.h
+++ b/src/include/replication/logical.h
@@ -29,6 +29,10 @@ typedef void (*LogicalOutputPluginWriterUpdateProgress) (struct LogicalDecodingC
TransactionId xid
);
+typedef struct LogicalDecodingContext LogicalDecodingContext;
+
+typedef bool (*LogicalDecodingXLogReadPageCB)(LogicalDecodingContext *ctx);
+
typedef struct LogicalDecodingContext
{
/* memory context this is all allocated in */
@@ -39,6 +43,7 @@ typedef struct LogicalDecodingContext
/* infrastructure pieces for decoding */
XLogReaderState *reader;
+ LogicalDecodingXLogReadPageCB read_page;
struct ReorderBuffer *reorder;
struct SnapBuild *snapshot_builder;
@@ -95,14 +100,14 @@ extern LogicalDecodingContext *CreateInitDecodingContext(char *plugin,
List *output_plugin_options,
bool need_full_snapshot,
XLogRecPtr restart_lsn,
- XLogPageReadCB read_page,
+ LogicalDecodingXLogReadPageCB read_page,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress);
extern LogicalDecodingContext *CreateDecodingContext(XLogRecPtr start_lsn,
List *output_plugin_options,
bool fast_forward,
- XLogPageReadCB read_page,
+ LogicalDecodingXLogReadPageCB read_page,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress);
diff --git a/src/include/replication/logicalfuncs.h b/src/include/replication/logicalfuncs.h
index ab7941a155..6a477d0930 100644
--- a/src/include/replication/logicalfuncs.h
+++ b/src/include/replication/logicalfuncs.h
@@ -11,9 +11,6 @@
#include "replication/logical.h"
-extern bool logical_read_local_xlog_page(XLogReaderState *state,
- XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr,
- char *cur_page);
+extern bool logical_read_local_xlog_page(LogicalDecodingContext *ctx);
#endif
--
2.18.2
----Next_Part(Tue_Jan_28_21_20_20_2020_871)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
filename="v13-0003-Remove-globals-readOff-readLen-and-readSegNo.patch"
^ permalink raw reply [nested|flat] 70+ messages in thread
* [PATCH v17 2/4] Move page-reader out of XLogReadRecord
@ 2019-09-10 03:58 Kyotaro Horiguchi <[email protected]>
0 siblings, 0 replies; 70+ messages in thread
From: Kyotaro Horiguchi @ 2019-09-10 03:58 UTC (permalink / raw)
This is the second step of removing callbacks from WAL record reader.
Since it is essential to take in additional data while reading a
record, the function have to ask caller for new data while keeping
working state. Thus the function is turned into a state machine.
---
src/backend/access/transam/twophase.c | 15 +-
src/backend/access/transam/xlog.c | 58 +-
src/backend/access/transam/xlogreader.c | 697 +++++++++++-------
src/backend/access/transam/xlogutils.c | 17 +-
src/backend/replication/logical/logical.c | 26 +-
.../replication/logical/logicalfuncs.c | 13 +-
src/backend/replication/slotfuncs.c | 18 +-
src/backend/replication/walsender.c | 32 +-
src/bin/pg_rewind/parsexlog.c | 84 ++-
src/bin/pg_waldump/pg_waldump.c | 36 +-
src/include/access/xlogreader.h | 121 ++-
src/include/access/xlogutils.h | 4 +-
src/include/pg_config_manual.h | 2 +-
src/include/replication/logical.h | 11 +-
14 files changed, 646 insertions(+), 488 deletions(-)
diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index 80d2d20d6c..7d5a96a5f5 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -1325,11 +1325,8 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
XLogReaderState *xlogreader;
char *errormsg;
- xlogreader = XLogReaderAllocate(wal_segment_size, NULL,
- XL_ROUTINE(.page_read = &read_local_xlog_page,
- .segment_open = &wal_segment_open,
- .segment_close = &wal_segment_close),
- NULL);
+ xlogreader = XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+
if (!xlogreader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
@@ -1337,7 +1334,13 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
errdetail("Failed while allocating a WAL reading processor.")));
XLogBeginRead(xlogreader, lsn);
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!read_local_xlog_page(xlogreader))
+ break;
+ }
+
if (record == NULL)
ereport(ERROR,
(errcode_for_file_access(),
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 606328a65a..670996ae67 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -836,13 +836,6 @@ static XLogSource currentSource = XLOG_FROM_ANY;
static bool lastSourceFailed = false;
static bool pendingWalRcvRestart = false;
-typedef struct XLogPageReadPrivate
-{
- int emode;
- bool fetching_ckpt; /* are we fetching a checkpoint record? */
- bool randAccess;
-} XLogPageReadPrivate;
-
/*
* These variables track when we last obtained some WAL data to process,
* and where we got it from. (XLogReceiptSource is initially the same as
@@ -917,8 +910,8 @@ static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath,
static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli,
XLogSource source, bool notfoundOk);
static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, XLogSource source);
-static bool XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
+static bool XLogPageRead(XLogReaderState *xlogreader,
+ bool fetching_ckpt, int emode, bool randAccess);
static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
bool fetching_ckpt, XLogRecPtr tliRecPtr);
static int emode_for_corrupt_record(int emode, XLogRecPtr RecPtr);
@@ -1231,8 +1224,7 @@ XLogInsertRecord(XLogRecData *rdata,
appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len);
if (!debug_reader)
- debug_reader = XLogReaderAllocate(wal_segment_size, NULL,
- XL_ROUTINE(), NULL);
+ debug_reader = XLogReaderAllocate(wal_segment_size, NULL, NULL);
if (!debug_reader)
{
@@ -4344,15 +4336,10 @@ CleanupBackupHistory(void)
* record is available.
*/
static XLogRecord *
-ReadRecord(XLogReaderState *xlogreader, int emode,
- bool fetching_ckpt)
+ReadRecord(XLogReaderState *xlogreader, int emode, bool fetching_ckpt)
{
XLogRecord *record;
- XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
-
- private->fetching_ckpt = fetching_ckpt;
- private->emode = emode;
- private->randAccess = (xlogreader->ReadRecPtr == InvalidXLogRecPtr);
+ bool randAccess = (xlogreader->ReadRecPtr == InvalidXLogRecPtr);
/* This is the first attempt to read this page. */
lastSourceFailed = false;
@@ -4360,8 +4347,16 @@ ReadRecord(XLogReaderState *xlogreader, int emode,
for (;;)
{
char *errormsg;
+ XLogReadRecordResult result;
+
+ while ((result = XLogReadRecord(xlogreader, &record, &errormsg))
+ == XLREAD_NEED_DATA)
+ {
+ if (!XLogPageRead(xlogreader, fetching_ckpt, emode, randAccess))
+ break;
+
+ }
- record = XLogReadRecord(xlogreader, &errormsg);
ReadRecPtr = xlogreader->ReadRecPtr;
EndRecPtr = xlogreader->EndRecPtr;
if (record == NULL)
@@ -6380,7 +6375,6 @@ StartupXLOG(void)
bool backupFromStandby = false;
DBState dbstate_at_startup;
XLogReaderState *xlogreader;
- XLogPageReadPrivate private;
bool promoted = false;
struct stat st;
@@ -6539,13 +6533,9 @@ StartupXLOG(void)
OwnLatch(&XLogCtl->recoveryWakeupLatch);
/* Set up XLOG reader facility */
- MemSet(&private, 0, sizeof(XLogPageReadPrivate));
xlogreader =
- XLogReaderAllocate(wal_segment_size, NULL,
- XL_ROUTINE(.page_read = &XLogPageRead,
- .segment_open = NULL,
- .segment_close = wal_segment_close),
- &private);
+ XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+
if (!xlogreader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
@@ -11987,12 +11977,13 @@ CancelBackup(void)
* sleep and retry.
*/
static bool
-XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetRecPtr, char *readBuf)
+XLogPageRead(XLogReaderState *xlogreader,
+ bool fetching_ckpt, int emode, bool randAccess)
{
- XLogPageReadPrivate *private =
- (XLogPageReadPrivate *) xlogreader->private_data;
- int emode = private->emode;
+ char *readBuf = xlogreader->readBuf;
+ XLogRecPtr targetPagePtr = xlogreader->readPagePtr;
+ int reqLen = xlogreader->readLen;
+ XLogRecPtr targetRecPtr = xlogreader->ReadRecPtr;
uint32 targetPageOff;
XLogSegNo targetSegNo PG_USED_FOR_ASSERTS_ONLY;
int r;
@@ -12035,8 +12026,8 @@ retry:
flushedUpto < targetPagePtr + reqLen))
{
if (!WaitForWALToBecomeAvailable(targetPagePtr + reqLen,
- private->randAccess,
- private->fetching_ckpt,
+ randAccess,
+ fetching_ckpt,
targetRecPtr))
{
if (readFile >= 0)
@@ -12141,6 +12132,7 @@ retry:
goto next_record_is_invalid;
}
+ Assert(xlogreader->readPagePtr == targetPagePtr);
xlogreader->readLen = readLen;
return true;
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index 7d04995b7b..ba3e4868f7 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -40,7 +40,7 @@ static bool XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr,
int reqLen, bool header_inclusive);
static void XLogReaderInvalReadState(XLogReaderState *state);
static bool ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
- XLogRecPtr PrevRecPtr, XLogRecord *record, bool randAccess);
+ XLogRecPtr PrevRecPtr, XLogRecord *record);
static bool ValidXLogRecord(XLogReaderState *state, XLogRecord *record,
XLogRecPtr recptr);
static void ResetDecoder(XLogReaderState *state);
@@ -73,7 +73,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...)
*/
XLogReaderState *
XLogReaderAllocate(int wal_segment_size, const char *waldir,
- XLogReaderRoutine *routine, void *private_data)
+ WALSegmentCleanupCB cleanup_cb)
{
XLogReaderState *state;
@@ -84,7 +84,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
return NULL;
/* initialize caller-provided support functions */
- state->routine = *routine;
+ state->cleanup_cb = cleanup_cb;
state->max_block_id = -1;
@@ -107,8 +107,6 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
WALOpenSegmentInit(&state->seg, &state->segcxt, wal_segment_size,
waldir);
- /* system_identifier initialized to zeroes above */
- state->private_data = private_data;
/* ReadRecPtr, EndRecPtr and readLen initialized to zeroes above */
state->errormsg_buf = palloc_extended(MAX_ERRORMSG_LEN + 1,
MCXT_ALLOC_NO_OOM);
@@ -140,8 +138,8 @@ XLogReaderFree(XLogReaderState *state)
{
int block_id;
- if (state->seg.ws_file != -1)
- state->routine.segment_close(state);
+ if (state->seg.ws_file >= 0)
+ state->cleanup_cb(state);
for (block_id = 0; block_id <= XLR_MAX_BLOCK_ID; block_id++)
{
@@ -246,6 +244,7 @@ XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
/* Begin at the passed-in record pointer. */
state->EndRecPtr = RecPtr;
state->ReadRecPtr = InvalidXLogRecPtr;
+ state->readRecordState = XLREAD_NEXT_RECORD;
}
/*
@@ -254,317 +253,449 @@ XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
* XLogBeginRead() or XLogFindNextRecord() must be called before the first call
* to XLogReadRecord().
*
- * If the page_read callback fails to read the requested data, NULL is
- * returned. The callback is expected to have reported the error; errormsg
- * is set to NULL.
+ * This function may return XLREAD_NEED_DATA several times before returning a
+ * result record. The caller shall read in some new data then call this
+ * function again with the same parameters.
*
- * If the reading fails for some other reason, NULL is also returned, and
- * *errormsg is set to a string with details of the failure.
+ * When a record is successfully read, returns XLREAD_SUCCESS with result
+ * record being stored in *record. Otherwise *record is NULL.
*
- * The returned pointer (or *errormsg) points to an internal buffer that's
- * valid until the next call to XLogReadRecord.
+ * Returns XLREAD_NEED_DATA if more data is needed to finish reading the
+ * current record. In that case, state->readPagePtr and state->readLen inform
+ * the desired position and minimum length of data needed. The caller shall
+ * read in the requested data and set state->readBuf to point to a buffer
+ * containing it. The caller must also set state->readPageTLI and
+ * state->readLen to indicate the timeline that it was read from, and the
+ * length of data that is now available (which must be >= given readLen),
+ * respectively.
+ *
+ * If invalid data is encountered, returns XLREAD_FAIL with *record being set to
+ * NULL. *errormsg is set to a string with details of the failure.
+ * The returned pointer (or *errormsg) points to an internal buffer that's valid
+ * until the next call to XLogReadRecord.
+ *
+ *
+ * This function runs a state machine consists of the following states.
+ *
+ * XLREAD_NEXT_RECORD :
+ * The initial state, if called with valid RecPtr, try to read a record at
+ * that position. If invalid RecPtr is given try to read a record just after
+ * the last one previously read.
+ * This state ens after setting ReadRecPtr. Then goes to XLREAD_TOT_LEN.
+ *
+ * XLREAD_TOT_LEN:
+ * Examining record header. Ends after reading record total
+ * length. recordRemainLen and recordGotLen are initialized.
+ *
+ * XLREAD_FIRST_FRAGMENT:
+ * Reading the first fragment. Ends with finishing reading a single
+ * record. Goes to XLREAD_NEXT_RECORD if that's all or
+ * XLREAD_CONTINUATION if we have continuation.
+
+ * XLREAD_CONTINUATION:
+ * Reading continuation of record. Ends with finishing the whole record then
+ * goes to XLREAD_NEXT_RECORD. During this state, recordRemainLen indicates
+ * how much is left and readRecordBuf holds the partially assert
+ * record.recordContRecPtr points to the beginning of the next page where to
+ * continue.
+ *
+ * If wrong data found in any state, the state machine stays at the current
+ * state. This behavior allows to continue reading a reacord switching among
+ * different souces, while streaming replication.
*/
-XLogRecord *
-XLogReadRecord(XLogReaderState *state, char **errormsg)
+XLogReadRecordResult
+XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
{
- XLogRecPtr RecPtr;
- XLogRecord *record;
- XLogRecPtr targetPagePtr;
- bool randAccess;
- uint32 len,
- total_len;
- uint32 targetRecOff;
- uint32 pageHeaderSize;
- bool gotheader;
+ XLogRecord *prec;
- /*
- * randAccess indicates whether to verify the previous-record pointer of
- * the record we're reading. We only do this if we're reading
- * sequentially, which is what we initially assume.
- */
- randAccess = false;
+ *record = NULL;
/* reset error state */
*errormsg = NULL;
state->errormsg_buf[0] = '\0';
- ResetDecoder(state);
-
- RecPtr = state->EndRecPtr;
-
- if (state->ReadRecPtr != InvalidXLogRecPtr)
- {
- /* read the record after the one we just read */
-
- /*
- * EndRecPtr is pointing to end+1 of the previous WAL record. If
- * we're at a page boundary, no more records can fit on the current
- * page. We must skip over the page header, but we can't do that until
- * we've read in the page, since the header size is variable.
- */
- }
- else
- {
- /*
- * Caller supplied a position to start at.
- *
- * In this case, EndRecPtr should already be pointing to a valid
- * record starting position.
- */
- Assert(XRecOffIsValid(RecPtr));
- randAccess = true;
- }
-
- state->currRecPtr = RecPtr;
-
- targetPagePtr = RecPtr - (RecPtr % XLOG_BLCKSZ);
- targetRecOff = RecPtr % XLOG_BLCKSZ;
-
- /*
- * Read the page containing the record into state->readBuf. Request enough
- * byte to cover the whole record header, or at least the part of it that
- * fits on the same page.
- */
- while (XLogNeedData(state, targetPagePtr,
- Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
- targetRecOff != 0))
- {
- if (!state->routine.page_read(state, state->readPagePtr, state->readLen,
- RecPtr, state->readBuf))
- break;
- }
-
- if (!state->page_verified)
- goto err;
-
- /*
- * We have at least the page header, so we can examine it now.
- */
- pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
- if (targetRecOff == 0)
- {
- /*
- * At page start, so skip over page header.
- */
- RecPtr += pageHeaderSize;
- targetRecOff = pageHeaderSize;
- }
- else if (targetRecOff < pageHeaderSize)
- {
- report_invalid_record(state, "invalid record offset at %X/%X",
- LSN_FORMAT_ARGS(RecPtr));
- goto err;
- }
-
- if ((((XLogPageHeader) state->readBuf)->xlp_info & XLP_FIRST_IS_CONTRECORD) &&
- targetRecOff == pageHeaderSize)
- {
- report_invalid_record(state, "contrecord is requested by %X/%X",
- LSN_FORMAT_ARGS(RecPtr));
- goto err;
- }
-
- /* XLogNeedData has verified the page header */
- Assert(pageHeaderSize <= state->readLen);
-
- /*
- * Read the record length.
- *
- * NB: Even though we use an XLogRecord pointer here, the whole record
- * header might not fit on this page. xl_tot_len is the first field of the
- * struct, so it must be on this page (the records are MAXALIGNed), but we
- * cannot access any other fields until we've verified that we got the
- * whole header.
- */
- record = (XLogRecord *) (state->readBuf + RecPtr % XLOG_BLCKSZ);
- total_len = record->xl_tot_len;
-
- /*
- * If the whole record header is on this page, validate it immediately.
- * Otherwise do just a basic sanity check on xl_tot_len, and validate the
- * rest of the header after reading it from the next page. The xl_tot_len
- * check is necessary here to ensure that we enter the "Need to reassemble
- * record" code path below; otherwise we might fail to apply
- * ValidXLogRecordHeader at all.
- */
- if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
- {
- if (!ValidXLogRecordHeader(state, RecPtr, state->ReadRecPtr, record,
- randAccess))
- goto err;
- gotheader = true;
- }
- else
- {
- /* XXX: more validation should be done here */
- if (total_len < SizeOfXLogRecord)
- {
- report_invalid_record(state,
- "invalid record length at %X/%X: wanted %u, got %u",
- LSN_FORMAT_ARGS(RecPtr),
- (uint32) SizeOfXLogRecord, total_len);
- goto err;
- }
- gotheader = false;
- }
-
- len = XLOG_BLCKSZ - RecPtr % XLOG_BLCKSZ;
- if (total_len > len)
+ switch (state->readRecordState)
{
- /* Need to reassemble record */
- char *contdata;
- XLogPageHeader pageHeader;
- char *buffer;
- uint32 gotlen;
-
- /*
- * Enlarge readRecordBuf as needed.
- */
- if (total_len > state->readRecordBufSize &&
- !allocate_recordbuf(state, total_len))
- {
- /* We treat this as a "bogus data" condition */
- report_invalid_record(state, "record length %u at %X/%X too long",
- total_len, LSN_FORMAT_ARGS(RecPtr));
- goto err;
- }
+ case XLREAD_NEXT_RECORD:
+ ResetDecoder(state);
- /* Copy the first fragment of the record from the first page. */
- memcpy(state->readRecordBuf,
- state->readBuf + RecPtr % XLOG_BLCKSZ, len);
- buffer = state->readRecordBuf + len;
- gotlen = len;
-
- do
- {
- int rest_len = total_len - gotlen;
-
- /* Calculate pointer to beginning of next page */
- targetPagePtr += XLOG_BLCKSZ;
-
- /* Wait for the next page to become available */
- while (XLogNeedData(state, targetPagePtr,
- Min(rest_len, XLOG_BLCKSZ),
- false))
+ if (state->ReadRecPtr != InvalidXLogRecPtr)
{
- if (!state->routine.page_read(state, state->readPagePtr,
- state->readLen,
- state->ReadRecPtr,
- state->readBuf))
- break;
+ /* read the record after the one we just read */
+
+ /*
+ * EndRecPtr is pointing to end+1 of the previous WAL record.
+ * If we're at a page boundary, no more records can fit on the
+ * current page. We must skip over the page header, but we
+ * can't do that until we've read in the page, since the header
+ * size is variable.
+ */
+ state->PrevRecPtr = state->ReadRecPtr;
+ state->ReadRecPtr = state->EndRecPtr;
+ }
+ else
+ {
+ /*
+ * Caller supplied a position to start at.
+ *
+ * In this case, EndRecPtr should already be pointing to a
+ * valid record starting position.
+ */
+ Assert(XRecOffIsValid(state->EndRecPtr));
+ state->ReadRecPtr = state->EndRecPtr;
+
+ /*
+ * We cannot verify the previous-record pointer when we're
+ * seeking to a particular record. Reset PrevRecPtr so that we
+ * won't try doing that.
+ */
+ state->PrevRecPtr = InvalidXLogRecPtr;
+ state->EndRecPtr = InvalidXLogRecPtr; /* to be tidy */
}
+ state->record_verified = false;
+ state->readRecordState = XLREAD_TOT_LEN;
+ /* fall through */
+
+ case XLREAD_TOT_LEN:
+ {
+ uint32 total_len;
+ uint32 pageHeaderSize;
+ XLogRecPtr targetPagePtr;
+ uint32 targetRecOff;
+ XLogPageHeader pageHeader;
+
+ targetPagePtr =
+ state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
+ targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+
+ /*
+ * Check if we have enough data. For the first record in the page,
+ * the requesting length doesn't contain page header.
+ */
+ if (XLogNeedData(state, targetPagePtr,
+ Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+ targetRecOff != 0))
+ return XLREAD_NEED_DATA;
+
+ /* error out if caller supplied bogus page */
if (!state->page_verified)
goto err;
- Assert(SizeOfXLogShortPHD <= state->readLen);
+ /* examine page header now. */
+ pageHeaderSize =
+ XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+ if (targetRecOff == 0)
+ {
+ /* At page start, so skip over page header. */
+ state->ReadRecPtr += pageHeaderSize;
+ targetRecOff = pageHeaderSize;
+ }
+ else if (targetRecOff < pageHeaderSize)
+ {
+ report_invalid_record(state, "invalid record offset at %X/%X",
+ LSN_FORMAT_ARGS(state->ReadRecPtr));
+ goto err;
+ }
- /* Check that the continuation on next page looks valid */
pageHeader = (XLogPageHeader) state->readBuf;
- if (!(pageHeader->xlp_info & XLP_FIRST_IS_CONTRECORD))
+ if ((pageHeader->xlp_info & XLP_FIRST_IS_CONTRECORD) &&
+ targetRecOff == pageHeaderSize)
{
- report_invalid_record(state,
- "there is no contrecord flag at %X/%X",
- LSN_FORMAT_ARGS(RecPtr));
+ report_invalid_record(state, "contrecord is requested by %X/%X",
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr);
goto err;
}
- /*
- * Cross-check that xlp_rem_len agrees with how much of the record
- * we expect there to be left.
- */
- if (pageHeader->xlp_rem_len == 0 ||
- total_len != (pageHeader->xlp_rem_len + gotlen))
- {
- report_invalid_record(state,
- "invalid contrecord length %u (expected %lld) at %X/%X",
- pageHeader->xlp_rem_len,
- ((long long) total_len) - gotlen,
- LSN_FORMAT_ARGS(RecPtr));
- goto err;
- }
-
- /* Append the continuation from this page to the buffer */
- pageHeaderSize = XLogPageHeaderSize(pageHeader);
-
+ /* XLogNeedData has verified the page header */
Assert(pageHeaderSize <= state->readLen);
- contdata = (char *) state->readBuf + pageHeaderSize;
- len = XLOG_BLCKSZ - pageHeaderSize;
- if (pageHeader->xlp_rem_len < len)
- len = pageHeader->xlp_rem_len;
+ /*
+ * Read the record length.
+ *
+ * NB: Even though we use an XLogRecord pointer here, the whole
+ * record header might not fit on this page. xl_tot_len is the first
+ * field of the struct, so it must be on this page (the records are
+ * MAXALIGNed), but we cannot access any other fields until we've
+ * verified that we got the whole header.
+ */
+ prec = (XLogRecord *) (state->readBuf +
+ state->ReadRecPtr % XLOG_BLCKSZ);
+ total_len = prec->xl_tot_len;
- Assert (pageHeaderSize + len <= state->readLen);
- memcpy(buffer, (char *) contdata, len);
- buffer += len;
- gotlen += len;
+ /*
+ * If the whole record header is on this page, validate it
+ * immediately. Otherwise do just a basic sanity check on
+ * xl_tot_len, and validate the rest of the header after reading it
+ * from the next page. The xl_tot_len check is necessary here to
+ * ensure that we enter the XLREAD_CONTINUATION state below;
+ * otherwise we might fail to apply ValidXLogRecordHeader at all.
+ */
+ if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
+ {
+ if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
+ state->PrevRecPtr, prec))
+ goto err;
- /* If we just reassembled the record header, validate it. */
- if (!gotheader)
+ state->record_verified = true;
+ }
+ else
{
- record = (XLogRecord *) state->readRecordBuf;
- if (!ValidXLogRecordHeader(state, RecPtr, state->ReadRecPtr,
- record, randAccess))
+ /* XXX: more validation should be done here */
+ if (total_len < SizeOfXLogRecord)
+ {
+ report_invalid_record(state,
+ "invalid record length at %X/%X: wanted %u, got %u",
+ LSN_FORMAT_ARGS(state->ReadRecPtr),
+ (uint32) SizeOfXLogRecord, total_len);
goto err;
- gotheader = true;
+ }
}
- } while (gotlen < total_len);
-
- Assert(gotheader);
- record = (XLogRecord *) state->readRecordBuf;
- if (!ValidXLogRecord(state, record, RecPtr))
- goto err;
+ /*
+ * Wait for the rest of the record, or the part of the record that
+ * fit on the first page if crossed a page boundary, to become
+ * available.
+ */
+ state->recordGotLen = 0;
+ state->recordRemainLen = total_len;
+ state->readRecordState = XLREAD_FIRST_FRAGMENT;
+ }
+ /* fall through */
- pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
- state->ReadRecPtr = RecPtr;
- state->EndRecPtr = targetPagePtr + pageHeaderSize
- + MAXALIGN(pageHeader->xlp_rem_len);
- }
- else
- {
- /* Wait for the record data to become available */
- while (XLogNeedData(state, targetPagePtr,
- Min(targetRecOff + total_len, XLOG_BLCKSZ), true))
+ case XLREAD_FIRST_FRAGMENT:
{
- if (!state->routine.page_read(state, state->readPagePtr,
- state->readLen,
- state->ReadRecPtr, state->readBuf))
+ uint32 total_len = state->recordRemainLen;
+ uint32 request_len;
+ uint32 record_len;
+ XLogRecPtr targetPagePtr;
+ uint32 targetRecOff;
+
+ /*
+ * Wait for the rest of the record on the first page to become
+ * available
+ */
+ targetPagePtr =
+ state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
+ targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+
+ request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ);
+ record_len = request_len - targetRecOff;
+
+ /* ReadRecPtr contains page header */
+ Assert (targetRecOff != 0);
+ if (XLogNeedData(state, targetPagePtr, request_len, true))
+ return XLREAD_NEED_DATA;
+
+ /* error out if caller supplied bogus page */
+ if (!state->page_verified)
+ goto err;
+
+ prec = (XLogRecord *) (state->readBuf + targetRecOff);
+
+ /* validate record header if not yet */
+ if (!state->record_verified && record_len >= SizeOfXLogRecord)
+ {
+ if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
+ state->PrevRecPtr, prec))
+ goto err;
+
+ state->record_verified = true;
+ }
+
+
+ if (total_len == record_len)
+ {
+ /* Record does not cross a page boundary */
+ Assert(state->record_verified);
+
+ if (!ValidXLogRecord(state, prec, state->ReadRecPtr))
+ goto err;
+
+ state->record_verified = true; /* to be tidy */
+
+ /* We already checked the header earlier */
+ state->EndRecPtr = state->ReadRecPtr + MAXALIGN(record_len);
+
+ *record = prec;
+ state->readRecordState = XLREAD_NEXT_RECORD;
break;
+ }
+
+ /*
+ * The record continues on the next page. Need to reassemble
+ * record
+ */
+ Assert(total_len > record_len);
+
+ /* Enlarge readRecordBuf as needed. */
+ if (total_len > state->readRecordBufSize &&
+ !allocate_recordbuf(state, total_len))
+ {
+ /* We treat this as a "bogus data" condition */
+ report_invalid_record(state,
+ "record length %u at %X/%X too long",
+ total_len,
+ LSN_FORMAT_ARGS(state->ReadRecPtr));
+ goto err;
+ }
+
+ /* Copy the first fragment of the record from the first page. */
+ memcpy(state->readRecordBuf, state->readBuf + targetRecOff,
+ record_len);
+ state->recordGotLen += record_len;
+ state->recordRemainLen -= record_len;
+
+ /* Calculate pointer to beginning of next page */
+ state->recordContRecPtr = state->ReadRecPtr + record_len;
+ Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0);
+
+ state->readRecordState = XLREAD_CONTINUATION;
}
+ /* fall through */
+
+ case XLREAD_CONTINUATION:
+ {
+ XLogPageHeader pageHeader;
+ uint32 pageHeaderSize;
+ XLogRecPtr targetPagePtr;
+
+ /* we enter this state only if we haven't read the whole record. */
+ Assert (state->recordRemainLen > 0);
+
+ while(state->recordRemainLen > 0)
+ {
+ char *contdata;
+ uint32 request_len;
+ uint32 record_len;
+
+ /* Wait for the next page to become available */
+ targetPagePtr = state->recordContRecPtr;
+
+ /* this request contains page header */
+ Assert (targetPagePtr != 0);
+ if (XLogNeedData(state, targetPagePtr,
+ Min(state->recordRemainLen, XLOG_BLCKSZ),
+ false))
+ return XLREAD_NEED_DATA;
+
+ if (!state->page_verified)
+ goto err;
+
+ Assert(SizeOfXLogShortPHD <= state->readLen);
- if (!state->page_verified)
- goto err;
+ /* Check that the continuation on next page looks valid */
+ pageHeader = (XLogPageHeader) state->readBuf;
+ if (!(pageHeader->xlp_info & XLP_FIRST_IS_CONTRECORD))
+ {
+ report_invalid_record(
+ state,
+ "there is no contrecord flag at %X/%X reading %X/%X",
+ (uint32) (state->recordContRecPtr >> 32),
+ (uint32) state->recordContRecPtr,
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr);
+ goto err;
+ }
- /* Record does not cross a page boundary */
- if (!ValidXLogRecord(state, record, RecPtr))
- goto err;
+ /*
+ * Cross-check that xlp_rem_len agrees with how much of the
+ * record we expect there to be left.
+ */
+ if (pageHeader->xlp_rem_len == 0 ||
+ pageHeader->xlp_rem_len != state->recordRemainLen)
+ {
+ report_invalid_record(
+ state,
+ "invalid contrecord length %u at %X/%X reading %X/%X, expected %u",
+ pageHeader->xlp_rem_len,
+ (uint32) (state->recordContRecPtr >> 32),
+ (uint32) state->recordContRecPtr,
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr,
+ state->recordRemainLen);
+ goto err;
+ }
- state->EndRecPtr = RecPtr + MAXALIGN(total_len);
+ /* Append the continuation from this page to the buffer */
+ pageHeaderSize = XLogPageHeaderSize(pageHeader);
- state->ReadRecPtr = RecPtr;
+ /*
+ * XLogNeedData should have ensured that the whole page header
+ * was read
+ */
+ Assert(state->readLen >= pageHeaderSize);
+
+ contdata = (char *) state->readBuf + pageHeaderSize;
+ record_len = XLOG_BLCKSZ - pageHeaderSize;
+ if (pageHeader->xlp_rem_len < record_len)
+ record_len = pageHeader->xlp_rem_len;
+
+ request_len = record_len + pageHeaderSize;
+
+ /* XLogNeedData should have ensured all needed data was read */
+ Assert (state->readLen >= request_len);
+
+ memcpy(state->readRecordBuf + state->recordGotLen,
+ (char *) contdata, record_len);
+ state->recordGotLen += record_len;
+ state->recordRemainLen -= record_len;
+
+ /* If we just reassembled the record header, validate it. */
+ if (!state->record_verified)
+ {
+ Assert(state->recordGotLen >= SizeOfXLogRecord);
+ if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
+ state->PrevRecPtr,
+ (XLogRecord *) state->readRecordBuf))
+ goto err;
+
+ state->record_verified = true;
+ }
+
+ /* Calculate pointer to beginning of next page, and continue */
+ state->recordContRecPtr += XLOG_BLCKSZ;
+ }
+
+ /* targetPagePtr is pointing the last-read page here */
+ prec = (XLogRecord *) state->readRecordBuf;
+ if (!ValidXLogRecord(state, prec, state->ReadRecPtr))
+ goto err;
+
+ pageHeaderSize =
+ XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+ state->EndRecPtr = targetPagePtr + pageHeaderSize
+ + MAXALIGN(pageHeader->xlp_rem_len);
+
+ *record = prec;
+ state->readRecordState = XLREAD_NEXT_RECORD;
+ break;
+ }
}
/*
* Special processing if it's an XLOG SWITCH record
*/
- if (record->xl_rmid == RM_XLOG_ID &&
- (record->xl_info & ~XLR_INFO_MASK) == XLOG_SWITCH)
+ if ((*record)->xl_rmid == RM_XLOG_ID &&
+ ((*record)->xl_info & ~XLR_INFO_MASK) == XLOG_SWITCH)
{
/* Pretend it extends to end of segment */
state->EndRecPtr += state->segcxt.ws_segsize - 1;
state->EndRecPtr -= XLogSegmentOffset(state->EndRecPtr, state->segcxt.ws_segsize);
}
- if (DecodeXLogRecord(state, record, errormsg))
- return record;
- else
- return NULL;
+ Assert (!*record || state->readLen >= 0);
+ if (DecodeXLogRecord(state, *record, errormsg))
+ return XLREAD_SUCCESS;
+
+ *record = NULL;
+ return XLREAD_FAIL;
err:
/*
- * Invalidate the read state. We might read from a different source after
+ * Invalidate the read page. We might read from a different source after
* failure.
*/
XLogReaderInvalReadState(state);
@@ -572,7 +703,8 @@ err:
if (state->errormsg_buf[0] != '\0')
*errormsg = state->errormsg_buf;
- return NULL;
+ *record = NULL;
+ return XLREAD_FAIL;
}
/*
@@ -724,11 +856,12 @@ XLogReaderInvalReadState(XLogReaderState *state)
*
* This is just a convenience subroutine to avoid duplicated code in
* XLogReadRecord. It's not intended for use from anywhere else.
+ *
+ * If PrevRecPtr is valid, the xl_prev is is cross-checked with it.
*/
static bool
ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
- XLogRecPtr PrevRecPtr, XLogRecord *record,
- bool randAccess)
+ XLogRecPtr PrevRecPtr, XLogRecord *record)
{
if (record->xl_tot_len < SizeOfXLogRecord)
{
@@ -745,7 +878,7 @@ ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
record->xl_rmid, LSN_FORMAT_ARGS(RecPtr));
return false;
}
- if (randAccess)
+ if (PrevRecPtr == InvalidXLogRecPtr)
{
/*
* We can't exactly verify the prev-link, but surely it should be less
@@ -975,11 +1108,14 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
* XLogReadRecord() will read the next valid record.
*/
XLogRecPtr
-XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
+XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
+ XLogFindNextRecordCB read_page, void *private)
{
XLogRecPtr tmpRecPtr;
XLogRecPtr found = InvalidXLogRecPtr;
XLogPageHeader header;
+ XLogRecord *record;
+ XLogReadRecordResult result;
char *errormsg;
Assert(!XLogRecPtrIsInvalid(RecPtr));
@@ -1012,9 +1148,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
while(XLogNeedData(state, targetPagePtr, targetRecOff,
targetRecOff != 0))
{
- if (!state->routine.page_read(state, state->readPagePtr,
- state->readLen,
- state->ReadRecPtr, state->readBuf))
+ if (!read_page(state, private))
break;
}
@@ -1066,8 +1200,16 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
* or we just jumped over the remaining data of a continuation.
*/
XLogBeginRead(state, tmpRecPtr);
- while (XLogReadRecord(state, &errormsg) != NULL)
+ while ((result = XLogReadRecord(state, &record, &errormsg)) !=
+ XLREAD_FAIL)
{
+ if (result == XLREAD_NEED_DATA)
+ {
+ if (!read_page(state, private))
+ goto err;
+ continue;
+ }
+
/* past the record we've found, break out */
if (RecPtr <= state->ReadRecPtr)
{
@@ -1087,9 +1229,9 @@ err:
#endif /* FRONTEND */
/*
- * Helper function to ease writing of XLogRoutine->page_read callbacks.
- * If this function is used, caller must supply a segment_open callback in
- * 'state', as that is used here.
+ * Helper function to ease writing of page_read callback.
+ * If this function is used, caller must supply a segment_open callback and
+ * segment_close callback as that is used here.
*
* Read 'count' bytes into 'buf', starting at location 'startptr', from WAL
* fetched from timeline 'tli'.
@@ -1102,6 +1244,7 @@ err:
*/
bool
WALRead(XLogReaderState *state,
+ WALSegmentOpenCB segopenfn, WALSegmentCloseCB segclosefn,
char *buf, XLogRecPtr startptr, Size count, TimeLineID tli,
WALReadError *errinfo)
{
@@ -1133,10 +1276,10 @@ WALRead(XLogReaderState *state,
XLogSegNo nextSegNo;
if (state->seg.ws_file >= 0)
- state->routine.segment_close(state);
+ segclosefn(state);
XLByteToSeg(recptr, nextSegNo, state->segcxt.ws_segsize);
- state->routine.segment_open(state, nextSegNo, &tli);
+ segopenfn(state, nextSegNo, &tli);
/* This shouldn't happen -- indicates a bug in segment_open */
Assert(state->seg.ws_file >= 0);
diff --git a/src/backend/access/transam/xlogutils.c b/src/backend/access/transam/xlogutils.c
index 46eda33f25..b003990745 100644
--- a/src/backend/access/transam/xlogutils.c
+++ b/src/backend/access/transam/xlogutils.c
@@ -686,8 +686,7 @@ XLogTruncateRelation(RelFileNode rnode, ForkNumber forkNum,
void
XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wantLength)
{
- const XLogRecPtr lastReadPage = state->seg.ws_segno *
- state->segcxt.ws_segsize + state->readLen;
+ const XLogRecPtr lastReadPage = state->readPagePtr;
Assert(wantPage != InvalidXLogRecPtr && wantPage % XLOG_BLCKSZ == 0);
Assert(wantLength <= XLOG_BLCKSZ);
@@ -702,7 +701,7 @@ XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wa
* current TLI has since become historical.
*/
if (lastReadPage == wantPage &&
- state->readLen != 0 &&
+ state->page_verified &&
lastReadPage + state->readLen >= wantPage + Min(wantLength, XLOG_BLCKSZ - 1))
return;
@@ -788,6 +787,7 @@ wal_segment_open(XLogReaderState *state, XLogSegNo nextSegNo,
char path[MAXPGPATH];
XLogFilePath(path, tli, nextSegNo, state->segcxt.ws_segsize);
+ elog(LOG, "HOGE: %lu, %d => %s", nextSegNo, tli, path);
state->seg.ws_file = BasicOpenFile(path, O_RDONLY | PG_BINARY);
if (state->seg.ws_file >= 0)
return;
@@ -825,9 +825,11 @@ wal_segment_close(XLogReaderState *state)
* loop for now.
*/
bool
-read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *cur_page)
+read_local_xlog_page(XLogReaderState *state)
{
+ XLogRecPtr targetPagePtr = state->readPagePtr;
+ int reqLen = state->readLen;
+ char *cur_page = state->readBuf;
XLogRecPtr read_upto,
loc;
TimeLineID tli;
@@ -940,11 +942,12 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
* as 'count', read the whole page anyway. It's guaranteed to be
* zero-padded up to the page boundary if it's incomplete.
*/
- if (!WALRead(state, cur_page, targetPagePtr, XLOG_BLCKSZ, tli,
- &errinfo))
+ if (!WALRead(state, wal_segment_open, wal_segment_close,
+ cur_page, targetPagePtr, XLOG_BLCKSZ, tli, &errinfo))
WALReadRaiseError(&errinfo);
/* number of valid bytes in the buffer */
+ state->readPagePtr = targetPagePtr;
state->readLen = count;
return true;
}
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index 37b75deb72..9cbd8afa49 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -148,7 +148,8 @@ StartupDecodingContext(List *output_plugin_options,
TransactionId xmin_horizon,
bool need_full_snapshot,
bool fast_forward,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress)
@@ -198,11 +199,12 @@ StartupDecodingContext(List *output_plugin_options,
ctx->slot = slot;
- ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, xl_routine, ctx);
+ ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, cleanup_cb);
if (!ctx->reader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
errmsg("out of memory")));
+ ctx->page_read = page_read;
ctx->reorder = ReorderBufferAllocate();
ctx->snapshot_builder =
@@ -319,7 +321,8 @@ CreateInitDecodingContext(const char *plugin,
List *output_plugin_options,
bool need_full_snapshot,
XLogRecPtr restart_lsn,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress)
@@ -422,7 +425,7 @@ CreateInitDecodingContext(const char *plugin,
ctx = StartupDecodingContext(NIL, restart_lsn, xmin_horizon,
need_full_snapshot, false,
- xl_routine, prepare_write, do_write,
+ page_read, cleanup_cb, prepare_write, do_write,
update_progress);
/* call output plugin initialization callback */
@@ -476,7 +479,8 @@ LogicalDecodingContext *
CreateDecodingContext(XLogRecPtr start_lsn,
List *output_plugin_options,
bool fast_forward,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress)
@@ -528,8 +532,8 @@ CreateDecodingContext(XLogRecPtr start_lsn,
ctx = StartupDecodingContext(output_plugin_options,
start_lsn, InvalidTransactionId, false,
- fast_forward, xl_routine, prepare_write,
- do_write, update_progress);
+ fast_forward, page_read, cleanup_cb,
+ prepare_write, do_write, update_progress);
/* call output plugin initialization callback */
old_context = MemoryContextSwitchTo(ctx->context);
@@ -585,7 +589,13 @@ DecodingContextFindStartpoint(LogicalDecodingContext *ctx)
char *err = NULL;
/* the read_page callback waits for new WAL */
- record = XLogReadRecord(ctx->reader, &err);
+ while (XLogReadRecord(ctx->reader, &record, &err) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!ctx->page_read(ctx->reader))
+ break;
+ }
+
if (err)
elog(ERROR, "%s", err);
if (!record)
diff --git a/src/backend/replication/logical/logicalfuncs.c b/src/backend/replication/logical/logicalfuncs.c
index f7e055874e..b69f6911c5 100644
--- a/src/backend/replication/logical/logicalfuncs.c
+++ b/src/backend/replication/logical/logicalfuncs.c
@@ -233,9 +233,8 @@ pg_logical_slot_get_changes_guts(FunctionCallInfo fcinfo, bool confirm, bool bin
ctx = CreateDecodingContext(InvalidXLogRecPtr,
options,
false,
- XL_ROUTINE(.page_read = read_local_xlog_page,
- .segment_open = wal_segment_open,
- .segment_close = wal_segment_close),
+ read_local_xlog_page,
+ wal_segment_close,
LogicalOutputPrepareWrite,
LogicalOutputWrite, NULL);
@@ -284,7 +283,13 @@ pg_logical_slot_get_changes_guts(FunctionCallInfo fcinfo, bool confirm, bool bin
XLogRecord *record;
char *errm = NULL;
- record = XLogReadRecord(ctx->reader, &errm);
+ while (XLogReadRecord(ctx->reader, &record, &errm) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!ctx->page_read(ctx->reader))
+ break;
+ }
+
if (errm)
elog(ERROR, "%s", errm);
diff --git a/src/backend/replication/slotfuncs.c b/src/backend/replication/slotfuncs.c
index 9817b44113..deb7cf15eb 100644
--- a/src/backend/replication/slotfuncs.c
+++ b/src/backend/replication/slotfuncs.c
@@ -153,9 +153,8 @@ create_logical_replication_slot(char *name, char *plugin,
ctx = CreateInitDecodingContext(plugin, NIL,
false, /* just catalogs is OK */
restart_lsn,
- XL_ROUTINE(.page_read = read_local_xlog_page,
- .segment_open = wal_segment_open,
- .segment_close = wal_segment_close),
+ read_local_xlog_page,
+ wal_segment_close,
NULL, NULL, NULL);
/*
@@ -512,9 +511,8 @@ pg_logical_replication_slot_advance(XLogRecPtr moveto)
ctx = CreateDecodingContext(InvalidXLogRecPtr,
NIL,
true, /* fast_forward */
- XL_ROUTINE(.page_read = read_local_xlog_page,
- .segment_open = wal_segment_open,
- .segment_close = wal_segment_close),
+ read_local_xlog_page,
+ wal_segment_close,
NULL, NULL, NULL);
/*
@@ -536,7 +534,13 @@ pg_logical_replication_slot_advance(XLogRecPtr moveto)
* Read records. No changes are generated in fast_forward mode,
* but snapbuilder/slot statuses are updated properly.
*/
- record = XLogReadRecord(ctx->reader, &errm);
+ while (XLogReadRecord(ctx->reader, &record, &errm) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!ctx->page_read(ctx->reader))
+ break;
+ }
+
if (errm)
elog(ERROR, "%s", errm);
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index c36b91ae5e..062093727d 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -580,10 +580,7 @@ StartReplication(StartReplicationCmd *cmd)
/* create xlogreader for physical replication */
xlogreader =
- XLogReaderAllocate(wal_segment_size, NULL,
- XL_ROUTINE(.segment_open = WalSndSegmentOpen,
- .segment_close = wal_segment_close),
- NULL);
+ XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
if (!xlogreader)
ereport(ERROR,
@@ -807,9 +804,11 @@ StartReplication(StartReplicationCmd *cmd)
* set every time WAL is flushed.
*/
static bool
-logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetRecPtr, char *cur_page)
+logical_read_xlog_page(XLogReaderState *state)
{
+ XLogRecPtr targetPagePtr = state->readPagePtr;
+ int reqLen = state->readLen;
+ char *cur_page = state->readBuf;
XLogRecPtr flushptr;
int count;
WALReadError errinfo;
@@ -837,7 +836,7 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
count = flushptr - targetPagePtr; /* part of the page available */
/* now actually read the data, we know it's there */
- if (!WALRead(state,
+ if (!WALRead(state, WalSndSegmentOpen, wal_segment_close,
cur_page,
targetPagePtr,
XLOG_BLCKSZ,
@@ -1011,9 +1010,8 @@ CreateReplicationSlot(CreateReplicationSlotCmd *cmd)
ctx = CreateInitDecodingContext(cmd->plugin, NIL, need_full_snapshot,
InvalidXLogRecPtr,
- XL_ROUTINE(.page_read = logical_read_xlog_page,
- .segment_open = WalSndSegmentOpen,
- .segment_close = wal_segment_close),
+ logical_read_xlog_page,
+ wal_segment_close,
WalSndPrepareWrite, WalSndWriteData,
WalSndUpdateProgress);
@@ -1171,9 +1169,8 @@ StartLogicalReplication(StartReplicationCmd *cmd)
*/
logical_decoding_ctx =
CreateDecodingContext(cmd->startpoint, cmd->options, false,
- XL_ROUTINE(.page_read = logical_read_xlog_page,
- .segment_open = WalSndSegmentOpen,
- .segment_close = wal_segment_close),
+ logical_read_xlog_page,
+ wal_segment_close,
WalSndPrepareWrite, WalSndWriteData,
WalSndUpdateProgress);
xlogreader = logical_decoding_ctx->reader;
@@ -2749,7 +2746,7 @@ XLogSendPhysical(void)
enlargeStringInfo(&output_message, nbytes);
retry:
- if (!WALRead(xlogreader,
+ if (!WALRead(xlogreader, WalSndSegmentOpen, wal_segment_close,
&output_message.data[output_message.len],
startptr,
nbytes,
@@ -2847,7 +2844,12 @@ XLogSendLogical(void)
*/
WalSndCaughtUp = false;
- record = XLogReadRecord(logical_decoding_ctx->reader, &errm);
+ while (XLogReadRecord(logical_decoding_ctx->reader, &record, &errm) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!logical_decoding_ctx->page_read(logical_decoding_ctx->reader))
+ break;
+ }
/* xlog record was invalid */
if (errm != NULL)
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index cf119848b0..712c85281c 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -41,15 +41,9 @@ static int xlogreadfd = -1;
static XLogSegNo xlogreadsegno = -1;
static char xlogfpath[MAXPGPATH];
-typedef struct XLogPageReadPrivate
-{
- const char *restoreCommand;
- int tliIndex;
-} XLogPageReadPrivate;
-
-static bool SimpleXLogPageRead(XLogReaderState *xlogreader,
- XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
+static bool SimpleXLogPageRead(XLogReaderState *xlogreader,
+ const char *datadir, int *tliIndex,
+ const char *restoreCommand);
/*
* Read WAL from the datadir/pg_wal, starting from 'startpoint' on timeline
@@ -66,20 +60,22 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
XLogRecord *record;
XLogReaderState *xlogreader;
char *errormsg;
- XLogPageReadPrivate private;
- private.tliIndex = tliIndex;
- private.restoreCommand = restoreCommand;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir,
- XL_ROUTINE(.page_read = &SimpleXLogPageRead),
- &private);
+ xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+
if (xlogreader == NULL)
pg_fatal("out of memory");
XLogBeginRead(xlogreader, startpoint);
do
{
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!SimpleXLogPageRead(xlogreader, datadir,
+ &tliIndex, restoreCommand))
+ break;
+ }
if (record == NULL)
{
@@ -123,19 +119,19 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex,
XLogRecord *record;
XLogReaderState *xlogreader;
char *errormsg;
- XLogPageReadPrivate private;
XLogRecPtr endptr;
- private.tliIndex = tliIndex;
- private.restoreCommand = restoreCommand;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir,
- XL_ROUTINE(.page_read = &SimpleXLogPageRead),
- &private);
+ xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
XLogBeginRead(xlogreader, ptr);
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!SimpleXLogPageRead(xlogreader, datadir, &tliIndex, restoreCommand))
+ break;
+ }
if (record == NULL)
{
if (errormsg)
@@ -170,7 +166,6 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
XLogRecPtr searchptr;
XLogReaderState *xlogreader;
char *errormsg;
- XLogPageReadPrivate private;
/*
* The given fork pointer points to the end of the last common record,
@@ -186,11 +181,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
forkptr += SizeOfXLogShortPHD;
}
- private.tliIndex = tliIndex;
- private.restoreCommand = restoreCommand;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir,
- XL_ROUTINE(.page_read = &SimpleXLogPageRead),
- &private);
+ xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
@@ -200,7 +191,13 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
uint8 info;
XLogBeginRead(xlogreader, searchptr);
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!SimpleXLogPageRead(xlogreader, datadir,
+ &tliIndex, restoreCommand))
+ break;
+ }
if (record == NULL)
{
@@ -247,10 +244,11 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
/* XLogReader callback function, to read a WAL page */
static bool
-SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *readBuf)
+SimpleXLogPageRead(XLogReaderState *xlogreader, const char *datadir,
+ int *tliIndex, const char *restoreCommand)
{
- XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
+ XLogRecPtr targetPagePtr = xlogreader->readPagePtr;
+ char *readBuf = xlogreader->readBuf;
uint32 targetPageOff;
XLogRecPtr targetSegEnd;
XLogSegNo targetSegNo;
@@ -283,14 +281,14 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
* be done both forward and backward, consider also switching timeline
* accordingly.
*/
- while (private->tliIndex < targetNentries - 1 &&
- targetHistory[private->tliIndex].end < targetSegEnd)
- private->tliIndex++;
- while (private->tliIndex > 0 &&
- targetHistory[private->tliIndex].begin >= targetSegEnd)
- private->tliIndex--;
+ while (*tliIndex < targetNentries - 1 &&
+ targetHistory[*tliIndex].end < targetSegEnd)
+ (*tliIndex)++;
+ while (*tliIndex > 0 &&
+ targetHistory[*tliIndex].begin >= targetSegEnd)
+ (*tliIndex)--;
- XLogFileName(xlogfname, targetHistory[private->tliIndex].tli,
+ XLogFileName(xlogfname, targetHistory[*tliIndex].tli,
xlogreadsegno, WalSegSz);
snprintf(xlogfpath, MAXPGPATH, "%s/" XLOGDIR "/%s",
@@ -303,7 +301,7 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
/*
* If we have no restore_command to execute, then exit.
*/
- if (private->restoreCommand == NULL)
+ if (restoreCommand == NULL)
{
pg_log_error("could not open file \"%s\": %m", xlogfpath);
xlogreader->readLen = -1;
@@ -317,7 +315,7 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
xlogreadfd = RestoreArchivedFile(xlogreader->segcxt.ws_dir,
xlogfname,
WalSegSz,
- private->restoreCommand);
+ restoreCommand);
if (xlogreadfd < 0)
{
@@ -359,7 +357,7 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
Assert(targetSegNo == xlogreadsegno);
- xlogreader->seg.ws_tli = targetHistory[private->tliIndex].tli;
+ xlogreader->seg.ws_tli = targetHistory[*tliIndex].tli;
xlogreader->readLen = XLOG_BLCKSZ;
return true;
}
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index 0f0e0723a3..f514de6b13 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -330,12 +330,17 @@ WALDumpCloseSegment(XLogReaderState *state)
state->seg.ws_file = -1;
}
-/* pg_waldump's XLogReaderRoutine->page_read callback */
+/*
+ * pg_waldump's WAL page rader, also used as page_read callback for
+ * XLogFindNextRecord
+ */
static bool
-WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetPtr, char *readBuff)
+WALDumpReadPage(XLogReaderState *state, void *priv)
{
- XLogDumpPrivate *private = state->private_data;
+ XLogRecPtr targetPagePtr = state->readPagePtr;
+ int reqLen = state->readLen;
+ char *readBuff = state->readBuf;
+ XLogDumpPrivate *private = (XLogDumpPrivate *) priv;
int count = XLOG_BLCKSZ;
WALReadError errinfo;
@@ -353,8 +358,8 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
}
}
- if (!WALRead(state, readBuff, targetPagePtr, count, private->timeline,
- &errinfo))
+ if (!WALRead(state, WALDumpOpenSegment, WALDumpCloseSegment,
+ readBuff, targetPagePtr, count, private->timeline, &errinfo))
{
WALOpenSegment *seg = &errinfo.wre_seg;
char fname[MAXPGPATH];
@@ -374,6 +379,7 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
(Size) errinfo.wre_req);
}
+ Assert(count >= state->readLen);
state->readLen = count;
return true;
}
@@ -1035,16 +1041,14 @@ main(int argc, char **argv)
/* we have everything we need, start reading */
xlogreader_state =
- XLogReaderAllocate(WalSegSz, waldir,
- XL_ROUTINE(.page_read = WALDumpReadPage,
- .segment_open = WALDumpOpenSegment,
- .segment_close = WALDumpCloseSegment),
- &private);
+ XLogReaderAllocate(WalSegSz, waldir, WALDumpCloseSegment);
+
if (!xlogreader_state)
fatal_error("out of memory");
/* first find a valid recptr to start from */
- first_record = XLogFindNextRecord(xlogreader_state, private.startptr);
+ first_record = XLogFindNextRecord(xlogreader_state, private.startptr,
+ &WALDumpReadPage, (void*) &private);
if (first_record == InvalidXLogRecPtr)
fatal_error("could not find a valid record after %X/%X",
@@ -1067,7 +1071,13 @@ main(int argc, char **argv)
for (;;)
{
/* try to read the next record */
- record = XLogReadRecord(xlogreader_state, &errormsg);
+ while (XLogReadRecord(xlogreader_state, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!WALDumpReadPage(xlogreader_state, (void *) &private))
+ break;
+ }
+
if (!record)
{
if (!config.follow || private.endptr_reached)
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index 23d9e70a59..4e9268b553 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -57,64 +57,15 @@ typedef struct WALSegmentContext
typedef struct XLogReaderState XLogReaderState;
-/* Function type definition for the read_page callback */
-typedef bool (*XLogPageReadCB) (XLogReaderState *xlogreader,
- XLogRecPtr targetPagePtr,
- int reqLen,
- XLogRecPtr targetRecPtr,
- char *readBuf);
+/* Function type definition for the segment cleanup callback */
+typedef void (*WALSegmentCleanupCB) (XLogReaderState *xlogreader);
+
+/* Function type definition for the open/close callbacks for WALRead() */
typedef void (*WALSegmentOpenCB) (XLogReaderState *xlogreader,
XLogSegNo nextSegNo,
TimeLineID *tli_p);
typedef void (*WALSegmentCloseCB) (XLogReaderState *xlogreader);
-typedef struct XLogReaderRoutine
-{
- /*
- * Data input callback
- *
- * This callback shall read at least reqLen valid bytes of the xlog page
- * starting at targetPagePtr, and store them in readBuf. The callback
- * shall return the number of bytes read (never more than XLOG_BLCKSZ), or
- * -1 on failure. The callback shall sleep, if necessary, to wait for the
- * requested bytes to become available. The callback will not be invoked
- * again for the same page unless more than the returned number of bytes
- * are needed.
- *
- * targetRecPtr is the position of the WAL record we're reading. Usually
- * it is equal to targetPagePtr + reqLen, but sometimes xlogreader needs
- * to read and verify the page or segment header, before it reads the
- * actual WAL record it's interested in. In that case, targetRecPtr can
- * be used to determine which timeline to read the page from.
- *
- * The callback shall set ->seg.ws_tli to the TLI of the file the page was
- * read from.
- */
- XLogPageReadCB page_read;
-
- /*
- * Callback to open the specified WAL segment for reading. ->seg.ws_file
- * shall be set to the file descriptor of the opened segment. In case of
- * failure, an error shall be raised by the callback and it shall not
- * return.
- *
- * "nextSegNo" is the number of the segment to be opened.
- *
- * "tli_p" is an input/output argument. WALRead() uses it to pass the
- * timeline in which the new segment should be found, but the callback can
- * use it to return the TLI that it actually opened.
- */
- WALSegmentOpenCB segment_open;
-
- /*
- * WAL segment close callback. ->seg.ws_file shall be set to a negative
- * number.
- */
- WALSegmentCloseCB segment_close;
-} XLogReaderRoutine;
-
-#define XL_ROUTINE(...) &(XLogReaderRoutine){__VA_ARGS__}
-
typedef struct
{
/* Is this block ref in use? */
@@ -144,12 +95,35 @@ typedef struct
uint16 data_bufsz;
} DecodedBkpBlock;
+/* Return code from XLogReadRecord */
+typedef enum XLogReadRecordResult
+{
+ XLREAD_SUCCESS, /* record is successfully read */
+ XLREAD_NEED_DATA, /* need more data. see XLogReadRecord. */
+ XLREAD_FAIL /* failed during reading a record */
+} XLogReadRecordResult;
+
+/*
+ * internal state of XLogReadRecord
+ *
+ * XLogReadState runs a state machine while reading a record. Theses states
+ * are not seen outside the function. Each state may repeat several times
+ * exiting requesting caller for new data. See the comment of XLogReadRecrod
+ * for details.
+ */
+typedef enum XLogReadRecordState {
+ XLREAD_NEXT_RECORD,
+ XLREAD_TOT_LEN,
+ XLREAD_FIRST_FRAGMENT,
+ XLREAD_CONTINUATION
+} XLogReadRecordState;
+
struct XLogReaderState
{
/*
* Operational callbacks
*/
- XLogReaderRoutine routine;
+ WALSegmentCleanupCB cleanup_cb;
/* ----------------------------------------
* Public parameters
@@ -162,18 +136,14 @@ struct XLogReaderState
*/
uint64 system_identifier;
- /*
- * Opaque data for callbacks to use. Not used by XLogReader.
- */
- void *private_data;
-
/*
* Start and end point of last record read. EndRecPtr is also used as the
* position to read next. Calling XLogBeginRead() sets EndRecPtr to the
* starting position and ReadRecPtr to invalid.
*/
- XLogRecPtr ReadRecPtr; /* start of last record read */
+ XLogRecPtr ReadRecPtr; /* start of last record read or being read */
XLogRecPtr EndRecPtr; /* end+1 of last record read */
+ XLogRecPtr PrevRecPtr; /* start of previous record read */
/* ----------------------------------------
* Communication with page reader
@@ -187,7 +157,9 @@ struct XLogReaderState
* the request, or -1 on error */
TimeLineID readPageTLI; /* TLI for data currently in readBuf */
char *readBuf; /* buffer to store data */
- bool page_verified; /* is the page on the buffer verified? */
+ bool page_verified; /* is the page header on the buffer verified? */
+ bool record_verified;/* is the current record header verified? */
+
/* ----------------------------------------
@@ -229,8 +201,6 @@ struct XLogReaderState
XLogRecPtr latestPagePtr;
TimeLineID latestPageTLI;
- /* beginning of the WAL record being read. */
- XLogRecPtr currRecPtr;
/* timeline to read it from, 0 if a lookup is required */
TimeLineID currTLI;
@@ -257,6 +227,15 @@ struct XLogReaderState
char *readRecordBuf;
uint32 readRecordBufSize;
+ /*
+ * XLogReadRecord() state
+ */
+ XLogReadRecordState readRecordState;/* state machine state */
+ int recordGotLen; /* amount of current record that has
+ * already been read */
+ int recordRemainLen; /* length of current record that remains */
+ XLogRecPtr recordContRecPtr; /* where the current record continues */
+
/* Buffer to hold error message */
char *errormsg_buf;
};
@@ -264,9 +243,7 @@ struct XLogReaderState
/* Get a new XLogReader */
extern XLogReaderState *XLogReaderAllocate(int wal_segment_size,
const char *waldir,
- XLogReaderRoutine *routine,
- void *private_data);
-extern XLogReaderRoutine *LocalXLogReaderRoutine(void);
+ WALSegmentCleanupCB cleanup_cb);
/* Free an XLogReader */
extern void XLogReaderFree(XLogReaderState *state);
@@ -274,12 +251,17 @@ extern void XLogReaderFree(XLogReaderState *state);
/* Position the XLogReader to given record */
extern void XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr);
#ifdef FRONTEND
-extern XLogRecPtr XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr);
+/* Function type definition for the read_page callback */
+typedef bool (*XLogFindNextRecordCB) (XLogReaderState *xlogreader,
+ void *private);
+extern XLogRecPtr XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
+ XLogFindNextRecordCB read_page, void *private);
#endif /* FRONTEND */
/* Read the next XLog record. Returns NULL on end-of-WAL or failure */
-extern struct XLogRecord *XLogReadRecord(XLogReaderState *state,
- char **errormsg);
+extern XLogReadRecordResult XLogReadRecord(XLogReaderState *state,
+ XLogRecord **record,
+ char **errormsg);
/* Validate a page */
extern bool XLogReaderValidatePageHeader(XLogReaderState *state,
@@ -299,6 +281,7 @@ typedef struct WALReadError
} WALReadError;
extern bool WALRead(XLogReaderState *state,
+ WALSegmentOpenCB segopenfn, WALSegmentCloseCB sgclosefn,
char *buf, XLogRecPtr startptr, Size count,
TimeLineID tli, WALReadError *errinfo);
diff --git a/src/include/access/xlogutils.h b/src/include/access/xlogutils.h
index 364a21c4ea..397fb27fc2 100644
--- a/src/include/access/xlogutils.h
+++ b/src/include/access/xlogutils.h
@@ -47,9 +47,7 @@ extern Buffer XLogReadBufferExtended(RelFileNode rnode, ForkNumber forknum,
extern Relation CreateFakeRelcacheEntry(RelFileNode rnode);
extern void FreeFakeRelcacheEntry(Relation fakerel);
-extern bool read_local_xlog_page(XLogReaderState *state,
- XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetRecPtr, char *cur_page);
+extern bool read_local_xlog_page(XLogReaderState *state);
extern void wal_segment_open(XLogReaderState *state,
XLogSegNo nextSegNo,
TimeLineID *tli_p);
diff --git a/src/include/pg_config_manual.h b/src/include/pg_config_manual.h
index f10ad0acd6..94749609b4 100644
--- a/src/include/pg_config_manual.h
+++ b/src/include/pg_config_manual.h
@@ -377,7 +377,7 @@
* Enable debugging print statements for WAL-related operations; see
* also the wal_debug GUC var.
*/
-/* #define WAL_DEBUG */
+#define WAL_DEBUG
/*
* Enable tracing of resource consumption during sort operations;
diff --git a/src/include/replication/logical.h b/src/include/replication/logical.h
index c253403372..b936f31393 100644
--- a/src/include/replication/logical.h
+++ b/src/include/replication/logical.h
@@ -29,6 +29,10 @@ typedef void (*LogicalOutputPluginWriterUpdateProgress) (struct LogicalDecodingC
TransactionId xid
);
+typedef struct LogicalDecodingContext LogicalDecodingContext;
+
+typedef bool (*LogicalDecodingXLogPageReadCB)(XLogReaderState *ctx);
+
typedef struct LogicalDecodingContext
{
/* memory context this is all allocated in */
@@ -39,6 +43,7 @@ typedef struct LogicalDecodingContext
/* infrastructure pieces for decoding */
XLogReaderState *reader;
+ LogicalDecodingXLogPageReadCB page_read;
struct ReorderBuffer *reorder;
struct SnapBuild *snapshot_builder;
@@ -105,14 +110,16 @@ extern LogicalDecodingContext *CreateInitDecodingContext(const char *plugin,
List *output_plugin_options,
bool need_full_snapshot,
XLogRecPtr restart_lsn,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress);
extern LogicalDecodingContext *CreateDecodingContext(XLogRecPtr start_lsn,
List *output_plugin_options,
bool fast_forward,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress);
--
2.27.0
----Next_Part(Thu_Mar__4_11_28_53_2021_014)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
filename="v17-0003-Remove-globals-readOff-readLen-and-readSegNo.patch"
^ permalink raw reply [nested|flat] 70+ messages in thread
* [PATCH v17 2/4] Move page-reader out of XLogReadRecord
@ 2019-09-10 03:58 Kyotaro Horiguchi <[email protected]>
0 siblings, 0 replies; 70+ messages in thread
From: Kyotaro Horiguchi @ 2019-09-10 03:58 UTC (permalink / raw)
This is the second step of removing callbacks from WAL record reader.
Since it is essential to take in additional data while reading a
record, the function have to ask caller for new data while keeping
working state. Thus the function is turned into a state machine.
---
src/backend/access/transam/twophase.c | 15 +-
src/backend/access/transam/xlog.c | 58 +-
src/backend/access/transam/xlogreader.c | 697 +++++++++++-------
src/backend/access/transam/xlogutils.c | 17 +-
src/backend/replication/logical/logical.c | 26 +-
.../replication/logical/logicalfuncs.c | 13 +-
src/backend/replication/slotfuncs.c | 18 +-
src/backend/replication/walsender.c | 32 +-
src/bin/pg_rewind/parsexlog.c | 84 ++-
src/bin/pg_waldump/pg_waldump.c | 36 +-
src/include/access/xlogreader.h | 121 ++-
src/include/access/xlogutils.h | 4 +-
src/include/pg_config_manual.h | 2 +-
src/include/replication/logical.h | 11 +-
14 files changed, 646 insertions(+), 488 deletions(-)
diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index 80d2d20d6c..7d5a96a5f5 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -1325,11 +1325,8 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
XLogReaderState *xlogreader;
char *errormsg;
- xlogreader = XLogReaderAllocate(wal_segment_size, NULL,
- XL_ROUTINE(.page_read = &read_local_xlog_page,
- .segment_open = &wal_segment_open,
- .segment_close = &wal_segment_close),
- NULL);
+ xlogreader = XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+
if (!xlogreader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
@@ -1337,7 +1334,13 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
errdetail("Failed while allocating a WAL reading processor.")));
XLogBeginRead(xlogreader, lsn);
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!read_local_xlog_page(xlogreader))
+ break;
+ }
+
if (record == NULL)
ereport(ERROR,
(errcode_for_file_access(),
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 606328a65a..670996ae67 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -836,13 +836,6 @@ static XLogSource currentSource = XLOG_FROM_ANY;
static bool lastSourceFailed = false;
static bool pendingWalRcvRestart = false;
-typedef struct XLogPageReadPrivate
-{
- int emode;
- bool fetching_ckpt; /* are we fetching a checkpoint record? */
- bool randAccess;
-} XLogPageReadPrivate;
-
/*
* These variables track when we last obtained some WAL data to process,
* and where we got it from. (XLogReceiptSource is initially the same as
@@ -917,8 +910,8 @@ static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath,
static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli,
XLogSource source, bool notfoundOk);
static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, XLogSource source);
-static bool XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
+static bool XLogPageRead(XLogReaderState *xlogreader,
+ bool fetching_ckpt, int emode, bool randAccess);
static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
bool fetching_ckpt, XLogRecPtr tliRecPtr);
static int emode_for_corrupt_record(int emode, XLogRecPtr RecPtr);
@@ -1231,8 +1224,7 @@ XLogInsertRecord(XLogRecData *rdata,
appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len);
if (!debug_reader)
- debug_reader = XLogReaderAllocate(wal_segment_size, NULL,
- XL_ROUTINE(), NULL);
+ debug_reader = XLogReaderAllocate(wal_segment_size, NULL, NULL);
if (!debug_reader)
{
@@ -4344,15 +4336,10 @@ CleanupBackupHistory(void)
* record is available.
*/
static XLogRecord *
-ReadRecord(XLogReaderState *xlogreader, int emode,
- bool fetching_ckpt)
+ReadRecord(XLogReaderState *xlogreader, int emode, bool fetching_ckpt)
{
XLogRecord *record;
- XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
-
- private->fetching_ckpt = fetching_ckpt;
- private->emode = emode;
- private->randAccess = (xlogreader->ReadRecPtr == InvalidXLogRecPtr);
+ bool randAccess = (xlogreader->ReadRecPtr == InvalidXLogRecPtr);
/* This is the first attempt to read this page. */
lastSourceFailed = false;
@@ -4360,8 +4347,16 @@ ReadRecord(XLogReaderState *xlogreader, int emode,
for (;;)
{
char *errormsg;
+ XLogReadRecordResult result;
+
+ while ((result = XLogReadRecord(xlogreader, &record, &errormsg))
+ == XLREAD_NEED_DATA)
+ {
+ if (!XLogPageRead(xlogreader, fetching_ckpt, emode, randAccess))
+ break;
+
+ }
- record = XLogReadRecord(xlogreader, &errormsg);
ReadRecPtr = xlogreader->ReadRecPtr;
EndRecPtr = xlogreader->EndRecPtr;
if (record == NULL)
@@ -6380,7 +6375,6 @@ StartupXLOG(void)
bool backupFromStandby = false;
DBState dbstate_at_startup;
XLogReaderState *xlogreader;
- XLogPageReadPrivate private;
bool promoted = false;
struct stat st;
@@ -6539,13 +6533,9 @@ StartupXLOG(void)
OwnLatch(&XLogCtl->recoveryWakeupLatch);
/* Set up XLOG reader facility */
- MemSet(&private, 0, sizeof(XLogPageReadPrivate));
xlogreader =
- XLogReaderAllocate(wal_segment_size, NULL,
- XL_ROUTINE(.page_read = &XLogPageRead,
- .segment_open = NULL,
- .segment_close = wal_segment_close),
- &private);
+ XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+
if (!xlogreader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
@@ -11987,12 +11977,13 @@ CancelBackup(void)
* sleep and retry.
*/
static bool
-XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetRecPtr, char *readBuf)
+XLogPageRead(XLogReaderState *xlogreader,
+ bool fetching_ckpt, int emode, bool randAccess)
{
- XLogPageReadPrivate *private =
- (XLogPageReadPrivate *) xlogreader->private_data;
- int emode = private->emode;
+ char *readBuf = xlogreader->readBuf;
+ XLogRecPtr targetPagePtr = xlogreader->readPagePtr;
+ int reqLen = xlogreader->readLen;
+ XLogRecPtr targetRecPtr = xlogreader->ReadRecPtr;
uint32 targetPageOff;
XLogSegNo targetSegNo PG_USED_FOR_ASSERTS_ONLY;
int r;
@@ -12035,8 +12026,8 @@ retry:
flushedUpto < targetPagePtr + reqLen))
{
if (!WaitForWALToBecomeAvailable(targetPagePtr + reqLen,
- private->randAccess,
- private->fetching_ckpt,
+ randAccess,
+ fetching_ckpt,
targetRecPtr))
{
if (readFile >= 0)
@@ -12141,6 +12132,7 @@ retry:
goto next_record_is_invalid;
}
+ Assert(xlogreader->readPagePtr == targetPagePtr);
xlogreader->readLen = readLen;
return true;
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index 7d04995b7b..ba3e4868f7 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -40,7 +40,7 @@ static bool XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr,
int reqLen, bool header_inclusive);
static void XLogReaderInvalReadState(XLogReaderState *state);
static bool ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
- XLogRecPtr PrevRecPtr, XLogRecord *record, bool randAccess);
+ XLogRecPtr PrevRecPtr, XLogRecord *record);
static bool ValidXLogRecord(XLogReaderState *state, XLogRecord *record,
XLogRecPtr recptr);
static void ResetDecoder(XLogReaderState *state);
@@ -73,7 +73,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...)
*/
XLogReaderState *
XLogReaderAllocate(int wal_segment_size, const char *waldir,
- XLogReaderRoutine *routine, void *private_data)
+ WALSegmentCleanupCB cleanup_cb)
{
XLogReaderState *state;
@@ -84,7 +84,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
return NULL;
/* initialize caller-provided support functions */
- state->routine = *routine;
+ state->cleanup_cb = cleanup_cb;
state->max_block_id = -1;
@@ -107,8 +107,6 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
WALOpenSegmentInit(&state->seg, &state->segcxt, wal_segment_size,
waldir);
- /* system_identifier initialized to zeroes above */
- state->private_data = private_data;
/* ReadRecPtr, EndRecPtr and readLen initialized to zeroes above */
state->errormsg_buf = palloc_extended(MAX_ERRORMSG_LEN + 1,
MCXT_ALLOC_NO_OOM);
@@ -140,8 +138,8 @@ XLogReaderFree(XLogReaderState *state)
{
int block_id;
- if (state->seg.ws_file != -1)
- state->routine.segment_close(state);
+ if (state->seg.ws_file >= 0)
+ state->cleanup_cb(state);
for (block_id = 0; block_id <= XLR_MAX_BLOCK_ID; block_id++)
{
@@ -246,6 +244,7 @@ XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
/* Begin at the passed-in record pointer. */
state->EndRecPtr = RecPtr;
state->ReadRecPtr = InvalidXLogRecPtr;
+ state->readRecordState = XLREAD_NEXT_RECORD;
}
/*
@@ -254,317 +253,449 @@ XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
* XLogBeginRead() or XLogFindNextRecord() must be called before the first call
* to XLogReadRecord().
*
- * If the page_read callback fails to read the requested data, NULL is
- * returned. The callback is expected to have reported the error; errormsg
- * is set to NULL.
+ * This function may return XLREAD_NEED_DATA several times before returning a
+ * result record. The caller shall read in some new data then call this
+ * function again with the same parameters.
*
- * If the reading fails for some other reason, NULL is also returned, and
- * *errormsg is set to a string with details of the failure.
+ * When a record is successfully read, returns XLREAD_SUCCESS with result
+ * record being stored in *record. Otherwise *record is NULL.
*
- * The returned pointer (or *errormsg) points to an internal buffer that's
- * valid until the next call to XLogReadRecord.
+ * Returns XLREAD_NEED_DATA if more data is needed to finish reading the
+ * current record. In that case, state->readPagePtr and state->readLen inform
+ * the desired position and minimum length of data needed. The caller shall
+ * read in the requested data and set state->readBuf to point to a buffer
+ * containing it. The caller must also set state->readPageTLI and
+ * state->readLen to indicate the timeline that it was read from, and the
+ * length of data that is now available (which must be >= given readLen),
+ * respectively.
+ *
+ * If invalid data is encountered, returns XLREAD_FAIL with *record being set to
+ * NULL. *errormsg is set to a string with details of the failure.
+ * The returned pointer (or *errormsg) points to an internal buffer that's valid
+ * until the next call to XLogReadRecord.
+ *
+ *
+ * This function runs a state machine consists of the following states.
+ *
+ * XLREAD_NEXT_RECORD :
+ * The initial state, if called with valid RecPtr, try to read a record at
+ * that position. If invalid RecPtr is given try to read a record just after
+ * the last one previously read.
+ * This state ens after setting ReadRecPtr. Then goes to XLREAD_TOT_LEN.
+ *
+ * XLREAD_TOT_LEN:
+ * Examining record header. Ends after reading record total
+ * length. recordRemainLen and recordGotLen are initialized.
+ *
+ * XLREAD_FIRST_FRAGMENT:
+ * Reading the first fragment. Ends with finishing reading a single
+ * record. Goes to XLREAD_NEXT_RECORD if that's all or
+ * XLREAD_CONTINUATION if we have continuation.
+
+ * XLREAD_CONTINUATION:
+ * Reading continuation of record. Ends with finishing the whole record then
+ * goes to XLREAD_NEXT_RECORD. During this state, recordRemainLen indicates
+ * how much is left and readRecordBuf holds the partially assert
+ * record.recordContRecPtr points to the beginning of the next page where to
+ * continue.
+ *
+ * If wrong data found in any state, the state machine stays at the current
+ * state. This behavior allows to continue reading a reacord switching among
+ * different souces, while streaming replication.
*/
-XLogRecord *
-XLogReadRecord(XLogReaderState *state, char **errormsg)
+XLogReadRecordResult
+XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
{
- XLogRecPtr RecPtr;
- XLogRecord *record;
- XLogRecPtr targetPagePtr;
- bool randAccess;
- uint32 len,
- total_len;
- uint32 targetRecOff;
- uint32 pageHeaderSize;
- bool gotheader;
+ XLogRecord *prec;
- /*
- * randAccess indicates whether to verify the previous-record pointer of
- * the record we're reading. We only do this if we're reading
- * sequentially, which is what we initially assume.
- */
- randAccess = false;
+ *record = NULL;
/* reset error state */
*errormsg = NULL;
state->errormsg_buf[0] = '\0';
- ResetDecoder(state);
-
- RecPtr = state->EndRecPtr;
-
- if (state->ReadRecPtr != InvalidXLogRecPtr)
- {
- /* read the record after the one we just read */
-
- /*
- * EndRecPtr is pointing to end+1 of the previous WAL record. If
- * we're at a page boundary, no more records can fit on the current
- * page. We must skip over the page header, but we can't do that until
- * we've read in the page, since the header size is variable.
- */
- }
- else
- {
- /*
- * Caller supplied a position to start at.
- *
- * In this case, EndRecPtr should already be pointing to a valid
- * record starting position.
- */
- Assert(XRecOffIsValid(RecPtr));
- randAccess = true;
- }
-
- state->currRecPtr = RecPtr;
-
- targetPagePtr = RecPtr - (RecPtr % XLOG_BLCKSZ);
- targetRecOff = RecPtr % XLOG_BLCKSZ;
-
- /*
- * Read the page containing the record into state->readBuf. Request enough
- * byte to cover the whole record header, or at least the part of it that
- * fits on the same page.
- */
- while (XLogNeedData(state, targetPagePtr,
- Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
- targetRecOff != 0))
- {
- if (!state->routine.page_read(state, state->readPagePtr, state->readLen,
- RecPtr, state->readBuf))
- break;
- }
-
- if (!state->page_verified)
- goto err;
-
- /*
- * We have at least the page header, so we can examine it now.
- */
- pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
- if (targetRecOff == 0)
- {
- /*
- * At page start, so skip over page header.
- */
- RecPtr += pageHeaderSize;
- targetRecOff = pageHeaderSize;
- }
- else if (targetRecOff < pageHeaderSize)
- {
- report_invalid_record(state, "invalid record offset at %X/%X",
- LSN_FORMAT_ARGS(RecPtr));
- goto err;
- }
-
- if ((((XLogPageHeader) state->readBuf)->xlp_info & XLP_FIRST_IS_CONTRECORD) &&
- targetRecOff == pageHeaderSize)
- {
- report_invalid_record(state, "contrecord is requested by %X/%X",
- LSN_FORMAT_ARGS(RecPtr));
- goto err;
- }
-
- /* XLogNeedData has verified the page header */
- Assert(pageHeaderSize <= state->readLen);
-
- /*
- * Read the record length.
- *
- * NB: Even though we use an XLogRecord pointer here, the whole record
- * header might not fit on this page. xl_tot_len is the first field of the
- * struct, so it must be on this page (the records are MAXALIGNed), but we
- * cannot access any other fields until we've verified that we got the
- * whole header.
- */
- record = (XLogRecord *) (state->readBuf + RecPtr % XLOG_BLCKSZ);
- total_len = record->xl_tot_len;
-
- /*
- * If the whole record header is on this page, validate it immediately.
- * Otherwise do just a basic sanity check on xl_tot_len, and validate the
- * rest of the header after reading it from the next page. The xl_tot_len
- * check is necessary here to ensure that we enter the "Need to reassemble
- * record" code path below; otherwise we might fail to apply
- * ValidXLogRecordHeader at all.
- */
- if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
- {
- if (!ValidXLogRecordHeader(state, RecPtr, state->ReadRecPtr, record,
- randAccess))
- goto err;
- gotheader = true;
- }
- else
- {
- /* XXX: more validation should be done here */
- if (total_len < SizeOfXLogRecord)
- {
- report_invalid_record(state,
- "invalid record length at %X/%X: wanted %u, got %u",
- LSN_FORMAT_ARGS(RecPtr),
- (uint32) SizeOfXLogRecord, total_len);
- goto err;
- }
- gotheader = false;
- }
-
- len = XLOG_BLCKSZ - RecPtr % XLOG_BLCKSZ;
- if (total_len > len)
+ switch (state->readRecordState)
{
- /* Need to reassemble record */
- char *contdata;
- XLogPageHeader pageHeader;
- char *buffer;
- uint32 gotlen;
-
- /*
- * Enlarge readRecordBuf as needed.
- */
- if (total_len > state->readRecordBufSize &&
- !allocate_recordbuf(state, total_len))
- {
- /* We treat this as a "bogus data" condition */
- report_invalid_record(state, "record length %u at %X/%X too long",
- total_len, LSN_FORMAT_ARGS(RecPtr));
- goto err;
- }
+ case XLREAD_NEXT_RECORD:
+ ResetDecoder(state);
- /* Copy the first fragment of the record from the first page. */
- memcpy(state->readRecordBuf,
- state->readBuf + RecPtr % XLOG_BLCKSZ, len);
- buffer = state->readRecordBuf + len;
- gotlen = len;
-
- do
- {
- int rest_len = total_len - gotlen;
-
- /* Calculate pointer to beginning of next page */
- targetPagePtr += XLOG_BLCKSZ;
-
- /* Wait for the next page to become available */
- while (XLogNeedData(state, targetPagePtr,
- Min(rest_len, XLOG_BLCKSZ),
- false))
+ if (state->ReadRecPtr != InvalidXLogRecPtr)
{
- if (!state->routine.page_read(state, state->readPagePtr,
- state->readLen,
- state->ReadRecPtr,
- state->readBuf))
- break;
+ /* read the record after the one we just read */
+
+ /*
+ * EndRecPtr is pointing to end+1 of the previous WAL record.
+ * If we're at a page boundary, no more records can fit on the
+ * current page. We must skip over the page header, but we
+ * can't do that until we've read in the page, since the header
+ * size is variable.
+ */
+ state->PrevRecPtr = state->ReadRecPtr;
+ state->ReadRecPtr = state->EndRecPtr;
+ }
+ else
+ {
+ /*
+ * Caller supplied a position to start at.
+ *
+ * In this case, EndRecPtr should already be pointing to a
+ * valid record starting position.
+ */
+ Assert(XRecOffIsValid(state->EndRecPtr));
+ state->ReadRecPtr = state->EndRecPtr;
+
+ /*
+ * We cannot verify the previous-record pointer when we're
+ * seeking to a particular record. Reset PrevRecPtr so that we
+ * won't try doing that.
+ */
+ state->PrevRecPtr = InvalidXLogRecPtr;
+ state->EndRecPtr = InvalidXLogRecPtr; /* to be tidy */
}
+ state->record_verified = false;
+ state->readRecordState = XLREAD_TOT_LEN;
+ /* fall through */
+
+ case XLREAD_TOT_LEN:
+ {
+ uint32 total_len;
+ uint32 pageHeaderSize;
+ XLogRecPtr targetPagePtr;
+ uint32 targetRecOff;
+ XLogPageHeader pageHeader;
+
+ targetPagePtr =
+ state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
+ targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+
+ /*
+ * Check if we have enough data. For the first record in the page,
+ * the requesting length doesn't contain page header.
+ */
+ if (XLogNeedData(state, targetPagePtr,
+ Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+ targetRecOff != 0))
+ return XLREAD_NEED_DATA;
+
+ /* error out if caller supplied bogus page */
if (!state->page_verified)
goto err;
- Assert(SizeOfXLogShortPHD <= state->readLen);
+ /* examine page header now. */
+ pageHeaderSize =
+ XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+ if (targetRecOff == 0)
+ {
+ /* At page start, so skip over page header. */
+ state->ReadRecPtr += pageHeaderSize;
+ targetRecOff = pageHeaderSize;
+ }
+ else if (targetRecOff < pageHeaderSize)
+ {
+ report_invalid_record(state, "invalid record offset at %X/%X",
+ LSN_FORMAT_ARGS(state->ReadRecPtr));
+ goto err;
+ }
- /* Check that the continuation on next page looks valid */
pageHeader = (XLogPageHeader) state->readBuf;
- if (!(pageHeader->xlp_info & XLP_FIRST_IS_CONTRECORD))
+ if ((pageHeader->xlp_info & XLP_FIRST_IS_CONTRECORD) &&
+ targetRecOff == pageHeaderSize)
{
- report_invalid_record(state,
- "there is no contrecord flag at %X/%X",
- LSN_FORMAT_ARGS(RecPtr));
+ report_invalid_record(state, "contrecord is requested by %X/%X",
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr);
goto err;
}
- /*
- * Cross-check that xlp_rem_len agrees with how much of the record
- * we expect there to be left.
- */
- if (pageHeader->xlp_rem_len == 0 ||
- total_len != (pageHeader->xlp_rem_len + gotlen))
- {
- report_invalid_record(state,
- "invalid contrecord length %u (expected %lld) at %X/%X",
- pageHeader->xlp_rem_len,
- ((long long) total_len) - gotlen,
- LSN_FORMAT_ARGS(RecPtr));
- goto err;
- }
-
- /* Append the continuation from this page to the buffer */
- pageHeaderSize = XLogPageHeaderSize(pageHeader);
-
+ /* XLogNeedData has verified the page header */
Assert(pageHeaderSize <= state->readLen);
- contdata = (char *) state->readBuf + pageHeaderSize;
- len = XLOG_BLCKSZ - pageHeaderSize;
- if (pageHeader->xlp_rem_len < len)
- len = pageHeader->xlp_rem_len;
+ /*
+ * Read the record length.
+ *
+ * NB: Even though we use an XLogRecord pointer here, the whole
+ * record header might not fit on this page. xl_tot_len is the first
+ * field of the struct, so it must be on this page (the records are
+ * MAXALIGNed), but we cannot access any other fields until we've
+ * verified that we got the whole header.
+ */
+ prec = (XLogRecord *) (state->readBuf +
+ state->ReadRecPtr % XLOG_BLCKSZ);
+ total_len = prec->xl_tot_len;
- Assert (pageHeaderSize + len <= state->readLen);
- memcpy(buffer, (char *) contdata, len);
- buffer += len;
- gotlen += len;
+ /*
+ * If the whole record header is on this page, validate it
+ * immediately. Otherwise do just a basic sanity check on
+ * xl_tot_len, and validate the rest of the header after reading it
+ * from the next page. The xl_tot_len check is necessary here to
+ * ensure that we enter the XLREAD_CONTINUATION state below;
+ * otherwise we might fail to apply ValidXLogRecordHeader at all.
+ */
+ if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
+ {
+ if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
+ state->PrevRecPtr, prec))
+ goto err;
- /* If we just reassembled the record header, validate it. */
- if (!gotheader)
+ state->record_verified = true;
+ }
+ else
{
- record = (XLogRecord *) state->readRecordBuf;
- if (!ValidXLogRecordHeader(state, RecPtr, state->ReadRecPtr,
- record, randAccess))
+ /* XXX: more validation should be done here */
+ if (total_len < SizeOfXLogRecord)
+ {
+ report_invalid_record(state,
+ "invalid record length at %X/%X: wanted %u, got %u",
+ LSN_FORMAT_ARGS(state->ReadRecPtr),
+ (uint32) SizeOfXLogRecord, total_len);
goto err;
- gotheader = true;
+ }
}
- } while (gotlen < total_len);
-
- Assert(gotheader);
- record = (XLogRecord *) state->readRecordBuf;
- if (!ValidXLogRecord(state, record, RecPtr))
- goto err;
+ /*
+ * Wait for the rest of the record, or the part of the record that
+ * fit on the first page if crossed a page boundary, to become
+ * available.
+ */
+ state->recordGotLen = 0;
+ state->recordRemainLen = total_len;
+ state->readRecordState = XLREAD_FIRST_FRAGMENT;
+ }
+ /* fall through */
- pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
- state->ReadRecPtr = RecPtr;
- state->EndRecPtr = targetPagePtr + pageHeaderSize
- + MAXALIGN(pageHeader->xlp_rem_len);
- }
- else
- {
- /* Wait for the record data to become available */
- while (XLogNeedData(state, targetPagePtr,
- Min(targetRecOff + total_len, XLOG_BLCKSZ), true))
+ case XLREAD_FIRST_FRAGMENT:
{
- if (!state->routine.page_read(state, state->readPagePtr,
- state->readLen,
- state->ReadRecPtr, state->readBuf))
+ uint32 total_len = state->recordRemainLen;
+ uint32 request_len;
+ uint32 record_len;
+ XLogRecPtr targetPagePtr;
+ uint32 targetRecOff;
+
+ /*
+ * Wait for the rest of the record on the first page to become
+ * available
+ */
+ targetPagePtr =
+ state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
+ targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+
+ request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ);
+ record_len = request_len - targetRecOff;
+
+ /* ReadRecPtr contains page header */
+ Assert (targetRecOff != 0);
+ if (XLogNeedData(state, targetPagePtr, request_len, true))
+ return XLREAD_NEED_DATA;
+
+ /* error out if caller supplied bogus page */
+ if (!state->page_verified)
+ goto err;
+
+ prec = (XLogRecord *) (state->readBuf + targetRecOff);
+
+ /* validate record header if not yet */
+ if (!state->record_verified && record_len >= SizeOfXLogRecord)
+ {
+ if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
+ state->PrevRecPtr, prec))
+ goto err;
+
+ state->record_verified = true;
+ }
+
+
+ if (total_len == record_len)
+ {
+ /* Record does not cross a page boundary */
+ Assert(state->record_verified);
+
+ if (!ValidXLogRecord(state, prec, state->ReadRecPtr))
+ goto err;
+
+ state->record_verified = true; /* to be tidy */
+
+ /* We already checked the header earlier */
+ state->EndRecPtr = state->ReadRecPtr + MAXALIGN(record_len);
+
+ *record = prec;
+ state->readRecordState = XLREAD_NEXT_RECORD;
break;
+ }
+
+ /*
+ * The record continues on the next page. Need to reassemble
+ * record
+ */
+ Assert(total_len > record_len);
+
+ /* Enlarge readRecordBuf as needed. */
+ if (total_len > state->readRecordBufSize &&
+ !allocate_recordbuf(state, total_len))
+ {
+ /* We treat this as a "bogus data" condition */
+ report_invalid_record(state,
+ "record length %u at %X/%X too long",
+ total_len,
+ LSN_FORMAT_ARGS(state->ReadRecPtr));
+ goto err;
+ }
+
+ /* Copy the first fragment of the record from the first page. */
+ memcpy(state->readRecordBuf, state->readBuf + targetRecOff,
+ record_len);
+ state->recordGotLen += record_len;
+ state->recordRemainLen -= record_len;
+
+ /* Calculate pointer to beginning of next page */
+ state->recordContRecPtr = state->ReadRecPtr + record_len;
+ Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0);
+
+ state->readRecordState = XLREAD_CONTINUATION;
}
+ /* fall through */
+
+ case XLREAD_CONTINUATION:
+ {
+ XLogPageHeader pageHeader;
+ uint32 pageHeaderSize;
+ XLogRecPtr targetPagePtr;
+
+ /* we enter this state only if we haven't read the whole record. */
+ Assert (state->recordRemainLen > 0);
+
+ while(state->recordRemainLen > 0)
+ {
+ char *contdata;
+ uint32 request_len;
+ uint32 record_len;
+
+ /* Wait for the next page to become available */
+ targetPagePtr = state->recordContRecPtr;
+
+ /* this request contains page header */
+ Assert (targetPagePtr != 0);
+ if (XLogNeedData(state, targetPagePtr,
+ Min(state->recordRemainLen, XLOG_BLCKSZ),
+ false))
+ return XLREAD_NEED_DATA;
+
+ if (!state->page_verified)
+ goto err;
+
+ Assert(SizeOfXLogShortPHD <= state->readLen);
- if (!state->page_verified)
- goto err;
+ /* Check that the continuation on next page looks valid */
+ pageHeader = (XLogPageHeader) state->readBuf;
+ if (!(pageHeader->xlp_info & XLP_FIRST_IS_CONTRECORD))
+ {
+ report_invalid_record(
+ state,
+ "there is no contrecord flag at %X/%X reading %X/%X",
+ (uint32) (state->recordContRecPtr >> 32),
+ (uint32) state->recordContRecPtr,
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr);
+ goto err;
+ }
- /* Record does not cross a page boundary */
- if (!ValidXLogRecord(state, record, RecPtr))
- goto err;
+ /*
+ * Cross-check that xlp_rem_len agrees with how much of the
+ * record we expect there to be left.
+ */
+ if (pageHeader->xlp_rem_len == 0 ||
+ pageHeader->xlp_rem_len != state->recordRemainLen)
+ {
+ report_invalid_record(
+ state,
+ "invalid contrecord length %u at %X/%X reading %X/%X, expected %u",
+ pageHeader->xlp_rem_len,
+ (uint32) (state->recordContRecPtr >> 32),
+ (uint32) state->recordContRecPtr,
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr,
+ state->recordRemainLen);
+ goto err;
+ }
- state->EndRecPtr = RecPtr + MAXALIGN(total_len);
+ /* Append the continuation from this page to the buffer */
+ pageHeaderSize = XLogPageHeaderSize(pageHeader);
- state->ReadRecPtr = RecPtr;
+ /*
+ * XLogNeedData should have ensured that the whole page header
+ * was read
+ */
+ Assert(state->readLen >= pageHeaderSize);
+
+ contdata = (char *) state->readBuf + pageHeaderSize;
+ record_len = XLOG_BLCKSZ - pageHeaderSize;
+ if (pageHeader->xlp_rem_len < record_len)
+ record_len = pageHeader->xlp_rem_len;
+
+ request_len = record_len + pageHeaderSize;
+
+ /* XLogNeedData should have ensured all needed data was read */
+ Assert (state->readLen >= request_len);
+
+ memcpy(state->readRecordBuf + state->recordGotLen,
+ (char *) contdata, record_len);
+ state->recordGotLen += record_len;
+ state->recordRemainLen -= record_len;
+
+ /* If we just reassembled the record header, validate it. */
+ if (!state->record_verified)
+ {
+ Assert(state->recordGotLen >= SizeOfXLogRecord);
+ if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
+ state->PrevRecPtr,
+ (XLogRecord *) state->readRecordBuf))
+ goto err;
+
+ state->record_verified = true;
+ }
+
+ /* Calculate pointer to beginning of next page, and continue */
+ state->recordContRecPtr += XLOG_BLCKSZ;
+ }
+
+ /* targetPagePtr is pointing the last-read page here */
+ prec = (XLogRecord *) state->readRecordBuf;
+ if (!ValidXLogRecord(state, prec, state->ReadRecPtr))
+ goto err;
+
+ pageHeaderSize =
+ XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+ state->EndRecPtr = targetPagePtr + pageHeaderSize
+ + MAXALIGN(pageHeader->xlp_rem_len);
+
+ *record = prec;
+ state->readRecordState = XLREAD_NEXT_RECORD;
+ break;
+ }
}
/*
* Special processing if it's an XLOG SWITCH record
*/
- if (record->xl_rmid == RM_XLOG_ID &&
- (record->xl_info & ~XLR_INFO_MASK) == XLOG_SWITCH)
+ if ((*record)->xl_rmid == RM_XLOG_ID &&
+ ((*record)->xl_info & ~XLR_INFO_MASK) == XLOG_SWITCH)
{
/* Pretend it extends to end of segment */
state->EndRecPtr += state->segcxt.ws_segsize - 1;
state->EndRecPtr -= XLogSegmentOffset(state->EndRecPtr, state->segcxt.ws_segsize);
}
- if (DecodeXLogRecord(state, record, errormsg))
- return record;
- else
- return NULL;
+ Assert (!*record || state->readLen >= 0);
+ if (DecodeXLogRecord(state, *record, errormsg))
+ return XLREAD_SUCCESS;
+
+ *record = NULL;
+ return XLREAD_FAIL;
err:
/*
- * Invalidate the read state. We might read from a different source after
+ * Invalidate the read page. We might read from a different source after
* failure.
*/
XLogReaderInvalReadState(state);
@@ -572,7 +703,8 @@ err:
if (state->errormsg_buf[0] != '\0')
*errormsg = state->errormsg_buf;
- return NULL;
+ *record = NULL;
+ return XLREAD_FAIL;
}
/*
@@ -724,11 +856,12 @@ XLogReaderInvalReadState(XLogReaderState *state)
*
* This is just a convenience subroutine to avoid duplicated code in
* XLogReadRecord. It's not intended for use from anywhere else.
+ *
+ * If PrevRecPtr is valid, the xl_prev is is cross-checked with it.
*/
static bool
ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
- XLogRecPtr PrevRecPtr, XLogRecord *record,
- bool randAccess)
+ XLogRecPtr PrevRecPtr, XLogRecord *record)
{
if (record->xl_tot_len < SizeOfXLogRecord)
{
@@ -745,7 +878,7 @@ ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
record->xl_rmid, LSN_FORMAT_ARGS(RecPtr));
return false;
}
- if (randAccess)
+ if (PrevRecPtr == InvalidXLogRecPtr)
{
/*
* We can't exactly verify the prev-link, but surely it should be less
@@ -975,11 +1108,14 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
* XLogReadRecord() will read the next valid record.
*/
XLogRecPtr
-XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
+XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
+ XLogFindNextRecordCB read_page, void *private)
{
XLogRecPtr tmpRecPtr;
XLogRecPtr found = InvalidXLogRecPtr;
XLogPageHeader header;
+ XLogRecord *record;
+ XLogReadRecordResult result;
char *errormsg;
Assert(!XLogRecPtrIsInvalid(RecPtr));
@@ -1012,9 +1148,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
while(XLogNeedData(state, targetPagePtr, targetRecOff,
targetRecOff != 0))
{
- if (!state->routine.page_read(state, state->readPagePtr,
- state->readLen,
- state->ReadRecPtr, state->readBuf))
+ if (!read_page(state, private))
break;
}
@@ -1066,8 +1200,16 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
* or we just jumped over the remaining data of a continuation.
*/
XLogBeginRead(state, tmpRecPtr);
- while (XLogReadRecord(state, &errormsg) != NULL)
+ while ((result = XLogReadRecord(state, &record, &errormsg)) !=
+ XLREAD_FAIL)
{
+ if (result == XLREAD_NEED_DATA)
+ {
+ if (!read_page(state, private))
+ goto err;
+ continue;
+ }
+
/* past the record we've found, break out */
if (RecPtr <= state->ReadRecPtr)
{
@@ -1087,9 +1229,9 @@ err:
#endif /* FRONTEND */
/*
- * Helper function to ease writing of XLogRoutine->page_read callbacks.
- * If this function is used, caller must supply a segment_open callback in
- * 'state', as that is used here.
+ * Helper function to ease writing of page_read callback.
+ * If this function is used, caller must supply a segment_open callback and
+ * segment_close callback as that is used here.
*
* Read 'count' bytes into 'buf', starting at location 'startptr', from WAL
* fetched from timeline 'tli'.
@@ -1102,6 +1244,7 @@ err:
*/
bool
WALRead(XLogReaderState *state,
+ WALSegmentOpenCB segopenfn, WALSegmentCloseCB segclosefn,
char *buf, XLogRecPtr startptr, Size count, TimeLineID tli,
WALReadError *errinfo)
{
@@ -1133,10 +1276,10 @@ WALRead(XLogReaderState *state,
XLogSegNo nextSegNo;
if (state->seg.ws_file >= 0)
- state->routine.segment_close(state);
+ segclosefn(state);
XLByteToSeg(recptr, nextSegNo, state->segcxt.ws_segsize);
- state->routine.segment_open(state, nextSegNo, &tli);
+ segopenfn(state, nextSegNo, &tli);
/* This shouldn't happen -- indicates a bug in segment_open */
Assert(state->seg.ws_file >= 0);
diff --git a/src/backend/access/transam/xlogutils.c b/src/backend/access/transam/xlogutils.c
index 46eda33f25..b003990745 100644
--- a/src/backend/access/transam/xlogutils.c
+++ b/src/backend/access/transam/xlogutils.c
@@ -686,8 +686,7 @@ XLogTruncateRelation(RelFileNode rnode, ForkNumber forkNum,
void
XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wantLength)
{
- const XLogRecPtr lastReadPage = state->seg.ws_segno *
- state->segcxt.ws_segsize + state->readLen;
+ const XLogRecPtr lastReadPage = state->readPagePtr;
Assert(wantPage != InvalidXLogRecPtr && wantPage % XLOG_BLCKSZ == 0);
Assert(wantLength <= XLOG_BLCKSZ);
@@ -702,7 +701,7 @@ XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wa
* current TLI has since become historical.
*/
if (lastReadPage == wantPage &&
- state->readLen != 0 &&
+ state->page_verified &&
lastReadPage + state->readLen >= wantPage + Min(wantLength, XLOG_BLCKSZ - 1))
return;
@@ -788,6 +787,7 @@ wal_segment_open(XLogReaderState *state, XLogSegNo nextSegNo,
char path[MAXPGPATH];
XLogFilePath(path, tli, nextSegNo, state->segcxt.ws_segsize);
+ elog(LOG, "HOGE: %lu, %d => %s", nextSegNo, tli, path);
state->seg.ws_file = BasicOpenFile(path, O_RDONLY | PG_BINARY);
if (state->seg.ws_file >= 0)
return;
@@ -825,9 +825,11 @@ wal_segment_close(XLogReaderState *state)
* loop for now.
*/
bool
-read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *cur_page)
+read_local_xlog_page(XLogReaderState *state)
{
+ XLogRecPtr targetPagePtr = state->readPagePtr;
+ int reqLen = state->readLen;
+ char *cur_page = state->readBuf;
XLogRecPtr read_upto,
loc;
TimeLineID tli;
@@ -940,11 +942,12 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
* as 'count', read the whole page anyway. It's guaranteed to be
* zero-padded up to the page boundary if it's incomplete.
*/
- if (!WALRead(state, cur_page, targetPagePtr, XLOG_BLCKSZ, tli,
- &errinfo))
+ if (!WALRead(state, wal_segment_open, wal_segment_close,
+ cur_page, targetPagePtr, XLOG_BLCKSZ, tli, &errinfo))
WALReadRaiseError(&errinfo);
/* number of valid bytes in the buffer */
+ state->readPagePtr = targetPagePtr;
state->readLen = count;
return true;
}
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index 37b75deb72..9cbd8afa49 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -148,7 +148,8 @@ StartupDecodingContext(List *output_plugin_options,
TransactionId xmin_horizon,
bool need_full_snapshot,
bool fast_forward,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress)
@@ -198,11 +199,12 @@ StartupDecodingContext(List *output_plugin_options,
ctx->slot = slot;
- ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, xl_routine, ctx);
+ ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, cleanup_cb);
if (!ctx->reader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
errmsg("out of memory")));
+ ctx->page_read = page_read;
ctx->reorder = ReorderBufferAllocate();
ctx->snapshot_builder =
@@ -319,7 +321,8 @@ CreateInitDecodingContext(const char *plugin,
List *output_plugin_options,
bool need_full_snapshot,
XLogRecPtr restart_lsn,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress)
@@ -422,7 +425,7 @@ CreateInitDecodingContext(const char *plugin,
ctx = StartupDecodingContext(NIL, restart_lsn, xmin_horizon,
need_full_snapshot, false,
- xl_routine, prepare_write, do_write,
+ page_read, cleanup_cb, prepare_write, do_write,
update_progress);
/* call output plugin initialization callback */
@@ -476,7 +479,8 @@ LogicalDecodingContext *
CreateDecodingContext(XLogRecPtr start_lsn,
List *output_plugin_options,
bool fast_forward,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress)
@@ -528,8 +532,8 @@ CreateDecodingContext(XLogRecPtr start_lsn,
ctx = StartupDecodingContext(output_plugin_options,
start_lsn, InvalidTransactionId, false,
- fast_forward, xl_routine, prepare_write,
- do_write, update_progress);
+ fast_forward, page_read, cleanup_cb,
+ prepare_write, do_write, update_progress);
/* call output plugin initialization callback */
old_context = MemoryContextSwitchTo(ctx->context);
@@ -585,7 +589,13 @@ DecodingContextFindStartpoint(LogicalDecodingContext *ctx)
char *err = NULL;
/* the read_page callback waits for new WAL */
- record = XLogReadRecord(ctx->reader, &err);
+ while (XLogReadRecord(ctx->reader, &record, &err) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!ctx->page_read(ctx->reader))
+ break;
+ }
+
if (err)
elog(ERROR, "%s", err);
if (!record)
diff --git a/src/backend/replication/logical/logicalfuncs.c b/src/backend/replication/logical/logicalfuncs.c
index f7e055874e..b69f6911c5 100644
--- a/src/backend/replication/logical/logicalfuncs.c
+++ b/src/backend/replication/logical/logicalfuncs.c
@@ -233,9 +233,8 @@ pg_logical_slot_get_changes_guts(FunctionCallInfo fcinfo, bool confirm, bool bin
ctx = CreateDecodingContext(InvalidXLogRecPtr,
options,
false,
- XL_ROUTINE(.page_read = read_local_xlog_page,
- .segment_open = wal_segment_open,
- .segment_close = wal_segment_close),
+ read_local_xlog_page,
+ wal_segment_close,
LogicalOutputPrepareWrite,
LogicalOutputWrite, NULL);
@@ -284,7 +283,13 @@ pg_logical_slot_get_changes_guts(FunctionCallInfo fcinfo, bool confirm, bool bin
XLogRecord *record;
char *errm = NULL;
- record = XLogReadRecord(ctx->reader, &errm);
+ while (XLogReadRecord(ctx->reader, &record, &errm) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!ctx->page_read(ctx->reader))
+ break;
+ }
+
if (errm)
elog(ERROR, "%s", errm);
diff --git a/src/backend/replication/slotfuncs.c b/src/backend/replication/slotfuncs.c
index 9817b44113..deb7cf15eb 100644
--- a/src/backend/replication/slotfuncs.c
+++ b/src/backend/replication/slotfuncs.c
@@ -153,9 +153,8 @@ create_logical_replication_slot(char *name, char *plugin,
ctx = CreateInitDecodingContext(plugin, NIL,
false, /* just catalogs is OK */
restart_lsn,
- XL_ROUTINE(.page_read = read_local_xlog_page,
- .segment_open = wal_segment_open,
- .segment_close = wal_segment_close),
+ read_local_xlog_page,
+ wal_segment_close,
NULL, NULL, NULL);
/*
@@ -512,9 +511,8 @@ pg_logical_replication_slot_advance(XLogRecPtr moveto)
ctx = CreateDecodingContext(InvalidXLogRecPtr,
NIL,
true, /* fast_forward */
- XL_ROUTINE(.page_read = read_local_xlog_page,
- .segment_open = wal_segment_open,
- .segment_close = wal_segment_close),
+ read_local_xlog_page,
+ wal_segment_close,
NULL, NULL, NULL);
/*
@@ -536,7 +534,13 @@ pg_logical_replication_slot_advance(XLogRecPtr moveto)
* Read records. No changes are generated in fast_forward mode,
* but snapbuilder/slot statuses are updated properly.
*/
- record = XLogReadRecord(ctx->reader, &errm);
+ while (XLogReadRecord(ctx->reader, &record, &errm) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!ctx->page_read(ctx->reader))
+ break;
+ }
+
if (errm)
elog(ERROR, "%s", errm);
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index c36b91ae5e..062093727d 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -580,10 +580,7 @@ StartReplication(StartReplicationCmd *cmd)
/* create xlogreader for physical replication */
xlogreader =
- XLogReaderAllocate(wal_segment_size, NULL,
- XL_ROUTINE(.segment_open = WalSndSegmentOpen,
- .segment_close = wal_segment_close),
- NULL);
+ XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
if (!xlogreader)
ereport(ERROR,
@@ -807,9 +804,11 @@ StartReplication(StartReplicationCmd *cmd)
* set every time WAL is flushed.
*/
static bool
-logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetRecPtr, char *cur_page)
+logical_read_xlog_page(XLogReaderState *state)
{
+ XLogRecPtr targetPagePtr = state->readPagePtr;
+ int reqLen = state->readLen;
+ char *cur_page = state->readBuf;
XLogRecPtr flushptr;
int count;
WALReadError errinfo;
@@ -837,7 +836,7 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
count = flushptr - targetPagePtr; /* part of the page available */
/* now actually read the data, we know it's there */
- if (!WALRead(state,
+ if (!WALRead(state, WalSndSegmentOpen, wal_segment_close,
cur_page,
targetPagePtr,
XLOG_BLCKSZ,
@@ -1011,9 +1010,8 @@ CreateReplicationSlot(CreateReplicationSlotCmd *cmd)
ctx = CreateInitDecodingContext(cmd->plugin, NIL, need_full_snapshot,
InvalidXLogRecPtr,
- XL_ROUTINE(.page_read = logical_read_xlog_page,
- .segment_open = WalSndSegmentOpen,
- .segment_close = wal_segment_close),
+ logical_read_xlog_page,
+ wal_segment_close,
WalSndPrepareWrite, WalSndWriteData,
WalSndUpdateProgress);
@@ -1171,9 +1169,8 @@ StartLogicalReplication(StartReplicationCmd *cmd)
*/
logical_decoding_ctx =
CreateDecodingContext(cmd->startpoint, cmd->options, false,
- XL_ROUTINE(.page_read = logical_read_xlog_page,
- .segment_open = WalSndSegmentOpen,
- .segment_close = wal_segment_close),
+ logical_read_xlog_page,
+ wal_segment_close,
WalSndPrepareWrite, WalSndWriteData,
WalSndUpdateProgress);
xlogreader = logical_decoding_ctx->reader;
@@ -2749,7 +2746,7 @@ XLogSendPhysical(void)
enlargeStringInfo(&output_message, nbytes);
retry:
- if (!WALRead(xlogreader,
+ if (!WALRead(xlogreader, WalSndSegmentOpen, wal_segment_close,
&output_message.data[output_message.len],
startptr,
nbytes,
@@ -2847,7 +2844,12 @@ XLogSendLogical(void)
*/
WalSndCaughtUp = false;
- record = XLogReadRecord(logical_decoding_ctx->reader, &errm);
+ while (XLogReadRecord(logical_decoding_ctx->reader, &record, &errm) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!logical_decoding_ctx->page_read(logical_decoding_ctx->reader))
+ break;
+ }
/* xlog record was invalid */
if (errm != NULL)
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index cf119848b0..712c85281c 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -41,15 +41,9 @@ static int xlogreadfd = -1;
static XLogSegNo xlogreadsegno = -1;
static char xlogfpath[MAXPGPATH];
-typedef struct XLogPageReadPrivate
-{
- const char *restoreCommand;
- int tliIndex;
-} XLogPageReadPrivate;
-
-static bool SimpleXLogPageRead(XLogReaderState *xlogreader,
- XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
+static bool SimpleXLogPageRead(XLogReaderState *xlogreader,
+ const char *datadir, int *tliIndex,
+ const char *restoreCommand);
/*
* Read WAL from the datadir/pg_wal, starting from 'startpoint' on timeline
@@ -66,20 +60,22 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
XLogRecord *record;
XLogReaderState *xlogreader;
char *errormsg;
- XLogPageReadPrivate private;
- private.tliIndex = tliIndex;
- private.restoreCommand = restoreCommand;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir,
- XL_ROUTINE(.page_read = &SimpleXLogPageRead),
- &private);
+ xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+
if (xlogreader == NULL)
pg_fatal("out of memory");
XLogBeginRead(xlogreader, startpoint);
do
{
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!SimpleXLogPageRead(xlogreader, datadir,
+ &tliIndex, restoreCommand))
+ break;
+ }
if (record == NULL)
{
@@ -123,19 +119,19 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex,
XLogRecord *record;
XLogReaderState *xlogreader;
char *errormsg;
- XLogPageReadPrivate private;
XLogRecPtr endptr;
- private.tliIndex = tliIndex;
- private.restoreCommand = restoreCommand;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir,
- XL_ROUTINE(.page_read = &SimpleXLogPageRead),
- &private);
+ xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
XLogBeginRead(xlogreader, ptr);
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!SimpleXLogPageRead(xlogreader, datadir, &tliIndex, restoreCommand))
+ break;
+ }
if (record == NULL)
{
if (errormsg)
@@ -170,7 +166,6 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
XLogRecPtr searchptr;
XLogReaderState *xlogreader;
char *errormsg;
- XLogPageReadPrivate private;
/*
* The given fork pointer points to the end of the last common record,
@@ -186,11 +181,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
forkptr += SizeOfXLogShortPHD;
}
- private.tliIndex = tliIndex;
- private.restoreCommand = restoreCommand;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir,
- XL_ROUTINE(.page_read = &SimpleXLogPageRead),
- &private);
+ xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
@@ -200,7 +191,13 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
uint8 info;
XLogBeginRead(xlogreader, searchptr);
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!SimpleXLogPageRead(xlogreader, datadir,
+ &tliIndex, restoreCommand))
+ break;
+ }
if (record == NULL)
{
@@ -247,10 +244,11 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
/* XLogReader callback function, to read a WAL page */
static bool
-SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *readBuf)
+SimpleXLogPageRead(XLogReaderState *xlogreader, const char *datadir,
+ int *tliIndex, const char *restoreCommand)
{
- XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
+ XLogRecPtr targetPagePtr = xlogreader->readPagePtr;
+ char *readBuf = xlogreader->readBuf;
uint32 targetPageOff;
XLogRecPtr targetSegEnd;
XLogSegNo targetSegNo;
@@ -283,14 +281,14 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
* be done both forward and backward, consider also switching timeline
* accordingly.
*/
- while (private->tliIndex < targetNentries - 1 &&
- targetHistory[private->tliIndex].end < targetSegEnd)
- private->tliIndex++;
- while (private->tliIndex > 0 &&
- targetHistory[private->tliIndex].begin >= targetSegEnd)
- private->tliIndex--;
+ while (*tliIndex < targetNentries - 1 &&
+ targetHistory[*tliIndex].end < targetSegEnd)
+ (*tliIndex)++;
+ while (*tliIndex > 0 &&
+ targetHistory[*tliIndex].begin >= targetSegEnd)
+ (*tliIndex)--;
- XLogFileName(xlogfname, targetHistory[private->tliIndex].tli,
+ XLogFileName(xlogfname, targetHistory[*tliIndex].tli,
xlogreadsegno, WalSegSz);
snprintf(xlogfpath, MAXPGPATH, "%s/" XLOGDIR "/%s",
@@ -303,7 +301,7 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
/*
* If we have no restore_command to execute, then exit.
*/
- if (private->restoreCommand == NULL)
+ if (restoreCommand == NULL)
{
pg_log_error("could not open file \"%s\": %m", xlogfpath);
xlogreader->readLen = -1;
@@ -317,7 +315,7 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
xlogreadfd = RestoreArchivedFile(xlogreader->segcxt.ws_dir,
xlogfname,
WalSegSz,
- private->restoreCommand);
+ restoreCommand);
if (xlogreadfd < 0)
{
@@ -359,7 +357,7 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
Assert(targetSegNo == xlogreadsegno);
- xlogreader->seg.ws_tli = targetHistory[private->tliIndex].tli;
+ xlogreader->seg.ws_tli = targetHistory[*tliIndex].tli;
xlogreader->readLen = XLOG_BLCKSZ;
return true;
}
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index 0f0e0723a3..f514de6b13 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -330,12 +330,17 @@ WALDumpCloseSegment(XLogReaderState *state)
state->seg.ws_file = -1;
}
-/* pg_waldump's XLogReaderRoutine->page_read callback */
+/*
+ * pg_waldump's WAL page rader, also used as page_read callback for
+ * XLogFindNextRecord
+ */
static bool
-WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetPtr, char *readBuff)
+WALDumpReadPage(XLogReaderState *state, void *priv)
{
- XLogDumpPrivate *private = state->private_data;
+ XLogRecPtr targetPagePtr = state->readPagePtr;
+ int reqLen = state->readLen;
+ char *readBuff = state->readBuf;
+ XLogDumpPrivate *private = (XLogDumpPrivate *) priv;
int count = XLOG_BLCKSZ;
WALReadError errinfo;
@@ -353,8 +358,8 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
}
}
- if (!WALRead(state, readBuff, targetPagePtr, count, private->timeline,
- &errinfo))
+ if (!WALRead(state, WALDumpOpenSegment, WALDumpCloseSegment,
+ readBuff, targetPagePtr, count, private->timeline, &errinfo))
{
WALOpenSegment *seg = &errinfo.wre_seg;
char fname[MAXPGPATH];
@@ -374,6 +379,7 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
(Size) errinfo.wre_req);
}
+ Assert(count >= state->readLen);
state->readLen = count;
return true;
}
@@ -1035,16 +1041,14 @@ main(int argc, char **argv)
/* we have everything we need, start reading */
xlogreader_state =
- XLogReaderAllocate(WalSegSz, waldir,
- XL_ROUTINE(.page_read = WALDumpReadPage,
- .segment_open = WALDumpOpenSegment,
- .segment_close = WALDumpCloseSegment),
- &private);
+ XLogReaderAllocate(WalSegSz, waldir, WALDumpCloseSegment);
+
if (!xlogreader_state)
fatal_error("out of memory");
/* first find a valid recptr to start from */
- first_record = XLogFindNextRecord(xlogreader_state, private.startptr);
+ first_record = XLogFindNextRecord(xlogreader_state, private.startptr,
+ &WALDumpReadPage, (void*) &private);
if (first_record == InvalidXLogRecPtr)
fatal_error("could not find a valid record after %X/%X",
@@ -1067,7 +1071,13 @@ main(int argc, char **argv)
for (;;)
{
/* try to read the next record */
- record = XLogReadRecord(xlogreader_state, &errormsg);
+ while (XLogReadRecord(xlogreader_state, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!WALDumpReadPage(xlogreader_state, (void *) &private))
+ break;
+ }
+
if (!record)
{
if (!config.follow || private.endptr_reached)
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index 23d9e70a59..4e9268b553 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -57,64 +57,15 @@ typedef struct WALSegmentContext
typedef struct XLogReaderState XLogReaderState;
-/* Function type definition for the read_page callback */
-typedef bool (*XLogPageReadCB) (XLogReaderState *xlogreader,
- XLogRecPtr targetPagePtr,
- int reqLen,
- XLogRecPtr targetRecPtr,
- char *readBuf);
+/* Function type definition for the segment cleanup callback */
+typedef void (*WALSegmentCleanupCB) (XLogReaderState *xlogreader);
+
+/* Function type definition for the open/close callbacks for WALRead() */
typedef void (*WALSegmentOpenCB) (XLogReaderState *xlogreader,
XLogSegNo nextSegNo,
TimeLineID *tli_p);
typedef void (*WALSegmentCloseCB) (XLogReaderState *xlogreader);
-typedef struct XLogReaderRoutine
-{
- /*
- * Data input callback
- *
- * This callback shall read at least reqLen valid bytes of the xlog page
- * starting at targetPagePtr, and store them in readBuf. The callback
- * shall return the number of bytes read (never more than XLOG_BLCKSZ), or
- * -1 on failure. The callback shall sleep, if necessary, to wait for the
- * requested bytes to become available. The callback will not be invoked
- * again for the same page unless more than the returned number of bytes
- * are needed.
- *
- * targetRecPtr is the position of the WAL record we're reading. Usually
- * it is equal to targetPagePtr + reqLen, but sometimes xlogreader needs
- * to read and verify the page or segment header, before it reads the
- * actual WAL record it's interested in. In that case, targetRecPtr can
- * be used to determine which timeline to read the page from.
- *
- * The callback shall set ->seg.ws_tli to the TLI of the file the page was
- * read from.
- */
- XLogPageReadCB page_read;
-
- /*
- * Callback to open the specified WAL segment for reading. ->seg.ws_file
- * shall be set to the file descriptor of the opened segment. In case of
- * failure, an error shall be raised by the callback and it shall not
- * return.
- *
- * "nextSegNo" is the number of the segment to be opened.
- *
- * "tli_p" is an input/output argument. WALRead() uses it to pass the
- * timeline in which the new segment should be found, but the callback can
- * use it to return the TLI that it actually opened.
- */
- WALSegmentOpenCB segment_open;
-
- /*
- * WAL segment close callback. ->seg.ws_file shall be set to a negative
- * number.
- */
- WALSegmentCloseCB segment_close;
-} XLogReaderRoutine;
-
-#define XL_ROUTINE(...) &(XLogReaderRoutine){__VA_ARGS__}
-
typedef struct
{
/* Is this block ref in use? */
@@ -144,12 +95,35 @@ typedef struct
uint16 data_bufsz;
} DecodedBkpBlock;
+/* Return code from XLogReadRecord */
+typedef enum XLogReadRecordResult
+{
+ XLREAD_SUCCESS, /* record is successfully read */
+ XLREAD_NEED_DATA, /* need more data. see XLogReadRecord. */
+ XLREAD_FAIL /* failed during reading a record */
+} XLogReadRecordResult;
+
+/*
+ * internal state of XLogReadRecord
+ *
+ * XLogReadState runs a state machine while reading a record. Theses states
+ * are not seen outside the function. Each state may repeat several times
+ * exiting requesting caller for new data. See the comment of XLogReadRecrod
+ * for details.
+ */
+typedef enum XLogReadRecordState {
+ XLREAD_NEXT_RECORD,
+ XLREAD_TOT_LEN,
+ XLREAD_FIRST_FRAGMENT,
+ XLREAD_CONTINUATION
+} XLogReadRecordState;
+
struct XLogReaderState
{
/*
* Operational callbacks
*/
- XLogReaderRoutine routine;
+ WALSegmentCleanupCB cleanup_cb;
/* ----------------------------------------
* Public parameters
@@ -162,18 +136,14 @@ struct XLogReaderState
*/
uint64 system_identifier;
- /*
- * Opaque data for callbacks to use. Not used by XLogReader.
- */
- void *private_data;
-
/*
* Start and end point of last record read. EndRecPtr is also used as the
* position to read next. Calling XLogBeginRead() sets EndRecPtr to the
* starting position and ReadRecPtr to invalid.
*/
- XLogRecPtr ReadRecPtr; /* start of last record read */
+ XLogRecPtr ReadRecPtr; /* start of last record read or being read */
XLogRecPtr EndRecPtr; /* end+1 of last record read */
+ XLogRecPtr PrevRecPtr; /* start of previous record read */
/* ----------------------------------------
* Communication with page reader
@@ -187,7 +157,9 @@ struct XLogReaderState
* the request, or -1 on error */
TimeLineID readPageTLI; /* TLI for data currently in readBuf */
char *readBuf; /* buffer to store data */
- bool page_verified; /* is the page on the buffer verified? */
+ bool page_verified; /* is the page header on the buffer verified? */
+ bool record_verified;/* is the current record header verified? */
+
/* ----------------------------------------
@@ -229,8 +201,6 @@ struct XLogReaderState
XLogRecPtr latestPagePtr;
TimeLineID latestPageTLI;
- /* beginning of the WAL record being read. */
- XLogRecPtr currRecPtr;
/* timeline to read it from, 0 if a lookup is required */
TimeLineID currTLI;
@@ -257,6 +227,15 @@ struct XLogReaderState
char *readRecordBuf;
uint32 readRecordBufSize;
+ /*
+ * XLogReadRecord() state
+ */
+ XLogReadRecordState readRecordState;/* state machine state */
+ int recordGotLen; /* amount of current record that has
+ * already been read */
+ int recordRemainLen; /* length of current record that remains */
+ XLogRecPtr recordContRecPtr; /* where the current record continues */
+
/* Buffer to hold error message */
char *errormsg_buf;
};
@@ -264,9 +243,7 @@ struct XLogReaderState
/* Get a new XLogReader */
extern XLogReaderState *XLogReaderAllocate(int wal_segment_size,
const char *waldir,
- XLogReaderRoutine *routine,
- void *private_data);
-extern XLogReaderRoutine *LocalXLogReaderRoutine(void);
+ WALSegmentCleanupCB cleanup_cb);
/* Free an XLogReader */
extern void XLogReaderFree(XLogReaderState *state);
@@ -274,12 +251,17 @@ extern void XLogReaderFree(XLogReaderState *state);
/* Position the XLogReader to given record */
extern void XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr);
#ifdef FRONTEND
-extern XLogRecPtr XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr);
+/* Function type definition for the read_page callback */
+typedef bool (*XLogFindNextRecordCB) (XLogReaderState *xlogreader,
+ void *private);
+extern XLogRecPtr XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
+ XLogFindNextRecordCB read_page, void *private);
#endif /* FRONTEND */
/* Read the next XLog record. Returns NULL on end-of-WAL or failure */
-extern struct XLogRecord *XLogReadRecord(XLogReaderState *state,
- char **errormsg);
+extern XLogReadRecordResult XLogReadRecord(XLogReaderState *state,
+ XLogRecord **record,
+ char **errormsg);
/* Validate a page */
extern bool XLogReaderValidatePageHeader(XLogReaderState *state,
@@ -299,6 +281,7 @@ typedef struct WALReadError
} WALReadError;
extern bool WALRead(XLogReaderState *state,
+ WALSegmentOpenCB segopenfn, WALSegmentCloseCB sgclosefn,
char *buf, XLogRecPtr startptr, Size count,
TimeLineID tli, WALReadError *errinfo);
diff --git a/src/include/access/xlogutils.h b/src/include/access/xlogutils.h
index 364a21c4ea..397fb27fc2 100644
--- a/src/include/access/xlogutils.h
+++ b/src/include/access/xlogutils.h
@@ -47,9 +47,7 @@ extern Buffer XLogReadBufferExtended(RelFileNode rnode, ForkNumber forknum,
extern Relation CreateFakeRelcacheEntry(RelFileNode rnode);
extern void FreeFakeRelcacheEntry(Relation fakerel);
-extern bool read_local_xlog_page(XLogReaderState *state,
- XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetRecPtr, char *cur_page);
+extern bool read_local_xlog_page(XLogReaderState *state);
extern void wal_segment_open(XLogReaderState *state,
XLogSegNo nextSegNo,
TimeLineID *tli_p);
diff --git a/src/include/pg_config_manual.h b/src/include/pg_config_manual.h
index f10ad0acd6..94749609b4 100644
--- a/src/include/pg_config_manual.h
+++ b/src/include/pg_config_manual.h
@@ -377,7 +377,7 @@
* Enable debugging print statements for WAL-related operations; see
* also the wal_debug GUC var.
*/
-/* #define WAL_DEBUG */
+#define WAL_DEBUG
/*
* Enable tracing of resource consumption during sort operations;
diff --git a/src/include/replication/logical.h b/src/include/replication/logical.h
index c253403372..b936f31393 100644
--- a/src/include/replication/logical.h
+++ b/src/include/replication/logical.h
@@ -29,6 +29,10 @@ typedef void (*LogicalOutputPluginWriterUpdateProgress) (struct LogicalDecodingC
TransactionId xid
);
+typedef struct LogicalDecodingContext LogicalDecodingContext;
+
+typedef bool (*LogicalDecodingXLogPageReadCB)(XLogReaderState *ctx);
+
typedef struct LogicalDecodingContext
{
/* memory context this is all allocated in */
@@ -39,6 +43,7 @@ typedef struct LogicalDecodingContext
/* infrastructure pieces for decoding */
XLogReaderState *reader;
+ LogicalDecodingXLogPageReadCB page_read;
struct ReorderBuffer *reorder;
struct SnapBuild *snapshot_builder;
@@ -105,14 +110,16 @@ extern LogicalDecodingContext *CreateInitDecodingContext(const char *plugin,
List *output_plugin_options,
bool need_full_snapshot,
XLogRecPtr restart_lsn,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress);
extern LogicalDecodingContext *CreateDecodingContext(XLogRecPtr start_lsn,
List *output_plugin_options,
bool fast_forward,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress);
--
2.27.0
----Next_Part(Thu_Mar__4_11_28_53_2021_014)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
filename="v17-0003-Remove-globals-readOff-readLen-and-readSegNo.patch"
^ permalink raw reply [nested|flat] 70+ messages in thread
* [PATCH v17 2/4] Move page-reader out of XLogReadRecord
@ 2019-09-10 03:58 Kyotaro Horiguchi <[email protected]>
0 siblings, 0 replies; 70+ messages in thread
From: Kyotaro Horiguchi @ 2019-09-10 03:58 UTC (permalink / raw)
This is the second step of removing callbacks from WAL record reader.
Since it is essential to take in additional data while reading a
record, the function have to ask caller for new data while keeping
working state. Thus the function is turned into a state machine.
---
src/backend/access/transam/twophase.c | 15 +-
src/backend/access/transam/xlog.c | 58 +-
src/backend/access/transam/xlogreader.c | 697 +++++++++++-------
src/backend/access/transam/xlogutils.c | 17 +-
src/backend/replication/logical/logical.c | 26 +-
.../replication/logical/logicalfuncs.c | 13 +-
src/backend/replication/slotfuncs.c | 18 +-
src/backend/replication/walsender.c | 32 +-
src/bin/pg_rewind/parsexlog.c | 84 ++-
src/bin/pg_waldump/pg_waldump.c | 36 +-
src/include/access/xlogreader.h | 121 ++-
src/include/access/xlogutils.h | 4 +-
src/include/pg_config_manual.h | 2 +-
src/include/replication/logical.h | 11 +-
14 files changed, 646 insertions(+), 488 deletions(-)
diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index 80d2d20d6c..7d5a96a5f5 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -1325,11 +1325,8 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
XLogReaderState *xlogreader;
char *errormsg;
- xlogreader = XLogReaderAllocate(wal_segment_size, NULL,
- XL_ROUTINE(.page_read = &read_local_xlog_page,
- .segment_open = &wal_segment_open,
- .segment_close = &wal_segment_close),
- NULL);
+ xlogreader = XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+
if (!xlogreader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
@@ -1337,7 +1334,13 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
errdetail("Failed while allocating a WAL reading processor.")));
XLogBeginRead(xlogreader, lsn);
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!read_local_xlog_page(xlogreader))
+ break;
+ }
+
if (record == NULL)
ereport(ERROR,
(errcode_for_file_access(),
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 606328a65a..670996ae67 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -836,13 +836,6 @@ static XLogSource currentSource = XLOG_FROM_ANY;
static bool lastSourceFailed = false;
static bool pendingWalRcvRestart = false;
-typedef struct XLogPageReadPrivate
-{
- int emode;
- bool fetching_ckpt; /* are we fetching a checkpoint record? */
- bool randAccess;
-} XLogPageReadPrivate;
-
/*
* These variables track when we last obtained some WAL data to process,
* and where we got it from. (XLogReceiptSource is initially the same as
@@ -917,8 +910,8 @@ static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath,
static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli,
XLogSource source, bool notfoundOk);
static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, XLogSource source);
-static bool XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
+static bool XLogPageRead(XLogReaderState *xlogreader,
+ bool fetching_ckpt, int emode, bool randAccess);
static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
bool fetching_ckpt, XLogRecPtr tliRecPtr);
static int emode_for_corrupt_record(int emode, XLogRecPtr RecPtr);
@@ -1231,8 +1224,7 @@ XLogInsertRecord(XLogRecData *rdata,
appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len);
if (!debug_reader)
- debug_reader = XLogReaderAllocate(wal_segment_size, NULL,
- XL_ROUTINE(), NULL);
+ debug_reader = XLogReaderAllocate(wal_segment_size, NULL, NULL);
if (!debug_reader)
{
@@ -4344,15 +4336,10 @@ CleanupBackupHistory(void)
* record is available.
*/
static XLogRecord *
-ReadRecord(XLogReaderState *xlogreader, int emode,
- bool fetching_ckpt)
+ReadRecord(XLogReaderState *xlogreader, int emode, bool fetching_ckpt)
{
XLogRecord *record;
- XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
-
- private->fetching_ckpt = fetching_ckpt;
- private->emode = emode;
- private->randAccess = (xlogreader->ReadRecPtr == InvalidXLogRecPtr);
+ bool randAccess = (xlogreader->ReadRecPtr == InvalidXLogRecPtr);
/* This is the first attempt to read this page. */
lastSourceFailed = false;
@@ -4360,8 +4347,16 @@ ReadRecord(XLogReaderState *xlogreader, int emode,
for (;;)
{
char *errormsg;
+ XLogReadRecordResult result;
+
+ while ((result = XLogReadRecord(xlogreader, &record, &errormsg))
+ == XLREAD_NEED_DATA)
+ {
+ if (!XLogPageRead(xlogreader, fetching_ckpt, emode, randAccess))
+ break;
+
+ }
- record = XLogReadRecord(xlogreader, &errormsg);
ReadRecPtr = xlogreader->ReadRecPtr;
EndRecPtr = xlogreader->EndRecPtr;
if (record == NULL)
@@ -6380,7 +6375,6 @@ StartupXLOG(void)
bool backupFromStandby = false;
DBState dbstate_at_startup;
XLogReaderState *xlogreader;
- XLogPageReadPrivate private;
bool promoted = false;
struct stat st;
@@ -6539,13 +6533,9 @@ StartupXLOG(void)
OwnLatch(&XLogCtl->recoveryWakeupLatch);
/* Set up XLOG reader facility */
- MemSet(&private, 0, sizeof(XLogPageReadPrivate));
xlogreader =
- XLogReaderAllocate(wal_segment_size, NULL,
- XL_ROUTINE(.page_read = &XLogPageRead,
- .segment_open = NULL,
- .segment_close = wal_segment_close),
- &private);
+ XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+
if (!xlogreader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
@@ -11987,12 +11977,13 @@ CancelBackup(void)
* sleep and retry.
*/
static bool
-XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetRecPtr, char *readBuf)
+XLogPageRead(XLogReaderState *xlogreader,
+ bool fetching_ckpt, int emode, bool randAccess)
{
- XLogPageReadPrivate *private =
- (XLogPageReadPrivate *) xlogreader->private_data;
- int emode = private->emode;
+ char *readBuf = xlogreader->readBuf;
+ XLogRecPtr targetPagePtr = xlogreader->readPagePtr;
+ int reqLen = xlogreader->readLen;
+ XLogRecPtr targetRecPtr = xlogreader->ReadRecPtr;
uint32 targetPageOff;
XLogSegNo targetSegNo PG_USED_FOR_ASSERTS_ONLY;
int r;
@@ -12035,8 +12026,8 @@ retry:
flushedUpto < targetPagePtr + reqLen))
{
if (!WaitForWALToBecomeAvailable(targetPagePtr + reqLen,
- private->randAccess,
- private->fetching_ckpt,
+ randAccess,
+ fetching_ckpt,
targetRecPtr))
{
if (readFile >= 0)
@@ -12141,6 +12132,7 @@ retry:
goto next_record_is_invalid;
}
+ Assert(xlogreader->readPagePtr == targetPagePtr);
xlogreader->readLen = readLen;
return true;
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index 7d04995b7b..ba3e4868f7 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -40,7 +40,7 @@ static bool XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr,
int reqLen, bool header_inclusive);
static void XLogReaderInvalReadState(XLogReaderState *state);
static bool ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
- XLogRecPtr PrevRecPtr, XLogRecord *record, bool randAccess);
+ XLogRecPtr PrevRecPtr, XLogRecord *record);
static bool ValidXLogRecord(XLogReaderState *state, XLogRecord *record,
XLogRecPtr recptr);
static void ResetDecoder(XLogReaderState *state);
@@ -73,7 +73,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...)
*/
XLogReaderState *
XLogReaderAllocate(int wal_segment_size, const char *waldir,
- XLogReaderRoutine *routine, void *private_data)
+ WALSegmentCleanupCB cleanup_cb)
{
XLogReaderState *state;
@@ -84,7 +84,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
return NULL;
/* initialize caller-provided support functions */
- state->routine = *routine;
+ state->cleanup_cb = cleanup_cb;
state->max_block_id = -1;
@@ -107,8 +107,6 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
WALOpenSegmentInit(&state->seg, &state->segcxt, wal_segment_size,
waldir);
- /* system_identifier initialized to zeroes above */
- state->private_data = private_data;
/* ReadRecPtr, EndRecPtr and readLen initialized to zeroes above */
state->errormsg_buf = palloc_extended(MAX_ERRORMSG_LEN + 1,
MCXT_ALLOC_NO_OOM);
@@ -140,8 +138,8 @@ XLogReaderFree(XLogReaderState *state)
{
int block_id;
- if (state->seg.ws_file != -1)
- state->routine.segment_close(state);
+ if (state->seg.ws_file >= 0)
+ state->cleanup_cb(state);
for (block_id = 0; block_id <= XLR_MAX_BLOCK_ID; block_id++)
{
@@ -246,6 +244,7 @@ XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
/* Begin at the passed-in record pointer. */
state->EndRecPtr = RecPtr;
state->ReadRecPtr = InvalidXLogRecPtr;
+ state->readRecordState = XLREAD_NEXT_RECORD;
}
/*
@@ -254,317 +253,449 @@ XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
* XLogBeginRead() or XLogFindNextRecord() must be called before the first call
* to XLogReadRecord().
*
- * If the page_read callback fails to read the requested data, NULL is
- * returned. The callback is expected to have reported the error; errormsg
- * is set to NULL.
+ * This function may return XLREAD_NEED_DATA several times before returning a
+ * result record. The caller shall read in some new data then call this
+ * function again with the same parameters.
*
- * If the reading fails for some other reason, NULL is also returned, and
- * *errormsg is set to a string with details of the failure.
+ * When a record is successfully read, returns XLREAD_SUCCESS with result
+ * record being stored in *record. Otherwise *record is NULL.
*
- * The returned pointer (or *errormsg) points to an internal buffer that's
- * valid until the next call to XLogReadRecord.
+ * Returns XLREAD_NEED_DATA if more data is needed to finish reading the
+ * current record. In that case, state->readPagePtr and state->readLen inform
+ * the desired position and minimum length of data needed. The caller shall
+ * read in the requested data and set state->readBuf to point to a buffer
+ * containing it. The caller must also set state->readPageTLI and
+ * state->readLen to indicate the timeline that it was read from, and the
+ * length of data that is now available (which must be >= given readLen),
+ * respectively.
+ *
+ * If invalid data is encountered, returns XLREAD_FAIL with *record being set to
+ * NULL. *errormsg is set to a string with details of the failure.
+ * The returned pointer (or *errormsg) points to an internal buffer that's valid
+ * until the next call to XLogReadRecord.
+ *
+ *
+ * This function runs a state machine consists of the following states.
+ *
+ * XLREAD_NEXT_RECORD :
+ * The initial state, if called with valid RecPtr, try to read a record at
+ * that position. If invalid RecPtr is given try to read a record just after
+ * the last one previously read.
+ * This state ens after setting ReadRecPtr. Then goes to XLREAD_TOT_LEN.
+ *
+ * XLREAD_TOT_LEN:
+ * Examining record header. Ends after reading record total
+ * length. recordRemainLen and recordGotLen are initialized.
+ *
+ * XLREAD_FIRST_FRAGMENT:
+ * Reading the first fragment. Ends with finishing reading a single
+ * record. Goes to XLREAD_NEXT_RECORD if that's all or
+ * XLREAD_CONTINUATION if we have continuation.
+
+ * XLREAD_CONTINUATION:
+ * Reading continuation of record. Ends with finishing the whole record then
+ * goes to XLREAD_NEXT_RECORD. During this state, recordRemainLen indicates
+ * how much is left and readRecordBuf holds the partially assert
+ * record.recordContRecPtr points to the beginning of the next page where to
+ * continue.
+ *
+ * If wrong data found in any state, the state machine stays at the current
+ * state. This behavior allows to continue reading a reacord switching among
+ * different souces, while streaming replication.
*/
-XLogRecord *
-XLogReadRecord(XLogReaderState *state, char **errormsg)
+XLogReadRecordResult
+XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
{
- XLogRecPtr RecPtr;
- XLogRecord *record;
- XLogRecPtr targetPagePtr;
- bool randAccess;
- uint32 len,
- total_len;
- uint32 targetRecOff;
- uint32 pageHeaderSize;
- bool gotheader;
+ XLogRecord *prec;
- /*
- * randAccess indicates whether to verify the previous-record pointer of
- * the record we're reading. We only do this if we're reading
- * sequentially, which is what we initially assume.
- */
- randAccess = false;
+ *record = NULL;
/* reset error state */
*errormsg = NULL;
state->errormsg_buf[0] = '\0';
- ResetDecoder(state);
-
- RecPtr = state->EndRecPtr;
-
- if (state->ReadRecPtr != InvalidXLogRecPtr)
- {
- /* read the record after the one we just read */
-
- /*
- * EndRecPtr is pointing to end+1 of the previous WAL record. If
- * we're at a page boundary, no more records can fit on the current
- * page. We must skip over the page header, but we can't do that until
- * we've read in the page, since the header size is variable.
- */
- }
- else
- {
- /*
- * Caller supplied a position to start at.
- *
- * In this case, EndRecPtr should already be pointing to a valid
- * record starting position.
- */
- Assert(XRecOffIsValid(RecPtr));
- randAccess = true;
- }
-
- state->currRecPtr = RecPtr;
-
- targetPagePtr = RecPtr - (RecPtr % XLOG_BLCKSZ);
- targetRecOff = RecPtr % XLOG_BLCKSZ;
-
- /*
- * Read the page containing the record into state->readBuf. Request enough
- * byte to cover the whole record header, or at least the part of it that
- * fits on the same page.
- */
- while (XLogNeedData(state, targetPagePtr,
- Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
- targetRecOff != 0))
- {
- if (!state->routine.page_read(state, state->readPagePtr, state->readLen,
- RecPtr, state->readBuf))
- break;
- }
-
- if (!state->page_verified)
- goto err;
-
- /*
- * We have at least the page header, so we can examine it now.
- */
- pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
- if (targetRecOff == 0)
- {
- /*
- * At page start, so skip over page header.
- */
- RecPtr += pageHeaderSize;
- targetRecOff = pageHeaderSize;
- }
- else if (targetRecOff < pageHeaderSize)
- {
- report_invalid_record(state, "invalid record offset at %X/%X",
- LSN_FORMAT_ARGS(RecPtr));
- goto err;
- }
-
- if ((((XLogPageHeader) state->readBuf)->xlp_info & XLP_FIRST_IS_CONTRECORD) &&
- targetRecOff == pageHeaderSize)
- {
- report_invalid_record(state, "contrecord is requested by %X/%X",
- LSN_FORMAT_ARGS(RecPtr));
- goto err;
- }
-
- /* XLogNeedData has verified the page header */
- Assert(pageHeaderSize <= state->readLen);
-
- /*
- * Read the record length.
- *
- * NB: Even though we use an XLogRecord pointer here, the whole record
- * header might not fit on this page. xl_tot_len is the first field of the
- * struct, so it must be on this page (the records are MAXALIGNed), but we
- * cannot access any other fields until we've verified that we got the
- * whole header.
- */
- record = (XLogRecord *) (state->readBuf + RecPtr % XLOG_BLCKSZ);
- total_len = record->xl_tot_len;
-
- /*
- * If the whole record header is on this page, validate it immediately.
- * Otherwise do just a basic sanity check on xl_tot_len, and validate the
- * rest of the header after reading it from the next page. The xl_tot_len
- * check is necessary here to ensure that we enter the "Need to reassemble
- * record" code path below; otherwise we might fail to apply
- * ValidXLogRecordHeader at all.
- */
- if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
- {
- if (!ValidXLogRecordHeader(state, RecPtr, state->ReadRecPtr, record,
- randAccess))
- goto err;
- gotheader = true;
- }
- else
- {
- /* XXX: more validation should be done here */
- if (total_len < SizeOfXLogRecord)
- {
- report_invalid_record(state,
- "invalid record length at %X/%X: wanted %u, got %u",
- LSN_FORMAT_ARGS(RecPtr),
- (uint32) SizeOfXLogRecord, total_len);
- goto err;
- }
- gotheader = false;
- }
-
- len = XLOG_BLCKSZ - RecPtr % XLOG_BLCKSZ;
- if (total_len > len)
+ switch (state->readRecordState)
{
- /* Need to reassemble record */
- char *contdata;
- XLogPageHeader pageHeader;
- char *buffer;
- uint32 gotlen;
-
- /*
- * Enlarge readRecordBuf as needed.
- */
- if (total_len > state->readRecordBufSize &&
- !allocate_recordbuf(state, total_len))
- {
- /* We treat this as a "bogus data" condition */
- report_invalid_record(state, "record length %u at %X/%X too long",
- total_len, LSN_FORMAT_ARGS(RecPtr));
- goto err;
- }
+ case XLREAD_NEXT_RECORD:
+ ResetDecoder(state);
- /* Copy the first fragment of the record from the first page. */
- memcpy(state->readRecordBuf,
- state->readBuf + RecPtr % XLOG_BLCKSZ, len);
- buffer = state->readRecordBuf + len;
- gotlen = len;
-
- do
- {
- int rest_len = total_len - gotlen;
-
- /* Calculate pointer to beginning of next page */
- targetPagePtr += XLOG_BLCKSZ;
-
- /* Wait for the next page to become available */
- while (XLogNeedData(state, targetPagePtr,
- Min(rest_len, XLOG_BLCKSZ),
- false))
+ if (state->ReadRecPtr != InvalidXLogRecPtr)
{
- if (!state->routine.page_read(state, state->readPagePtr,
- state->readLen,
- state->ReadRecPtr,
- state->readBuf))
- break;
+ /* read the record after the one we just read */
+
+ /*
+ * EndRecPtr is pointing to end+1 of the previous WAL record.
+ * If we're at a page boundary, no more records can fit on the
+ * current page. We must skip over the page header, but we
+ * can't do that until we've read in the page, since the header
+ * size is variable.
+ */
+ state->PrevRecPtr = state->ReadRecPtr;
+ state->ReadRecPtr = state->EndRecPtr;
+ }
+ else
+ {
+ /*
+ * Caller supplied a position to start at.
+ *
+ * In this case, EndRecPtr should already be pointing to a
+ * valid record starting position.
+ */
+ Assert(XRecOffIsValid(state->EndRecPtr));
+ state->ReadRecPtr = state->EndRecPtr;
+
+ /*
+ * We cannot verify the previous-record pointer when we're
+ * seeking to a particular record. Reset PrevRecPtr so that we
+ * won't try doing that.
+ */
+ state->PrevRecPtr = InvalidXLogRecPtr;
+ state->EndRecPtr = InvalidXLogRecPtr; /* to be tidy */
}
+ state->record_verified = false;
+ state->readRecordState = XLREAD_TOT_LEN;
+ /* fall through */
+
+ case XLREAD_TOT_LEN:
+ {
+ uint32 total_len;
+ uint32 pageHeaderSize;
+ XLogRecPtr targetPagePtr;
+ uint32 targetRecOff;
+ XLogPageHeader pageHeader;
+
+ targetPagePtr =
+ state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
+ targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+
+ /*
+ * Check if we have enough data. For the first record in the page,
+ * the requesting length doesn't contain page header.
+ */
+ if (XLogNeedData(state, targetPagePtr,
+ Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+ targetRecOff != 0))
+ return XLREAD_NEED_DATA;
+
+ /* error out if caller supplied bogus page */
if (!state->page_verified)
goto err;
- Assert(SizeOfXLogShortPHD <= state->readLen);
+ /* examine page header now. */
+ pageHeaderSize =
+ XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+ if (targetRecOff == 0)
+ {
+ /* At page start, so skip over page header. */
+ state->ReadRecPtr += pageHeaderSize;
+ targetRecOff = pageHeaderSize;
+ }
+ else if (targetRecOff < pageHeaderSize)
+ {
+ report_invalid_record(state, "invalid record offset at %X/%X",
+ LSN_FORMAT_ARGS(state->ReadRecPtr));
+ goto err;
+ }
- /* Check that the continuation on next page looks valid */
pageHeader = (XLogPageHeader) state->readBuf;
- if (!(pageHeader->xlp_info & XLP_FIRST_IS_CONTRECORD))
+ if ((pageHeader->xlp_info & XLP_FIRST_IS_CONTRECORD) &&
+ targetRecOff == pageHeaderSize)
{
- report_invalid_record(state,
- "there is no contrecord flag at %X/%X",
- LSN_FORMAT_ARGS(RecPtr));
+ report_invalid_record(state, "contrecord is requested by %X/%X",
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr);
goto err;
}
- /*
- * Cross-check that xlp_rem_len agrees with how much of the record
- * we expect there to be left.
- */
- if (pageHeader->xlp_rem_len == 0 ||
- total_len != (pageHeader->xlp_rem_len + gotlen))
- {
- report_invalid_record(state,
- "invalid contrecord length %u (expected %lld) at %X/%X",
- pageHeader->xlp_rem_len,
- ((long long) total_len) - gotlen,
- LSN_FORMAT_ARGS(RecPtr));
- goto err;
- }
-
- /* Append the continuation from this page to the buffer */
- pageHeaderSize = XLogPageHeaderSize(pageHeader);
-
+ /* XLogNeedData has verified the page header */
Assert(pageHeaderSize <= state->readLen);
- contdata = (char *) state->readBuf + pageHeaderSize;
- len = XLOG_BLCKSZ - pageHeaderSize;
- if (pageHeader->xlp_rem_len < len)
- len = pageHeader->xlp_rem_len;
+ /*
+ * Read the record length.
+ *
+ * NB: Even though we use an XLogRecord pointer here, the whole
+ * record header might not fit on this page. xl_tot_len is the first
+ * field of the struct, so it must be on this page (the records are
+ * MAXALIGNed), but we cannot access any other fields until we've
+ * verified that we got the whole header.
+ */
+ prec = (XLogRecord *) (state->readBuf +
+ state->ReadRecPtr % XLOG_BLCKSZ);
+ total_len = prec->xl_tot_len;
- Assert (pageHeaderSize + len <= state->readLen);
- memcpy(buffer, (char *) contdata, len);
- buffer += len;
- gotlen += len;
+ /*
+ * If the whole record header is on this page, validate it
+ * immediately. Otherwise do just a basic sanity check on
+ * xl_tot_len, and validate the rest of the header after reading it
+ * from the next page. The xl_tot_len check is necessary here to
+ * ensure that we enter the XLREAD_CONTINUATION state below;
+ * otherwise we might fail to apply ValidXLogRecordHeader at all.
+ */
+ if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
+ {
+ if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
+ state->PrevRecPtr, prec))
+ goto err;
- /* If we just reassembled the record header, validate it. */
- if (!gotheader)
+ state->record_verified = true;
+ }
+ else
{
- record = (XLogRecord *) state->readRecordBuf;
- if (!ValidXLogRecordHeader(state, RecPtr, state->ReadRecPtr,
- record, randAccess))
+ /* XXX: more validation should be done here */
+ if (total_len < SizeOfXLogRecord)
+ {
+ report_invalid_record(state,
+ "invalid record length at %X/%X: wanted %u, got %u",
+ LSN_FORMAT_ARGS(state->ReadRecPtr),
+ (uint32) SizeOfXLogRecord, total_len);
goto err;
- gotheader = true;
+ }
}
- } while (gotlen < total_len);
-
- Assert(gotheader);
- record = (XLogRecord *) state->readRecordBuf;
- if (!ValidXLogRecord(state, record, RecPtr))
- goto err;
+ /*
+ * Wait for the rest of the record, or the part of the record that
+ * fit on the first page if crossed a page boundary, to become
+ * available.
+ */
+ state->recordGotLen = 0;
+ state->recordRemainLen = total_len;
+ state->readRecordState = XLREAD_FIRST_FRAGMENT;
+ }
+ /* fall through */
- pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
- state->ReadRecPtr = RecPtr;
- state->EndRecPtr = targetPagePtr + pageHeaderSize
- + MAXALIGN(pageHeader->xlp_rem_len);
- }
- else
- {
- /* Wait for the record data to become available */
- while (XLogNeedData(state, targetPagePtr,
- Min(targetRecOff + total_len, XLOG_BLCKSZ), true))
+ case XLREAD_FIRST_FRAGMENT:
{
- if (!state->routine.page_read(state, state->readPagePtr,
- state->readLen,
- state->ReadRecPtr, state->readBuf))
+ uint32 total_len = state->recordRemainLen;
+ uint32 request_len;
+ uint32 record_len;
+ XLogRecPtr targetPagePtr;
+ uint32 targetRecOff;
+
+ /*
+ * Wait for the rest of the record on the first page to become
+ * available
+ */
+ targetPagePtr =
+ state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
+ targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+
+ request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ);
+ record_len = request_len - targetRecOff;
+
+ /* ReadRecPtr contains page header */
+ Assert (targetRecOff != 0);
+ if (XLogNeedData(state, targetPagePtr, request_len, true))
+ return XLREAD_NEED_DATA;
+
+ /* error out if caller supplied bogus page */
+ if (!state->page_verified)
+ goto err;
+
+ prec = (XLogRecord *) (state->readBuf + targetRecOff);
+
+ /* validate record header if not yet */
+ if (!state->record_verified && record_len >= SizeOfXLogRecord)
+ {
+ if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
+ state->PrevRecPtr, prec))
+ goto err;
+
+ state->record_verified = true;
+ }
+
+
+ if (total_len == record_len)
+ {
+ /* Record does not cross a page boundary */
+ Assert(state->record_verified);
+
+ if (!ValidXLogRecord(state, prec, state->ReadRecPtr))
+ goto err;
+
+ state->record_verified = true; /* to be tidy */
+
+ /* We already checked the header earlier */
+ state->EndRecPtr = state->ReadRecPtr + MAXALIGN(record_len);
+
+ *record = prec;
+ state->readRecordState = XLREAD_NEXT_RECORD;
break;
+ }
+
+ /*
+ * The record continues on the next page. Need to reassemble
+ * record
+ */
+ Assert(total_len > record_len);
+
+ /* Enlarge readRecordBuf as needed. */
+ if (total_len > state->readRecordBufSize &&
+ !allocate_recordbuf(state, total_len))
+ {
+ /* We treat this as a "bogus data" condition */
+ report_invalid_record(state,
+ "record length %u at %X/%X too long",
+ total_len,
+ LSN_FORMAT_ARGS(state->ReadRecPtr));
+ goto err;
+ }
+
+ /* Copy the first fragment of the record from the first page. */
+ memcpy(state->readRecordBuf, state->readBuf + targetRecOff,
+ record_len);
+ state->recordGotLen += record_len;
+ state->recordRemainLen -= record_len;
+
+ /* Calculate pointer to beginning of next page */
+ state->recordContRecPtr = state->ReadRecPtr + record_len;
+ Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0);
+
+ state->readRecordState = XLREAD_CONTINUATION;
}
+ /* fall through */
+
+ case XLREAD_CONTINUATION:
+ {
+ XLogPageHeader pageHeader;
+ uint32 pageHeaderSize;
+ XLogRecPtr targetPagePtr;
+
+ /* we enter this state only if we haven't read the whole record. */
+ Assert (state->recordRemainLen > 0);
+
+ while(state->recordRemainLen > 0)
+ {
+ char *contdata;
+ uint32 request_len;
+ uint32 record_len;
+
+ /* Wait for the next page to become available */
+ targetPagePtr = state->recordContRecPtr;
+
+ /* this request contains page header */
+ Assert (targetPagePtr != 0);
+ if (XLogNeedData(state, targetPagePtr,
+ Min(state->recordRemainLen, XLOG_BLCKSZ),
+ false))
+ return XLREAD_NEED_DATA;
+
+ if (!state->page_verified)
+ goto err;
+
+ Assert(SizeOfXLogShortPHD <= state->readLen);
- if (!state->page_verified)
- goto err;
+ /* Check that the continuation on next page looks valid */
+ pageHeader = (XLogPageHeader) state->readBuf;
+ if (!(pageHeader->xlp_info & XLP_FIRST_IS_CONTRECORD))
+ {
+ report_invalid_record(
+ state,
+ "there is no contrecord flag at %X/%X reading %X/%X",
+ (uint32) (state->recordContRecPtr >> 32),
+ (uint32) state->recordContRecPtr,
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr);
+ goto err;
+ }
- /* Record does not cross a page boundary */
- if (!ValidXLogRecord(state, record, RecPtr))
- goto err;
+ /*
+ * Cross-check that xlp_rem_len agrees with how much of the
+ * record we expect there to be left.
+ */
+ if (pageHeader->xlp_rem_len == 0 ||
+ pageHeader->xlp_rem_len != state->recordRemainLen)
+ {
+ report_invalid_record(
+ state,
+ "invalid contrecord length %u at %X/%X reading %X/%X, expected %u",
+ pageHeader->xlp_rem_len,
+ (uint32) (state->recordContRecPtr >> 32),
+ (uint32) state->recordContRecPtr,
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr,
+ state->recordRemainLen);
+ goto err;
+ }
- state->EndRecPtr = RecPtr + MAXALIGN(total_len);
+ /* Append the continuation from this page to the buffer */
+ pageHeaderSize = XLogPageHeaderSize(pageHeader);
- state->ReadRecPtr = RecPtr;
+ /*
+ * XLogNeedData should have ensured that the whole page header
+ * was read
+ */
+ Assert(state->readLen >= pageHeaderSize);
+
+ contdata = (char *) state->readBuf + pageHeaderSize;
+ record_len = XLOG_BLCKSZ - pageHeaderSize;
+ if (pageHeader->xlp_rem_len < record_len)
+ record_len = pageHeader->xlp_rem_len;
+
+ request_len = record_len + pageHeaderSize;
+
+ /* XLogNeedData should have ensured all needed data was read */
+ Assert (state->readLen >= request_len);
+
+ memcpy(state->readRecordBuf + state->recordGotLen,
+ (char *) contdata, record_len);
+ state->recordGotLen += record_len;
+ state->recordRemainLen -= record_len;
+
+ /* If we just reassembled the record header, validate it. */
+ if (!state->record_verified)
+ {
+ Assert(state->recordGotLen >= SizeOfXLogRecord);
+ if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
+ state->PrevRecPtr,
+ (XLogRecord *) state->readRecordBuf))
+ goto err;
+
+ state->record_verified = true;
+ }
+
+ /* Calculate pointer to beginning of next page, and continue */
+ state->recordContRecPtr += XLOG_BLCKSZ;
+ }
+
+ /* targetPagePtr is pointing the last-read page here */
+ prec = (XLogRecord *) state->readRecordBuf;
+ if (!ValidXLogRecord(state, prec, state->ReadRecPtr))
+ goto err;
+
+ pageHeaderSize =
+ XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+ state->EndRecPtr = targetPagePtr + pageHeaderSize
+ + MAXALIGN(pageHeader->xlp_rem_len);
+
+ *record = prec;
+ state->readRecordState = XLREAD_NEXT_RECORD;
+ break;
+ }
}
/*
* Special processing if it's an XLOG SWITCH record
*/
- if (record->xl_rmid == RM_XLOG_ID &&
- (record->xl_info & ~XLR_INFO_MASK) == XLOG_SWITCH)
+ if ((*record)->xl_rmid == RM_XLOG_ID &&
+ ((*record)->xl_info & ~XLR_INFO_MASK) == XLOG_SWITCH)
{
/* Pretend it extends to end of segment */
state->EndRecPtr += state->segcxt.ws_segsize - 1;
state->EndRecPtr -= XLogSegmentOffset(state->EndRecPtr, state->segcxt.ws_segsize);
}
- if (DecodeXLogRecord(state, record, errormsg))
- return record;
- else
- return NULL;
+ Assert (!*record || state->readLen >= 0);
+ if (DecodeXLogRecord(state, *record, errormsg))
+ return XLREAD_SUCCESS;
+
+ *record = NULL;
+ return XLREAD_FAIL;
err:
/*
- * Invalidate the read state. We might read from a different source after
+ * Invalidate the read page. We might read from a different source after
* failure.
*/
XLogReaderInvalReadState(state);
@@ -572,7 +703,8 @@ err:
if (state->errormsg_buf[0] != '\0')
*errormsg = state->errormsg_buf;
- return NULL;
+ *record = NULL;
+ return XLREAD_FAIL;
}
/*
@@ -724,11 +856,12 @@ XLogReaderInvalReadState(XLogReaderState *state)
*
* This is just a convenience subroutine to avoid duplicated code in
* XLogReadRecord. It's not intended for use from anywhere else.
+ *
+ * If PrevRecPtr is valid, the xl_prev is is cross-checked with it.
*/
static bool
ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
- XLogRecPtr PrevRecPtr, XLogRecord *record,
- bool randAccess)
+ XLogRecPtr PrevRecPtr, XLogRecord *record)
{
if (record->xl_tot_len < SizeOfXLogRecord)
{
@@ -745,7 +878,7 @@ ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
record->xl_rmid, LSN_FORMAT_ARGS(RecPtr));
return false;
}
- if (randAccess)
+ if (PrevRecPtr == InvalidXLogRecPtr)
{
/*
* We can't exactly verify the prev-link, but surely it should be less
@@ -975,11 +1108,14 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
* XLogReadRecord() will read the next valid record.
*/
XLogRecPtr
-XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
+XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
+ XLogFindNextRecordCB read_page, void *private)
{
XLogRecPtr tmpRecPtr;
XLogRecPtr found = InvalidXLogRecPtr;
XLogPageHeader header;
+ XLogRecord *record;
+ XLogReadRecordResult result;
char *errormsg;
Assert(!XLogRecPtrIsInvalid(RecPtr));
@@ -1012,9 +1148,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
while(XLogNeedData(state, targetPagePtr, targetRecOff,
targetRecOff != 0))
{
- if (!state->routine.page_read(state, state->readPagePtr,
- state->readLen,
- state->ReadRecPtr, state->readBuf))
+ if (!read_page(state, private))
break;
}
@@ -1066,8 +1200,16 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
* or we just jumped over the remaining data of a continuation.
*/
XLogBeginRead(state, tmpRecPtr);
- while (XLogReadRecord(state, &errormsg) != NULL)
+ while ((result = XLogReadRecord(state, &record, &errormsg)) !=
+ XLREAD_FAIL)
{
+ if (result == XLREAD_NEED_DATA)
+ {
+ if (!read_page(state, private))
+ goto err;
+ continue;
+ }
+
/* past the record we've found, break out */
if (RecPtr <= state->ReadRecPtr)
{
@@ -1087,9 +1229,9 @@ err:
#endif /* FRONTEND */
/*
- * Helper function to ease writing of XLogRoutine->page_read callbacks.
- * If this function is used, caller must supply a segment_open callback in
- * 'state', as that is used here.
+ * Helper function to ease writing of page_read callback.
+ * If this function is used, caller must supply a segment_open callback and
+ * segment_close callback as that is used here.
*
* Read 'count' bytes into 'buf', starting at location 'startptr', from WAL
* fetched from timeline 'tli'.
@@ -1102,6 +1244,7 @@ err:
*/
bool
WALRead(XLogReaderState *state,
+ WALSegmentOpenCB segopenfn, WALSegmentCloseCB segclosefn,
char *buf, XLogRecPtr startptr, Size count, TimeLineID tli,
WALReadError *errinfo)
{
@@ -1133,10 +1276,10 @@ WALRead(XLogReaderState *state,
XLogSegNo nextSegNo;
if (state->seg.ws_file >= 0)
- state->routine.segment_close(state);
+ segclosefn(state);
XLByteToSeg(recptr, nextSegNo, state->segcxt.ws_segsize);
- state->routine.segment_open(state, nextSegNo, &tli);
+ segopenfn(state, nextSegNo, &tli);
/* This shouldn't happen -- indicates a bug in segment_open */
Assert(state->seg.ws_file >= 0);
diff --git a/src/backend/access/transam/xlogutils.c b/src/backend/access/transam/xlogutils.c
index 46eda33f25..b003990745 100644
--- a/src/backend/access/transam/xlogutils.c
+++ b/src/backend/access/transam/xlogutils.c
@@ -686,8 +686,7 @@ XLogTruncateRelation(RelFileNode rnode, ForkNumber forkNum,
void
XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wantLength)
{
- const XLogRecPtr lastReadPage = state->seg.ws_segno *
- state->segcxt.ws_segsize + state->readLen;
+ const XLogRecPtr lastReadPage = state->readPagePtr;
Assert(wantPage != InvalidXLogRecPtr && wantPage % XLOG_BLCKSZ == 0);
Assert(wantLength <= XLOG_BLCKSZ);
@@ -702,7 +701,7 @@ XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wa
* current TLI has since become historical.
*/
if (lastReadPage == wantPage &&
- state->readLen != 0 &&
+ state->page_verified &&
lastReadPage + state->readLen >= wantPage + Min(wantLength, XLOG_BLCKSZ - 1))
return;
@@ -788,6 +787,7 @@ wal_segment_open(XLogReaderState *state, XLogSegNo nextSegNo,
char path[MAXPGPATH];
XLogFilePath(path, tli, nextSegNo, state->segcxt.ws_segsize);
+ elog(LOG, "HOGE: %lu, %d => %s", nextSegNo, tli, path);
state->seg.ws_file = BasicOpenFile(path, O_RDONLY | PG_BINARY);
if (state->seg.ws_file >= 0)
return;
@@ -825,9 +825,11 @@ wal_segment_close(XLogReaderState *state)
* loop for now.
*/
bool
-read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *cur_page)
+read_local_xlog_page(XLogReaderState *state)
{
+ XLogRecPtr targetPagePtr = state->readPagePtr;
+ int reqLen = state->readLen;
+ char *cur_page = state->readBuf;
XLogRecPtr read_upto,
loc;
TimeLineID tli;
@@ -940,11 +942,12 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
* as 'count', read the whole page anyway. It's guaranteed to be
* zero-padded up to the page boundary if it's incomplete.
*/
- if (!WALRead(state, cur_page, targetPagePtr, XLOG_BLCKSZ, tli,
- &errinfo))
+ if (!WALRead(state, wal_segment_open, wal_segment_close,
+ cur_page, targetPagePtr, XLOG_BLCKSZ, tli, &errinfo))
WALReadRaiseError(&errinfo);
/* number of valid bytes in the buffer */
+ state->readPagePtr = targetPagePtr;
state->readLen = count;
return true;
}
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index 37b75deb72..9cbd8afa49 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -148,7 +148,8 @@ StartupDecodingContext(List *output_plugin_options,
TransactionId xmin_horizon,
bool need_full_snapshot,
bool fast_forward,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress)
@@ -198,11 +199,12 @@ StartupDecodingContext(List *output_plugin_options,
ctx->slot = slot;
- ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, xl_routine, ctx);
+ ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, cleanup_cb);
if (!ctx->reader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
errmsg("out of memory")));
+ ctx->page_read = page_read;
ctx->reorder = ReorderBufferAllocate();
ctx->snapshot_builder =
@@ -319,7 +321,8 @@ CreateInitDecodingContext(const char *plugin,
List *output_plugin_options,
bool need_full_snapshot,
XLogRecPtr restart_lsn,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress)
@@ -422,7 +425,7 @@ CreateInitDecodingContext(const char *plugin,
ctx = StartupDecodingContext(NIL, restart_lsn, xmin_horizon,
need_full_snapshot, false,
- xl_routine, prepare_write, do_write,
+ page_read, cleanup_cb, prepare_write, do_write,
update_progress);
/* call output plugin initialization callback */
@@ -476,7 +479,8 @@ LogicalDecodingContext *
CreateDecodingContext(XLogRecPtr start_lsn,
List *output_plugin_options,
bool fast_forward,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress)
@@ -528,8 +532,8 @@ CreateDecodingContext(XLogRecPtr start_lsn,
ctx = StartupDecodingContext(output_plugin_options,
start_lsn, InvalidTransactionId, false,
- fast_forward, xl_routine, prepare_write,
- do_write, update_progress);
+ fast_forward, page_read, cleanup_cb,
+ prepare_write, do_write, update_progress);
/* call output plugin initialization callback */
old_context = MemoryContextSwitchTo(ctx->context);
@@ -585,7 +589,13 @@ DecodingContextFindStartpoint(LogicalDecodingContext *ctx)
char *err = NULL;
/* the read_page callback waits for new WAL */
- record = XLogReadRecord(ctx->reader, &err);
+ while (XLogReadRecord(ctx->reader, &record, &err) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!ctx->page_read(ctx->reader))
+ break;
+ }
+
if (err)
elog(ERROR, "%s", err);
if (!record)
diff --git a/src/backend/replication/logical/logicalfuncs.c b/src/backend/replication/logical/logicalfuncs.c
index f7e055874e..b69f6911c5 100644
--- a/src/backend/replication/logical/logicalfuncs.c
+++ b/src/backend/replication/logical/logicalfuncs.c
@@ -233,9 +233,8 @@ pg_logical_slot_get_changes_guts(FunctionCallInfo fcinfo, bool confirm, bool bin
ctx = CreateDecodingContext(InvalidXLogRecPtr,
options,
false,
- XL_ROUTINE(.page_read = read_local_xlog_page,
- .segment_open = wal_segment_open,
- .segment_close = wal_segment_close),
+ read_local_xlog_page,
+ wal_segment_close,
LogicalOutputPrepareWrite,
LogicalOutputWrite, NULL);
@@ -284,7 +283,13 @@ pg_logical_slot_get_changes_guts(FunctionCallInfo fcinfo, bool confirm, bool bin
XLogRecord *record;
char *errm = NULL;
- record = XLogReadRecord(ctx->reader, &errm);
+ while (XLogReadRecord(ctx->reader, &record, &errm) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!ctx->page_read(ctx->reader))
+ break;
+ }
+
if (errm)
elog(ERROR, "%s", errm);
diff --git a/src/backend/replication/slotfuncs.c b/src/backend/replication/slotfuncs.c
index 9817b44113..deb7cf15eb 100644
--- a/src/backend/replication/slotfuncs.c
+++ b/src/backend/replication/slotfuncs.c
@@ -153,9 +153,8 @@ create_logical_replication_slot(char *name, char *plugin,
ctx = CreateInitDecodingContext(plugin, NIL,
false, /* just catalogs is OK */
restart_lsn,
- XL_ROUTINE(.page_read = read_local_xlog_page,
- .segment_open = wal_segment_open,
- .segment_close = wal_segment_close),
+ read_local_xlog_page,
+ wal_segment_close,
NULL, NULL, NULL);
/*
@@ -512,9 +511,8 @@ pg_logical_replication_slot_advance(XLogRecPtr moveto)
ctx = CreateDecodingContext(InvalidXLogRecPtr,
NIL,
true, /* fast_forward */
- XL_ROUTINE(.page_read = read_local_xlog_page,
- .segment_open = wal_segment_open,
- .segment_close = wal_segment_close),
+ read_local_xlog_page,
+ wal_segment_close,
NULL, NULL, NULL);
/*
@@ -536,7 +534,13 @@ pg_logical_replication_slot_advance(XLogRecPtr moveto)
* Read records. No changes are generated in fast_forward mode,
* but snapbuilder/slot statuses are updated properly.
*/
- record = XLogReadRecord(ctx->reader, &errm);
+ while (XLogReadRecord(ctx->reader, &record, &errm) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!ctx->page_read(ctx->reader))
+ break;
+ }
+
if (errm)
elog(ERROR, "%s", errm);
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index c36b91ae5e..062093727d 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -580,10 +580,7 @@ StartReplication(StartReplicationCmd *cmd)
/* create xlogreader for physical replication */
xlogreader =
- XLogReaderAllocate(wal_segment_size, NULL,
- XL_ROUTINE(.segment_open = WalSndSegmentOpen,
- .segment_close = wal_segment_close),
- NULL);
+ XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
if (!xlogreader)
ereport(ERROR,
@@ -807,9 +804,11 @@ StartReplication(StartReplicationCmd *cmd)
* set every time WAL is flushed.
*/
static bool
-logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetRecPtr, char *cur_page)
+logical_read_xlog_page(XLogReaderState *state)
{
+ XLogRecPtr targetPagePtr = state->readPagePtr;
+ int reqLen = state->readLen;
+ char *cur_page = state->readBuf;
XLogRecPtr flushptr;
int count;
WALReadError errinfo;
@@ -837,7 +836,7 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
count = flushptr - targetPagePtr; /* part of the page available */
/* now actually read the data, we know it's there */
- if (!WALRead(state,
+ if (!WALRead(state, WalSndSegmentOpen, wal_segment_close,
cur_page,
targetPagePtr,
XLOG_BLCKSZ,
@@ -1011,9 +1010,8 @@ CreateReplicationSlot(CreateReplicationSlotCmd *cmd)
ctx = CreateInitDecodingContext(cmd->plugin, NIL, need_full_snapshot,
InvalidXLogRecPtr,
- XL_ROUTINE(.page_read = logical_read_xlog_page,
- .segment_open = WalSndSegmentOpen,
- .segment_close = wal_segment_close),
+ logical_read_xlog_page,
+ wal_segment_close,
WalSndPrepareWrite, WalSndWriteData,
WalSndUpdateProgress);
@@ -1171,9 +1169,8 @@ StartLogicalReplication(StartReplicationCmd *cmd)
*/
logical_decoding_ctx =
CreateDecodingContext(cmd->startpoint, cmd->options, false,
- XL_ROUTINE(.page_read = logical_read_xlog_page,
- .segment_open = WalSndSegmentOpen,
- .segment_close = wal_segment_close),
+ logical_read_xlog_page,
+ wal_segment_close,
WalSndPrepareWrite, WalSndWriteData,
WalSndUpdateProgress);
xlogreader = logical_decoding_ctx->reader;
@@ -2749,7 +2746,7 @@ XLogSendPhysical(void)
enlargeStringInfo(&output_message, nbytes);
retry:
- if (!WALRead(xlogreader,
+ if (!WALRead(xlogreader, WalSndSegmentOpen, wal_segment_close,
&output_message.data[output_message.len],
startptr,
nbytes,
@@ -2847,7 +2844,12 @@ XLogSendLogical(void)
*/
WalSndCaughtUp = false;
- record = XLogReadRecord(logical_decoding_ctx->reader, &errm);
+ while (XLogReadRecord(logical_decoding_ctx->reader, &record, &errm) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!logical_decoding_ctx->page_read(logical_decoding_ctx->reader))
+ break;
+ }
/* xlog record was invalid */
if (errm != NULL)
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index cf119848b0..712c85281c 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -41,15 +41,9 @@ static int xlogreadfd = -1;
static XLogSegNo xlogreadsegno = -1;
static char xlogfpath[MAXPGPATH];
-typedef struct XLogPageReadPrivate
-{
- const char *restoreCommand;
- int tliIndex;
-} XLogPageReadPrivate;
-
-static bool SimpleXLogPageRead(XLogReaderState *xlogreader,
- XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
+static bool SimpleXLogPageRead(XLogReaderState *xlogreader,
+ const char *datadir, int *tliIndex,
+ const char *restoreCommand);
/*
* Read WAL from the datadir/pg_wal, starting from 'startpoint' on timeline
@@ -66,20 +60,22 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
XLogRecord *record;
XLogReaderState *xlogreader;
char *errormsg;
- XLogPageReadPrivate private;
- private.tliIndex = tliIndex;
- private.restoreCommand = restoreCommand;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir,
- XL_ROUTINE(.page_read = &SimpleXLogPageRead),
- &private);
+ xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+
if (xlogreader == NULL)
pg_fatal("out of memory");
XLogBeginRead(xlogreader, startpoint);
do
{
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!SimpleXLogPageRead(xlogreader, datadir,
+ &tliIndex, restoreCommand))
+ break;
+ }
if (record == NULL)
{
@@ -123,19 +119,19 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex,
XLogRecord *record;
XLogReaderState *xlogreader;
char *errormsg;
- XLogPageReadPrivate private;
XLogRecPtr endptr;
- private.tliIndex = tliIndex;
- private.restoreCommand = restoreCommand;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir,
- XL_ROUTINE(.page_read = &SimpleXLogPageRead),
- &private);
+ xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
XLogBeginRead(xlogreader, ptr);
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!SimpleXLogPageRead(xlogreader, datadir, &tliIndex, restoreCommand))
+ break;
+ }
if (record == NULL)
{
if (errormsg)
@@ -170,7 +166,6 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
XLogRecPtr searchptr;
XLogReaderState *xlogreader;
char *errormsg;
- XLogPageReadPrivate private;
/*
* The given fork pointer points to the end of the last common record,
@@ -186,11 +181,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
forkptr += SizeOfXLogShortPHD;
}
- private.tliIndex = tliIndex;
- private.restoreCommand = restoreCommand;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir,
- XL_ROUTINE(.page_read = &SimpleXLogPageRead),
- &private);
+ xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
@@ -200,7 +191,13 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
uint8 info;
XLogBeginRead(xlogreader, searchptr);
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!SimpleXLogPageRead(xlogreader, datadir,
+ &tliIndex, restoreCommand))
+ break;
+ }
if (record == NULL)
{
@@ -247,10 +244,11 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
/* XLogReader callback function, to read a WAL page */
static bool
-SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *readBuf)
+SimpleXLogPageRead(XLogReaderState *xlogreader, const char *datadir,
+ int *tliIndex, const char *restoreCommand)
{
- XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
+ XLogRecPtr targetPagePtr = xlogreader->readPagePtr;
+ char *readBuf = xlogreader->readBuf;
uint32 targetPageOff;
XLogRecPtr targetSegEnd;
XLogSegNo targetSegNo;
@@ -283,14 +281,14 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
* be done both forward and backward, consider also switching timeline
* accordingly.
*/
- while (private->tliIndex < targetNentries - 1 &&
- targetHistory[private->tliIndex].end < targetSegEnd)
- private->tliIndex++;
- while (private->tliIndex > 0 &&
- targetHistory[private->tliIndex].begin >= targetSegEnd)
- private->tliIndex--;
+ while (*tliIndex < targetNentries - 1 &&
+ targetHistory[*tliIndex].end < targetSegEnd)
+ (*tliIndex)++;
+ while (*tliIndex > 0 &&
+ targetHistory[*tliIndex].begin >= targetSegEnd)
+ (*tliIndex)--;
- XLogFileName(xlogfname, targetHistory[private->tliIndex].tli,
+ XLogFileName(xlogfname, targetHistory[*tliIndex].tli,
xlogreadsegno, WalSegSz);
snprintf(xlogfpath, MAXPGPATH, "%s/" XLOGDIR "/%s",
@@ -303,7 +301,7 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
/*
* If we have no restore_command to execute, then exit.
*/
- if (private->restoreCommand == NULL)
+ if (restoreCommand == NULL)
{
pg_log_error("could not open file \"%s\": %m", xlogfpath);
xlogreader->readLen = -1;
@@ -317,7 +315,7 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
xlogreadfd = RestoreArchivedFile(xlogreader->segcxt.ws_dir,
xlogfname,
WalSegSz,
- private->restoreCommand);
+ restoreCommand);
if (xlogreadfd < 0)
{
@@ -359,7 +357,7 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
Assert(targetSegNo == xlogreadsegno);
- xlogreader->seg.ws_tli = targetHistory[private->tliIndex].tli;
+ xlogreader->seg.ws_tli = targetHistory[*tliIndex].tli;
xlogreader->readLen = XLOG_BLCKSZ;
return true;
}
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index 0f0e0723a3..f514de6b13 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -330,12 +330,17 @@ WALDumpCloseSegment(XLogReaderState *state)
state->seg.ws_file = -1;
}
-/* pg_waldump's XLogReaderRoutine->page_read callback */
+/*
+ * pg_waldump's WAL page rader, also used as page_read callback for
+ * XLogFindNextRecord
+ */
static bool
-WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetPtr, char *readBuff)
+WALDumpReadPage(XLogReaderState *state, void *priv)
{
- XLogDumpPrivate *private = state->private_data;
+ XLogRecPtr targetPagePtr = state->readPagePtr;
+ int reqLen = state->readLen;
+ char *readBuff = state->readBuf;
+ XLogDumpPrivate *private = (XLogDumpPrivate *) priv;
int count = XLOG_BLCKSZ;
WALReadError errinfo;
@@ -353,8 +358,8 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
}
}
- if (!WALRead(state, readBuff, targetPagePtr, count, private->timeline,
- &errinfo))
+ if (!WALRead(state, WALDumpOpenSegment, WALDumpCloseSegment,
+ readBuff, targetPagePtr, count, private->timeline, &errinfo))
{
WALOpenSegment *seg = &errinfo.wre_seg;
char fname[MAXPGPATH];
@@ -374,6 +379,7 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
(Size) errinfo.wre_req);
}
+ Assert(count >= state->readLen);
state->readLen = count;
return true;
}
@@ -1035,16 +1041,14 @@ main(int argc, char **argv)
/* we have everything we need, start reading */
xlogreader_state =
- XLogReaderAllocate(WalSegSz, waldir,
- XL_ROUTINE(.page_read = WALDumpReadPage,
- .segment_open = WALDumpOpenSegment,
- .segment_close = WALDumpCloseSegment),
- &private);
+ XLogReaderAllocate(WalSegSz, waldir, WALDumpCloseSegment);
+
if (!xlogreader_state)
fatal_error("out of memory");
/* first find a valid recptr to start from */
- first_record = XLogFindNextRecord(xlogreader_state, private.startptr);
+ first_record = XLogFindNextRecord(xlogreader_state, private.startptr,
+ &WALDumpReadPage, (void*) &private);
if (first_record == InvalidXLogRecPtr)
fatal_error("could not find a valid record after %X/%X",
@@ -1067,7 +1071,13 @@ main(int argc, char **argv)
for (;;)
{
/* try to read the next record */
- record = XLogReadRecord(xlogreader_state, &errormsg);
+ while (XLogReadRecord(xlogreader_state, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!WALDumpReadPage(xlogreader_state, (void *) &private))
+ break;
+ }
+
if (!record)
{
if (!config.follow || private.endptr_reached)
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index 23d9e70a59..4e9268b553 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -57,64 +57,15 @@ typedef struct WALSegmentContext
typedef struct XLogReaderState XLogReaderState;
-/* Function type definition for the read_page callback */
-typedef bool (*XLogPageReadCB) (XLogReaderState *xlogreader,
- XLogRecPtr targetPagePtr,
- int reqLen,
- XLogRecPtr targetRecPtr,
- char *readBuf);
+/* Function type definition for the segment cleanup callback */
+typedef void (*WALSegmentCleanupCB) (XLogReaderState *xlogreader);
+
+/* Function type definition for the open/close callbacks for WALRead() */
typedef void (*WALSegmentOpenCB) (XLogReaderState *xlogreader,
XLogSegNo nextSegNo,
TimeLineID *tli_p);
typedef void (*WALSegmentCloseCB) (XLogReaderState *xlogreader);
-typedef struct XLogReaderRoutine
-{
- /*
- * Data input callback
- *
- * This callback shall read at least reqLen valid bytes of the xlog page
- * starting at targetPagePtr, and store them in readBuf. The callback
- * shall return the number of bytes read (never more than XLOG_BLCKSZ), or
- * -1 on failure. The callback shall sleep, if necessary, to wait for the
- * requested bytes to become available. The callback will not be invoked
- * again for the same page unless more than the returned number of bytes
- * are needed.
- *
- * targetRecPtr is the position of the WAL record we're reading. Usually
- * it is equal to targetPagePtr + reqLen, but sometimes xlogreader needs
- * to read and verify the page or segment header, before it reads the
- * actual WAL record it's interested in. In that case, targetRecPtr can
- * be used to determine which timeline to read the page from.
- *
- * The callback shall set ->seg.ws_tli to the TLI of the file the page was
- * read from.
- */
- XLogPageReadCB page_read;
-
- /*
- * Callback to open the specified WAL segment for reading. ->seg.ws_file
- * shall be set to the file descriptor of the opened segment. In case of
- * failure, an error shall be raised by the callback and it shall not
- * return.
- *
- * "nextSegNo" is the number of the segment to be opened.
- *
- * "tli_p" is an input/output argument. WALRead() uses it to pass the
- * timeline in which the new segment should be found, but the callback can
- * use it to return the TLI that it actually opened.
- */
- WALSegmentOpenCB segment_open;
-
- /*
- * WAL segment close callback. ->seg.ws_file shall be set to a negative
- * number.
- */
- WALSegmentCloseCB segment_close;
-} XLogReaderRoutine;
-
-#define XL_ROUTINE(...) &(XLogReaderRoutine){__VA_ARGS__}
-
typedef struct
{
/* Is this block ref in use? */
@@ -144,12 +95,35 @@ typedef struct
uint16 data_bufsz;
} DecodedBkpBlock;
+/* Return code from XLogReadRecord */
+typedef enum XLogReadRecordResult
+{
+ XLREAD_SUCCESS, /* record is successfully read */
+ XLREAD_NEED_DATA, /* need more data. see XLogReadRecord. */
+ XLREAD_FAIL /* failed during reading a record */
+} XLogReadRecordResult;
+
+/*
+ * internal state of XLogReadRecord
+ *
+ * XLogReadState runs a state machine while reading a record. Theses states
+ * are not seen outside the function. Each state may repeat several times
+ * exiting requesting caller for new data. See the comment of XLogReadRecrod
+ * for details.
+ */
+typedef enum XLogReadRecordState {
+ XLREAD_NEXT_RECORD,
+ XLREAD_TOT_LEN,
+ XLREAD_FIRST_FRAGMENT,
+ XLREAD_CONTINUATION
+} XLogReadRecordState;
+
struct XLogReaderState
{
/*
* Operational callbacks
*/
- XLogReaderRoutine routine;
+ WALSegmentCleanupCB cleanup_cb;
/* ----------------------------------------
* Public parameters
@@ -162,18 +136,14 @@ struct XLogReaderState
*/
uint64 system_identifier;
- /*
- * Opaque data for callbacks to use. Not used by XLogReader.
- */
- void *private_data;
-
/*
* Start and end point of last record read. EndRecPtr is also used as the
* position to read next. Calling XLogBeginRead() sets EndRecPtr to the
* starting position and ReadRecPtr to invalid.
*/
- XLogRecPtr ReadRecPtr; /* start of last record read */
+ XLogRecPtr ReadRecPtr; /* start of last record read or being read */
XLogRecPtr EndRecPtr; /* end+1 of last record read */
+ XLogRecPtr PrevRecPtr; /* start of previous record read */
/* ----------------------------------------
* Communication with page reader
@@ -187,7 +157,9 @@ struct XLogReaderState
* the request, or -1 on error */
TimeLineID readPageTLI; /* TLI for data currently in readBuf */
char *readBuf; /* buffer to store data */
- bool page_verified; /* is the page on the buffer verified? */
+ bool page_verified; /* is the page header on the buffer verified? */
+ bool record_verified;/* is the current record header verified? */
+
/* ----------------------------------------
@@ -229,8 +201,6 @@ struct XLogReaderState
XLogRecPtr latestPagePtr;
TimeLineID latestPageTLI;
- /* beginning of the WAL record being read. */
- XLogRecPtr currRecPtr;
/* timeline to read it from, 0 if a lookup is required */
TimeLineID currTLI;
@@ -257,6 +227,15 @@ struct XLogReaderState
char *readRecordBuf;
uint32 readRecordBufSize;
+ /*
+ * XLogReadRecord() state
+ */
+ XLogReadRecordState readRecordState;/* state machine state */
+ int recordGotLen; /* amount of current record that has
+ * already been read */
+ int recordRemainLen; /* length of current record that remains */
+ XLogRecPtr recordContRecPtr; /* where the current record continues */
+
/* Buffer to hold error message */
char *errormsg_buf;
};
@@ -264,9 +243,7 @@ struct XLogReaderState
/* Get a new XLogReader */
extern XLogReaderState *XLogReaderAllocate(int wal_segment_size,
const char *waldir,
- XLogReaderRoutine *routine,
- void *private_data);
-extern XLogReaderRoutine *LocalXLogReaderRoutine(void);
+ WALSegmentCleanupCB cleanup_cb);
/* Free an XLogReader */
extern void XLogReaderFree(XLogReaderState *state);
@@ -274,12 +251,17 @@ extern void XLogReaderFree(XLogReaderState *state);
/* Position the XLogReader to given record */
extern void XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr);
#ifdef FRONTEND
-extern XLogRecPtr XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr);
+/* Function type definition for the read_page callback */
+typedef bool (*XLogFindNextRecordCB) (XLogReaderState *xlogreader,
+ void *private);
+extern XLogRecPtr XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
+ XLogFindNextRecordCB read_page, void *private);
#endif /* FRONTEND */
/* Read the next XLog record. Returns NULL on end-of-WAL or failure */
-extern struct XLogRecord *XLogReadRecord(XLogReaderState *state,
- char **errormsg);
+extern XLogReadRecordResult XLogReadRecord(XLogReaderState *state,
+ XLogRecord **record,
+ char **errormsg);
/* Validate a page */
extern bool XLogReaderValidatePageHeader(XLogReaderState *state,
@@ -299,6 +281,7 @@ typedef struct WALReadError
} WALReadError;
extern bool WALRead(XLogReaderState *state,
+ WALSegmentOpenCB segopenfn, WALSegmentCloseCB sgclosefn,
char *buf, XLogRecPtr startptr, Size count,
TimeLineID tli, WALReadError *errinfo);
diff --git a/src/include/access/xlogutils.h b/src/include/access/xlogutils.h
index 364a21c4ea..397fb27fc2 100644
--- a/src/include/access/xlogutils.h
+++ b/src/include/access/xlogutils.h
@@ -47,9 +47,7 @@ extern Buffer XLogReadBufferExtended(RelFileNode rnode, ForkNumber forknum,
extern Relation CreateFakeRelcacheEntry(RelFileNode rnode);
extern void FreeFakeRelcacheEntry(Relation fakerel);
-extern bool read_local_xlog_page(XLogReaderState *state,
- XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetRecPtr, char *cur_page);
+extern bool read_local_xlog_page(XLogReaderState *state);
extern void wal_segment_open(XLogReaderState *state,
XLogSegNo nextSegNo,
TimeLineID *tli_p);
diff --git a/src/include/pg_config_manual.h b/src/include/pg_config_manual.h
index f10ad0acd6..94749609b4 100644
--- a/src/include/pg_config_manual.h
+++ b/src/include/pg_config_manual.h
@@ -377,7 +377,7 @@
* Enable debugging print statements for WAL-related operations; see
* also the wal_debug GUC var.
*/
-/* #define WAL_DEBUG */
+#define WAL_DEBUG
/*
* Enable tracing of resource consumption during sort operations;
diff --git a/src/include/replication/logical.h b/src/include/replication/logical.h
index c253403372..b936f31393 100644
--- a/src/include/replication/logical.h
+++ b/src/include/replication/logical.h
@@ -29,6 +29,10 @@ typedef void (*LogicalOutputPluginWriterUpdateProgress) (struct LogicalDecodingC
TransactionId xid
);
+typedef struct LogicalDecodingContext LogicalDecodingContext;
+
+typedef bool (*LogicalDecodingXLogPageReadCB)(XLogReaderState *ctx);
+
typedef struct LogicalDecodingContext
{
/* memory context this is all allocated in */
@@ -39,6 +43,7 @@ typedef struct LogicalDecodingContext
/* infrastructure pieces for decoding */
XLogReaderState *reader;
+ LogicalDecodingXLogPageReadCB page_read;
struct ReorderBuffer *reorder;
struct SnapBuild *snapshot_builder;
@@ -105,14 +110,16 @@ extern LogicalDecodingContext *CreateInitDecodingContext(const char *plugin,
List *output_plugin_options,
bool need_full_snapshot,
XLogRecPtr restart_lsn,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress);
extern LogicalDecodingContext *CreateDecodingContext(XLogRecPtr start_lsn,
List *output_plugin_options,
bool fast_forward,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress);
--
2.27.0
----Next_Part(Thu_Mar__4_11_28_53_2021_014)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
filename="v17-0003-Remove-globals-readOff-readLen-and-readSegNo.patch"
^ permalink raw reply [nested|flat] 70+ messages in thread
* [PATCH v17 2/4] Move page-reader out of XLogReadRecord
@ 2019-09-10 03:58 Kyotaro Horiguchi <[email protected]>
0 siblings, 0 replies; 70+ messages in thread
From: Kyotaro Horiguchi @ 2019-09-10 03:58 UTC (permalink / raw)
This is the second step of removing callbacks from WAL record reader.
Since it is essential to take in additional data while reading a
record, the function have to ask caller for new data while keeping
working state. Thus the function is turned into a state machine.
---
src/backend/access/transam/twophase.c | 15 +-
src/backend/access/transam/xlog.c | 58 +-
src/backend/access/transam/xlogreader.c | 697 +++++++++++-------
src/backend/access/transam/xlogutils.c | 17 +-
src/backend/replication/logical/logical.c | 26 +-
.../replication/logical/logicalfuncs.c | 13 +-
src/backend/replication/slotfuncs.c | 18 +-
src/backend/replication/walsender.c | 32 +-
src/bin/pg_rewind/parsexlog.c | 84 ++-
src/bin/pg_waldump/pg_waldump.c | 36 +-
src/include/access/xlogreader.h | 121 ++-
src/include/access/xlogutils.h | 4 +-
src/include/pg_config_manual.h | 2 +-
src/include/replication/logical.h | 11 +-
14 files changed, 646 insertions(+), 488 deletions(-)
diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index 80d2d20d6c..7d5a96a5f5 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -1325,11 +1325,8 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
XLogReaderState *xlogreader;
char *errormsg;
- xlogreader = XLogReaderAllocate(wal_segment_size, NULL,
- XL_ROUTINE(.page_read = &read_local_xlog_page,
- .segment_open = &wal_segment_open,
- .segment_close = &wal_segment_close),
- NULL);
+ xlogreader = XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+
if (!xlogreader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
@@ -1337,7 +1334,13 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
errdetail("Failed while allocating a WAL reading processor.")));
XLogBeginRead(xlogreader, lsn);
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!read_local_xlog_page(xlogreader))
+ break;
+ }
+
if (record == NULL)
ereport(ERROR,
(errcode_for_file_access(),
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 606328a65a..670996ae67 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -836,13 +836,6 @@ static XLogSource currentSource = XLOG_FROM_ANY;
static bool lastSourceFailed = false;
static bool pendingWalRcvRestart = false;
-typedef struct XLogPageReadPrivate
-{
- int emode;
- bool fetching_ckpt; /* are we fetching a checkpoint record? */
- bool randAccess;
-} XLogPageReadPrivate;
-
/*
* These variables track when we last obtained some WAL data to process,
* and where we got it from. (XLogReceiptSource is initially the same as
@@ -917,8 +910,8 @@ static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath,
static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli,
XLogSource source, bool notfoundOk);
static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, XLogSource source);
-static bool XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
+static bool XLogPageRead(XLogReaderState *xlogreader,
+ bool fetching_ckpt, int emode, bool randAccess);
static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
bool fetching_ckpt, XLogRecPtr tliRecPtr);
static int emode_for_corrupt_record(int emode, XLogRecPtr RecPtr);
@@ -1231,8 +1224,7 @@ XLogInsertRecord(XLogRecData *rdata,
appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len);
if (!debug_reader)
- debug_reader = XLogReaderAllocate(wal_segment_size, NULL,
- XL_ROUTINE(), NULL);
+ debug_reader = XLogReaderAllocate(wal_segment_size, NULL, NULL);
if (!debug_reader)
{
@@ -4344,15 +4336,10 @@ CleanupBackupHistory(void)
* record is available.
*/
static XLogRecord *
-ReadRecord(XLogReaderState *xlogreader, int emode,
- bool fetching_ckpt)
+ReadRecord(XLogReaderState *xlogreader, int emode, bool fetching_ckpt)
{
XLogRecord *record;
- XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
-
- private->fetching_ckpt = fetching_ckpt;
- private->emode = emode;
- private->randAccess = (xlogreader->ReadRecPtr == InvalidXLogRecPtr);
+ bool randAccess = (xlogreader->ReadRecPtr == InvalidXLogRecPtr);
/* This is the first attempt to read this page. */
lastSourceFailed = false;
@@ -4360,8 +4347,16 @@ ReadRecord(XLogReaderState *xlogreader, int emode,
for (;;)
{
char *errormsg;
+ XLogReadRecordResult result;
+
+ while ((result = XLogReadRecord(xlogreader, &record, &errormsg))
+ == XLREAD_NEED_DATA)
+ {
+ if (!XLogPageRead(xlogreader, fetching_ckpt, emode, randAccess))
+ break;
+
+ }
- record = XLogReadRecord(xlogreader, &errormsg);
ReadRecPtr = xlogreader->ReadRecPtr;
EndRecPtr = xlogreader->EndRecPtr;
if (record == NULL)
@@ -6380,7 +6375,6 @@ StartupXLOG(void)
bool backupFromStandby = false;
DBState dbstate_at_startup;
XLogReaderState *xlogreader;
- XLogPageReadPrivate private;
bool promoted = false;
struct stat st;
@@ -6539,13 +6533,9 @@ StartupXLOG(void)
OwnLatch(&XLogCtl->recoveryWakeupLatch);
/* Set up XLOG reader facility */
- MemSet(&private, 0, sizeof(XLogPageReadPrivate));
xlogreader =
- XLogReaderAllocate(wal_segment_size, NULL,
- XL_ROUTINE(.page_read = &XLogPageRead,
- .segment_open = NULL,
- .segment_close = wal_segment_close),
- &private);
+ XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+
if (!xlogreader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
@@ -11987,12 +11977,13 @@ CancelBackup(void)
* sleep and retry.
*/
static bool
-XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetRecPtr, char *readBuf)
+XLogPageRead(XLogReaderState *xlogreader,
+ bool fetching_ckpt, int emode, bool randAccess)
{
- XLogPageReadPrivate *private =
- (XLogPageReadPrivate *) xlogreader->private_data;
- int emode = private->emode;
+ char *readBuf = xlogreader->readBuf;
+ XLogRecPtr targetPagePtr = xlogreader->readPagePtr;
+ int reqLen = xlogreader->readLen;
+ XLogRecPtr targetRecPtr = xlogreader->ReadRecPtr;
uint32 targetPageOff;
XLogSegNo targetSegNo PG_USED_FOR_ASSERTS_ONLY;
int r;
@@ -12035,8 +12026,8 @@ retry:
flushedUpto < targetPagePtr + reqLen))
{
if (!WaitForWALToBecomeAvailable(targetPagePtr + reqLen,
- private->randAccess,
- private->fetching_ckpt,
+ randAccess,
+ fetching_ckpt,
targetRecPtr))
{
if (readFile >= 0)
@@ -12141,6 +12132,7 @@ retry:
goto next_record_is_invalid;
}
+ Assert(xlogreader->readPagePtr == targetPagePtr);
xlogreader->readLen = readLen;
return true;
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index 7d04995b7b..ba3e4868f7 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -40,7 +40,7 @@ static bool XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr,
int reqLen, bool header_inclusive);
static void XLogReaderInvalReadState(XLogReaderState *state);
static bool ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
- XLogRecPtr PrevRecPtr, XLogRecord *record, bool randAccess);
+ XLogRecPtr PrevRecPtr, XLogRecord *record);
static bool ValidXLogRecord(XLogReaderState *state, XLogRecord *record,
XLogRecPtr recptr);
static void ResetDecoder(XLogReaderState *state);
@@ -73,7 +73,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...)
*/
XLogReaderState *
XLogReaderAllocate(int wal_segment_size, const char *waldir,
- XLogReaderRoutine *routine, void *private_data)
+ WALSegmentCleanupCB cleanup_cb)
{
XLogReaderState *state;
@@ -84,7 +84,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
return NULL;
/* initialize caller-provided support functions */
- state->routine = *routine;
+ state->cleanup_cb = cleanup_cb;
state->max_block_id = -1;
@@ -107,8 +107,6 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
WALOpenSegmentInit(&state->seg, &state->segcxt, wal_segment_size,
waldir);
- /* system_identifier initialized to zeroes above */
- state->private_data = private_data;
/* ReadRecPtr, EndRecPtr and readLen initialized to zeroes above */
state->errormsg_buf = palloc_extended(MAX_ERRORMSG_LEN + 1,
MCXT_ALLOC_NO_OOM);
@@ -140,8 +138,8 @@ XLogReaderFree(XLogReaderState *state)
{
int block_id;
- if (state->seg.ws_file != -1)
- state->routine.segment_close(state);
+ if (state->seg.ws_file >= 0)
+ state->cleanup_cb(state);
for (block_id = 0; block_id <= XLR_MAX_BLOCK_ID; block_id++)
{
@@ -246,6 +244,7 @@ XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
/* Begin at the passed-in record pointer. */
state->EndRecPtr = RecPtr;
state->ReadRecPtr = InvalidXLogRecPtr;
+ state->readRecordState = XLREAD_NEXT_RECORD;
}
/*
@@ -254,317 +253,449 @@ XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
* XLogBeginRead() or XLogFindNextRecord() must be called before the first call
* to XLogReadRecord().
*
- * If the page_read callback fails to read the requested data, NULL is
- * returned. The callback is expected to have reported the error; errormsg
- * is set to NULL.
+ * This function may return XLREAD_NEED_DATA several times before returning a
+ * result record. The caller shall read in some new data then call this
+ * function again with the same parameters.
*
- * If the reading fails for some other reason, NULL is also returned, and
- * *errormsg is set to a string with details of the failure.
+ * When a record is successfully read, returns XLREAD_SUCCESS with result
+ * record being stored in *record. Otherwise *record is NULL.
*
- * The returned pointer (or *errormsg) points to an internal buffer that's
- * valid until the next call to XLogReadRecord.
+ * Returns XLREAD_NEED_DATA if more data is needed to finish reading the
+ * current record. In that case, state->readPagePtr and state->readLen inform
+ * the desired position and minimum length of data needed. The caller shall
+ * read in the requested data and set state->readBuf to point to a buffer
+ * containing it. The caller must also set state->readPageTLI and
+ * state->readLen to indicate the timeline that it was read from, and the
+ * length of data that is now available (which must be >= given readLen),
+ * respectively.
+ *
+ * If invalid data is encountered, returns XLREAD_FAIL with *record being set to
+ * NULL. *errormsg is set to a string with details of the failure.
+ * The returned pointer (or *errormsg) points to an internal buffer that's valid
+ * until the next call to XLogReadRecord.
+ *
+ *
+ * This function runs a state machine consists of the following states.
+ *
+ * XLREAD_NEXT_RECORD :
+ * The initial state, if called with valid RecPtr, try to read a record at
+ * that position. If invalid RecPtr is given try to read a record just after
+ * the last one previously read.
+ * This state ens after setting ReadRecPtr. Then goes to XLREAD_TOT_LEN.
+ *
+ * XLREAD_TOT_LEN:
+ * Examining record header. Ends after reading record total
+ * length. recordRemainLen and recordGotLen are initialized.
+ *
+ * XLREAD_FIRST_FRAGMENT:
+ * Reading the first fragment. Ends with finishing reading a single
+ * record. Goes to XLREAD_NEXT_RECORD if that's all or
+ * XLREAD_CONTINUATION if we have continuation.
+
+ * XLREAD_CONTINUATION:
+ * Reading continuation of record. Ends with finishing the whole record then
+ * goes to XLREAD_NEXT_RECORD. During this state, recordRemainLen indicates
+ * how much is left and readRecordBuf holds the partially assert
+ * record.recordContRecPtr points to the beginning of the next page where to
+ * continue.
+ *
+ * If wrong data found in any state, the state machine stays at the current
+ * state. This behavior allows to continue reading a reacord switching among
+ * different souces, while streaming replication.
*/
-XLogRecord *
-XLogReadRecord(XLogReaderState *state, char **errormsg)
+XLogReadRecordResult
+XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
{
- XLogRecPtr RecPtr;
- XLogRecord *record;
- XLogRecPtr targetPagePtr;
- bool randAccess;
- uint32 len,
- total_len;
- uint32 targetRecOff;
- uint32 pageHeaderSize;
- bool gotheader;
+ XLogRecord *prec;
- /*
- * randAccess indicates whether to verify the previous-record pointer of
- * the record we're reading. We only do this if we're reading
- * sequentially, which is what we initially assume.
- */
- randAccess = false;
+ *record = NULL;
/* reset error state */
*errormsg = NULL;
state->errormsg_buf[0] = '\0';
- ResetDecoder(state);
-
- RecPtr = state->EndRecPtr;
-
- if (state->ReadRecPtr != InvalidXLogRecPtr)
- {
- /* read the record after the one we just read */
-
- /*
- * EndRecPtr is pointing to end+1 of the previous WAL record. If
- * we're at a page boundary, no more records can fit on the current
- * page. We must skip over the page header, but we can't do that until
- * we've read in the page, since the header size is variable.
- */
- }
- else
- {
- /*
- * Caller supplied a position to start at.
- *
- * In this case, EndRecPtr should already be pointing to a valid
- * record starting position.
- */
- Assert(XRecOffIsValid(RecPtr));
- randAccess = true;
- }
-
- state->currRecPtr = RecPtr;
-
- targetPagePtr = RecPtr - (RecPtr % XLOG_BLCKSZ);
- targetRecOff = RecPtr % XLOG_BLCKSZ;
-
- /*
- * Read the page containing the record into state->readBuf. Request enough
- * byte to cover the whole record header, or at least the part of it that
- * fits on the same page.
- */
- while (XLogNeedData(state, targetPagePtr,
- Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
- targetRecOff != 0))
- {
- if (!state->routine.page_read(state, state->readPagePtr, state->readLen,
- RecPtr, state->readBuf))
- break;
- }
-
- if (!state->page_verified)
- goto err;
-
- /*
- * We have at least the page header, so we can examine it now.
- */
- pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
- if (targetRecOff == 0)
- {
- /*
- * At page start, so skip over page header.
- */
- RecPtr += pageHeaderSize;
- targetRecOff = pageHeaderSize;
- }
- else if (targetRecOff < pageHeaderSize)
- {
- report_invalid_record(state, "invalid record offset at %X/%X",
- LSN_FORMAT_ARGS(RecPtr));
- goto err;
- }
-
- if ((((XLogPageHeader) state->readBuf)->xlp_info & XLP_FIRST_IS_CONTRECORD) &&
- targetRecOff == pageHeaderSize)
- {
- report_invalid_record(state, "contrecord is requested by %X/%X",
- LSN_FORMAT_ARGS(RecPtr));
- goto err;
- }
-
- /* XLogNeedData has verified the page header */
- Assert(pageHeaderSize <= state->readLen);
-
- /*
- * Read the record length.
- *
- * NB: Even though we use an XLogRecord pointer here, the whole record
- * header might not fit on this page. xl_tot_len is the first field of the
- * struct, so it must be on this page (the records are MAXALIGNed), but we
- * cannot access any other fields until we've verified that we got the
- * whole header.
- */
- record = (XLogRecord *) (state->readBuf + RecPtr % XLOG_BLCKSZ);
- total_len = record->xl_tot_len;
-
- /*
- * If the whole record header is on this page, validate it immediately.
- * Otherwise do just a basic sanity check on xl_tot_len, and validate the
- * rest of the header after reading it from the next page. The xl_tot_len
- * check is necessary here to ensure that we enter the "Need to reassemble
- * record" code path below; otherwise we might fail to apply
- * ValidXLogRecordHeader at all.
- */
- if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
- {
- if (!ValidXLogRecordHeader(state, RecPtr, state->ReadRecPtr, record,
- randAccess))
- goto err;
- gotheader = true;
- }
- else
- {
- /* XXX: more validation should be done here */
- if (total_len < SizeOfXLogRecord)
- {
- report_invalid_record(state,
- "invalid record length at %X/%X: wanted %u, got %u",
- LSN_FORMAT_ARGS(RecPtr),
- (uint32) SizeOfXLogRecord, total_len);
- goto err;
- }
- gotheader = false;
- }
-
- len = XLOG_BLCKSZ - RecPtr % XLOG_BLCKSZ;
- if (total_len > len)
+ switch (state->readRecordState)
{
- /* Need to reassemble record */
- char *contdata;
- XLogPageHeader pageHeader;
- char *buffer;
- uint32 gotlen;
-
- /*
- * Enlarge readRecordBuf as needed.
- */
- if (total_len > state->readRecordBufSize &&
- !allocate_recordbuf(state, total_len))
- {
- /* We treat this as a "bogus data" condition */
- report_invalid_record(state, "record length %u at %X/%X too long",
- total_len, LSN_FORMAT_ARGS(RecPtr));
- goto err;
- }
+ case XLREAD_NEXT_RECORD:
+ ResetDecoder(state);
- /* Copy the first fragment of the record from the first page. */
- memcpy(state->readRecordBuf,
- state->readBuf + RecPtr % XLOG_BLCKSZ, len);
- buffer = state->readRecordBuf + len;
- gotlen = len;
-
- do
- {
- int rest_len = total_len - gotlen;
-
- /* Calculate pointer to beginning of next page */
- targetPagePtr += XLOG_BLCKSZ;
-
- /* Wait for the next page to become available */
- while (XLogNeedData(state, targetPagePtr,
- Min(rest_len, XLOG_BLCKSZ),
- false))
+ if (state->ReadRecPtr != InvalidXLogRecPtr)
{
- if (!state->routine.page_read(state, state->readPagePtr,
- state->readLen,
- state->ReadRecPtr,
- state->readBuf))
- break;
+ /* read the record after the one we just read */
+
+ /*
+ * EndRecPtr is pointing to end+1 of the previous WAL record.
+ * If we're at a page boundary, no more records can fit on the
+ * current page. We must skip over the page header, but we
+ * can't do that until we've read in the page, since the header
+ * size is variable.
+ */
+ state->PrevRecPtr = state->ReadRecPtr;
+ state->ReadRecPtr = state->EndRecPtr;
+ }
+ else
+ {
+ /*
+ * Caller supplied a position to start at.
+ *
+ * In this case, EndRecPtr should already be pointing to a
+ * valid record starting position.
+ */
+ Assert(XRecOffIsValid(state->EndRecPtr));
+ state->ReadRecPtr = state->EndRecPtr;
+
+ /*
+ * We cannot verify the previous-record pointer when we're
+ * seeking to a particular record. Reset PrevRecPtr so that we
+ * won't try doing that.
+ */
+ state->PrevRecPtr = InvalidXLogRecPtr;
+ state->EndRecPtr = InvalidXLogRecPtr; /* to be tidy */
}
+ state->record_verified = false;
+ state->readRecordState = XLREAD_TOT_LEN;
+ /* fall through */
+
+ case XLREAD_TOT_LEN:
+ {
+ uint32 total_len;
+ uint32 pageHeaderSize;
+ XLogRecPtr targetPagePtr;
+ uint32 targetRecOff;
+ XLogPageHeader pageHeader;
+
+ targetPagePtr =
+ state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
+ targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+
+ /*
+ * Check if we have enough data. For the first record in the page,
+ * the requesting length doesn't contain page header.
+ */
+ if (XLogNeedData(state, targetPagePtr,
+ Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+ targetRecOff != 0))
+ return XLREAD_NEED_DATA;
+
+ /* error out if caller supplied bogus page */
if (!state->page_verified)
goto err;
- Assert(SizeOfXLogShortPHD <= state->readLen);
+ /* examine page header now. */
+ pageHeaderSize =
+ XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+ if (targetRecOff == 0)
+ {
+ /* At page start, so skip over page header. */
+ state->ReadRecPtr += pageHeaderSize;
+ targetRecOff = pageHeaderSize;
+ }
+ else if (targetRecOff < pageHeaderSize)
+ {
+ report_invalid_record(state, "invalid record offset at %X/%X",
+ LSN_FORMAT_ARGS(state->ReadRecPtr));
+ goto err;
+ }
- /* Check that the continuation on next page looks valid */
pageHeader = (XLogPageHeader) state->readBuf;
- if (!(pageHeader->xlp_info & XLP_FIRST_IS_CONTRECORD))
+ if ((pageHeader->xlp_info & XLP_FIRST_IS_CONTRECORD) &&
+ targetRecOff == pageHeaderSize)
{
- report_invalid_record(state,
- "there is no contrecord flag at %X/%X",
- LSN_FORMAT_ARGS(RecPtr));
+ report_invalid_record(state, "contrecord is requested by %X/%X",
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr);
goto err;
}
- /*
- * Cross-check that xlp_rem_len agrees with how much of the record
- * we expect there to be left.
- */
- if (pageHeader->xlp_rem_len == 0 ||
- total_len != (pageHeader->xlp_rem_len + gotlen))
- {
- report_invalid_record(state,
- "invalid contrecord length %u (expected %lld) at %X/%X",
- pageHeader->xlp_rem_len,
- ((long long) total_len) - gotlen,
- LSN_FORMAT_ARGS(RecPtr));
- goto err;
- }
-
- /* Append the continuation from this page to the buffer */
- pageHeaderSize = XLogPageHeaderSize(pageHeader);
-
+ /* XLogNeedData has verified the page header */
Assert(pageHeaderSize <= state->readLen);
- contdata = (char *) state->readBuf + pageHeaderSize;
- len = XLOG_BLCKSZ - pageHeaderSize;
- if (pageHeader->xlp_rem_len < len)
- len = pageHeader->xlp_rem_len;
+ /*
+ * Read the record length.
+ *
+ * NB: Even though we use an XLogRecord pointer here, the whole
+ * record header might not fit on this page. xl_tot_len is the first
+ * field of the struct, so it must be on this page (the records are
+ * MAXALIGNed), but we cannot access any other fields until we've
+ * verified that we got the whole header.
+ */
+ prec = (XLogRecord *) (state->readBuf +
+ state->ReadRecPtr % XLOG_BLCKSZ);
+ total_len = prec->xl_tot_len;
- Assert (pageHeaderSize + len <= state->readLen);
- memcpy(buffer, (char *) contdata, len);
- buffer += len;
- gotlen += len;
+ /*
+ * If the whole record header is on this page, validate it
+ * immediately. Otherwise do just a basic sanity check on
+ * xl_tot_len, and validate the rest of the header after reading it
+ * from the next page. The xl_tot_len check is necessary here to
+ * ensure that we enter the XLREAD_CONTINUATION state below;
+ * otherwise we might fail to apply ValidXLogRecordHeader at all.
+ */
+ if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
+ {
+ if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
+ state->PrevRecPtr, prec))
+ goto err;
- /* If we just reassembled the record header, validate it. */
- if (!gotheader)
+ state->record_verified = true;
+ }
+ else
{
- record = (XLogRecord *) state->readRecordBuf;
- if (!ValidXLogRecordHeader(state, RecPtr, state->ReadRecPtr,
- record, randAccess))
+ /* XXX: more validation should be done here */
+ if (total_len < SizeOfXLogRecord)
+ {
+ report_invalid_record(state,
+ "invalid record length at %X/%X: wanted %u, got %u",
+ LSN_FORMAT_ARGS(state->ReadRecPtr),
+ (uint32) SizeOfXLogRecord, total_len);
goto err;
- gotheader = true;
+ }
}
- } while (gotlen < total_len);
-
- Assert(gotheader);
- record = (XLogRecord *) state->readRecordBuf;
- if (!ValidXLogRecord(state, record, RecPtr))
- goto err;
+ /*
+ * Wait for the rest of the record, or the part of the record that
+ * fit on the first page if crossed a page boundary, to become
+ * available.
+ */
+ state->recordGotLen = 0;
+ state->recordRemainLen = total_len;
+ state->readRecordState = XLREAD_FIRST_FRAGMENT;
+ }
+ /* fall through */
- pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
- state->ReadRecPtr = RecPtr;
- state->EndRecPtr = targetPagePtr + pageHeaderSize
- + MAXALIGN(pageHeader->xlp_rem_len);
- }
- else
- {
- /* Wait for the record data to become available */
- while (XLogNeedData(state, targetPagePtr,
- Min(targetRecOff + total_len, XLOG_BLCKSZ), true))
+ case XLREAD_FIRST_FRAGMENT:
{
- if (!state->routine.page_read(state, state->readPagePtr,
- state->readLen,
- state->ReadRecPtr, state->readBuf))
+ uint32 total_len = state->recordRemainLen;
+ uint32 request_len;
+ uint32 record_len;
+ XLogRecPtr targetPagePtr;
+ uint32 targetRecOff;
+
+ /*
+ * Wait for the rest of the record on the first page to become
+ * available
+ */
+ targetPagePtr =
+ state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
+ targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+
+ request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ);
+ record_len = request_len - targetRecOff;
+
+ /* ReadRecPtr contains page header */
+ Assert (targetRecOff != 0);
+ if (XLogNeedData(state, targetPagePtr, request_len, true))
+ return XLREAD_NEED_DATA;
+
+ /* error out if caller supplied bogus page */
+ if (!state->page_verified)
+ goto err;
+
+ prec = (XLogRecord *) (state->readBuf + targetRecOff);
+
+ /* validate record header if not yet */
+ if (!state->record_verified && record_len >= SizeOfXLogRecord)
+ {
+ if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
+ state->PrevRecPtr, prec))
+ goto err;
+
+ state->record_verified = true;
+ }
+
+
+ if (total_len == record_len)
+ {
+ /* Record does not cross a page boundary */
+ Assert(state->record_verified);
+
+ if (!ValidXLogRecord(state, prec, state->ReadRecPtr))
+ goto err;
+
+ state->record_verified = true; /* to be tidy */
+
+ /* We already checked the header earlier */
+ state->EndRecPtr = state->ReadRecPtr + MAXALIGN(record_len);
+
+ *record = prec;
+ state->readRecordState = XLREAD_NEXT_RECORD;
break;
+ }
+
+ /*
+ * The record continues on the next page. Need to reassemble
+ * record
+ */
+ Assert(total_len > record_len);
+
+ /* Enlarge readRecordBuf as needed. */
+ if (total_len > state->readRecordBufSize &&
+ !allocate_recordbuf(state, total_len))
+ {
+ /* We treat this as a "bogus data" condition */
+ report_invalid_record(state,
+ "record length %u at %X/%X too long",
+ total_len,
+ LSN_FORMAT_ARGS(state->ReadRecPtr));
+ goto err;
+ }
+
+ /* Copy the first fragment of the record from the first page. */
+ memcpy(state->readRecordBuf, state->readBuf + targetRecOff,
+ record_len);
+ state->recordGotLen += record_len;
+ state->recordRemainLen -= record_len;
+
+ /* Calculate pointer to beginning of next page */
+ state->recordContRecPtr = state->ReadRecPtr + record_len;
+ Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0);
+
+ state->readRecordState = XLREAD_CONTINUATION;
}
+ /* fall through */
+
+ case XLREAD_CONTINUATION:
+ {
+ XLogPageHeader pageHeader;
+ uint32 pageHeaderSize;
+ XLogRecPtr targetPagePtr;
+
+ /* we enter this state only if we haven't read the whole record. */
+ Assert (state->recordRemainLen > 0);
+
+ while(state->recordRemainLen > 0)
+ {
+ char *contdata;
+ uint32 request_len;
+ uint32 record_len;
+
+ /* Wait for the next page to become available */
+ targetPagePtr = state->recordContRecPtr;
+
+ /* this request contains page header */
+ Assert (targetPagePtr != 0);
+ if (XLogNeedData(state, targetPagePtr,
+ Min(state->recordRemainLen, XLOG_BLCKSZ),
+ false))
+ return XLREAD_NEED_DATA;
+
+ if (!state->page_verified)
+ goto err;
+
+ Assert(SizeOfXLogShortPHD <= state->readLen);
- if (!state->page_verified)
- goto err;
+ /* Check that the continuation on next page looks valid */
+ pageHeader = (XLogPageHeader) state->readBuf;
+ if (!(pageHeader->xlp_info & XLP_FIRST_IS_CONTRECORD))
+ {
+ report_invalid_record(
+ state,
+ "there is no contrecord flag at %X/%X reading %X/%X",
+ (uint32) (state->recordContRecPtr >> 32),
+ (uint32) state->recordContRecPtr,
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr);
+ goto err;
+ }
- /* Record does not cross a page boundary */
- if (!ValidXLogRecord(state, record, RecPtr))
- goto err;
+ /*
+ * Cross-check that xlp_rem_len agrees with how much of the
+ * record we expect there to be left.
+ */
+ if (pageHeader->xlp_rem_len == 0 ||
+ pageHeader->xlp_rem_len != state->recordRemainLen)
+ {
+ report_invalid_record(
+ state,
+ "invalid contrecord length %u at %X/%X reading %X/%X, expected %u",
+ pageHeader->xlp_rem_len,
+ (uint32) (state->recordContRecPtr >> 32),
+ (uint32) state->recordContRecPtr,
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr,
+ state->recordRemainLen);
+ goto err;
+ }
- state->EndRecPtr = RecPtr + MAXALIGN(total_len);
+ /* Append the continuation from this page to the buffer */
+ pageHeaderSize = XLogPageHeaderSize(pageHeader);
- state->ReadRecPtr = RecPtr;
+ /*
+ * XLogNeedData should have ensured that the whole page header
+ * was read
+ */
+ Assert(state->readLen >= pageHeaderSize);
+
+ contdata = (char *) state->readBuf + pageHeaderSize;
+ record_len = XLOG_BLCKSZ - pageHeaderSize;
+ if (pageHeader->xlp_rem_len < record_len)
+ record_len = pageHeader->xlp_rem_len;
+
+ request_len = record_len + pageHeaderSize;
+
+ /* XLogNeedData should have ensured all needed data was read */
+ Assert (state->readLen >= request_len);
+
+ memcpy(state->readRecordBuf + state->recordGotLen,
+ (char *) contdata, record_len);
+ state->recordGotLen += record_len;
+ state->recordRemainLen -= record_len;
+
+ /* If we just reassembled the record header, validate it. */
+ if (!state->record_verified)
+ {
+ Assert(state->recordGotLen >= SizeOfXLogRecord);
+ if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
+ state->PrevRecPtr,
+ (XLogRecord *) state->readRecordBuf))
+ goto err;
+
+ state->record_verified = true;
+ }
+
+ /* Calculate pointer to beginning of next page, and continue */
+ state->recordContRecPtr += XLOG_BLCKSZ;
+ }
+
+ /* targetPagePtr is pointing the last-read page here */
+ prec = (XLogRecord *) state->readRecordBuf;
+ if (!ValidXLogRecord(state, prec, state->ReadRecPtr))
+ goto err;
+
+ pageHeaderSize =
+ XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+ state->EndRecPtr = targetPagePtr + pageHeaderSize
+ + MAXALIGN(pageHeader->xlp_rem_len);
+
+ *record = prec;
+ state->readRecordState = XLREAD_NEXT_RECORD;
+ break;
+ }
}
/*
* Special processing if it's an XLOG SWITCH record
*/
- if (record->xl_rmid == RM_XLOG_ID &&
- (record->xl_info & ~XLR_INFO_MASK) == XLOG_SWITCH)
+ if ((*record)->xl_rmid == RM_XLOG_ID &&
+ ((*record)->xl_info & ~XLR_INFO_MASK) == XLOG_SWITCH)
{
/* Pretend it extends to end of segment */
state->EndRecPtr += state->segcxt.ws_segsize - 1;
state->EndRecPtr -= XLogSegmentOffset(state->EndRecPtr, state->segcxt.ws_segsize);
}
- if (DecodeXLogRecord(state, record, errormsg))
- return record;
- else
- return NULL;
+ Assert (!*record || state->readLen >= 0);
+ if (DecodeXLogRecord(state, *record, errormsg))
+ return XLREAD_SUCCESS;
+
+ *record = NULL;
+ return XLREAD_FAIL;
err:
/*
- * Invalidate the read state. We might read from a different source after
+ * Invalidate the read page. We might read from a different source after
* failure.
*/
XLogReaderInvalReadState(state);
@@ -572,7 +703,8 @@ err:
if (state->errormsg_buf[0] != '\0')
*errormsg = state->errormsg_buf;
- return NULL;
+ *record = NULL;
+ return XLREAD_FAIL;
}
/*
@@ -724,11 +856,12 @@ XLogReaderInvalReadState(XLogReaderState *state)
*
* This is just a convenience subroutine to avoid duplicated code in
* XLogReadRecord. It's not intended for use from anywhere else.
+ *
+ * If PrevRecPtr is valid, the xl_prev is is cross-checked with it.
*/
static bool
ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
- XLogRecPtr PrevRecPtr, XLogRecord *record,
- bool randAccess)
+ XLogRecPtr PrevRecPtr, XLogRecord *record)
{
if (record->xl_tot_len < SizeOfXLogRecord)
{
@@ -745,7 +878,7 @@ ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
record->xl_rmid, LSN_FORMAT_ARGS(RecPtr));
return false;
}
- if (randAccess)
+ if (PrevRecPtr == InvalidXLogRecPtr)
{
/*
* We can't exactly verify the prev-link, but surely it should be less
@@ -975,11 +1108,14 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
* XLogReadRecord() will read the next valid record.
*/
XLogRecPtr
-XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
+XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
+ XLogFindNextRecordCB read_page, void *private)
{
XLogRecPtr tmpRecPtr;
XLogRecPtr found = InvalidXLogRecPtr;
XLogPageHeader header;
+ XLogRecord *record;
+ XLogReadRecordResult result;
char *errormsg;
Assert(!XLogRecPtrIsInvalid(RecPtr));
@@ -1012,9 +1148,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
while(XLogNeedData(state, targetPagePtr, targetRecOff,
targetRecOff != 0))
{
- if (!state->routine.page_read(state, state->readPagePtr,
- state->readLen,
- state->ReadRecPtr, state->readBuf))
+ if (!read_page(state, private))
break;
}
@@ -1066,8 +1200,16 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
* or we just jumped over the remaining data of a continuation.
*/
XLogBeginRead(state, tmpRecPtr);
- while (XLogReadRecord(state, &errormsg) != NULL)
+ while ((result = XLogReadRecord(state, &record, &errormsg)) !=
+ XLREAD_FAIL)
{
+ if (result == XLREAD_NEED_DATA)
+ {
+ if (!read_page(state, private))
+ goto err;
+ continue;
+ }
+
/* past the record we've found, break out */
if (RecPtr <= state->ReadRecPtr)
{
@@ -1087,9 +1229,9 @@ err:
#endif /* FRONTEND */
/*
- * Helper function to ease writing of XLogRoutine->page_read callbacks.
- * If this function is used, caller must supply a segment_open callback in
- * 'state', as that is used here.
+ * Helper function to ease writing of page_read callback.
+ * If this function is used, caller must supply a segment_open callback and
+ * segment_close callback as that is used here.
*
* Read 'count' bytes into 'buf', starting at location 'startptr', from WAL
* fetched from timeline 'tli'.
@@ -1102,6 +1244,7 @@ err:
*/
bool
WALRead(XLogReaderState *state,
+ WALSegmentOpenCB segopenfn, WALSegmentCloseCB segclosefn,
char *buf, XLogRecPtr startptr, Size count, TimeLineID tli,
WALReadError *errinfo)
{
@@ -1133,10 +1276,10 @@ WALRead(XLogReaderState *state,
XLogSegNo nextSegNo;
if (state->seg.ws_file >= 0)
- state->routine.segment_close(state);
+ segclosefn(state);
XLByteToSeg(recptr, nextSegNo, state->segcxt.ws_segsize);
- state->routine.segment_open(state, nextSegNo, &tli);
+ segopenfn(state, nextSegNo, &tli);
/* This shouldn't happen -- indicates a bug in segment_open */
Assert(state->seg.ws_file >= 0);
diff --git a/src/backend/access/transam/xlogutils.c b/src/backend/access/transam/xlogutils.c
index 46eda33f25..b003990745 100644
--- a/src/backend/access/transam/xlogutils.c
+++ b/src/backend/access/transam/xlogutils.c
@@ -686,8 +686,7 @@ XLogTruncateRelation(RelFileNode rnode, ForkNumber forkNum,
void
XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wantLength)
{
- const XLogRecPtr lastReadPage = state->seg.ws_segno *
- state->segcxt.ws_segsize + state->readLen;
+ const XLogRecPtr lastReadPage = state->readPagePtr;
Assert(wantPage != InvalidXLogRecPtr && wantPage % XLOG_BLCKSZ == 0);
Assert(wantLength <= XLOG_BLCKSZ);
@@ -702,7 +701,7 @@ XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wa
* current TLI has since become historical.
*/
if (lastReadPage == wantPage &&
- state->readLen != 0 &&
+ state->page_verified &&
lastReadPage + state->readLen >= wantPage + Min(wantLength, XLOG_BLCKSZ - 1))
return;
@@ -788,6 +787,7 @@ wal_segment_open(XLogReaderState *state, XLogSegNo nextSegNo,
char path[MAXPGPATH];
XLogFilePath(path, tli, nextSegNo, state->segcxt.ws_segsize);
+ elog(LOG, "HOGE: %lu, %d => %s", nextSegNo, tli, path);
state->seg.ws_file = BasicOpenFile(path, O_RDONLY | PG_BINARY);
if (state->seg.ws_file >= 0)
return;
@@ -825,9 +825,11 @@ wal_segment_close(XLogReaderState *state)
* loop for now.
*/
bool
-read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *cur_page)
+read_local_xlog_page(XLogReaderState *state)
{
+ XLogRecPtr targetPagePtr = state->readPagePtr;
+ int reqLen = state->readLen;
+ char *cur_page = state->readBuf;
XLogRecPtr read_upto,
loc;
TimeLineID tli;
@@ -940,11 +942,12 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
* as 'count', read the whole page anyway. It's guaranteed to be
* zero-padded up to the page boundary if it's incomplete.
*/
- if (!WALRead(state, cur_page, targetPagePtr, XLOG_BLCKSZ, tli,
- &errinfo))
+ if (!WALRead(state, wal_segment_open, wal_segment_close,
+ cur_page, targetPagePtr, XLOG_BLCKSZ, tli, &errinfo))
WALReadRaiseError(&errinfo);
/* number of valid bytes in the buffer */
+ state->readPagePtr = targetPagePtr;
state->readLen = count;
return true;
}
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index 37b75deb72..9cbd8afa49 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -148,7 +148,8 @@ StartupDecodingContext(List *output_plugin_options,
TransactionId xmin_horizon,
bool need_full_snapshot,
bool fast_forward,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress)
@@ -198,11 +199,12 @@ StartupDecodingContext(List *output_plugin_options,
ctx->slot = slot;
- ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, xl_routine, ctx);
+ ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, cleanup_cb);
if (!ctx->reader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
errmsg("out of memory")));
+ ctx->page_read = page_read;
ctx->reorder = ReorderBufferAllocate();
ctx->snapshot_builder =
@@ -319,7 +321,8 @@ CreateInitDecodingContext(const char *plugin,
List *output_plugin_options,
bool need_full_snapshot,
XLogRecPtr restart_lsn,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress)
@@ -422,7 +425,7 @@ CreateInitDecodingContext(const char *plugin,
ctx = StartupDecodingContext(NIL, restart_lsn, xmin_horizon,
need_full_snapshot, false,
- xl_routine, prepare_write, do_write,
+ page_read, cleanup_cb, prepare_write, do_write,
update_progress);
/* call output plugin initialization callback */
@@ -476,7 +479,8 @@ LogicalDecodingContext *
CreateDecodingContext(XLogRecPtr start_lsn,
List *output_plugin_options,
bool fast_forward,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress)
@@ -528,8 +532,8 @@ CreateDecodingContext(XLogRecPtr start_lsn,
ctx = StartupDecodingContext(output_plugin_options,
start_lsn, InvalidTransactionId, false,
- fast_forward, xl_routine, prepare_write,
- do_write, update_progress);
+ fast_forward, page_read, cleanup_cb,
+ prepare_write, do_write, update_progress);
/* call output plugin initialization callback */
old_context = MemoryContextSwitchTo(ctx->context);
@@ -585,7 +589,13 @@ DecodingContextFindStartpoint(LogicalDecodingContext *ctx)
char *err = NULL;
/* the read_page callback waits for new WAL */
- record = XLogReadRecord(ctx->reader, &err);
+ while (XLogReadRecord(ctx->reader, &record, &err) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!ctx->page_read(ctx->reader))
+ break;
+ }
+
if (err)
elog(ERROR, "%s", err);
if (!record)
diff --git a/src/backend/replication/logical/logicalfuncs.c b/src/backend/replication/logical/logicalfuncs.c
index f7e055874e..b69f6911c5 100644
--- a/src/backend/replication/logical/logicalfuncs.c
+++ b/src/backend/replication/logical/logicalfuncs.c
@@ -233,9 +233,8 @@ pg_logical_slot_get_changes_guts(FunctionCallInfo fcinfo, bool confirm, bool bin
ctx = CreateDecodingContext(InvalidXLogRecPtr,
options,
false,
- XL_ROUTINE(.page_read = read_local_xlog_page,
- .segment_open = wal_segment_open,
- .segment_close = wal_segment_close),
+ read_local_xlog_page,
+ wal_segment_close,
LogicalOutputPrepareWrite,
LogicalOutputWrite, NULL);
@@ -284,7 +283,13 @@ pg_logical_slot_get_changes_guts(FunctionCallInfo fcinfo, bool confirm, bool bin
XLogRecord *record;
char *errm = NULL;
- record = XLogReadRecord(ctx->reader, &errm);
+ while (XLogReadRecord(ctx->reader, &record, &errm) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!ctx->page_read(ctx->reader))
+ break;
+ }
+
if (errm)
elog(ERROR, "%s", errm);
diff --git a/src/backend/replication/slotfuncs.c b/src/backend/replication/slotfuncs.c
index 9817b44113..deb7cf15eb 100644
--- a/src/backend/replication/slotfuncs.c
+++ b/src/backend/replication/slotfuncs.c
@@ -153,9 +153,8 @@ create_logical_replication_slot(char *name, char *plugin,
ctx = CreateInitDecodingContext(plugin, NIL,
false, /* just catalogs is OK */
restart_lsn,
- XL_ROUTINE(.page_read = read_local_xlog_page,
- .segment_open = wal_segment_open,
- .segment_close = wal_segment_close),
+ read_local_xlog_page,
+ wal_segment_close,
NULL, NULL, NULL);
/*
@@ -512,9 +511,8 @@ pg_logical_replication_slot_advance(XLogRecPtr moveto)
ctx = CreateDecodingContext(InvalidXLogRecPtr,
NIL,
true, /* fast_forward */
- XL_ROUTINE(.page_read = read_local_xlog_page,
- .segment_open = wal_segment_open,
- .segment_close = wal_segment_close),
+ read_local_xlog_page,
+ wal_segment_close,
NULL, NULL, NULL);
/*
@@ -536,7 +534,13 @@ pg_logical_replication_slot_advance(XLogRecPtr moveto)
* Read records. No changes are generated in fast_forward mode,
* but snapbuilder/slot statuses are updated properly.
*/
- record = XLogReadRecord(ctx->reader, &errm);
+ while (XLogReadRecord(ctx->reader, &record, &errm) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!ctx->page_read(ctx->reader))
+ break;
+ }
+
if (errm)
elog(ERROR, "%s", errm);
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index c36b91ae5e..062093727d 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -580,10 +580,7 @@ StartReplication(StartReplicationCmd *cmd)
/* create xlogreader for physical replication */
xlogreader =
- XLogReaderAllocate(wal_segment_size, NULL,
- XL_ROUTINE(.segment_open = WalSndSegmentOpen,
- .segment_close = wal_segment_close),
- NULL);
+ XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
if (!xlogreader)
ereport(ERROR,
@@ -807,9 +804,11 @@ StartReplication(StartReplicationCmd *cmd)
* set every time WAL is flushed.
*/
static bool
-logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetRecPtr, char *cur_page)
+logical_read_xlog_page(XLogReaderState *state)
{
+ XLogRecPtr targetPagePtr = state->readPagePtr;
+ int reqLen = state->readLen;
+ char *cur_page = state->readBuf;
XLogRecPtr flushptr;
int count;
WALReadError errinfo;
@@ -837,7 +836,7 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
count = flushptr - targetPagePtr; /* part of the page available */
/* now actually read the data, we know it's there */
- if (!WALRead(state,
+ if (!WALRead(state, WalSndSegmentOpen, wal_segment_close,
cur_page,
targetPagePtr,
XLOG_BLCKSZ,
@@ -1011,9 +1010,8 @@ CreateReplicationSlot(CreateReplicationSlotCmd *cmd)
ctx = CreateInitDecodingContext(cmd->plugin, NIL, need_full_snapshot,
InvalidXLogRecPtr,
- XL_ROUTINE(.page_read = logical_read_xlog_page,
- .segment_open = WalSndSegmentOpen,
- .segment_close = wal_segment_close),
+ logical_read_xlog_page,
+ wal_segment_close,
WalSndPrepareWrite, WalSndWriteData,
WalSndUpdateProgress);
@@ -1171,9 +1169,8 @@ StartLogicalReplication(StartReplicationCmd *cmd)
*/
logical_decoding_ctx =
CreateDecodingContext(cmd->startpoint, cmd->options, false,
- XL_ROUTINE(.page_read = logical_read_xlog_page,
- .segment_open = WalSndSegmentOpen,
- .segment_close = wal_segment_close),
+ logical_read_xlog_page,
+ wal_segment_close,
WalSndPrepareWrite, WalSndWriteData,
WalSndUpdateProgress);
xlogreader = logical_decoding_ctx->reader;
@@ -2749,7 +2746,7 @@ XLogSendPhysical(void)
enlargeStringInfo(&output_message, nbytes);
retry:
- if (!WALRead(xlogreader,
+ if (!WALRead(xlogreader, WalSndSegmentOpen, wal_segment_close,
&output_message.data[output_message.len],
startptr,
nbytes,
@@ -2847,7 +2844,12 @@ XLogSendLogical(void)
*/
WalSndCaughtUp = false;
- record = XLogReadRecord(logical_decoding_ctx->reader, &errm);
+ while (XLogReadRecord(logical_decoding_ctx->reader, &record, &errm) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!logical_decoding_ctx->page_read(logical_decoding_ctx->reader))
+ break;
+ }
/* xlog record was invalid */
if (errm != NULL)
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index cf119848b0..712c85281c 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -41,15 +41,9 @@ static int xlogreadfd = -1;
static XLogSegNo xlogreadsegno = -1;
static char xlogfpath[MAXPGPATH];
-typedef struct XLogPageReadPrivate
-{
- const char *restoreCommand;
- int tliIndex;
-} XLogPageReadPrivate;
-
-static bool SimpleXLogPageRead(XLogReaderState *xlogreader,
- XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
+static bool SimpleXLogPageRead(XLogReaderState *xlogreader,
+ const char *datadir, int *tliIndex,
+ const char *restoreCommand);
/*
* Read WAL from the datadir/pg_wal, starting from 'startpoint' on timeline
@@ -66,20 +60,22 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
XLogRecord *record;
XLogReaderState *xlogreader;
char *errormsg;
- XLogPageReadPrivate private;
- private.tliIndex = tliIndex;
- private.restoreCommand = restoreCommand;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir,
- XL_ROUTINE(.page_read = &SimpleXLogPageRead),
- &private);
+ xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+
if (xlogreader == NULL)
pg_fatal("out of memory");
XLogBeginRead(xlogreader, startpoint);
do
{
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!SimpleXLogPageRead(xlogreader, datadir,
+ &tliIndex, restoreCommand))
+ break;
+ }
if (record == NULL)
{
@@ -123,19 +119,19 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex,
XLogRecord *record;
XLogReaderState *xlogreader;
char *errormsg;
- XLogPageReadPrivate private;
XLogRecPtr endptr;
- private.tliIndex = tliIndex;
- private.restoreCommand = restoreCommand;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir,
- XL_ROUTINE(.page_read = &SimpleXLogPageRead),
- &private);
+ xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
XLogBeginRead(xlogreader, ptr);
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!SimpleXLogPageRead(xlogreader, datadir, &tliIndex, restoreCommand))
+ break;
+ }
if (record == NULL)
{
if (errormsg)
@@ -170,7 +166,6 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
XLogRecPtr searchptr;
XLogReaderState *xlogreader;
char *errormsg;
- XLogPageReadPrivate private;
/*
* The given fork pointer points to the end of the last common record,
@@ -186,11 +181,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
forkptr += SizeOfXLogShortPHD;
}
- private.tliIndex = tliIndex;
- private.restoreCommand = restoreCommand;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir,
- XL_ROUTINE(.page_read = &SimpleXLogPageRead),
- &private);
+ xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
@@ -200,7 +191,13 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
uint8 info;
XLogBeginRead(xlogreader, searchptr);
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!SimpleXLogPageRead(xlogreader, datadir,
+ &tliIndex, restoreCommand))
+ break;
+ }
if (record == NULL)
{
@@ -247,10 +244,11 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
/* XLogReader callback function, to read a WAL page */
static bool
-SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *readBuf)
+SimpleXLogPageRead(XLogReaderState *xlogreader, const char *datadir,
+ int *tliIndex, const char *restoreCommand)
{
- XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
+ XLogRecPtr targetPagePtr = xlogreader->readPagePtr;
+ char *readBuf = xlogreader->readBuf;
uint32 targetPageOff;
XLogRecPtr targetSegEnd;
XLogSegNo targetSegNo;
@@ -283,14 +281,14 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
* be done both forward and backward, consider also switching timeline
* accordingly.
*/
- while (private->tliIndex < targetNentries - 1 &&
- targetHistory[private->tliIndex].end < targetSegEnd)
- private->tliIndex++;
- while (private->tliIndex > 0 &&
- targetHistory[private->tliIndex].begin >= targetSegEnd)
- private->tliIndex--;
+ while (*tliIndex < targetNentries - 1 &&
+ targetHistory[*tliIndex].end < targetSegEnd)
+ (*tliIndex)++;
+ while (*tliIndex > 0 &&
+ targetHistory[*tliIndex].begin >= targetSegEnd)
+ (*tliIndex)--;
- XLogFileName(xlogfname, targetHistory[private->tliIndex].tli,
+ XLogFileName(xlogfname, targetHistory[*tliIndex].tli,
xlogreadsegno, WalSegSz);
snprintf(xlogfpath, MAXPGPATH, "%s/" XLOGDIR "/%s",
@@ -303,7 +301,7 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
/*
* If we have no restore_command to execute, then exit.
*/
- if (private->restoreCommand == NULL)
+ if (restoreCommand == NULL)
{
pg_log_error("could not open file \"%s\": %m", xlogfpath);
xlogreader->readLen = -1;
@@ -317,7 +315,7 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
xlogreadfd = RestoreArchivedFile(xlogreader->segcxt.ws_dir,
xlogfname,
WalSegSz,
- private->restoreCommand);
+ restoreCommand);
if (xlogreadfd < 0)
{
@@ -359,7 +357,7 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
Assert(targetSegNo == xlogreadsegno);
- xlogreader->seg.ws_tli = targetHistory[private->tliIndex].tli;
+ xlogreader->seg.ws_tli = targetHistory[*tliIndex].tli;
xlogreader->readLen = XLOG_BLCKSZ;
return true;
}
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index 0f0e0723a3..f514de6b13 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -330,12 +330,17 @@ WALDumpCloseSegment(XLogReaderState *state)
state->seg.ws_file = -1;
}
-/* pg_waldump's XLogReaderRoutine->page_read callback */
+/*
+ * pg_waldump's WAL page rader, also used as page_read callback for
+ * XLogFindNextRecord
+ */
static bool
-WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetPtr, char *readBuff)
+WALDumpReadPage(XLogReaderState *state, void *priv)
{
- XLogDumpPrivate *private = state->private_data;
+ XLogRecPtr targetPagePtr = state->readPagePtr;
+ int reqLen = state->readLen;
+ char *readBuff = state->readBuf;
+ XLogDumpPrivate *private = (XLogDumpPrivate *) priv;
int count = XLOG_BLCKSZ;
WALReadError errinfo;
@@ -353,8 +358,8 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
}
}
- if (!WALRead(state, readBuff, targetPagePtr, count, private->timeline,
- &errinfo))
+ if (!WALRead(state, WALDumpOpenSegment, WALDumpCloseSegment,
+ readBuff, targetPagePtr, count, private->timeline, &errinfo))
{
WALOpenSegment *seg = &errinfo.wre_seg;
char fname[MAXPGPATH];
@@ -374,6 +379,7 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
(Size) errinfo.wre_req);
}
+ Assert(count >= state->readLen);
state->readLen = count;
return true;
}
@@ -1035,16 +1041,14 @@ main(int argc, char **argv)
/* we have everything we need, start reading */
xlogreader_state =
- XLogReaderAllocate(WalSegSz, waldir,
- XL_ROUTINE(.page_read = WALDumpReadPage,
- .segment_open = WALDumpOpenSegment,
- .segment_close = WALDumpCloseSegment),
- &private);
+ XLogReaderAllocate(WalSegSz, waldir, WALDumpCloseSegment);
+
if (!xlogreader_state)
fatal_error("out of memory");
/* first find a valid recptr to start from */
- first_record = XLogFindNextRecord(xlogreader_state, private.startptr);
+ first_record = XLogFindNextRecord(xlogreader_state, private.startptr,
+ &WALDumpReadPage, (void*) &private);
if (first_record == InvalidXLogRecPtr)
fatal_error("could not find a valid record after %X/%X",
@@ -1067,7 +1071,13 @@ main(int argc, char **argv)
for (;;)
{
/* try to read the next record */
- record = XLogReadRecord(xlogreader_state, &errormsg);
+ while (XLogReadRecord(xlogreader_state, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!WALDumpReadPage(xlogreader_state, (void *) &private))
+ break;
+ }
+
if (!record)
{
if (!config.follow || private.endptr_reached)
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index 23d9e70a59..4e9268b553 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -57,64 +57,15 @@ typedef struct WALSegmentContext
typedef struct XLogReaderState XLogReaderState;
-/* Function type definition for the read_page callback */
-typedef bool (*XLogPageReadCB) (XLogReaderState *xlogreader,
- XLogRecPtr targetPagePtr,
- int reqLen,
- XLogRecPtr targetRecPtr,
- char *readBuf);
+/* Function type definition for the segment cleanup callback */
+typedef void (*WALSegmentCleanupCB) (XLogReaderState *xlogreader);
+
+/* Function type definition for the open/close callbacks for WALRead() */
typedef void (*WALSegmentOpenCB) (XLogReaderState *xlogreader,
XLogSegNo nextSegNo,
TimeLineID *tli_p);
typedef void (*WALSegmentCloseCB) (XLogReaderState *xlogreader);
-typedef struct XLogReaderRoutine
-{
- /*
- * Data input callback
- *
- * This callback shall read at least reqLen valid bytes of the xlog page
- * starting at targetPagePtr, and store them in readBuf. The callback
- * shall return the number of bytes read (never more than XLOG_BLCKSZ), or
- * -1 on failure. The callback shall sleep, if necessary, to wait for the
- * requested bytes to become available. The callback will not be invoked
- * again for the same page unless more than the returned number of bytes
- * are needed.
- *
- * targetRecPtr is the position of the WAL record we're reading. Usually
- * it is equal to targetPagePtr + reqLen, but sometimes xlogreader needs
- * to read and verify the page or segment header, before it reads the
- * actual WAL record it's interested in. In that case, targetRecPtr can
- * be used to determine which timeline to read the page from.
- *
- * The callback shall set ->seg.ws_tli to the TLI of the file the page was
- * read from.
- */
- XLogPageReadCB page_read;
-
- /*
- * Callback to open the specified WAL segment for reading. ->seg.ws_file
- * shall be set to the file descriptor of the opened segment. In case of
- * failure, an error shall be raised by the callback and it shall not
- * return.
- *
- * "nextSegNo" is the number of the segment to be opened.
- *
- * "tli_p" is an input/output argument. WALRead() uses it to pass the
- * timeline in which the new segment should be found, but the callback can
- * use it to return the TLI that it actually opened.
- */
- WALSegmentOpenCB segment_open;
-
- /*
- * WAL segment close callback. ->seg.ws_file shall be set to a negative
- * number.
- */
- WALSegmentCloseCB segment_close;
-} XLogReaderRoutine;
-
-#define XL_ROUTINE(...) &(XLogReaderRoutine){__VA_ARGS__}
-
typedef struct
{
/* Is this block ref in use? */
@@ -144,12 +95,35 @@ typedef struct
uint16 data_bufsz;
} DecodedBkpBlock;
+/* Return code from XLogReadRecord */
+typedef enum XLogReadRecordResult
+{
+ XLREAD_SUCCESS, /* record is successfully read */
+ XLREAD_NEED_DATA, /* need more data. see XLogReadRecord. */
+ XLREAD_FAIL /* failed during reading a record */
+} XLogReadRecordResult;
+
+/*
+ * internal state of XLogReadRecord
+ *
+ * XLogReadState runs a state machine while reading a record. Theses states
+ * are not seen outside the function. Each state may repeat several times
+ * exiting requesting caller for new data. See the comment of XLogReadRecrod
+ * for details.
+ */
+typedef enum XLogReadRecordState {
+ XLREAD_NEXT_RECORD,
+ XLREAD_TOT_LEN,
+ XLREAD_FIRST_FRAGMENT,
+ XLREAD_CONTINUATION
+} XLogReadRecordState;
+
struct XLogReaderState
{
/*
* Operational callbacks
*/
- XLogReaderRoutine routine;
+ WALSegmentCleanupCB cleanup_cb;
/* ----------------------------------------
* Public parameters
@@ -162,18 +136,14 @@ struct XLogReaderState
*/
uint64 system_identifier;
- /*
- * Opaque data for callbacks to use. Not used by XLogReader.
- */
- void *private_data;
-
/*
* Start and end point of last record read. EndRecPtr is also used as the
* position to read next. Calling XLogBeginRead() sets EndRecPtr to the
* starting position and ReadRecPtr to invalid.
*/
- XLogRecPtr ReadRecPtr; /* start of last record read */
+ XLogRecPtr ReadRecPtr; /* start of last record read or being read */
XLogRecPtr EndRecPtr; /* end+1 of last record read */
+ XLogRecPtr PrevRecPtr; /* start of previous record read */
/* ----------------------------------------
* Communication with page reader
@@ -187,7 +157,9 @@ struct XLogReaderState
* the request, or -1 on error */
TimeLineID readPageTLI; /* TLI for data currently in readBuf */
char *readBuf; /* buffer to store data */
- bool page_verified; /* is the page on the buffer verified? */
+ bool page_verified; /* is the page header on the buffer verified? */
+ bool record_verified;/* is the current record header verified? */
+
/* ----------------------------------------
@@ -229,8 +201,6 @@ struct XLogReaderState
XLogRecPtr latestPagePtr;
TimeLineID latestPageTLI;
- /* beginning of the WAL record being read. */
- XLogRecPtr currRecPtr;
/* timeline to read it from, 0 if a lookup is required */
TimeLineID currTLI;
@@ -257,6 +227,15 @@ struct XLogReaderState
char *readRecordBuf;
uint32 readRecordBufSize;
+ /*
+ * XLogReadRecord() state
+ */
+ XLogReadRecordState readRecordState;/* state machine state */
+ int recordGotLen; /* amount of current record that has
+ * already been read */
+ int recordRemainLen; /* length of current record that remains */
+ XLogRecPtr recordContRecPtr; /* where the current record continues */
+
/* Buffer to hold error message */
char *errormsg_buf;
};
@@ -264,9 +243,7 @@ struct XLogReaderState
/* Get a new XLogReader */
extern XLogReaderState *XLogReaderAllocate(int wal_segment_size,
const char *waldir,
- XLogReaderRoutine *routine,
- void *private_data);
-extern XLogReaderRoutine *LocalXLogReaderRoutine(void);
+ WALSegmentCleanupCB cleanup_cb);
/* Free an XLogReader */
extern void XLogReaderFree(XLogReaderState *state);
@@ -274,12 +251,17 @@ extern void XLogReaderFree(XLogReaderState *state);
/* Position the XLogReader to given record */
extern void XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr);
#ifdef FRONTEND
-extern XLogRecPtr XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr);
+/* Function type definition for the read_page callback */
+typedef bool (*XLogFindNextRecordCB) (XLogReaderState *xlogreader,
+ void *private);
+extern XLogRecPtr XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
+ XLogFindNextRecordCB read_page, void *private);
#endif /* FRONTEND */
/* Read the next XLog record. Returns NULL on end-of-WAL or failure */
-extern struct XLogRecord *XLogReadRecord(XLogReaderState *state,
- char **errormsg);
+extern XLogReadRecordResult XLogReadRecord(XLogReaderState *state,
+ XLogRecord **record,
+ char **errormsg);
/* Validate a page */
extern bool XLogReaderValidatePageHeader(XLogReaderState *state,
@@ -299,6 +281,7 @@ typedef struct WALReadError
} WALReadError;
extern bool WALRead(XLogReaderState *state,
+ WALSegmentOpenCB segopenfn, WALSegmentCloseCB sgclosefn,
char *buf, XLogRecPtr startptr, Size count,
TimeLineID tli, WALReadError *errinfo);
diff --git a/src/include/access/xlogutils.h b/src/include/access/xlogutils.h
index 364a21c4ea..397fb27fc2 100644
--- a/src/include/access/xlogutils.h
+++ b/src/include/access/xlogutils.h
@@ -47,9 +47,7 @@ extern Buffer XLogReadBufferExtended(RelFileNode rnode, ForkNumber forknum,
extern Relation CreateFakeRelcacheEntry(RelFileNode rnode);
extern void FreeFakeRelcacheEntry(Relation fakerel);
-extern bool read_local_xlog_page(XLogReaderState *state,
- XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetRecPtr, char *cur_page);
+extern bool read_local_xlog_page(XLogReaderState *state);
extern void wal_segment_open(XLogReaderState *state,
XLogSegNo nextSegNo,
TimeLineID *tli_p);
diff --git a/src/include/pg_config_manual.h b/src/include/pg_config_manual.h
index f10ad0acd6..94749609b4 100644
--- a/src/include/pg_config_manual.h
+++ b/src/include/pg_config_manual.h
@@ -377,7 +377,7 @@
* Enable debugging print statements for WAL-related operations; see
* also the wal_debug GUC var.
*/
-/* #define WAL_DEBUG */
+#define WAL_DEBUG
/*
* Enable tracing of resource consumption during sort operations;
diff --git a/src/include/replication/logical.h b/src/include/replication/logical.h
index c253403372..b936f31393 100644
--- a/src/include/replication/logical.h
+++ b/src/include/replication/logical.h
@@ -29,6 +29,10 @@ typedef void (*LogicalOutputPluginWriterUpdateProgress) (struct LogicalDecodingC
TransactionId xid
);
+typedef struct LogicalDecodingContext LogicalDecodingContext;
+
+typedef bool (*LogicalDecodingXLogPageReadCB)(XLogReaderState *ctx);
+
typedef struct LogicalDecodingContext
{
/* memory context this is all allocated in */
@@ -39,6 +43,7 @@ typedef struct LogicalDecodingContext
/* infrastructure pieces for decoding */
XLogReaderState *reader;
+ LogicalDecodingXLogPageReadCB page_read;
struct ReorderBuffer *reorder;
struct SnapBuild *snapshot_builder;
@@ -105,14 +110,16 @@ extern LogicalDecodingContext *CreateInitDecodingContext(const char *plugin,
List *output_plugin_options,
bool need_full_snapshot,
XLogRecPtr restart_lsn,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress);
extern LogicalDecodingContext *CreateDecodingContext(XLogRecPtr start_lsn,
List *output_plugin_options,
bool fast_forward,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress);
--
2.27.0
----Next_Part(Thu_Mar__4_11_28_53_2021_014)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
filename="v17-0003-Remove-globals-readOff-readLen-and-readSegNo.patch"
^ permalink raw reply [nested|flat] 70+ messages in thread
* [PATCH v17 2/4] Move page-reader out of XLogReadRecord
@ 2019-09-10 03:58 Kyotaro Horiguchi <[email protected]>
0 siblings, 0 replies; 70+ messages in thread
From: Kyotaro Horiguchi @ 2019-09-10 03:58 UTC (permalink / raw)
This is the second step of removing callbacks from WAL record reader.
Since it is essential to take in additional data while reading a
record, the function have to ask caller for new data while keeping
working state. Thus the function is turned into a state machine.
---
src/backend/access/transam/twophase.c | 15 +-
src/backend/access/transam/xlog.c | 58 +-
src/backend/access/transam/xlogreader.c | 697 +++++++++++-------
src/backend/access/transam/xlogutils.c | 17 +-
src/backend/replication/logical/logical.c | 26 +-
.../replication/logical/logicalfuncs.c | 13 +-
src/backend/replication/slotfuncs.c | 18 +-
src/backend/replication/walsender.c | 32 +-
src/bin/pg_rewind/parsexlog.c | 84 ++-
src/bin/pg_waldump/pg_waldump.c | 36 +-
src/include/access/xlogreader.h | 121 ++-
src/include/access/xlogutils.h | 4 +-
src/include/pg_config_manual.h | 2 +-
src/include/replication/logical.h | 11 +-
14 files changed, 646 insertions(+), 488 deletions(-)
diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index 80d2d20d6c..7d5a96a5f5 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -1325,11 +1325,8 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
XLogReaderState *xlogreader;
char *errormsg;
- xlogreader = XLogReaderAllocate(wal_segment_size, NULL,
- XL_ROUTINE(.page_read = &read_local_xlog_page,
- .segment_open = &wal_segment_open,
- .segment_close = &wal_segment_close),
- NULL);
+ xlogreader = XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+
if (!xlogreader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
@@ -1337,7 +1334,13 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
errdetail("Failed while allocating a WAL reading processor.")));
XLogBeginRead(xlogreader, lsn);
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!read_local_xlog_page(xlogreader))
+ break;
+ }
+
if (record == NULL)
ereport(ERROR,
(errcode_for_file_access(),
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 606328a65a..670996ae67 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -836,13 +836,6 @@ static XLogSource currentSource = XLOG_FROM_ANY;
static bool lastSourceFailed = false;
static bool pendingWalRcvRestart = false;
-typedef struct XLogPageReadPrivate
-{
- int emode;
- bool fetching_ckpt; /* are we fetching a checkpoint record? */
- bool randAccess;
-} XLogPageReadPrivate;
-
/*
* These variables track when we last obtained some WAL data to process,
* and where we got it from. (XLogReceiptSource is initially the same as
@@ -917,8 +910,8 @@ static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath,
static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli,
XLogSource source, bool notfoundOk);
static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, XLogSource source);
-static bool XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
+static bool XLogPageRead(XLogReaderState *xlogreader,
+ bool fetching_ckpt, int emode, bool randAccess);
static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
bool fetching_ckpt, XLogRecPtr tliRecPtr);
static int emode_for_corrupt_record(int emode, XLogRecPtr RecPtr);
@@ -1231,8 +1224,7 @@ XLogInsertRecord(XLogRecData *rdata,
appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len);
if (!debug_reader)
- debug_reader = XLogReaderAllocate(wal_segment_size, NULL,
- XL_ROUTINE(), NULL);
+ debug_reader = XLogReaderAllocate(wal_segment_size, NULL, NULL);
if (!debug_reader)
{
@@ -4344,15 +4336,10 @@ CleanupBackupHistory(void)
* record is available.
*/
static XLogRecord *
-ReadRecord(XLogReaderState *xlogreader, int emode,
- bool fetching_ckpt)
+ReadRecord(XLogReaderState *xlogreader, int emode, bool fetching_ckpt)
{
XLogRecord *record;
- XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
-
- private->fetching_ckpt = fetching_ckpt;
- private->emode = emode;
- private->randAccess = (xlogreader->ReadRecPtr == InvalidXLogRecPtr);
+ bool randAccess = (xlogreader->ReadRecPtr == InvalidXLogRecPtr);
/* This is the first attempt to read this page. */
lastSourceFailed = false;
@@ -4360,8 +4347,16 @@ ReadRecord(XLogReaderState *xlogreader, int emode,
for (;;)
{
char *errormsg;
+ XLogReadRecordResult result;
+
+ while ((result = XLogReadRecord(xlogreader, &record, &errormsg))
+ == XLREAD_NEED_DATA)
+ {
+ if (!XLogPageRead(xlogreader, fetching_ckpt, emode, randAccess))
+ break;
+
+ }
- record = XLogReadRecord(xlogreader, &errormsg);
ReadRecPtr = xlogreader->ReadRecPtr;
EndRecPtr = xlogreader->EndRecPtr;
if (record == NULL)
@@ -6380,7 +6375,6 @@ StartupXLOG(void)
bool backupFromStandby = false;
DBState dbstate_at_startup;
XLogReaderState *xlogreader;
- XLogPageReadPrivate private;
bool promoted = false;
struct stat st;
@@ -6539,13 +6533,9 @@ StartupXLOG(void)
OwnLatch(&XLogCtl->recoveryWakeupLatch);
/* Set up XLOG reader facility */
- MemSet(&private, 0, sizeof(XLogPageReadPrivate));
xlogreader =
- XLogReaderAllocate(wal_segment_size, NULL,
- XL_ROUTINE(.page_read = &XLogPageRead,
- .segment_open = NULL,
- .segment_close = wal_segment_close),
- &private);
+ XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+
if (!xlogreader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
@@ -11987,12 +11977,13 @@ CancelBackup(void)
* sleep and retry.
*/
static bool
-XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetRecPtr, char *readBuf)
+XLogPageRead(XLogReaderState *xlogreader,
+ bool fetching_ckpt, int emode, bool randAccess)
{
- XLogPageReadPrivate *private =
- (XLogPageReadPrivate *) xlogreader->private_data;
- int emode = private->emode;
+ char *readBuf = xlogreader->readBuf;
+ XLogRecPtr targetPagePtr = xlogreader->readPagePtr;
+ int reqLen = xlogreader->readLen;
+ XLogRecPtr targetRecPtr = xlogreader->ReadRecPtr;
uint32 targetPageOff;
XLogSegNo targetSegNo PG_USED_FOR_ASSERTS_ONLY;
int r;
@@ -12035,8 +12026,8 @@ retry:
flushedUpto < targetPagePtr + reqLen))
{
if (!WaitForWALToBecomeAvailable(targetPagePtr + reqLen,
- private->randAccess,
- private->fetching_ckpt,
+ randAccess,
+ fetching_ckpt,
targetRecPtr))
{
if (readFile >= 0)
@@ -12141,6 +12132,7 @@ retry:
goto next_record_is_invalid;
}
+ Assert(xlogreader->readPagePtr == targetPagePtr);
xlogreader->readLen = readLen;
return true;
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index 7d04995b7b..ba3e4868f7 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -40,7 +40,7 @@ static bool XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr,
int reqLen, bool header_inclusive);
static void XLogReaderInvalReadState(XLogReaderState *state);
static bool ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
- XLogRecPtr PrevRecPtr, XLogRecord *record, bool randAccess);
+ XLogRecPtr PrevRecPtr, XLogRecord *record);
static bool ValidXLogRecord(XLogReaderState *state, XLogRecord *record,
XLogRecPtr recptr);
static void ResetDecoder(XLogReaderState *state);
@@ -73,7 +73,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...)
*/
XLogReaderState *
XLogReaderAllocate(int wal_segment_size, const char *waldir,
- XLogReaderRoutine *routine, void *private_data)
+ WALSegmentCleanupCB cleanup_cb)
{
XLogReaderState *state;
@@ -84,7 +84,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
return NULL;
/* initialize caller-provided support functions */
- state->routine = *routine;
+ state->cleanup_cb = cleanup_cb;
state->max_block_id = -1;
@@ -107,8 +107,6 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
WALOpenSegmentInit(&state->seg, &state->segcxt, wal_segment_size,
waldir);
- /* system_identifier initialized to zeroes above */
- state->private_data = private_data;
/* ReadRecPtr, EndRecPtr and readLen initialized to zeroes above */
state->errormsg_buf = palloc_extended(MAX_ERRORMSG_LEN + 1,
MCXT_ALLOC_NO_OOM);
@@ -140,8 +138,8 @@ XLogReaderFree(XLogReaderState *state)
{
int block_id;
- if (state->seg.ws_file != -1)
- state->routine.segment_close(state);
+ if (state->seg.ws_file >= 0)
+ state->cleanup_cb(state);
for (block_id = 0; block_id <= XLR_MAX_BLOCK_ID; block_id++)
{
@@ -246,6 +244,7 @@ XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
/* Begin at the passed-in record pointer. */
state->EndRecPtr = RecPtr;
state->ReadRecPtr = InvalidXLogRecPtr;
+ state->readRecordState = XLREAD_NEXT_RECORD;
}
/*
@@ -254,317 +253,449 @@ XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
* XLogBeginRead() or XLogFindNextRecord() must be called before the first call
* to XLogReadRecord().
*
- * If the page_read callback fails to read the requested data, NULL is
- * returned. The callback is expected to have reported the error; errormsg
- * is set to NULL.
+ * This function may return XLREAD_NEED_DATA several times before returning a
+ * result record. The caller shall read in some new data then call this
+ * function again with the same parameters.
*
- * If the reading fails for some other reason, NULL is also returned, and
- * *errormsg is set to a string with details of the failure.
+ * When a record is successfully read, returns XLREAD_SUCCESS with result
+ * record being stored in *record. Otherwise *record is NULL.
*
- * The returned pointer (or *errormsg) points to an internal buffer that's
- * valid until the next call to XLogReadRecord.
+ * Returns XLREAD_NEED_DATA if more data is needed to finish reading the
+ * current record. In that case, state->readPagePtr and state->readLen inform
+ * the desired position and minimum length of data needed. The caller shall
+ * read in the requested data and set state->readBuf to point to a buffer
+ * containing it. The caller must also set state->readPageTLI and
+ * state->readLen to indicate the timeline that it was read from, and the
+ * length of data that is now available (which must be >= given readLen),
+ * respectively.
+ *
+ * If invalid data is encountered, returns XLREAD_FAIL with *record being set to
+ * NULL. *errormsg is set to a string with details of the failure.
+ * The returned pointer (or *errormsg) points to an internal buffer that's valid
+ * until the next call to XLogReadRecord.
+ *
+ *
+ * This function runs a state machine consists of the following states.
+ *
+ * XLREAD_NEXT_RECORD :
+ * The initial state, if called with valid RecPtr, try to read a record at
+ * that position. If invalid RecPtr is given try to read a record just after
+ * the last one previously read.
+ * This state ens after setting ReadRecPtr. Then goes to XLREAD_TOT_LEN.
+ *
+ * XLREAD_TOT_LEN:
+ * Examining record header. Ends after reading record total
+ * length. recordRemainLen and recordGotLen are initialized.
+ *
+ * XLREAD_FIRST_FRAGMENT:
+ * Reading the first fragment. Ends with finishing reading a single
+ * record. Goes to XLREAD_NEXT_RECORD if that's all or
+ * XLREAD_CONTINUATION if we have continuation.
+
+ * XLREAD_CONTINUATION:
+ * Reading continuation of record. Ends with finishing the whole record then
+ * goes to XLREAD_NEXT_RECORD. During this state, recordRemainLen indicates
+ * how much is left and readRecordBuf holds the partially assert
+ * record.recordContRecPtr points to the beginning of the next page where to
+ * continue.
+ *
+ * If wrong data found in any state, the state machine stays at the current
+ * state. This behavior allows to continue reading a reacord switching among
+ * different souces, while streaming replication.
*/
-XLogRecord *
-XLogReadRecord(XLogReaderState *state, char **errormsg)
+XLogReadRecordResult
+XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
{
- XLogRecPtr RecPtr;
- XLogRecord *record;
- XLogRecPtr targetPagePtr;
- bool randAccess;
- uint32 len,
- total_len;
- uint32 targetRecOff;
- uint32 pageHeaderSize;
- bool gotheader;
+ XLogRecord *prec;
- /*
- * randAccess indicates whether to verify the previous-record pointer of
- * the record we're reading. We only do this if we're reading
- * sequentially, which is what we initially assume.
- */
- randAccess = false;
+ *record = NULL;
/* reset error state */
*errormsg = NULL;
state->errormsg_buf[0] = '\0';
- ResetDecoder(state);
-
- RecPtr = state->EndRecPtr;
-
- if (state->ReadRecPtr != InvalidXLogRecPtr)
- {
- /* read the record after the one we just read */
-
- /*
- * EndRecPtr is pointing to end+1 of the previous WAL record. If
- * we're at a page boundary, no more records can fit on the current
- * page. We must skip over the page header, but we can't do that until
- * we've read in the page, since the header size is variable.
- */
- }
- else
- {
- /*
- * Caller supplied a position to start at.
- *
- * In this case, EndRecPtr should already be pointing to a valid
- * record starting position.
- */
- Assert(XRecOffIsValid(RecPtr));
- randAccess = true;
- }
-
- state->currRecPtr = RecPtr;
-
- targetPagePtr = RecPtr - (RecPtr % XLOG_BLCKSZ);
- targetRecOff = RecPtr % XLOG_BLCKSZ;
-
- /*
- * Read the page containing the record into state->readBuf. Request enough
- * byte to cover the whole record header, or at least the part of it that
- * fits on the same page.
- */
- while (XLogNeedData(state, targetPagePtr,
- Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
- targetRecOff != 0))
- {
- if (!state->routine.page_read(state, state->readPagePtr, state->readLen,
- RecPtr, state->readBuf))
- break;
- }
-
- if (!state->page_verified)
- goto err;
-
- /*
- * We have at least the page header, so we can examine it now.
- */
- pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
- if (targetRecOff == 0)
- {
- /*
- * At page start, so skip over page header.
- */
- RecPtr += pageHeaderSize;
- targetRecOff = pageHeaderSize;
- }
- else if (targetRecOff < pageHeaderSize)
- {
- report_invalid_record(state, "invalid record offset at %X/%X",
- LSN_FORMAT_ARGS(RecPtr));
- goto err;
- }
-
- if ((((XLogPageHeader) state->readBuf)->xlp_info & XLP_FIRST_IS_CONTRECORD) &&
- targetRecOff == pageHeaderSize)
- {
- report_invalid_record(state, "contrecord is requested by %X/%X",
- LSN_FORMAT_ARGS(RecPtr));
- goto err;
- }
-
- /* XLogNeedData has verified the page header */
- Assert(pageHeaderSize <= state->readLen);
-
- /*
- * Read the record length.
- *
- * NB: Even though we use an XLogRecord pointer here, the whole record
- * header might not fit on this page. xl_tot_len is the first field of the
- * struct, so it must be on this page (the records are MAXALIGNed), but we
- * cannot access any other fields until we've verified that we got the
- * whole header.
- */
- record = (XLogRecord *) (state->readBuf + RecPtr % XLOG_BLCKSZ);
- total_len = record->xl_tot_len;
-
- /*
- * If the whole record header is on this page, validate it immediately.
- * Otherwise do just a basic sanity check on xl_tot_len, and validate the
- * rest of the header after reading it from the next page. The xl_tot_len
- * check is necessary here to ensure that we enter the "Need to reassemble
- * record" code path below; otherwise we might fail to apply
- * ValidXLogRecordHeader at all.
- */
- if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
- {
- if (!ValidXLogRecordHeader(state, RecPtr, state->ReadRecPtr, record,
- randAccess))
- goto err;
- gotheader = true;
- }
- else
- {
- /* XXX: more validation should be done here */
- if (total_len < SizeOfXLogRecord)
- {
- report_invalid_record(state,
- "invalid record length at %X/%X: wanted %u, got %u",
- LSN_FORMAT_ARGS(RecPtr),
- (uint32) SizeOfXLogRecord, total_len);
- goto err;
- }
- gotheader = false;
- }
-
- len = XLOG_BLCKSZ - RecPtr % XLOG_BLCKSZ;
- if (total_len > len)
+ switch (state->readRecordState)
{
- /* Need to reassemble record */
- char *contdata;
- XLogPageHeader pageHeader;
- char *buffer;
- uint32 gotlen;
-
- /*
- * Enlarge readRecordBuf as needed.
- */
- if (total_len > state->readRecordBufSize &&
- !allocate_recordbuf(state, total_len))
- {
- /* We treat this as a "bogus data" condition */
- report_invalid_record(state, "record length %u at %X/%X too long",
- total_len, LSN_FORMAT_ARGS(RecPtr));
- goto err;
- }
+ case XLREAD_NEXT_RECORD:
+ ResetDecoder(state);
- /* Copy the first fragment of the record from the first page. */
- memcpy(state->readRecordBuf,
- state->readBuf + RecPtr % XLOG_BLCKSZ, len);
- buffer = state->readRecordBuf + len;
- gotlen = len;
-
- do
- {
- int rest_len = total_len - gotlen;
-
- /* Calculate pointer to beginning of next page */
- targetPagePtr += XLOG_BLCKSZ;
-
- /* Wait for the next page to become available */
- while (XLogNeedData(state, targetPagePtr,
- Min(rest_len, XLOG_BLCKSZ),
- false))
+ if (state->ReadRecPtr != InvalidXLogRecPtr)
{
- if (!state->routine.page_read(state, state->readPagePtr,
- state->readLen,
- state->ReadRecPtr,
- state->readBuf))
- break;
+ /* read the record after the one we just read */
+
+ /*
+ * EndRecPtr is pointing to end+1 of the previous WAL record.
+ * If we're at a page boundary, no more records can fit on the
+ * current page. We must skip over the page header, but we
+ * can't do that until we've read in the page, since the header
+ * size is variable.
+ */
+ state->PrevRecPtr = state->ReadRecPtr;
+ state->ReadRecPtr = state->EndRecPtr;
+ }
+ else
+ {
+ /*
+ * Caller supplied a position to start at.
+ *
+ * In this case, EndRecPtr should already be pointing to a
+ * valid record starting position.
+ */
+ Assert(XRecOffIsValid(state->EndRecPtr));
+ state->ReadRecPtr = state->EndRecPtr;
+
+ /*
+ * We cannot verify the previous-record pointer when we're
+ * seeking to a particular record. Reset PrevRecPtr so that we
+ * won't try doing that.
+ */
+ state->PrevRecPtr = InvalidXLogRecPtr;
+ state->EndRecPtr = InvalidXLogRecPtr; /* to be tidy */
}
+ state->record_verified = false;
+ state->readRecordState = XLREAD_TOT_LEN;
+ /* fall through */
+
+ case XLREAD_TOT_LEN:
+ {
+ uint32 total_len;
+ uint32 pageHeaderSize;
+ XLogRecPtr targetPagePtr;
+ uint32 targetRecOff;
+ XLogPageHeader pageHeader;
+
+ targetPagePtr =
+ state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
+ targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+
+ /*
+ * Check if we have enough data. For the first record in the page,
+ * the requesting length doesn't contain page header.
+ */
+ if (XLogNeedData(state, targetPagePtr,
+ Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+ targetRecOff != 0))
+ return XLREAD_NEED_DATA;
+
+ /* error out if caller supplied bogus page */
if (!state->page_verified)
goto err;
- Assert(SizeOfXLogShortPHD <= state->readLen);
+ /* examine page header now. */
+ pageHeaderSize =
+ XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+ if (targetRecOff == 0)
+ {
+ /* At page start, so skip over page header. */
+ state->ReadRecPtr += pageHeaderSize;
+ targetRecOff = pageHeaderSize;
+ }
+ else if (targetRecOff < pageHeaderSize)
+ {
+ report_invalid_record(state, "invalid record offset at %X/%X",
+ LSN_FORMAT_ARGS(state->ReadRecPtr));
+ goto err;
+ }
- /* Check that the continuation on next page looks valid */
pageHeader = (XLogPageHeader) state->readBuf;
- if (!(pageHeader->xlp_info & XLP_FIRST_IS_CONTRECORD))
+ if ((pageHeader->xlp_info & XLP_FIRST_IS_CONTRECORD) &&
+ targetRecOff == pageHeaderSize)
{
- report_invalid_record(state,
- "there is no contrecord flag at %X/%X",
- LSN_FORMAT_ARGS(RecPtr));
+ report_invalid_record(state, "contrecord is requested by %X/%X",
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr);
goto err;
}
- /*
- * Cross-check that xlp_rem_len agrees with how much of the record
- * we expect there to be left.
- */
- if (pageHeader->xlp_rem_len == 0 ||
- total_len != (pageHeader->xlp_rem_len + gotlen))
- {
- report_invalid_record(state,
- "invalid contrecord length %u (expected %lld) at %X/%X",
- pageHeader->xlp_rem_len,
- ((long long) total_len) - gotlen,
- LSN_FORMAT_ARGS(RecPtr));
- goto err;
- }
-
- /* Append the continuation from this page to the buffer */
- pageHeaderSize = XLogPageHeaderSize(pageHeader);
-
+ /* XLogNeedData has verified the page header */
Assert(pageHeaderSize <= state->readLen);
- contdata = (char *) state->readBuf + pageHeaderSize;
- len = XLOG_BLCKSZ - pageHeaderSize;
- if (pageHeader->xlp_rem_len < len)
- len = pageHeader->xlp_rem_len;
+ /*
+ * Read the record length.
+ *
+ * NB: Even though we use an XLogRecord pointer here, the whole
+ * record header might not fit on this page. xl_tot_len is the first
+ * field of the struct, so it must be on this page (the records are
+ * MAXALIGNed), but we cannot access any other fields until we've
+ * verified that we got the whole header.
+ */
+ prec = (XLogRecord *) (state->readBuf +
+ state->ReadRecPtr % XLOG_BLCKSZ);
+ total_len = prec->xl_tot_len;
- Assert (pageHeaderSize + len <= state->readLen);
- memcpy(buffer, (char *) contdata, len);
- buffer += len;
- gotlen += len;
+ /*
+ * If the whole record header is on this page, validate it
+ * immediately. Otherwise do just a basic sanity check on
+ * xl_tot_len, and validate the rest of the header after reading it
+ * from the next page. The xl_tot_len check is necessary here to
+ * ensure that we enter the XLREAD_CONTINUATION state below;
+ * otherwise we might fail to apply ValidXLogRecordHeader at all.
+ */
+ if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
+ {
+ if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
+ state->PrevRecPtr, prec))
+ goto err;
- /* If we just reassembled the record header, validate it. */
- if (!gotheader)
+ state->record_verified = true;
+ }
+ else
{
- record = (XLogRecord *) state->readRecordBuf;
- if (!ValidXLogRecordHeader(state, RecPtr, state->ReadRecPtr,
- record, randAccess))
+ /* XXX: more validation should be done here */
+ if (total_len < SizeOfXLogRecord)
+ {
+ report_invalid_record(state,
+ "invalid record length at %X/%X: wanted %u, got %u",
+ LSN_FORMAT_ARGS(state->ReadRecPtr),
+ (uint32) SizeOfXLogRecord, total_len);
goto err;
- gotheader = true;
+ }
}
- } while (gotlen < total_len);
-
- Assert(gotheader);
- record = (XLogRecord *) state->readRecordBuf;
- if (!ValidXLogRecord(state, record, RecPtr))
- goto err;
+ /*
+ * Wait for the rest of the record, or the part of the record that
+ * fit on the first page if crossed a page boundary, to become
+ * available.
+ */
+ state->recordGotLen = 0;
+ state->recordRemainLen = total_len;
+ state->readRecordState = XLREAD_FIRST_FRAGMENT;
+ }
+ /* fall through */
- pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
- state->ReadRecPtr = RecPtr;
- state->EndRecPtr = targetPagePtr + pageHeaderSize
- + MAXALIGN(pageHeader->xlp_rem_len);
- }
- else
- {
- /* Wait for the record data to become available */
- while (XLogNeedData(state, targetPagePtr,
- Min(targetRecOff + total_len, XLOG_BLCKSZ), true))
+ case XLREAD_FIRST_FRAGMENT:
{
- if (!state->routine.page_read(state, state->readPagePtr,
- state->readLen,
- state->ReadRecPtr, state->readBuf))
+ uint32 total_len = state->recordRemainLen;
+ uint32 request_len;
+ uint32 record_len;
+ XLogRecPtr targetPagePtr;
+ uint32 targetRecOff;
+
+ /*
+ * Wait for the rest of the record on the first page to become
+ * available
+ */
+ targetPagePtr =
+ state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
+ targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+
+ request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ);
+ record_len = request_len - targetRecOff;
+
+ /* ReadRecPtr contains page header */
+ Assert (targetRecOff != 0);
+ if (XLogNeedData(state, targetPagePtr, request_len, true))
+ return XLREAD_NEED_DATA;
+
+ /* error out if caller supplied bogus page */
+ if (!state->page_verified)
+ goto err;
+
+ prec = (XLogRecord *) (state->readBuf + targetRecOff);
+
+ /* validate record header if not yet */
+ if (!state->record_verified && record_len >= SizeOfXLogRecord)
+ {
+ if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
+ state->PrevRecPtr, prec))
+ goto err;
+
+ state->record_verified = true;
+ }
+
+
+ if (total_len == record_len)
+ {
+ /* Record does not cross a page boundary */
+ Assert(state->record_verified);
+
+ if (!ValidXLogRecord(state, prec, state->ReadRecPtr))
+ goto err;
+
+ state->record_verified = true; /* to be tidy */
+
+ /* We already checked the header earlier */
+ state->EndRecPtr = state->ReadRecPtr + MAXALIGN(record_len);
+
+ *record = prec;
+ state->readRecordState = XLREAD_NEXT_RECORD;
break;
+ }
+
+ /*
+ * The record continues on the next page. Need to reassemble
+ * record
+ */
+ Assert(total_len > record_len);
+
+ /* Enlarge readRecordBuf as needed. */
+ if (total_len > state->readRecordBufSize &&
+ !allocate_recordbuf(state, total_len))
+ {
+ /* We treat this as a "bogus data" condition */
+ report_invalid_record(state,
+ "record length %u at %X/%X too long",
+ total_len,
+ LSN_FORMAT_ARGS(state->ReadRecPtr));
+ goto err;
+ }
+
+ /* Copy the first fragment of the record from the first page. */
+ memcpy(state->readRecordBuf, state->readBuf + targetRecOff,
+ record_len);
+ state->recordGotLen += record_len;
+ state->recordRemainLen -= record_len;
+
+ /* Calculate pointer to beginning of next page */
+ state->recordContRecPtr = state->ReadRecPtr + record_len;
+ Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0);
+
+ state->readRecordState = XLREAD_CONTINUATION;
}
+ /* fall through */
+
+ case XLREAD_CONTINUATION:
+ {
+ XLogPageHeader pageHeader;
+ uint32 pageHeaderSize;
+ XLogRecPtr targetPagePtr;
+
+ /* we enter this state only if we haven't read the whole record. */
+ Assert (state->recordRemainLen > 0);
+
+ while(state->recordRemainLen > 0)
+ {
+ char *contdata;
+ uint32 request_len;
+ uint32 record_len;
+
+ /* Wait for the next page to become available */
+ targetPagePtr = state->recordContRecPtr;
+
+ /* this request contains page header */
+ Assert (targetPagePtr != 0);
+ if (XLogNeedData(state, targetPagePtr,
+ Min(state->recordRemainLen, XLOG_BLCKSZ),
+ false))
+ return XLREAD_NEED_DATA;
+
+ if (!state->page_verified)
+ goto err;
+
+ Assert(SizeOfXLogShortPHD <= state->readLen);
- if (!state->page_verified)
- goto err;
+ /* Check that the continuation on next page looks valid */
+ pageHeader = (XLogPageHeader) state->readBuf;
+ if (!(pageHeader->xlp_info & XLP_FIRST_IS_CONTRECORD))
+ {
+ report_invalid_record(
+ state,
+ "there is no contrecord flag at %X/%X reading %X/%X",
+ (uint32) (state->recordContRecPtr >> 32),
+ (uint32) state->recordContRecPtr,
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr);
+ goto err;
+ }
- /* Record does not cross a page boundary */
- if (!ValidXLogRecord(state, record, RecPtr))
- goto err;
+ /*
+ * Cross-check that xlp_rem_len agrees with how much of the
+ * record we expect there to be left.
+ */
+ if (pageHeader->xlp_rem_len == 0 ||
+ pageHeader->xlp_rem_len != state->recordRemainLen)
+ {
+ report_invalid_record(
+ state,
+ "invalid contrecord length %u at %X/%X reading %X/%X, expected %u",
+ pageHeader->xlp_rem_len,
+ (uint32) (state->recordContRecPtr >> 32),
+ (uint32) state->recordContRecPtr,
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr,
+ state->recordRemainLen);
+ goto err;
+ }
- state->EndRecPtr = RecPtr + MAXALIGN(total_len);
+ /* Append the continuation from this page to the buffer */
+ pageHeaderSize = XLogPageHeaderSize(pageHeader);
- state->ReadRecPtr = RecPtr;
+ /*
+ * XLogNeedData should have ensured that the whole page header
+ * was read
+ */
+ Assert(state->readLen >= pageHeaderSize);
+
+ contdata = (char *) state->readBuf + pageHeaderSize;
+ record_len = XLOG_BLCKSZ - pageHeaderSize;
+ if (pageHeader->xlp_rem_len < record_len)
+ record_len = pageHeader->xlp_rem_len;
+
+ request_len = record_len + pageHeaderSize;
+
+ /* XLogNeedData should have ensured all needed data was read */
+ Assert (state->readLen >= request_len);
+
+ memcpy(state->readRecordBuf + state->recordGotLen,
+ (char *) contdata, record_len);
+ state->recordGotLen += record_len;
+ state->recordRemainLen -= record_len;
+
+ /* If we just reassembled the record header, validate it. */
+ if (!state->record_verified)
+ {
+ Assert(state->recordGotLen >= SizeOfXLogRecord);
+ if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
+ state->PrevRecPtr,
+ (XLogRecord *) state->readRecordBuf))
+ goto err;
+
+ state->record_verified = true;
+ }
+
+ /* Calculate pointer to beginning of next page, and continue */
+ state->recordContRecPtr += XLOG_BLCKSZ;
+ }
+
+ /* targetPagePtr is pointing the last-read page here */
+ prec = (XLogRecord *) state->readRecordBuf;
+ if (!ValidXLogRecord(state, prec, state->ReadRecPtr))
+ goto err;
+
+ pageHeaderSize =
+ XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+ state->EndRecPtr = targetPagePtr + pageHeaderSize
+ + MAXALIGN(pageHeader->xlp_rem_len);
+
+ *record = prec;
+ state->readRecordState = XLREAD_NEXT_RECORD;
+ break;
+ }
}
/*
* Special processing if it's an XLOG SWITCH record
*/
- if (record->xl_rmid == RM_XLOG_ID &&
- (record->xl_info & ~XLR_INFO_MASK) == XLOG_SWITCH)
+ if ((*record)->xl_rmid == RM_XLOG_ID &&
+ ((*record)->xl_info & ~XLR_INFO_MASK) == XLOG_SWITCH)
{
/* Pretend it extends to end of segment */
state->EndRecPtr += state->segcxt.ws_segsize - 1;
state->EndRecPtr -= XLogSegmentOffset(state->EndRecPtr, state->segcxt.ws_segsize);
}
- if (DecodeXLogRecord(state, record, errormsg))
- return record;
- else
- return NULL;
+ Assert (!*record || state->readLen >= 0);
+ if (DecodeXLogRecord(state, *record, errormsg))
+ return XLREAD_SUCCESS;
+
+ *record = NULL;
+ return XLREAD_FAIL;
err:
/*
- * Invalidate the read state. We might read from a different source after
+ * Invalidate the read page. We might read from a different source after
* failure.
*/
XLogReaderInvalReadState(state);
@@ -572,7 +703,8 @@ err:
if (state->errormsg_buf[0] != '\0')
*errormsg = state->errormsg_buf;
- return NULL;
+ *record = NULL;
+ return XLREAD_FAIL;
}
/*
@@ -724,11 +856,12 @@ XLogReaderInvalReadState(XLogReaderState *state)
*
* This is just a convenience subroutine to avoid duplicated code in
* XLogReadRecord. It's not intended for use from anywhere else.
+ *
+ * If PrevRecPtr is valid, the xl_prev is is cross-checked with it.
*/
static bool
ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
- XLogRecPtr PrevRecPtr, XLogRecord *record,
- bool randAccess)
+ XLogRecPtr PrevRecPtr, XLogRecord *record)
{
if (record->xl_tot_len < SizeOfXLogRecord)
{
@@ -745,7 +878,7 @@ ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
record->xl_rmid, LSN_FORMAT_ARGS(RecPtr));
return false;
}
- if (randAccess)
+ if (PrevRecPtr == InvalidXLogRecPtr)
{
/*
* We can't exactly verify the prev-link, but surely it should be less
@@ -975,11 +1108,14 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
* XLogReadRecord() will read the next valid record.
*/
XLogRecPtr
-XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
+XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
+ XLogFindNextRecordCB read_page, void *private)
{
XLogRecPtr tmpRecPtr;
XLogRecPtr found = InvalidXLogRecPtr;
XLogPageHeader header;
+ XLogRecord *record;
+ XLogReadRecordResult result;
char *errormsg;
Assert(!XLogRecPtrIsInvalid(RecPtr));
@@ -1012,9 +1148,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
while(XLogNeedData(state, targetPagePtr, targetRecOff,
targetRecOff != 0))
{
- if (!state->routine.page_read(state, state->readPagePtr,
- state->readLen,
- state->ReadRecPtr, state->readBuf))
+ if (!read_page(state, private))
break;
}
@@ -1066,8 +1200,16 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
* or we just jumped over the remaining data of a continuation.
*/
XLogBeginRead(state, tmpRecPtr);
- while (XLogReadRecord(state, &errormsg) != NULL)
+ while ((result = XLogReadRecord(state, &record, &errormsg)) !=
+ XLREAD_FAIL)
{
+ if (result == XLREAD_NEED_DATA)
+ {
+ if (!read_page(state, private))
+ goto err;
+ continue;
+ }
+
/* past the record we've found, break out */
if (RecPtr <= state->ReadRecPtr)
{
@@ -1087,9 +1229,9 @@ err:
#endif /* FRONTEND */
/*
- * Helper function to ease writing of XLogRoutine->page_read callbacks.
- * If this function is used, caller must supply a segment_open callback in
- * 'state', as that is used here.
+ * Helper function to ease writing of page_read callback.
+ * If this function is used, caller must supply a segment_open callback and
+ * segment_close callback as that is used here.
*
* Read 'count' bytes into 'buf', starting at location 'startptr', from WAL
* fetched from timeline 'tli'.
@@ -1102,6 +1244,7 @@ err:
*/
bool
WALRead(XLogReaderState *state,
+ WALSegmentOpenCB segopenfn, WALSegmentCloseCB segclosefn,
char *buf, XLogRecPtr startptr, Size count, TimeLineID tli,
WALReadError *errinfo)
{
@@ -1133,10 +1276,10 @@ WALRead(XLogReaderState *state,
XLogSegNo nextSegNo;
if (state->seg.ws_file >= 0)
- state->routine.segment_close(state);
+ segclosefn(state);
XLByteToSeg(recptr, nextSegNo, state->segcxt.ws_segsize);
- state->routine.segment_open(state, nextSegNo, &tli);
+ segopenfn(state, nextSegNo, &tli);
/* This shouldn't happen -- indicates a bug in segment_open */
Assert(state->seg.ws_file >= 0);
diff --git a/src/backend/access/transam/xlogutils.c b/src/backend/access/transam/xlogutils.c
index 46eda33f25..b003990745 100644
--- a/src/backend/access/transam/xlogutils.c
+++ b/src/backend/access/transam/xlogutils.c
@@ -686,8 +686,7 @@ XLogTruncateRelation(RelFileNode rnode, ForkNumber forkNum,
void
XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wantLength)
{
- const XLogRecPtr lastReadPage = state->seg.ws_segno *
- state->segcxt.ws_segsize + state->readLen;
+ const XLogRecPtr lastReadPage = state->readPagePtr;
Assert(wantPage != InvalidXLogRecPtr && wantPage % XLOG_BLCKSZ == 0);
Assert(wantLength <= XLOG_BLCKSZ);
@@ -702,7 +701,7 @@ XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wa
* current TLI has since become historical.
*/
if (lastReadPage == wantPage &&
- state->readLen != 0 &&
+ state->page_verified &&
lastReadPage + state->readLen >= wantPage + Min(wantLength, XLOG_BLCKSZ - 1))
return;
@@ -788,6 +787,7 @@ wal_segment_open(XLogReaderState *state, XLogSegNo nextSegNo,
char path[MAXPGPATH];
XLogFilePath(path, tli, nextSegNo, state->segcxt.ws_segsize);
+ elog(LOG, "HOGE: %lu, %d => %s", nextSegNo, tli, path);
state->seg.ws_file = BasicOpenFile(path, O_RDONLY | PG_BINARY);
if (state->seg.ws_file >= 0)
return;
@@ -825,9 +825,11 @@ wal_segment_close(XLogReaderState *state)
* loop for now.
*/
bool
-read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *cur_page)
+read_local_xlog_page(XLogReaderState *state)
{
+ XLogRecPtr targetPagePtr = state->readPagePtr;
+ int reqLen = state->readLen;
+ char *cur_page = state->readBuf;
XLogRecPtr read_upto,
loc;
TimeLineID tli;
@@ -940,11 +942,12 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
* as 'count', read the whole page anyway. It's guaranteed to be
* zero-padded up to the page boundary if it's incomplete.
*/
- if (!WALRead(state, cur_page, targetPagePtr, XLOG_BLCKSZ, tli,
- &errinfo))
+ if (!WALRead(state, wal_segment_open, wal_segment_close,
+ cur_page, targetPagePtr, XLOG_BLCKSZ, tli, &errinfo))
WALReadRaiseError(&errinfo);
/* number of valid bytes in the buffer */
+ state->readPagePtr = targetPagePtr;
state->readLen = count;
return true;
}
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index 37b75deb72..9cbd8afa49 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -148,7 +148,8 @@ StartupDecodingContext(List *output_plugin_options,
TransactionId xmin_horizon,
bool need_full_snapshot,
bool fast_forward,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress)
@@ -198,11 +199,12 @@ StartupDecodingContext(List *output_plugin_options,
ctx->slot = slot;
- ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, xl_routine, ctx);
+ ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, cleanup_cb);
if (!ctx->reader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
errmsg("out of memory")));
+ ctx->page_read = page_read;
ctx->reorder = ReorderBufferAllocate();
ctx->snapshot_builder =
@@ -319,7 +321,8 @@ CreateInitDecodingContext(const char *plugin,
List *output_plugin_options,
bool need_full_snapshot,
XLogRecPtr restart_lsn,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress)
@@ -422,7 +425,7 @@ CreateInitDecodingContext(const char *plugin,
ctx = StartupDecodingContext(NIL, restart_lsn, xmin_horizon,
need_full_snapshot, false,
- xl_routine, prepare_write, do_write,
+ page_read, cleanup_cb, prepare_write, do_write,
update_progress);
/* call output plugin initialization callback */
@@ -476,7 +479,8 @@ LogicalDecodingContext *
CreateDecodingContext(XLogRecPtr start_lsn,
List *output_plugin_options,
bool fast_forward,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress)
@@ -528,8 +532,8 @@ CreateDecodingContext(XLogRecPtr start_lsn,
ctx = StartupDecodingContext(output_plugin_options,
start_lsn, InvalidTransactionId, false,
- fast_forward, xl_routine, prepare_write,
- do_write, update_progress);
+ fast_forward, page_read, cleanup_cb,
+ prepare_write, do_write, update_progress);
/* call output plugin initialization callback */
old_context = MemoryContextSwitchTo(ctx->context);
@@ -585,7 +589,13 @@ DecodingContextFindStartpoint(LogicalDecodingContext *ctx)
char *err = NULL;
/* the read_page callback waits for new WAL */
- record = XLogReadRecord(ctx->reader, &err);
+ while (XLogReadRecord(ctx->reader, &record, &err) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!ctx->page_read(ctx->reader))
+ break;
+ }
+
if (err)
elog(ERROR, "%s", err);
if (!record)
diff --git a/src/backend/replication/logical/logicalfuncs.c b/src/backend/replication/logical/logicalfuncs.c
index f7e055874e..b69f6911c5 100644
--- a/src/backend/replication/logical/logicalfuncs.c
+++ b/src/backend/replication/logical/logicalfuncs.c
@@ -233,9 +233,8 @@ pg_logical_slot_get_changes_guts(FunctionCallInfo fcinfo, bool confirm, bool bin
ctx = CreateDecodingContext(InvalidXLogRecPtr,
options,
false,
- XL_ROUTINE(.page_read = read_local_xlog_page,
- .segment_open = wal_segment_open,
- .segment_close = wal_segment_close),
+ read_local_xlog_page,
+ wal_segment_close,
LogicalOutputPrepareWrite,
LogicalOutputWrite, NULL);
@@ -284,7 +283,13 @@ pg_logical_slot_get_changes_guts(FunctionCallInfo fcinfo, bool confirm, bool bin
XLogRecord *record;
char *errm = NULL;
- record = XLogReadRecord(ctx->reader, &errm);
+ while (XLogReadRecord(ctx->reader, &record, &errm) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!ctx->page_read(ctx->reader))
+ break;
+ }
+
if (errm)
elog(ERROR, "%s", errm);
diff --git a/src/backend/replication/slotfuncs.c b/src/backend/replication/slotfuncs.c
index 9817b44113..deb7cf15eb 100644
--- a/src/backend/replication/slotfuncs.c
+++ b/src/backend/replication/slotfuncs.c
@@ -153,9 +153,8 @@ create_logical_replication_slot(char *name, char *plugin,
ctx = CreateInitDecodingContext(plugin, NIL,
false, /* just catalogs is OK */
restart_lsn,
- XL_ROUTINE(.page_read = read_local_xlog_page,
- .segment_open = wal_segment_open,
- .segment_close = wal_segment_close),
+ read_local_xlog_page,
+ wal_segment_close,
NULL, NULL, NULL);
/*
@@ -512,9 +511,8 @@ pg_logical_replication_slot_advance(XLogRecPtr moveto)
ctx = CreateDecodingContext(InvalidXLogRecPtr,
NIL,
true, /* fast_forward */
- XL_ROUTINE(.page_read = read_local_xlog_page,
- .segment_open = wal_segment_open,
- .segment_close = wal_segment_close),
+ read_local_xlog_page,
+ wal_segment_close,
NULL, NULL, NULL);
/*
@@ -536,7 +534,13 @@ pg_logical_replication_slot_advance(XLogRecPtr moveto)
* Read records. No changes are generated in fast_forward mode,
* but snapbuilder/slot statuses are updated properly.
*/
- record = XLogReadRecord(ctx->reader, &errm);
+ while (XLogReadRecord(ctx->reader, &record, &errm) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!ctx->page_read(ctx->reader))
+ break;
+ }
+
if (errm)
elog(ERROR, "%s", errm);
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index c36b91ae5e..062093727d 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -580,10 +580,7 @@ StartReplication(StartReplicationCmd *cmd)
/* create xlogreader for physical replication */
xlogreader =
- XLogReaderAllocate(wal_segment_size, NULL,
- XL_ROUTINE(.segment_open = WalSndSegmentOpen,
- .segment_close = wal_segment_close),
- NULL);
+ XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
if (!xlogreader)
ereport(ERROR,
@@ -807,9 +804,11 @@ StartReplication(StartReplicationCmd *cmd)
* set every time WAL is flushed.
*/
static bool
-logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetRecPtr, char *cur_page)
+logical_read_xlog_page(XLogReaderState *state)
{
+ XLogRecPtr targetPagePtr = state->readPagePtr;
+ int reqLen = state->readLen;
+ char *cur_page = state->readBuf;
XLogRecPtr flushptr;
int count;
WALReadError errinfo;
@@ -837,7 +836,7 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
count = flushptr - targetPagePtr; /* part of the page available */
/* now actually read the data, we know it's there */
- if (!WALRead(state,
+ if (!WALRead(state, WalSndSegmentOpen, wal_segment_close,
cur_page,
targetPagePtr,
XLOG_BLCKSZ,
@@ -1011,9 +1010,8 @@ CreateReplicationSlot(CreateReplicationSlotCmd *cmd)
ctx = CreateInitDecodingContext(cmd->plugin, NIL, need_full_snapshot,
InvalidXLogRecPtr,
- XL_ROUTINE(.page_read = logical_read_xlog_page,
- .segment_open = WalSndSegmentOpen,
- .segment_close = wal_segment_close),
+ logical_read_xlog_page,
+ wal_segment_close,
WalSndPrepareWrite, WalSndWriteData,
WalSndUpdateProgress);
@@ -1171,9 +1169,8 @@ StartLogicalReplication(StartReplicationCmd *cmd)
*/
logical_decoding_ctx =
CreateDecodingContext(cmd->startpoint, cmd->options, false,
- XL_ROUTINE(.page_read = logical_read_xlog_page,
- .segment_open = WalSndSegmentOpen,
- .segment_close = wal_segment_close),
+ logical_read_xlog_page,
+ wal_segment_close,
WalSndPrepareWrite, WalSndWriteData,
WalSndUpdateProgress);
xlogreader = logical_decoding_ctx->reader;
@@ -2749,7 +2746,7 @@ XLogSendPhysical(void)
enlargeStringInfo(&output_message, nbytes);
retry:
- if (!WALRead(xlogreader,
+ if (!WALRead(xlogreader, WalSndSegmentOpen, wal_segment_close,
&output_message.data[output_message.len],
startptr,
nbytes,
@@ -2847,7 +2844,12 @@ XLogSendLogical(void)
*/
WalSndCaughtUp = false;
- record = XLogReadRecord(logical_decoding_ctx->reader, &errm);
+ while (XLogReadRecord(logical_decoding_ctx->reader, &record, &errm) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!logical_decoding_ctx->page_read(logical_decoding_ctx->reader))
+ break;
+ }
/* xlog record was invalid */
if (errm != NULL)
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index cf119848b0..712c85281c 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -41,15 +41,9 @@ static int xlogreadfd = -1;
static XLogSegNo xlogreadsegno = -1;
static char xlogfpath[MAXPGPATH];
-typedef struct XLogPageReadPrivate
-{
- const char *restoreCommand;
- int tliIndex;
-} XLogPageReadPrivate;
-
-static bool SimpleXLogPageRead(XLogReaderState *xlogreader,
- XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
+static bool SimpleXLogPageRead(XLogReaderState *xlogreader,
+ const char *datadir, int *tliIndex,
+ const char *restoreCommand);
/*
* Read WAL from the datadir/pg_wal, starting from 'startpoint' on timeline
@@ -66,20 +60,22 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
XLogRecord *record;
XLogReaderState *xlogreader;
char *errormsg;
- XLogPageReadPrivate private;
- private.tliIndex = tliIndex;
- private.restoreCommand = restoreCommand;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir,
- XL_ROUTINE(.page_read = &SimpleXLogPageRead),
- &private);
+ xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+
if (xlogreader == NULL)
pg_fatal("out of memory");
XLogBeginRead(xlogreader, startpoint);
do
{
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!SimpleXLogPageRead(xlogreader, datadir,
+ &tliIndex, restoreCommand))
+ break;
+ }
if (record == NULL)
{
@@ -123,19 +119,19 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex,
XLogRecord *record;
XLogReaderState *xlogreader;
char *errormsg;
- XLogPageReadPrivate private;
XLogRecPtr endptr;
- private.tliIndex = tliIndex;
- private.restoreCommand = restoreCommand;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir,
- XL_ROUTINE(.page_read = &SimpleXLogPageRead),
- &private);
+ xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
XLogBeginRead(xlogreader, ptr);
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!SimpleXLogPageRead(xlogreader, datadir, &tliIndex, restoreCommand))
+ break;
+ }
if (record == NULL)
{
if (errormsg)
@@ -170,7 +166,6 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
XLogRecPtr searchptr;
XLogReaderState *xlogreader;
char *errormsg;
- XLogPageReadPrivate private;
/*
* The given fork pointer points to the end of the last common record,
@@ -186,11 +181,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
forkptr += SizeOfXLogShortPHD;
}
- private.tliIndex = tliIndex;
- private.restoreCommand = restoreCommand;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir,
- XL_ROUTINE(.page_read = &SimpleXLogPageRead),
- &private);
+ xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
@@ -200,7 +191,13 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
uint8 info;
XLogBeginRead(xlogreader, searchptr);
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!SimpleXLogPageRead(xlogreader, datadir,
+ &tliIndex, restoreCommand))
+ break;
+ }
if (record == NULL)
{
@@ -247,10 +244,11 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
/* XLogReader callback function, to read a WAL page */
static bool
-SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *readBuf)
+SimpleXLogPageRead(XLogReaderState *xlogreader, const char *datadir,
+ int *tliIndex, const char *restoreCommand)
{
- XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
+ XLogRecPtr targetPagePtr = xlogreader->readPagePtr;
+ char *readBuf = xlogreader->readBuf;
uint32 targetPageOff;
XLogRecPtr targetSegEnd;
XLogSegNo targetSegNo;
@@ -283,14 +281,14 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
* be done both forward and backward, consider also switching timeline
* accordingly.
*/
- while (private->tliIndex < targetNentries - 1 &&
- targetHistory[private->tliIndex].end < targetSegEnd)
- private->tliIndex++;
- while (private->tliIndex > 0 &&
- targetHistory[private->tliIndex].begin >= targetSegEnd)
- private->tliIndex--;
+ while (*tliIndex < targetNentries - 1 &&
+ targetHistory[*tliIndex].end < targetSegEnd)
+ (*tliIndex)++;
+ while (*tliIndex > 0 &&
+ targetHistory[*tliIndex].begin >= targetSegEnd)
+ (*tliIndex)--;
- XLogFileName(xlogfname, targetHistory[private->tliIndex].tli,
+ XLogFileName(xlogfname, targetHistory[*tliIndex].tli,
xlogreadsegno, WalSegSz);
snprintf(xlogfpath, MAXPGPATH, "%s/" XLOGDIR "/%s",
@@ -303,7 +301,7 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
/*
* If we have no restore_command to execute, then exit.
*/
- if (private->restoreCommand == NULL)
+ if (restoreCommand == NULL)
{
pg_log_error("could not open file \"%s\": %m", xlogfpath);
xlogreader->readLen = -1;
@@ -317,7 +315,7 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
xlogreadfd = RestoreArchivedFile(xlogreader->segcxt.ws_dir,
xlogfname,
WalSegSz,
- private->restoreCommand);
+ restoreCommand);
if (xlogreadfd < 0)
{
@@ -359,7 +357,7 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
Assert(targetSegNo == xlogreadsegno);
- xlogreader->seg.ws_tli = targetHistory[private->tliIndex].tli;
+ xlogreader->seg.ws_tli = targetHistory[*tliIndex].tli;
xlogreader->readLen = XLOG_BLCKSZ;
return true;
}
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index 0f0e0723a3..f514de6b13 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -330,12 +330,17 @@ WALDumpCloseSegment(XLogReaderState *state)
state->seg.ws_file = -1;
}
-/* pg_waldump's XLogReaderRoutine->page_read callback */
+/*
+ * pg_waldump's WAL page rader, also used as page_read callback for
+ * XLogFindNextRecord
+ */
static bool
-WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetPtr, char *readBuff)
+WALDumpReadPage(XLogReaderState *state, void *priv)
{
- XLogDumpPrivate *private = state->private_data;
+ XLogRecPtr targetPagePtr = state->readPagePtr;
+ int reqLen = state->readLen;
+ char *readBuff = state->readBuf;
+ XLogDumpPrivate *private = (XLogDumpPrivate *) priv;
int count = XLOG_BLCKSZ;
WALReadError errinfo;
@@ -353,8 +358,8 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
}
}
- if (!WALRead(state, readBuff, targetPagePtr, count, private->timeline,
- &errinfo))
+ if (!WALRead(state, WALDumpOpenSegment, WALDumpCloseSegment,
+ readBuff, targetPagePtr, count, private->timeline, &errinfo))
{
WALOpenSegment *seg = &errinfo.wre_seg;
char fname[MAXPGPATH];
@@ -374,6 +379,7 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
(Size) errinfo.wre_req);
}
+ Assert(count >= state->readLen);
state->readLen = count;
return true;
}
@@ -1035,16 +1041,14 @@ main(int argc, char **argv)
/* we have everything we need, start reading */
xlogreader_state =
- XLogReaderAllocate(WalSegSz, waldir,
- XL_ROUTINE(.page_read = WALDumpReadPage,
- .segment_open = WALDumpOpenSegment,
- .segment_close = WALDumpCloseSegment),
- &private);
+ XLogReaderAllocate(WalSegSz, waldir, WALDumpCloseSegment);
+
if (!xlogreader_state)
fatal_error("out of memory");
/* first find a valid recptr to start from */
- first_record = XLogFindNextRecord(xlogreader_state, private.startptr);
+ first_record = XLogFindNextRecord(xlogreader_state, private.startptr,
+ &WALDumpReadPage, (void*) &private);
if (first_record == InvalidXLogRecPtr)
fatal_error("could not find a valid record after %X/%X",
@@ -1067,7 +1071,13 @@ main(int argc, char **argv)
for (;;)
{
/* try to read the next record */
- record = XLogReadRecord(xlogreader_state, &errormsg);
+ while (XLogReadRecord(xlogreader_state, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!WALDumpReadPage(xlogreader_state, (void *) &private))
+ break;
+ }
+
if (!record)
{
if (!config.follow || private.endptr_reached)
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index 23d9e70a59..4e9268b553 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -57,64 +57,15 @@ typedef struct WALSegmentContext
typedef struct XLogReaderState XLogReaderState;
-/* Function type definition for the read_page callback */
-typedef bool (*XLogPageReadCB) (XLogReaderState *xlogreader,
- XLogRecPtr targetPagePtr,
- int reqLen,
- XLogRecPtr targetRecPtr,
- char *readBuf);
+/* Function type definition for the segment cleanup callback */
+typedef void (*WALSegmentCleanupCB) (XLogReaderState *xlogreader);
+
+/* Function type definition for the open/close callbacks for WALRead() */
typedef void (*WALSegmentOpenCB) (XLogReaderState *xlogreader,
XLogSegNo nextSegNo,
TimeLineID *tli_p);
typedef void (*WALSegmentCloseCB) (XLogReaderState *xlogreader);
-typedef struct XLogReaderRoutine
-{
- /*
- * Data input callback
- *
- * This callback shall read at least reqLen valid bytes of the xlog page
- * starting at targetPagePtr, and store them in readBuf. The callback
- * shall return the number of bytes read (never more than XLOG_BLCKSZ), or
- * -1 on failure. The callback shall sleep, if necessary, to wait for the
- * requested bytes to become available. The callback will not be invoked
- * again for the same page unless more than the returned number of bytes
- * are needed.
- *
- * targetRecPtr is the position of the WAL record we're reading. Usually
- * it is equal to targetPagePtr + reqLen, but sometimes xlogreader needs
- * to read and verify the page or segment header, before it reads the
- * actual WAL record it's interested in. In that case, targetRecPtr can
- * be used to determine which timeline to read the page from.
- *
- * The callback shall set ->seg.ws_tli to the TLI of the file the page was
- * read from.
- */
- XLogPageReadCB page_read;
-
- /*
- * Callback to open the specified WAL segment for reading. ->seg.ws_file
- * shall be set to the file descriptor of the opened segment. In case of
- * failure, an error shall be raised by the callback and it shall not
- * return.
- *
- * "nextSegNo" is the number of the segment to be opened.
- *
- * "tli_p" is an input/output argument. WALRead() uses it to pass the
- * timeline in which the new segment should be found, but the callback can
- * use it to return the TLI that it actually opened.
- */
- WALSegmentOpenCB segment_open;
-
- /*
- * WAL segment close callback. ->seg.ws_file shall be set to a negative
- * number.
- */
- WALSegmentCloseCB segment_close;
-} XLogReaderRoutine;
-
-#define XL_ROUTINE(...) &(XLogReaderRoutine){__VA_ARGS__}
-
typedef struct
{
/* Is this block ref in use? */
@@ -144,12 +95,35 @@ typedef struct
uint16 data_bufsz;
} DecodedBkpBlock;
+/* Return code from XLogReadRecord */
+typedef enum XLogReadRecordResult
+{
+ XLREAD_SUCCESS, /* record is successfully read */
+ XLREAD_NEED_DATA, /* need more data. see XLogReadRecord. */
+ XLREAD_FAIL /* failed during reading a record */
+} XLogReadRecordResult;
+
+/*
+ * internal state of XLogReadRecord
+ *
+ * XLogReadState runs a state machine while reading a record. Theses states
+ * are not seen outside the function. Each state may repeat several times
+ * exiting requesting caller for new data. See the comment of XLogReadRecrod
+ * for details.
+ */
+typedef enum XLogReadRecordState {
+ XLREAD_NEXT_RECORD,
+ XLREAD_TOT_LEN,
+ XLREAD_FIRST_FRAGMENT,
+ XLREAD_CONTINUATION
+} XLogReadRecordState;
+
struct XLogReaderState
{
/*
* Operational callbacks
*/
- XLogReaderRoutine routine;
+ WALSegmentCleanupCB cleanup_cb;
/* ----------------------------------------
* Public parameters
@@ -162,18 +136,14 @@ struct XLogReaderState
*/
uint64 system_identifier;
- /*
- * Opaque data for callbacks to use. Not used by XLogReader.
- */
- void *private_data;
-
/*
* Start and end point of last record read. EndRecPtr is also used as the
* position to read next. Calling XLogBeginRead() sets EndRecPtr to the
* starting position and ReadRecPtr to invalid.
*/
- XLogRecPtr ReadRecPtr; /* start of last record read */
+ XLogRecPtr ReadRecPtr; /* start of last record read or being read */
XLogRecPtr EndRecPtr; /* end+1 of last record read */
+ XLogRecPtr PrevRecPtr; /* start of previous record read */
/* ----------------------------------------
* Communication with page reader
@@ -187,7 +157,9 @@ struct XLogReaderState
* the request, or -1 on error */
TimeLineID readPageTLI; /* TLI for data currently in readBuf */
char *readBuf; /* buffer to store data */
- bool page_verified; /* is the page on the buffer verified? */
+ bool page_verified; /* is the page header on the buffer verified? */
+ bool record_verified;/* is the current record header verified? */
+
/* ----------------------------------------
@@ -229,8 +201,6 @@ struct XLogReaderState
XLogRecPtr latestPagePtr;
TimeLineID latestPageTLI;
- /* beginning of the WAL record being read. */
- XLogRecPtr currRecPtr;
/* timeline to read it from, 0 if a lookup is required */
TimeLineID currTLI;
@@ -257,6 +227,15 @@ struct XLogReaderState
char *readRecordBuf;
uint32 readRecordBufSize;
+ /*
+ * XLogReadRecord() state
+ */
+ XLogReadRecordState readRecordState;/* state machine state */
+ int recordGotLen; /* amount of current record that has
+ * already been read */
+ int recordRemainLen; /* length of current record that remains */
+ XLogRecPtr recordContRecPtr; /* where the current record continues */
+
/* Buffer to hold error message */
char *errormsg_buf;
};
@@ -264,9 +243,7 @@ struct XLogReaderState
/* Get a new XLogReader */
extern XLogReaderState *XLogReaderAllocate(int wal_segment_size,
const char *waldir,
- XLogReaderRoutine *routine,
- void *private_data);
-extern XLogReaderRoutine *LocalXLogReaderRoutine(void);
+ WALSegmentCleanupCB cleanup_cb);
/* Free an XLogReader */
extern void XLogReaderFree(XLogReaderState *state);
@@ -274,12 +251,17 @@ extern void XLogReaderFree(XLogReaderState *state);
/* Position the XLogReader to given record */
extern void XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr);
#ifdef FRONTEND
-extern XLogRecPtr XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr);
+/* Function type definition for the read_page callback */
+typedef bool (*XLogFindNextRecordCB) (XLogReaderState *xlogreader,
+ void *private);
+extern XLogRecPtr XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
+ XLogFindNextRecordCB read_page, void *private);
#endif /* FRONTEND */
/* Read the next XLog record. Returns NULL on end-of-WAL or failure */
-extern struct XLogRecord *XLogReadRecord(XLogReaderState *state,
- char **errormsg);
+extern XLogReadRecordResult XLogReadRecord(XLogReaderState *state,
+ XLogRecord **record,
+ char **errormsg);
/* Validate a page */
extern bool XLogReaderValidatePageHeader(XLogReaderState *state,
@@ -299,6 +281,7 @@ typedef struct WALReadError
} WALReadError;
extern bool WALRead(XLogReaderState *state,
+ WALSegmentOpenCB segopenfn, WALSegmentCloseCB sgclosefn,
char *buf, XLogRecPtr startptr, Size count,
TimeLineID tli, WALReadError *errinfo);
diff --git a/src/include/access/xlogutils.h b/src/include/access/xlogutils.h
index 364a21c4ea..397fb27fc2 100644
--- a/src/include/access/xlogutils.h
+++ b/src/include/access/xlogutils.h
@@ -47,9 +47,7 @@ extern Buffer XLogReadBufferExtended(RelFileNode rnode, ForkNumber forknum,
extern Relation CreateFakeRelcacheEntry(RelFileNode rnode);
extern void FreeFakeRelcacheEntry(Relation fakerel);
-extern bool read_local_xlog_page(XLogReaderState *state,
- XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetRecPtr, char *cur_page);
+extern bool read_local_xlog_page(XLogReaderState *state);
extern void wal_segment_open(XLogReaderState *state,
XLogSegNo nextSegNo,
TimeLineID *tli_p);
diff --git a/src/include/pg_config_manual.h b/src/include/pg_config_manual.h
index f10ad0acd6..94749609b4 100644
--- a/src/include/pg_config_manual.h
+++ b/src/include/pg_config_manual.h
@@ -377,7 +377,7 @@
* Enable debugging print statements for WAL-related operations; see
* also the wal_debug GUC var.
*/
-/* #define WAL_DEBUG */
+#define WAL_DEBUG
/*
* Enable tracing of resource consumption during sort operations;
diff --git a/src/include/replication/logical.h b/src/include/replication/logical.h
index c253403372..b936f31393 100644
--- a/src/include/replication/logical.h
+++ b/src/include/replication/logical.h
@@ -29,6 +29,10 @@ typedef void (*LogicalOutputPluginWriterUpdateProgress) (struct LogicalDecodingC
TransactionId xid
);
+typedef struct LogicalDecodingContext LogicalDecodingContext;
+
+typedef bool (*LogicalDecodingXLogPageReadCB)(XLogReaderState *ctx);
+
typedef struct LogicalDecodingContext
{
/* memory context this is all allocated in */
@@ -39,6 +43,7 @@ typedef struct LogicalDecodingContext
/* infrastructure pieces for decoding */
XLogReaderState *reader;
+ LogicalDecodingXLogPageReadCB page_read;
struct ReorderBuffer *reorder;
struct SnapBuild *snapshot_builder;
@@ -105,14 +110,16 @@ extern LogicalDecodingContext *CreateInitDecodingContext(const char *plugin,
List *output_plugin_options,
bool need_full_snapshot,
XLogRecPtr restart_lsn,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress);
extern LogicalDecodingContext *CreateDecodingContext(XLogRecPtr start_lsn,
List *output_plugin_options,
bool fast_forward,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress);
--
2.27.0
----Next_Part(Thu_Mar__4_11_28_53_2021_014)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
filename="v17-0003-Remove-globals-readOff-readLen-and-readSegNo.patch"
^ permalink raw reply [nested|flat] 70+ messages in thread
* [PATCH v15 2/4] Move page-reader out of XLogReadRecord
@ 2019-09-10 03:58 Kyotaro Horiguchi <[email protected]>
0 siblings, 0 replies; 70+ messages in thread
From: Kyotaro Horiguchi @ 2019-09-10 03:58 UTC (permalink / raw)
This is the second step of removing callbacks from WAL record reader.
Since it is essential to take in additional data while reading a
record, the function have to ask caller for new data while keeping
working state. Thus the function is turned into a state machine.
---
src/backend/access/transam/twophase.c | 15 +-
src/backend/access/transam/xlog.c | 58 +-
src/backend/access/transam/xlogreader.c | 700 +++++++++++-------
src/backend/access/transam/xlogutils.c | 17 +-
src/backend/replication/logical/logical.c | 26 +-
.../replication/logical/logicalfuncs.c | 13 +-
src/backend/replication/slotfuncs.c | 18 +-
src/backend/replication/walsender.c | 32 +-
src/bin/pg_rewind/parsexlog.c | 84 +--
src/bin/pg_waldump/pg_waldump.c | 36 +-
src/include/access/xlogreader.h | 121 ++-
src/include/access/xlogutils.h | 4 +-
src/include/replication/logical.h | 11 +-
13 files changed, 648 insertions(+), 487 deletions(-)
diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index 9b2e59bf0e..b0d60a0d0f 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -1330,11 +1330,8 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
XLogReaderState *xlogreader;
char *errormsg;
- xlogreader = XLogReaderAllocate(wal_segment_size, NULL,
- XL_ROUTINE(.page_read = &read_local_xlog_page,
- .segment_open = &wal_segment_open,
- .segment_close = &wal_segment_close),
- NULL);
+ xlogreader = XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+
if (!xlogreader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
@@ -1342,7 +1339,13 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
errdetail("Failed while allocating a WAL reading processor.")));
XLogBeginRead(xlogreader, lsn);
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!read_local_xlog_page(xlogreader))
+ break;
+ }
+
if (record == NULL)
ereport(ERROR,
(errcode_for_file_access(),
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index e570e56a24..f9b0108602 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -831,13 +831,6 @@ static XLogSource currentSource = XLOG_FROM_ANY;
static bool lastSourceFailed = false;
static bool pendingWalRcvRestart = false;
-typedef struct XLogPageReadPrivate
-{
- int emode;
- bool fetching_ckpt; /* are we fetching a checkpoint record? */
- bool randAccess;
-} XLogPageReadPrivate;
-
/*
* These variables track when we last obtained some WAL data to process,
* and where we got it from. (XLogReceiptSource is initially the same as
@@ -912,8 +905,8 @@ static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath,
static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli,
XLogSource source, bool notfoundOk);
static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, XLogSource source);
-static bool XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
+static bool XLogPageRead(XLogReaderState *xlogreader,
+ bool fetching_ckpt, int emode, bool randAccess);
static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
bool fetching_ckpt, XLogRecPtr tliRecPtr);
static int emode_for_corrupt_record(int emode, XLogRecPtr RecPtr);
@@ -1225,8 +1218,7 @@ XLogInsertRecord(XLogRecData *rdata,
appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len);
if (!debug_reader)
- debug_reader = XLogReaderAllocate(wal_segment_size, NULL,
- XL_ROUTINE(), NULL);
+ debug_reader = XLogReaderAllocate(wal_segment_size, NULL NULL);
if (!debug_reader)
{
@@ -4326,15 +4318,10 @@ CleanupBackupHistory(void)
* record is available.
*/
static XLogRecord *
-ReadRecord(XLogReaderState *xlogreader, int emode,
- bool fetching_ckpt)
+ReadRecord(XLogReaderState *xlogreader, int emode, bool fetching_ckpt)
{
XLogRecord *record;
- XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
-
- private->fetching_ckpt = fetching_ckpt;
- private->emode = emode;
- private->randAccess = (xlogreader->ReadRecPtr == InvalidXLogRecPtr);
+ bool randAccess = (xlogreader->ReadRecPtr == InvalidXLogRecPtr);
/* This is the first attempt to read this page. */
lastSourceFailed = false;
@@ -4342,8 +4329,16 @@ ReadRecord(XLogReaderState *xlogreader, int emode,
for (;;)
{
char *errormsg;
+ XLogReadRecordResult result;
+
+ while ((result = XLogReadRecord(xlogreader, &record, &errormsg))
+ == XLREAD_NEED_DATA)
+ {
+ if (!XLogPageRead(xlogreader, fetching_ckpt, emode, randAccess))
+ break;
+
+ }
- record = XLogReadRecord(xlogreader, &errormsg);
ReadRecPtr = xlogreader->ReadRecPtr;
EndRecPtr = xlogreader->EndRecPtr;
if (record == NULL)
@@ -6320,7 +6315,6 @@ StartupXLOG(void)
bool backupFromStandby = false;
DBState dbstate_at_startup;
XLogReaderState *xlogreader;
- XLogPageReadPrivate private;
bool fast_promoted = false;
struct stat st;
@@ -6480,13 +6474,9 @@ StartupXLOG(void)
OwnLatch(&XLogCtl->recoveryWakeupLatch);
/* Set up XLOG reader facility */
- MemSet(&private, 0, sizeof(XLogPageReadPrivate));
xlogreader =
- XLogReaderAllocate(wal_segment_size, NULL,
- XL_ROUTINE(.page_read = &XLogPageRead,
- .segment_open = NULL,
- .segment_close = wal_segment_close),
- &private);
+ XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+
if (!xlogreader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
@@ -11860,12 +11850,13 @@ CancelBackup(void)
* sleep and retry.
*/
static bool
-XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetRecPtr, char *readBuf)
+XLogPageRead(XLogReaderState *xlogreader,
+ bool fetching_ckpt, int emode, bool randAccess)
{
- XLogPageReadPrivate *private =
- (XLogPageReadPrivate *) xlogreader->private_data;
- int emode = private->emode;
+ char *readBuf = xlogreader->readBuf;
+ XLogRecPtr targetPagePtr = xlogreader->readPagePtr;
+ int reqLen = xlogreader->readLen;
+ XLogRecPtr targetRecPtr = xlogreader->ReadRecPtr;
uint32 targetPageOff;
XLogSegNo targetSegNo PG_USED_FOR_ASSERTS_ONLY;
int r;
@@ -11908,8 +11899,8 @@ retry:
flushedUpto < targetPagePtr + reqLen))
{
if (!WaitForWALToBecomeAvailable(targetPagePtr + reqLen,
- private->randAccess,
- private->fetching_ckpt,
+ randAccess,
+ fetching_ckpt,
targetRecPtr))
{
if (readFile >= 0)
@@ -12014,6 +12005,7 @@ retry:
goto next_record_is_invalid;
}
+ Assert(xlogreader->readPagePtr == targetPagePtr);
xlogreader->readLen = readLen;
return true;
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index 3d599325ee..9281b57379 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -40,7 +40,7 @@ static bool XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr,
int reqLen, bool header_inclusive);
static void XLogReaderInvalReadState(XLogReaderState *state);
static bool ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
- XLogRecPtr PrevRecPtr, XLogRecord *record, bool randAccess);
+ XLogRecPtr PrevRecPtr, XLogRecord *record);
static bool ValidXLogRecord(XLogReaderState *state, XLogRecord *record,
XLogRecPtr recptr);
static void ResetDecoder(XLogReaderState *state);
@@ -73,7 +73,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...)
*/
XLogReaderState *
XLogReaderAllocate(int wal_segment_size, const char *waldir,
- XLogReaderRoutine *routine, void *private_data)
+ WALSegmentCleanupCB cleanup_cb)
{
XLogReaderState *state;
@@ -84,7 +84,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
return NULL;
/* initialize caller-provided support functions */
- state->routine = *routine;
+ state->cleanup_cb = cleanup_cb;
state->max_block_id = -1;
@@ -107,8 +107,6 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
WALOpenSegmentInit(&state->seg, &state->segcxt, wal_segment_size,
waldir);
- /* system_identifier initialized to zeroes above */
- state->private_data = private_data;
/* ReadRecPtr, EndRecPtr and readLen initialized to zeroes above */
state->errormsg_buf = palloc_extended(MAX_ERRORMSG_LEN + 1,
MCXT_ALLOC_NO_OOM);
@@ -140,8 +138,8 @@ XLogReaderFree(XLogReaderState *state)
{
int block_id;
- if (state->seg.ws_file != -1)
- state->routine.segment_close(state);
+ if (state->seg.ws_file >= 0)
+ state->cleanup_cb(state);
for (block_id = 0; block_id <= XLR_MAX_BLOCK_ID; block_id++)
{
@@ -246,6 +244,7 @@ XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
/* Begin at the passed-in record pointer. */
state->EndRecPtr = RecPtr;
state->ReadRecPtr = InvalidXLogRecPtr;
+ state->readRecordState = XLREAD_NEXT_RECORD;
}
/*
@@ -254,317 +253,452 @@ XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
* XLogBeginRead() or XLogFindNextRecord() must be called before the first call
* to XLogReadRecord().
*
- * If the page_read callback fails to read the requested data, NULL is
- * returned. The callback is expected to have reported the error; errormsg
- * is set to NULL.
+ * This function may return XLREAD_NEED_DATA several times before returning a
+ * result record. The caller shall read in some new data then call this
+ * function again with the same parameters.
*
- * If the reading fails for some other reason, NULL is also returned, and
- * *errormsg is set to a string with details of the failure.
+ * When a record is successfully read, returns XLREAD_SUCCESS with result
+ * record being stored in *record. Otherwise *record is NULL.
*
- * The returned pointer (or *errormsg) points to an internal buffer that's
- * valid until the next call to XLogReadRecord.
+ * Returns XLREAD_NEED_DATA if more data is needed to finish reading the
+ * current record. In that case, state->readPagePtr and state->readLen inform
+ * the desired position and minimum length of data needed. The caller shall
+ * read in the requested data and set state->readBuf to point to a buffer
+ * containing it. The caller must also set state->readPageTLI and
+ * state->readLen to indicate the timeline that it was read from, and the
+ * length of data that is now available (which must be >= given readLen),
+ * respectively.
+ *
+ * If invalid data is encountered, returns XLREAD_FAIL with *record being set to
+ * NULL. *errormsg is set to a string with details of the failure.
+ * The returned pointer (or *errormsg) points to an internal buffer that's valid
+ * until the next call to XLogReadRecord.
+ *
+ *
+ * This function runs a state machine consists of the following states.
+ *
+ * XLREAD_NEXT_RECORD :
+ * The initial state, if called with valid RecPtr, try to read a record at
+ * that position. If invalid RecPtr is given try to read a record just after
+ * the last one previously read.
+ * This state ens after setting ReadRecPtr. Then goes to XLREAD_TOT_LEN.
+ *
+ * XLREAD_TOT_LEN:
+ * Examining record header. Ends after reading record total
+ * length. recordRemainLen and recordGotLen are initialized.
+ *
+ * XLREAD_FIRST_FRAGMENT:
+ * Reading the first fragment. Ends with finishing reading a single
+ * record. Goes to XLREAD_NEXT_RECORD if that's all or
+ * XLREAD_CONTINUATION if we have continuation.
+
+ * XLREAD_CONTINUATION:
+ * Reading continuation of record. Ends with finishing the whole record then
+ * goes to XLREAD_NEXT_RECORD. During this state, recordRemainLen indicates
+ * how much is left and readRecordBuf holds the partially assert
+ * record.recordContRecPtr points to the beginning of the next page where to
+ * continue.
+ *
+ * If wrong data found in any state, the state machine stays at the current
+ * state. This behavior allows to continue reading a reacord switching among
+ * different souces, while streaming replication.
*/
-XLogRecord *
-XLogReadRecord(XLogReaderState *state, char **errormsg)
+XLogReadRecordResult
+XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
{
- XLogRecPtr RecPtr;
- XLogRecord *record;
- XLogRecPtr targetPagePtr;
- bool randAccess;
- uint32 len,
- total_len;
- uint32 targetRecOff;
- uint32 pageHeaderSize;
- bool gotheader;
+ XLogRecord *prec;
- /*
- * randAccess indicates whether to verify the previous-record pointer of
- * the record we're reading. We only do this if we're reading
- * sequentially, which is what we initially assume.
- */
- randAccess = false;
+ *record = NULL;
/* reset error state */
*errormsg = NULL;
state->errormsg_buf[0] = '\0';
- ResetDecoder(state);
-
- RecPtr = state->EndRecPtr;
-
- if (state->ReadRecPtr != InvalidXLogRecPtr)
- {
- /* read the record after the one we just read */
-
- /*
- * EndRecPtr is pointing to end+1 of the previous WAL record. If
- * we're at a page boundary, no more records can fit on the current
- * page. We must skip over the page header, but we can't do that until
- * we've read in the page, since the header size is variable.
- */
- }
- else
- {
- /*
- * Caller supplied a position to start at.
- *
- * In this case, EndRecPtr should already be pointing to a valid
- * record starting position.
- */
- Assert(XRecOffIsValid(RecPtr));
- randAccess = true;
- }
-
- state->currRecPtr = RecPtr;
-
- targetPagePtr = RecPtr - (RecPtr % XLOG_BLCKSZ);
- targetRecOff = RecPtr % XLOG_BLCKSZ;
-
- /*
- * Read the page containing the record into state->readBuf. Request enough
- * byte to cover the whole record header, or at least the part of it that
- * fits on the same page.
- */
- while (XLogNeedData(state, targetPagePtr,
- Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
- targetRecOff != 0))
- {
- if (!state->routine.page_read(state, state->readPagePtr, state->readLen,
- RecPtr, state->readBuf))
- break;
- }
-
- if (!state->page_verified)
- goto err;
-
- /*
- * We have at least the page header, so we can examine it now.
- */
- pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
- if (targetRecOff == 0)
- {
- /*
- * At page start, so skip over page header.
- */
- RecPtr += pageHeaderSize;
- targetRecOff = pageHeaderSize;
- }
- else if (targetRecOff < pageHeaderSize)
- {
- report_invalid_record(state, "invalid record offset at %X/%X",
- (uint32) (RecPtr >> 32), (uint32) RecPtr);
- goto err;
- }
-
- if ((((XLogPageHeader) state->readBuf)->xlp_info & XLP_FIRST_IS_CONTRECORD) &&
- targetRecOff == pageHeaderSize)
- {
- report_invalid_record(state, "contrecord is requested by %X/%X",
- (uint32) (RecPtr >> 32), (uint32) RecPtr);
- goto err;
- }
-
- /* XLogNeedData has verified the page header */
- Assert(pageHeaderSize <= state->readLen);
-
- /*
- * Read the record length.
- *
- * NB: Even though we use an XLogRecord pointer here, the whole record
- * header might not fit on this page. xl_tot_len is the first field of the
- * struct, so it must be on this page (the records are MAXALIGNed), but we
- * cannot access any other fields until we've verified that we got the
- * whole header.
- */
- record = (XLogRecord *) (state->readBuf + RecPtr % XLOG_BLCKSZ);
- total_len = record->xl_tot_len;
-
- /*
- * If the whole record header is on this page, validate it immediately.
- * Otherwise do just a basic sanity check on xl_tot_len, and validate the
- * rest of the header after reading it from the next page. The xl_tot_len
- * check is necessary here to ensure that we enter the "Need to reassemble
- * record" code path below; otherwise we might fail to apply
- * ValidXLogRecordHeader at all.
- */
- if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
- {
- if (!ValidXLogRecordHeader(state, RecPtr, state->ReadRecPtr, record,
- randAccess))
- goto err;
- gotheader = true;
- }
- else
- {
- /* XXX: more validation should be done here */
- if (total_len < SizeOfXLogRecord)
- {
- report_invalid_record(state,
- "invalid record length at %X/%X: wanted %u, got %u",
- (uint32) (RecPtr >> 32), (uint32) RecPtr,
- (uint32) SizeOfXLogRecord, total_len);
- goto err;
- }
- gotheader = false;
- }
-
- len = XLOG_BLCKSZ - RecPtr % XLOG_BLCKSZ;
- if (total_len > len)
+ switch (state->readRecordState)
{
- /* Need to reassemble record */
- char *contdata;
- XLogPageHeader pageHeader;
- char *buffer;
- uint32 gotlen;
-
- /*
- * Enlarge readRecordBuf as needed.
- */
- if (total_len > state->readRecordBufSize &&
- !allocate_recordbuf(state, total_len))
- {
- /* We treat this as a "bogus data" condition */
- report_invalid_record(state, "record length %u at %X/%X too long",
- total_len,
- (uint32) (RecPtr >> 32), (uint32) RecPtr);
- goto err;
- }
+ case XLREAD_NEXT_RECORD:
+ ResetDecoder(state);
- /* Copy the first fragment of the record from the first page. */
- memcpy(state->readRecordBuf,
- state->readBuf + RecPtr % XLOG_BLCKSZ, len);
- buffer = state->readRecordBuf + len;
- gotlen = len;
-
- do
- {
- int rest_len = total_len - gotlen;
-
- /* Calculate pointer to beginning of next page */
- targetPagePtr += XLOG_BLCKSZ;
-
- /* Wait for the next page to become available */
- while (XLogNeedData(state, targetPagePtr,
- Min(rest_len, XLOG_BLCKSZ),
- false))
+ if (state->ReadRecPtr != InvalidXLogRecPtr)
{
- if (!state->routine.page_read(state, state->readPagePtr,
- state->readLen,
- state->ReadRecPtr,
- state->readBuf))
- break;
+ /* read the record after the one we just read */
+
+ /*
+ * EndRecPtr is pointing to end+1 of the previous WAL record.
+ * If we're at a page boundary, no more records can fit on the
+ * current page. We must skip over the page header, but we
+ * can't do that until we've read in the page, since the header
+ * size is variable.
+ */
+ state->PrevRecPtr = state->ReadRecPtr;
+ state->ReadRecPtr = state->EndRecPtr;
+ }
+ else
+ {
+ /*
+ * Caller supplied a position to start at.
+ *
+ * In this case, EndRecPtr should already be pointing to a
+ * valid record starting position.
+ */
+ Assert(XRecOffIsValid(state->EndRecPtr));
+ state->ReadRecPtr = state->EndRecPtr;
+
+ /*
+ * We cannot verify the previous-record pointer when we're
+ * seeking to a particular record. Reset PrevRecPtr so that we
+ * won't try doing that.
+ */
+ state->PrevRecPtr = InvalidXLogRecPtr;
+ state->EndRecPtr = InvalidXLogRecPtr; /* to be tidy */
}
+ state->record_verified = false;
+ state->readRecordState = XLREAD_TOT_LEN;
+ /* fall through */
+
+ case XLREAD_TOT_LEN:
+ {
+ uint32 total_len;
+ uint32 pageHeaderSize;
+ XLogRecPtr targetPagePtr;
+ uint32 targetRecOff;
+ XLogPageHeader pageHeader;
+
+ targetPagePtr =
+ state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
+ targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+
+ /*
+ * Check if we have enough data. For the first record in the page,
+ * the requesting length doesn't contain page header.
+ */
+ if (XLogNeedData(state, targetPagePtr,
+ Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+ targetRecOff != 0))
+ return XLREAD_NEED_DATA;
+
+ /* error out if caller supplied bogus page */
if (!state->page_verified)
goto err;
- Assert(SizeOfXLogShortPHD <= state->readLen);
+ /* examine page header now. */
+ pageHeaderSize =
+ XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+ if (targetRecOff == 0)
+ {
+ /* At page start, so skip over page header. */
+ state->ReadRecPtr += pageHeaderSize;
+ targetRecOff = pageHeaderSize;
+ }
+ else if (targetRecOff < pageHeaderSize)
+ {
+ report_invalid_record(state, "invalid record offset at %X/%X",
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr);
+ goto err;
+ }
- /* Check that the continuation on next page looks valid */
pageHeader = (XLogPageHeader) state->readBuf;
- if (!(pageHeader->xlp_info & XLP_FIRST_IS_CONTRECORD))
+ if ((pageHeader->xlp_info & XLP_FIRST_IS_CONTRECORD) &&
+ targetRecOff == pageHeaderSize)
{
- report_invalid_record(state,
- "there is no contrecord flag at %X/%X",
- (uint32) (RecPtr >> 32), (uint32) RecPtr);
+ report_invalid_record(state, "contrecord is requested by %X/%X",
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr);
goto err;
}
- /*
- * Cross-check that xlp_rem_len agrees with how much of the record
- * we expect there to be left.
- */
- if (pageHeader->xlp_rem_len == 0 ||
- total_len != (pageHeader->xlp_rem_len + gotlen))
- {
- report_invalid_record(state,
- "invalid contrecord length %u at %X/%X",
- pageHeader->xlp_rem_len,
- (uint32) (RecPtr >> 32), (uint32) RecPtr);
- goto err;
- }
-
- /* Append the continuation from this page to the buffer */
- pageHeaderSize = XLogPageHeaderSize(pageHeader);
-
+ /* XLogNeedData has verified the page header */
Assert(pageHeaderSize <= state->readLen);
- contdata = (char *) state->readBuf + pageHeaderSize;
- len = XLOG_BLCKSZ - pageHeaderSize;
- if (pageHeader->xlp_rem_len < len)
- len = pageHeader->xlp_rem_len;
+ /*
+ * Read the record length.
+ *
+ * NB: Even though we use an XLogRecord pointer here, the whole
+ * record header might not fit on this page. xl_tot_len is the first
+ * field of the struct, so it must be on this page (the records are
+ * MAXALIGNed), but we cannot access any other fields until we've
+ * verified that we got the whole header.
+ */
+ prec = (XLogRecord *) (state->readBuf +
+ state->ReadRecPtr % XLOG_BLCKSZ);
+ total_len = prec->xl_tot_len;
- Assert (pageHeaderSize + len <= state->readLen);
- memcpy(buffer, (char *) contdata, len);
- buffer += len;
- gotlen += len;
+ /*
+ * If the whole record header is on this page, validate it
+ * immediately. Otherwise do just a basic sanity check on
+ * xl_tot_len, and validate the rest of the header after reading it
+ * from the next page. The xl_tot_len check is necessary here to
+ * ensure that we enter the XLREAD_CONTINUATION state below;
+ * otherwise we might fail to apply ValidXLogRecordHeader at all.
+ */
+ if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
+ {
+ if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
+ state->PrevRecPtr, prec))
+ goto err;
- /* If we just reassembled the record header, validate it. */
- if (!gotheader)
+ state->record_verified = true;
+ }
+ else
{
- record = (XLogRecord *) state->readRecordBuf;
- if (!ValidXLogRecordHeader(state, RecPtr, state->ReadRecPtr,
- record, randAccess))
+ /* XXX: more validation should be done here */
+ if (total_len < SizeOfXLogRecord)
+ {
+ report_invalid_record(state,
+ "invalid record length at %X/%X: wanted %u, got %u",
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr,
+ (uint32) SizeOfXLogRecord, total_len);
goto err;
- gotheader = true;
+ }
}
- } while (gotlen < total_len);
-
- Assert(gotheader);
- record = (XLogRecord *) state->readRecordBuf;
- if (!ValidXLogRecord(state, record, RecPtr))
- goto err;
+ /*
+ * Wait for the rest of the record, or the part of the record that
+ * fit on the first page if crossed a page boundary, to become
+ * available.
+ */
+ state->recordGotLen = 0;
+ state->recordRemainLen = total_len;
+ state->readRecordState = XLREAD_FIRST_FRAGMENT;
+ }
+ /* fall through */
- pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
- state->ReadRecPtr = RecPtr;
- state->EndRecPtr = targetPagePtr + pageHeaderSize
- + MAXALIGN(pageHeader->xlp_rem_len);
- }
- else
- {
- /* Wait for the record data to become available */
- while (XLogNeedData(state, targetPagePtr,
- Min(targetRecOff + total_len, XLOG_BLCKSZ), true))
+ case XLREAD_FIRST_FRAGMENT:
{
- if (!state->routine.page_read(state, state->readPagePtr,
- state->readLen,
- state->ReadRecPtr, state->readBuf))
+ uint32 total_len = state->recordRemainLen;
+ uint32 request_len;
+ uint32 record_len;
+ XLogRecPtr targetPagePtr;
+ uint32 targetRecOff;
+
+ /*
+ * Wait for the rest of the record on the first page to become
+ * available
+ */
+ targetPagePtr =
+ state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
+ targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+
+ request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ);
+ record_len = request_len - targetRecOff;
+
+ /* ReadRecPtr contains page header */
+ Assert (targetRecOff != 0);
+ if (XLogNeedData(state, targetPagePtr, request_len, true))
+ return XLREAD_NEED_DATA;
+
+ /* error out if caller supplied bogus page */
+ if (!state->page_verified)
+ goto err;
+
+ prec = (XLogRecord *) (state->readBuf + targetRecOff);
+
+ /* validate record header if not yet */
+ if (!state->record_verified && record_len >= SizeOfXLogRecord)
+ {
+ if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
+ state->PrevRecPtr, prec))
+ goto err;
+
+ state->record_verified = true;
+ }
+
+
+ if (total_len == record_len)
+ {
+ /* Record does not cross a page boundary */
+ Assert(state->record_verified);
+
+ if (!ValidXLogRecord(state, prec, state->ReadRecPtr))
+ goto err;
+
+ state->record_verified = true; /* to be tidy */
+
+ /* We already checked the header earlier */
+ state->EndRecPtr = state->ReadRecPtr + MAXALIGN(record_len);
+
+ *record = prec;
+ state->readRecordState = XLREAD_NEXT_RECORD;
break;
+ }
+
+ /*
+ * The record continues on the next page. Need to reassemble
+ * record
+ */
+ Assert(total_len > record_len);
+
+ /* Enlarge readRecordBuf as needed. */
+ if (total_len > state->readRecordBufSize &&
+ !allocate_recordbuf(state, total_len))
+ {
+ /* We treat this as a "bogus data" condition */
+ report_invalid_record(state,
+ "record length %u at %X/%X too long",
+ total_len,
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr);
+ goto err;
+ }
+
+ /* Copy the first fragment of the record from the first page. */
+ memcpy(state->readRecordBuf, state->readBuf + targetRecOff,
+ record_len);
+ state->recordGotLen += record_len;
+ state->recordRemainLen -= record_len;
+
+ /* Calculate pointer to beginning of next page */
+ state->recordContRecPtr = state->ReadRecPtr + record_len;
+ Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0);
+
+ state->readRecordState = XLREAD_CONTINUATION;
}
+ /* fall through */
+
+ case XLREAD_CONTINUATION:
+ {
+ XLogPageHeader pageHeader;
+ uint32 pageHeaderSize;
+ XLogRecPtr targetPagePtr;
+
+ /* we enter this state only if we haven't read the whole record. */
+ Assert (state->recordRemainLen > 0);
+
+ while(state->recordRemainLen > 0)
+ {
+ char *contdata;
+ uint32 request_len;
+ uint32 record_len;
+
+ /* Wait for the next page to become available */
+ targetPagePtr = state->recordContRecPtr;
+
+ /* this request contains page header */
+ Assert (targetPagePtr != 0);
+ if (XLogNeedData(state, targetPagePtr,
+ Min(state->recordRemainLen, XLOG_BLCKSZ),
+ false))
+ return XLREAD_NEED_DATA;
+
+ if (!state->page_verified)
+ goto err;
+
+ Assert(SizeOfXLogShortPHD <= state->readLen);
- if (!state->page_verified)
- goto err;
+ /* Check that the continuation on next page looks valid */
+ pageHeader = (XLogPageHeader) state->readBuf;
+ if (!(pageHeader->xlp_info & XLP_FIRST_IS_CONTRECORD))
+ {
+ report_invalid_record(
+ state,
+ "there is no contrecord flag at %X/%X reading %X/%X",
+ (uint32) (state->recordContRecPtr >> 32),
+ (uint32) state->recordContRecPtr,
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr);
+ goto err;
+ }
- /* Record does not cross a page boundary */
- if (!ValidXLogRecord(state, record, RecPtr))
- goto err;
+ /*
+ * Cross-check that xlp_rem_len agrees with how much of the
+ * record we expect there to be left.
+ */
+ if (pageHeader->xlp_rem_len == 0 ||
+ pageHeader->xlp_rem_len != state->recordRemainLen)
+ {
+ report_invalid_record(
+ state,
+ "invalid contrecord length %u at %X/%X reading %X/%X, expected %u",
+ pageHeader->xlp_rem_len,
+ (uint32) (state->recordContRecPtr >> 32),
+ (uint32) state->recordContRecPtr,
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr,
+ state->recordRemainLen);
+ goto err;
+ }
- state->EndRecPtr = RecPtr + MAXALIGN(total_len);
+ /* Append the continuation from this page to the buffer */
+ pageHeaderSize = XLogPageHeaderSize(pageHeader);
- state->ReadRecPtr = RecPtr;
+ /*
+ * XLogNeedData should have ensured that the whole page header
+ * was read
+ */
+ Assert(state->readLen >= pageHeaderSize);
+
+ contdata = (char *) state->readBuf + pageHeaderSize;
+ record_len = XLOG_BLCKSZ - pageHeaderSize;
+ if (pageHeader->xlp_rem_len < record_len)
+ record_len = pageHeader->xlp_rem_len;
+
+ request_len = record_len + pageHeaderSize;
+
+ /* XLogNeedData should have ensured all needed data was read */
+ Assert (state->readLen >= request_len);
+
+ memcpy(state->readRecordBuf + state->recordGotLen,
+ (char *) contdata, record_len);
+ state->recordGotLen += record_len;
+ state->recordRemainLen -= record_len;
+
+ /* If we just reassembled the record header, validate it. */
+ if (!state->record_verified)
+ {
+ Assert(state->recordGotLen >= SizeOfXLogRecord);
+ if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
+ state->PrevRecPtr,
+ (XLogRecord *) state->readRecordBuf))
+ goto err;
+
+ state->record_verified = true;
+ }
+
+ /* Calculate pointer to beginning of next page, and continue */
+ state->recordContRecPtr += XLOG_BLCKSZ;
+ }
+
+ /* targetPagePtr is pointing the last-read page here */
+ prec = (XLogRecord *) state->readRecordBuf;
+ if (!ValidXLogRecord(state, prec, state->ReadRecPtr))
+ goto err;
+
+ pageHeaderSize =
+ XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+ state->EndRecPtr = targetPagePtr + pageHeaderSize
+ + MAXALIGN(pageHeader->xlp_rem_len);
+
+ *record = prec;
+ state->readRecordState = XLREAD_NEXT_RECORD;
+ break;
+ }
}
/*
* Special processing if it's an XLOG SWITCH record
*/
- if (record->xl_rmid == RM_XLOG_ID &&
- (record->xl_info & ~XLR_INFO_MASK) == XLOG_SWITCH)
+ if ((*record)->xl_rmid == RM_XLOG_ID &&
+ ((*record)->xl_info & ~XLR_INFO_MASK) == XLOG_SWITCH)
{
/* Pretend it extends to end of segment */
state->EndRecPtr += state->segcxt.ws_segsize - 1;
state->EndRecPtr -= XLogSegmentOffset(state->EndRecPtr, state->segcxt.ws_segsize);
}
- if (DecodeXLogRecord(state, record, errormsg))
- return record;
- else
- return NULL;
+ Assert (!*record || state->readLen >= 0);
+ if (DecodeXLogRecord(state, *record, errormsg))
+ return XLREAD_SUCCESS;
+
+ *record = NULL;
+ return XLREAD_FAIL;
err:
/*
- * Invalidate the read state. We might read from a different source after
+ * Invalidate the read page. We might read from a different source after
* failure.
*/
XLogReaderInvalReadState(state);
@@ -572,7 +706,8 @@ err:
if (state->errormsg_buf[0] != '\0')
*errormsg = state->errormsg_buf;
- return NULL;
+ *record = NULL;
+ return XLREAD_FAIL;
}
/*
@@ -724,11 +859,12 @@ XLogReaderInvalReadState(XLogReaderState *state)
*
* This is just a convenience subroutine to avoid duplicated code in
* XLogReadRecord. It's not intended for use from anywhere else.
+ *
+ * If PrevRecPtr is valid, the xl_prev is is cross-checked with it.
*/
static bool
ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
- XLogRecPtr PrevRecPtr, XLogRecord *record,
- bool randAccess)
+ XLogRecPtr PrevRecPtr, XLogRecord *record)
{
if (record->xl_tot_len < SizeOfXLogRecord)
{
@@ -746,7 +882,7 @@ ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
(uint32) RecPtr);
return false;
}
- if (randAccess)
+ if (PrevRecPtr == InvalidXLogRecPtr)
{
/*
* We can't exactly verify the prev-link, but surely it should be less
@@ -978,11 +1114,14 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
* XLogReadRecord() will read the next valid record.
*/
XLogRecPtr
-XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
+XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
+ XLogFindNextRecordCB read_page, void *private)
{
XLogRecPtr tmpRecPtr;
XLogRecPtr found = InvalidXLogRecPtr;
XLogPageHeader header;
+ XLogRecord *record;
+ XLogReadRecordResult result;
char *errormsg;
Assert(!XLogRecPtrIsInvalid(RecPtr));
@@ -1015,9 +1154,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
while(XLogNeedData(state, targetPagePtr, targetRecOff,
targetRecOff != 0))
{
- if (!state->routine.page_read(state, state->readPagePtr,
- state->readLen,
- state->ReadRecPtr, state->readBuf))
+ if (!read_page(state, private))
break;
}
@@ -1069,8 +1206,16 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
* or we just jumped over the remaining data of a continuation.
*/
XLogBeginRead(state, tmpRecPtr);
- while (XLogReadRecord(state, &errormsg) != NULL)
+ while ((result = XLogReadRecord(state, &record, &errormsg)) !=
+ XLREAD_FAIL)
{
+ if (result == XLREAD_NEED_DATA)
+ {
+ if (!read_page(state, private))
+ goto err;
+ continue;
+ }
+
/* past the record we've found, break out */
if (RecPtr <= state->ReadRecPtr)
{
@@ -1090,9 +1235,9 @@ err:
#endif /* FRONTEND */
/*
- * Helper function to ease writing of XLogRoutine->page_read callbacks.
- * If this function is used, caller must supply a segment_open callback in
- * 'state', as that is used here.
+ * Helper function to ease writing of page_read callback.
+ * If this function is used, caller must supply a segment_open callback and
+ * segment_close callback as that is used here.
*
* Read 'count' bytes into 'buf', starting at location 'startptr', from WAL
* fetched from timeline 'tli'.
@@ -1105,6 +1250,7 @@ err:
*/
bool
WALRead(XLogReaderState *state,
+ WALSegmentOpenCB segopenfn, WALSegmentCloseCB segclosefn,
char *buf, XLogRecPtr startptr, Size count, TimeLineID tli,
WALReadError *errinfo)
{
@@ -1136,10 +1282,10 @@ WALRead(XLogReaderState *state,
XLogSegNo nextSegNo;
if (state->seg.ws_file >= 0)
- state->routine.segment_close(state);
+ segclosefn(state);
XLByteToSeg(recptr, nextSegNo, state->segcxt.ws_segsize);
- state->routine.segment_open(state, nextSegNo, &tli);
+ segopenfn(state, nextSegNo, &tli);
/* This shouldn't happen -- indicates a bug in segment_open */
Assert(state->seg.ws_file >= 0);
diff --git a/src/backend/access/transam/xlogutils.c b/src/backend/access/transam/xlogutils.c
index 1b10b6df20..303333571f 100644
--- a/src/backend/access/transam/xlogutils.c
+++ b/src/backend/access/transam/xlogutils.c
@@ -689,8 +689,7 @@ XLogTruncateRelation(RelFileNode rnode, ForkNumber forkNum,
void
XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wantLength)
{
- const XLogRecPtr lastReadPage = state->seg.ws_segno *
- state->segcxt.ws_segsize + state->readLen;
+ const XLogRecPtr lastReadPage = state->readPagePtr;
Assert(wantPage != InvalidXLogRecPtr && wantPage % XLOG_BLCKSZ == 0);
Assert(wantLength <= XLOG_BLCKSZ);
@@ -705,7 +704,7 @@ XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wa
* current TLI has since become historical.
*/
if (lastReadPage == wantPage &&
- state->readLen != 0 &&
+ state->page_verified &&
lastReadPage + state->readLen >= wantPage + Min(wantLength, XLOG_BLCKSZ - 1))
return;
@@ -792,6 +791,7 @@ wal_segment_open(XLogReaderState *state, XLogSegNo nextSegNo,
char path[MAXPGPATH];
XLogFilePath(path, tli, nextSegNo, state->segcxt.ws_segsize);
+ elog(LOG, "HOGE: %lu, %d => %s", nextSegNo, tli, path);
state->seg.ws_file = BasicOpenFile(path, O_RDONLY | PG_BINARY);
if (state->seg.ws_file >= 0)
return;
@@ -829,9 +829,11 @@ wal_segment_close(XLogReaderState *state)
* loop for now.
*/
bool
-read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *cur_page)
+read_local_xlog_page(XLogReaderState *state)
{
+ XLogRecPtr targetPagePtr = state->readPagePtr;
+ int reqLen = state->readLen;
+ char *cur_page = state->readBuf;
XLogRecPtr read_upto,
loc;
TimeLineID tli;
@@ -944,11 +946,12 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
* as 'count', read the whole page anyway. It's guaranteed to be
* zero-padded up to the page boundary if it's incomplete.
*/
- if (!WALRead(state, cur_page, targetPagePtr, XLOG_BLCKSZ, tli,
- &errinfo))
+ if (!WALRead(state, wal_segment_open, wal_segment_close,
+ cur_page, targetPagePtr, XLOG_BLCKSZ, tli, &errinfo))
WALReadRaiseError(&errinfo);
/* number of valid bytes in the buffer */
+ state->readPagePtr = targetPagePtr;
state->readLen = count;
return true;
}
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index 61902be3b0..a15b0b3355 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -120,7 +120,8 @@ StartupDecodingContext(List *output_plugin_options,
TransactionId xmin_horizon,
bool need_full_snapshot,
bool fast_forward,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress)
@@ -169,11 +170,12 @@ StartupDecodingContext(List *output_plugin_options,
ctx->slot = slot;
- ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, xl_routine, ctx);
+ ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, cleanup_cb);
if (!ctx->reader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
errmsg("out of memory")));
+ ctx->page_read = page_read;
ctx->reorder = ReorderBufferAllocate();
ctx->snapshot_builder =
@@ -231,7 +233,8 @@ CreateInitDecodingContext(char *plugin,
List *output_plugin_options,
bool need_full_snapshot,
XLogRecPtr restart_lsn,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress)
@@ -328,7 +331,7 @@ CreateInitDecodingContext(char *plugin,
ctx = StartupDecodingContext(NIL, restart_lsn, xmin_horizon,
need_full_snapshot, false,
- xl_routine, prepare_write, do_write,
+ page_read, cleanup_cb, prepare_write, do_write,
update_progress);
/* call output plugin initialization callback */
@@ -376,7 +379,8 @@ LogicalDecodingContext *
CreateDecodingContext(XLogRecPtr start_lsn,
List *output_plugin_options,
bool fast_forward,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress)
@@ -429,8 +433,8 @@ CreateDecodingContext(XLogRecPtr start_lsn,
ctx = StartupDecodingContext(output_plugin_options,
start_lsn, InvalidTransactionId, false,
- fast_forward, xl_routine, prepare_write,
- do_write, update_progress);
+ fast_forward, page_read, cleanup_cb,
+ prepare_write, do_write, update_progress);
/* call output plugin initialization callback */
old_context = MemoryContextSwitchTo(ctx->context);
@@ -483,7 +487,13 @@ DecodingContextFindStartpoint(LogicalDecodingContext *ctx)
char *err = NULL;
/* the read_page callback waits for new WAL */
- record = XLogReadRecord(ctx->reader, &err);
+ while (XLogReadRecord(ctx->reader, &record, &err) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!ctx->page_read(ctx->reader))
+ break;
+ }
+
if (err)
elog(ERROR, "%s", err);
if (!record)
diff --git a/src/backend/replication/logical/logicalfuncs.c b/src/backend/replication/logical/logicalfuncs.c
index b99c94e848..fcc81b358e 100644
--- a/src/backend/replication/logical/logicalfuncs.c
+++ b/src/backend/replication/logical/logicalfuncs.c
@@ -233,9 +233,8 @@ pg_logical_slot_get_changes_guts(FunctionCallInfo fcinfo, bool confirm, bool bin
ctx = CreateDecodingContext(InvalidXLogRecPtr,
options,
false,
- XL_ROUTINE(.page_read = read_local_xlog_page,
- .segment_open = wal_segment_open,
- .segment_close = wal_segment_close),
+ read_local_xlog_page,
+ wal_segment_close,
LogicalOutputPrepareWrite,
LogicalOutputWrite, NULL);
@@ -284,7 +283,13 @@ pg_logical_slot_get_changes_guts(FunctionCallInfo fcinfo, bool confirm, bool bin
XLogRecord *record;
char *errm = NULL;
- record = XLogReadRecord(ctx->reader, &errm);
+ while (XLogReadRecord(ctx->reader, &record, &errm) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!ctx->page_read(ctx->reader))
+ break;
+ }
+
if (errm)
elog(ERROR, "%s", errm);
diff --git a/src/backend/replication/slotfuncs.c b/src/backend/replication/slotfuncs.c
index 88033a79b2..8a678074d5 100644
--- a/src/backend/replication/slotfuncs.c
+++ b/src/backend/replication/slotfuncs.c
@@ -152,9 +152,8 @@ create_logical_replication_slot(char *name, char *plugin,
ctx = CreateInitDecodingContext(plugin, NIL,
false, /* just catalogs is OK */
restart_lsn,
- XL_ROUTINE(.page_read = read_local_xlog_page,
- .segment_open = wal_segment_open,
- .segment_close = wal_segment_close),
+ read_local_xlog_page,
+ wal_segment_close,
NULL, NULL, NULL);
/*
@@ -486,9 +485,8 @@ pg_logical_replication_slot_advance(XLogRecPtr moveto)
ctx = CreateDecodingContext(InvalidXLogRecPtr,
NIL,
true, /* fast_forward */
- XL_ROUTINE(.page_read = read_local_xlog_page,
- .segment_open = wal_segment_open,
- .segment_close = wal_segment_close),
+ read_local_xlog_page,
+ wal_segment_close,
NULL, NULL, NULL);
/*
@@ -513,7 +511,13 @@ pg_logical_replication_slot_advance(XLogRecPtr moveto)
* Read records. No changes are generated in fast_forward mode,
* but snapbuilder/slot statuses are updated properly.
*/
- record = XLogReadRecord(ctx->reader, &errm);
+ while (XLogReadRecord(ctx->reader, &record, &errm) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!ctx->page_read(ctx->reader))
+ break;
+ }
+
if (errm)
elog(ERROR, "%s", errm);
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index bf2711bddd..2fd24a9b14 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -580,10 +580,7 @@ StartReplication(StartReplicationCmd *cmd)
/* create xlogreader for physical replication */
xlogreader =
- XLogReaderAllocate(wal_segment_size, NULL,
- XL_ROUTINE(.segment_open = WalSndSegmentOpen,
- .segment_close = wal_segment_close),
- NULL);
+ XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
if (!xlogreader)
ereport(ERROR,
@@ -812,9 +809,11 @@ StartReplication(StartReplicationCmd *cmd)
* set every time WAL is flushed.
*/
static bool
-logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetRecPtr, char *cur_page)
+logical_read_xlog_page(XLogReaderState *state)
{
+ XLogRecPtr targetPagePtr = state->readPagePtr;
+ int reqLen = state->readLen;
+ char *cur_page = state->readBuf;
XLogRecPtr flushptr;
int count;
WALReadError errinfo;
@@ -842,7 +841,7 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
count = flushptr - targetPagePtr; /* part of the page available */
/* now actually read the data, we know it's there */
- if (!WALRead(state,
+ if (!WALRead(state, WalSndSegmentOpen, wal_segment_close,
cur_page,
targetPagePtr,
XLOG_BLCKSZ,
@@ -1014,9 +1013,8 @@ CreateReplicationSlot(CreateReplicationSlotCmd *cmd)
ctx = CreateInitDecodingContext(cmd->plugin, NIL, need_full_snapshot,
InvalidXLogRecPtr,
- XL_ROUTINE(.page_read = logical_read_xlog_page,
- .segment_open = WalSndSegmentOpen,
- .segment_close = wal_segment_close),
+ logical_read_xlog_page,
+ wal_segment_close,
WalSndPrepareWrite, WalSndWriteData,
WalSndUpdateProgress);
@@ -1179,9 +1177,8 @@ StartLogicalReplication(StartReplicationCmd *cmd)
*/
logical_decoding_ctx =
CreateDecodingContext(cmd->startpoint, cmd->options, false,
- XL_ROUTINE(.page_read = logical_read_xlog_page,
- .segment_open = WalSndSegmentOpen,
- .segment_close = wal_segment_close),
+ logical_read_xlog_page,
+ wal_segment_close,
WalSndPrepareWrite, WalSndWriteData,
WalSndUpdateProgress);
xlogreader = logical_decoding_ctx->reader;
@@ -2761,7 +2758,7 @@ XLogSendPhysical(void)
enlargeStringInfo(&output_message, nbytes);
retry:
- if (!WALRead(xlogreader,
+ if (!WALRead(xlogreader, WalSndSegmentOpen, wal_segment_close,
&output_message.data[output_message.len],
startptr,
nbytes,
@@ -2859,7 +2856,12 @@ XLogSendLogical(void)
*/
WalSndCaughtUp = false;
- record = XLogReadRecord(logical_decoding_ctx->reader, &errm);
+ while (XLogReadRecord(logical_decoding_ctx->reader, &record, &errm) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!logical_decoding_ctx->page_read(logical_decoding_ctx->reader))
+ break;
+ }
/* xlog record was invalid */
if (errm != NULL)
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index 0b7e73ae79..3dd6df4be6 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -40,15 +40,9 @@ static int xlogreadfd = -1;
static XLogSegNo xlogreadsegno = -1;
static char xlogfpath[MAXPGPATH];
-typedef struct XLogPageReadPrivate
-{
- const char *restoreCommand;
- int tliIndex;
-} XLogPageReadPrivate;
-
-static bool SimpleXLogPageRead(XLogReaderState *xlogreader,
- XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
+static bool SimpleXLogPageRead(XLogReaderState *xlogreader,
+ const char *datadir, int *tliIndex,
+ const char *restoreCommand);
/*
* Read WAL from the datadir/pg_wal, starting from 'startpoint' on timeline
@@ -62,20 +56,22 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
XLogRecord *record;
XLogReaderState *xlogreader;
char *errormsg;
- XLogPageReadPrivate private;
- private.tliIndex = tliIndex;
- private.restoreCommand = restoreCommand;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir,
- XL_ROUTINE(.page_read = &SimpleXLogPageRead),
- &private);
+ xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+
if (xlogreader == NULL)
pg_fatal("out of memory");
XLogBeginRead(xlogreader, startpoint);
do
{
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!SimpleXLogPageRead(xlogreader, datadir,
+ &tliIndex, restoreCommand))
+ break;
+ }
if (record == NULL)
{
@@ -113,19 +109,19 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex,
XLogRecord *record;
XLogReaderState *xlogreader;
char *errormsg;
- XLogPageReadPrivate private;
XLogRecPtr endptr;
- private.tliIndex = tliIndex;
- private.restoreCommand = restoreCommand;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir,
- XL_ROUTINE(.page_read = &SimpleXLogPageRead),
- &private);
+ xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
XLogBeginRead(xlogreader, ptr);
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!SimpleXLogPageRead(xlogreader, datadir, &tliIndex, restoreCommand))
+ break;
+ }
if (record == NULL)
{
if (errormsg)
@@ -160,7 +156,6 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
XLogRecPtr searchptr;
XLogReaderState *xlogreader;
char *errormsg;
- XLogPageReadPrivate private;
/*
* The given fork pointer points to the end of the last common record,
@@ -176,11 +171,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
forkptr += SizeOfXLogShortPHD;
}
- private.tliIndex = tliIndex;
- private.restoreCommand = restoreCommand;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir,
- XL_ROUTINE(.page_read = &SimpleXLogPageRead),
- &private);
+ xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
@@ -190,7 +181,13 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
uint8 info;
XLogBeginRead(xlogreader, searchptr);
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!SimpleXLogPageRead(xlogreader, datadir,
+ &tliIndex, restoreCommand))
+ break;
+ }
if (record == NULL)
{
@@ -237,10 +234,11 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
/* XLogReader callback function, to read a WAL page */
static bool
-SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *readBuf)
+SimpleXLogPageRead(XLogReaderState *xlogreader, const char *datadir,
+ int *tliIndex, const char *restoreCommand)
{
- XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
+ XLogRecPtr targetPagePtr = xlogreader->readPagePtr;
+ char *readBuf = xlogreader->readBuf;
uint32 targetPageOff;
XLogRecPtr targetSegEnd;
XLogSegNo targetSegNo;
@@ -273,14 +271,14 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
* be done both forward and backward, consider also switching timeline
* accordingly.
*/
- while (private->tliIndex < targetNentries - 1 &&
- targetHistory[private->tliIndex].end < targetSegEnd)
- private->tliIndex++;
- while (private->tliIndex > 0 &&
- targetHistory[private->tliIndex].begin >= targetSegEnd)
- private->tliIndex--;
+ while (*tliIndex < targetNentries - 1 &&
+ targetHistory[*tliIndex].end < targetSegEnd)
+ (*tliIndex)++;
+ while (*tliIndex > 0 &&
+ targetHistory[*tliIndex].begin >= targetSegEnd)
+ (*tliIndex)--;
- XLogFileName(xlogfname, targetHistory[private->tliIndex].tli,
+ XLogFileName(xlogfname, targetHistory[*tliIndex].tli,
xlogreadsegno, WalSegSz);
snprintf(xlogfpath, MAXPGPATH, "%s/" XLOGDIR "/%s",
@@ -293,7 +291,7 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
/*
* If we have no restore_command to execute, then exit.
*/
- if (private->restoreCommand == NULL)
+ if (restoreCommand == NULL)
{
pg_log_error("could not open file \"%s\": %m", xlogfpath);
xlogreader->readLen = -1;
@@ -307,7 +305,7 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
xlogreadfd = RestoreArchivedFile(xlogreader->segcxt.ws_dir,
xlogfname,
WalSegSz,
- private->restoreCommand);
+ restoreCommand);
if (xlogreadfd < 0)
{
@@ -349,7 +347,7 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
Assert(targetSegNo == xlogreadsegno);
- xlogreader->seg.ws_tli = targetHistory[private->tliIndex].tli;
+ xlogreader->seg.ws_tli = targetHistory[*tliIndex].tli;
xlogreader->readLen = XLOG_BLCKSZ;
return true;
}
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index ab0c28d9b7..02e71138cc 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -330,12 +330,17 @@ WALDumpCloseSegment(XLogReaderState *state)
state->seg.ws_file = -1;
}
-/* pg_waldump's XLogReaderRoutine->page_read callback */
+/*
+ * pg_waldump's WAL page rader, also used as page_read callback for
+ * XLogFindNextRecord
+ */
static bool
-WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetPtr, char *readBuff)
+WALDumpReadPage(XLogReaderState *state, void *priv)
{
- XLogDumpPrivate *private = state->private_data;
+ XLogRecPtr targetPagePtr = state->readPagePtr;
+ int reqLen = state->readLen;
+ char *readBuff = state->readBuf;
+ XLogDumpPrivate *private = (XLogDumpPrivate *) priv;
int count = XLOG_BLCKSZ;
WALReadError errinfo;
@@ -353,8 +358,8 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
}
}
- if (!WALRead(state, readBuff, targetPagePtr, count, private->timeline,
- &errinfo))
+ if (!WALRead(state, WALDumpOpenSegment, WALDumpCloseSegment,
+ readBuff, targetPagePtr, count, private->timeline, &errinfo))
{
WALOpenSegment *seg = &errinfo.wre_seg;
char fname[MAXPGPATH];
@@ -374,6 +379,7 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
(Size) errinfo.wre_req);
}
+ Assert(count >= state->readLen);
state->readLen = count;
return true;
}
@@ -1042,16 +1048,14 @@ main(int argc, char **argv)
/* we have everything we need, start reading */
xlogreader_state =
- XLogReaderAllocate(WalSegSz, waldir,
- XL_ROUTINE(.page_read = WALDumpReadPage,
- .segment_open = WALDumpOpenSegment,
- .segment_close = WALDumpCloseSegment),
- &private);
+ XLogReaderAllocate(WalSegSz, waldir, WALDumpCloseSegment);
+
if (!xlogreader_state)
fatal_error("out of memory");
/* first find a valid recptr to start from */
- first_record = XLogFindNextRecord(xlogreader_state, private.startptr);
+ first_record = XLogFindNextRecord(xlogreader_state, private.startptr,
+ &WALDumpReadPage, (void*) &private);
if (first_record == InvalidXLogRecPtr)
fatal_error("could not find a valid record after %X/%X",
@@ -1075,7 +1079,13 @@ main(int argc, char **argv)
for (;;)
{
/* try to read the next record */
- record = XLogReadRecord(xlogreader_state, &errormsg);
+ while (XLogReadRecord(xlogreader_state, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!WALDumpReadPage(xlogreader_state, (void *) &private))
+ break;
+ }
+
if (!record)
{
if (!config.follow || private.endptr_reached)
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index 6c8848e14f..6a7fe140cb 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -57,64 +57,15 @@ typedef struct WALSegmentContext
typedef struct XLogReaderState XLogReaderState;
-/* Function type definition for the read_page callback */
-typedef bool (*XLogPageReadCB) (XLogReaderState *xlogreader,
- XLogRecPtr targetPagePtr,
- int reqLen,
- XLogRecPtr targetRecPtr,
- char *readBuf);
+/* Function type definition for the segment cleanup callback */
+typedef void (*WALSegmentCleanupCB) (XLogReaderState *xlogreader);
+
+/* Function type definition for the open/close callbacks for WALRead() */
typedef void (*WALSegmentOpenCB) (XLogReaderState *xlogreader,
XLogSegNo nextSegNo,
TimeLineID *tli_p);
typedef void (*WALSegmentCloseCB) (XLogReaderState *xlogreader);
-typedef struct XLogReaderRoutine
-{
- /*
- * Data input callback
- *
- * This callback shall read at least reqLen valid bytes of the xlog page
- * starting at targetPagePtr, and store them in readBuf. The callback
- * shall return the number of bytes read (never more than XLOG_BLCKSZ), or
- * -1 on failure. The callback shall sleep, if necessary, to wait for the
- * requested bytes to become available. The callback will not be invoked
- * again for the same page unless more than the returned number of bytes
- * are needed.
- *
- * targetRecPtr is the position of the WAL record we're reading. Usually
- * it is equal to targetPagePtr + reqLen, but sometimes xlogreader needs
- * to read and verify the page or segment header, before it reads the
- * actual WAL record it's interested in. In that case, targetRecPtr can
- * be used to determine which timeline to read the page from.
- *
- * The callback shall set ->seg.ws_tli to the TLI of the file the page was
- * read from.
- */
- XLogPageReadCB page_read;
-
- /*
- * Callback to open the specified WAL segment for reading. ->seg.ws_file
- * shall be set to the file descriptor of the opened segment. In case of
- * failure, an error shall be raised by the callback and it shall not
- * return.
- *
- * "nextSegNo" is the number of the segment to be opened.
- *
- * "tli_p" is an input/output argument. WALRead() uses it to pass the
- * timeline in which the new segment should be found, but the callback can
- * use it to return the TLI that it actually opened.
- */
- WALSegmentOpenCB segment_open;
-
- /*
- * WAL segment close callback. ->seg.ws_file shall be set to a negative
- * number.
- */
- WALSegmentCloseCB segment_close;
-} XLogReaderRoutine;
-
-#define XL_ROUTINE(...) &(XLogReaderRoutine){__VA_ARGS__}
-
typedef struct
{
/* Is this block ref in use? */
@@ -144,12 +95,35 @@ typedef struct
uint16 data_bufsz;
} DecodedBkpBlock;
+/* Return code from XLogReadRecord */
+typedef enum XLogReadRecordResult
+{
+ XLREAD_SUCCESS, /* record is successfully read */
+ XLREAD_NEED_DATA, /* need more data. see XLogReadRecord. */
+ XLREAD_FAIL /* failed during reading a record */
+} XLogReadRecordResult;
+
+/*
+ * internal state of XLogReadRecord
+ *
+ * XLogReadState runs a state machine while reading a record. Theses states
+ * are not seen outside the function. Each state may repeat several times
+ * exiting requesting caller for new data. See the comment of XLogReadRecrod
+ * for details.
+ */
+typedef enum XLogReadRecordState {
+ XLREAD_NEXT_RECORD,
+ XLREAD_TOT_LEN,
+ XLREAD_FIRST_FRAGMENT,
+ XLREAD_CONTINUATION
+} XLogReadRecordState;
+
struct XLogReaderState
{
/*
* Operational callbacks
*/
- XLogReaderRoutine routine;
+ WALSegmentCleanupCB cleanup_cb;
/* ----------------------------------------
* Public parameters
@@ -162,18 +136,14 @@ struct XLogReaderState
*/
uint64 system_identifier;
- /*
- * Opaque data for callbacks to use. Not used by XLogReader.
- */
- void *private_data;
-
/*
* Start and end point of last record read. EndRecPtr is also used as the
* position to read next. Calling XLogBeginRead() sets EndRecPtr to the
* starting position and ReadRecPtr to invalid.
*/
- XLogRecPtr ReadRecPtr; /* start of last record read */
+ XLogRecPtr ReadRecPtr; /* start of last record read or being read */
XLogRecPtr EndRecPtr; /* end+1 of last record read */
+ XLogRecPtr PrevRecPtr; /* start of previous record read */
/* ----------------------------------------
* Communication with page reader
@@ -187,7 +157,9 @@ struct XLogReaderState
* the request, or -1 on error */
TimeLineID readPageTLI; /* TLI for data currently in readBuf */
char *readBuf; /* buffer to store data */
- bool page_verified; /* is the page on the buffer verified? */
+ bool page_verified; /* is the page header on the buffer verified? */
+ bool record_verified;/* is the current record header verified? */
+
/* ----------------------------------------
@@ -227,8 +199,6 @@ struct XLogReaderState
XLogRecPtr latestPagePtr;
TimeLineID latestPageTLI;
- /* beginning of the WAL record being read. */
- XLogRecPtr currRecPtr;
/* timeline to read it from, 0 if a lookup is required */
TimeLineID currTLI;
@@ -255,6 +225,15 @@ struct XLogReaderState
char *readRecordBuf;
uint32 readRecordBufSize;
+ /*
+ * XLogReadRecord() state
+ */
+ XLogReadRecordState readRecordState;/* state machine state */
+ int recordGotLen; /* amount of current record that has
+ * already been read */
+ int recordRemainLen; /* length of current record that remains */
+ XLogRecPtr recordContRecPtr; /* where the current record continues */
+
/* Buffer to hold error message */
char *errormsg_buf;
};
@@ -262,9 +241,7 @@ struct XLogReaderState
/* Get a new XLogReader */
extern XLogReaderState *XLogReaderAllocate(int wal_segment_size,
const char *waldir,
- XLogReaderRoutine *routine,
- void *private_data);
-extern XLogReaderRoutine *LocalXLogReaderRoutine(void);
+ WALSegmentCleanupCB cleanup_cb);
/* Free an XLogReader */
extern void XLogReaderFree(XLogReaderState *state);
@@ -272,12 +249,17 @@ extern void XLogReaderFree(XLogReaderState *state);
/* Position the XLogReader to given record */
extern void XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr);
#ifdef FRONTEND
-extern XLogRecPtr XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr);
+/* Function type definition for the read_page callback */
+typedef bool (*XLogFindNextRecordCB) (XLogReaderState *xlogreader,
+ void *private);
+extern XLogRecPtr XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
+ XLogFindNextRecordCB read_page, void *private);
#endif /* FRONTEND */
/* Read the next XLog record. Returns NULL on end-of-WAL or failure */
-extern struct XLogRecord *XLogReadRecord(XLogReaderState *state,
- char **errormsg);
+extern XLogReadRecordResult XLogReadRecord(XLogReaderState *state,
+ XLogRecord **record,
+ char **errormsg);
/* Validate a page */
extern bool XLogReaderValidatePageHeader(XLogReaderState *state,
@@ -297,6 +279,7 @@ typedef struct WALReadError
} WALReadError;
extern bool WALRead(XLogReaderState *state,
+ WALSegmentOpenCB segopenfn, WALSegmentCloseCB sgclosefn,
char *buf, XLogRecPtr startptr, Size count,
TimeLineID tli, WALReadError *errinfo);
diff --git a/src/include/access/xlogutils.h b/src/include/access/xlogutils.h
index c287edf206..482e4ced69 100644
--- a/src/include/access/xlogutils.h
+++ b/src/include/access/xlogutils.h
@@ -47,9 +47,7 @@ extern Buffer XLogReadBufferExtended(RelFileNode rnode, ForkNumber forknum,
extern Relation CreateFakeRelcacheEntry(RelFileNode rnode);
extern void FreeFakeRelcacheEntry(Relation fakerel);
-extern bool read_local_xlog_page(XLogReaderState *state,
- XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetRecPtr, char *cur_page);
+extern bool read_local_xlog_page(XLogReaderState *state);
extern void wal_segment_open(XLogReaderState *state,
XLogSegNo nextSegNo,
TimeLineID *tli_p);
diff --git a/src/include/replication/logical.h b/src/include/replication/logical.h
index c2f2475e5d..a743db49cf 100644
--- a/src/include/replication/logical.h
+++ b/src/include/replication/logical.h
@@ -29,6 +29,10 @@ typedef void (*LogicalOutputPluginWriterUpdateProgress) (struct LogicalDecodingC
TransactionId xid
);
+typedef struct LogicalDecodingContext LogicalDecodingContext;
+
+typedef bool (*LogicalDecodingXLogPageReadCB)(XLogReaderState *ctx);
+
typedef struct LogicalDecodingContext
{
/* memory context this is all allocated in */
@@ -39,6 +43,7 @@ typedef struct LogicalDecodingContext
/* infrastructure pieces for decoding */
XLogReaderState *reader;
+ LogicalDecodingXLogPageReadCB page_read;
struct ReorderBuffer *reorder;
struct SnapBuild *snapshot_builder;
@@ -95,14 +100,16 @@ extern LogicalDecodingContext *CreateInitDecodingContext(char *plugin,
List *output_plugin_options,
bool need_full_snapshot,
XLogRecPtr restart_lsn,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress);
extern LogicalDecodingContext *CreateDecodingContext(XLogRecPtr start_lsn,
List *output_plugin_options,
bool fast_forward,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress);
--
2.18.4
----Next_Part(Thu_Jul__2_13_53_30_2020_072)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
filename="v15-0003-Remove-globals-readOff-readLen-and-readSegNo.patch"
^ permalink raw reply [nested|flat] 70+ messages in thread
* [PATCH v15 2/4] Move page-reader out of XLogReadRecord
@ 2019-09-10 03:58 Kyotaro Horiguchi <[email protected]>
0 siblings, 0 replies; 70+ messages in thread
From: Kyotaro Horiguchi @ 2019-09-10 03:58 UTC (permalink / raw)
This is the second step of removing callbacks from WAL record reader.
Since it is essential to take in additional data while reading a
record, the function have to ask caller for new data while keeping
working state. Thus the function is turned into a state machine.
---
src/backend/access/transam/twophase.c | 15 +-
src/backend/access/transam/xlog.c | 58 +-
src/backend/access/transam/xlogreader.c | 700 +++++++++++-------
src/backend/access/transam/xlogutils.c | 17 +-
src/backend/replication/logical/logical.c | 26 +-
.../replication/logical/logicalfuncs.c | 13 +-
src/backend/replication/slotfuncs.c | 18 +-
src/backend/replication/walsender.c | 32 +-
src/bin/pg_rewind/parsexlog.c | 84 +--
src/bin/pg_waldump/pg_waldump.c | 36 +-
src/include/access/xlogreader.h | 121 ++-
src/include/access/xlogutils.h | 4 +-
src/include/replication/logical.h | 11 +-
13 files changed, 648 insertions(+), 487 deletions(-)
diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index 9b2e59bf0e..b0d60a0d0f 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -1330,11 +1330,8 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
XLogReaderState *xlogreader;
char *errormsg;
- xlogreader = XLogReaderAllocate(wal_segment_size, NULL,
- XL_ROUTINE(.page_read = &read_local_xlog_page,
- .segment_open = &wal_segment_open,
- .segment_close = &wal_segment_close),
- NULL);
+ xlogreader = XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+
if (!xlogreader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
@@ -1342,7 +1339,13 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
errdetail("Failed while allocating a WAL reading processor.")));
XLogBeginRead(xlogreader, lsn);
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!read_local_xlog_page(xlogreader))
+ break;
+ }
+
if (record == NULL)
ereport(ERROR,
(errcode_for_file_access(),
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index e570e56a24..f9b0108602 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -831,13 +831,6 @@ static XLogSource currentSource = XLOG_FROM_ANY;
static bool lastSourceFailed = false;
static bool pendingWalRcvRestart = false;
-typedef struct XLogPageReadPrivate
-{
- int emode;
- bool fetching_ckpt; /* are we fetching a checkpoint record? */
- bool randAccess;
-} XLogPageReadPrivate;
-
/*
* These variables track when we last obtained some WAL data to process,
* and where we got it from. (XLogReceiptSource is initially the same as
@@ -912,8 +905,8 @@ static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath,
static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli,
XLogSource source, bool notfoundOk);
static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, XLogSource source);
-static bool XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
+static bool XLogPageRead(XLogReaderState *xlogreader,
+ bool fetching_ckpt, int emode, bool randAccess);
static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
bool fetching_ckpt, XLogRecPtr tliRecPtr);
static int emode_for_corrupt_record(int emode, XLogRecPtr RecPtr);
@@ -1225,8 +1218,7 @@ XLogInsertRecord(XLogRecData *rdata,
appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len);
if (!debug_reader)
- debug_reader = XLogReaderAllocate(wal_segment_size, NULL,
- XL_ROUTINE(), NULL);
+ debug_reader = XLogReaderAllocate(wal_segment_size, NULL NULL);
if (!debug_reader)
{
@@ -4326,15 +4318,10 @@ CleanupBackupHistory(void)
* record is available.
*/
static XLogRecord *
-ReadRecord(XLogReaderState *xlogreader, int emode,
- bool fetching_ckpt)
+ReadRecord(XLogReaderState *xlogreader, int emode, bool fetching_ckpt)
{
XLogRecord *record;
- XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
-
- private->fetching_ckpt = fetching_ckpt;
- private->emode = emode;
- private->randAccess = (xlogreader->ReadRecPtr == InvalidXLogRecPtr);
+ bool randAccess = (xlogreader->ReadRecPtr == InvalidXLogRecPtr);
/* This is the first attempt to read this page. */
lastSourceFailed = false;
@@ -4342,8 +4329,16 @@ ReadRecord(XLogReaderState *xlogreader, int emode,
for (;;)
{
char *errormsg;
+ XLogReadRecordResult result;
+
+ while ((result = XLogReadRecord(xlogreader, &record, &errormsg))
+ == XLREAD_NEED_DATA)
+ {
+ if (!XLogPageRead(xlogreader, fetching_ckpt, emode, randAccess))
+ break;
+
+ }
- record = XLogReadRecord(xlogreader, &errormsg);
ReadRecPtr = xlogreader->ReadRecPtr;
EndRecPtr = xlogreader->EndRecPtr;
if (record == NULL)
@@ -6320,7 +6315,6 @@ StartupXLOG(void)
bool backupFromStandby = false;
DBState dbstate_at_startup;
XLogReaderState *xlogreader;
- XLogPageReadPrivate private;
bool fast_promoted = false;
struct stat st;
@@ -6480,13 +6474,9 @@ StartupXLOG(void)
OwnLatch(&XLogCtl->recoveryWakeupLatch);
/* Set up XLOG reader facility */
- MemSet(&private, 0, sizeof(XLogPageReadPrivate));
xlogreader =
- XLogReaderAllocate(wal_segment_size, NULL,
- XL_ROUTINE(.page_read = &XLogPageRead,
- .segment_open = NULL,
- .segment_close = wal_segment_close),
- &private);
+ XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+
if (!xlogreader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
@@ -11860,12 +11850,13 @@ CancelBackup(void)
* sleep and retry.
*/
static bool
-XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetRecPtr, char *readBuf)
+XLogPageRead(XLogReaderState *xlogreader,
+ bool fetching_ckpt, int emode, bool randAccess)
{
- XLogPageReadPrivate *private =
- (XLogPageReadPrivate *) xlogreader->private_data;
- int emode = private->emode;
+ char *readBuf = xlogreader->readBuf;
+ XLogRecPtr targetPagePtr = xlogreader->readPagePtr;
+ int reqLen = xlogreader->readLen;
+ XLogRecPtr targetRecPtr = xlogreader->ReadRecPtr;
uint32 targetPageOff;
XLogSegNo targetSegNo PG_USED_FOR_ASSERTS_ONLY;
int r;
@@ -11908,8 +11899,8 @@ retry:
flushedUpto < targetPagePtr + reqLen))
{
if (!WaitForWALToBecomeAvailable(targetPagePtr + reqLen,
- private->randAccess,
- private->fetching_ckpt,
+ randAccess,
+ fetching_ckpt,
targetRecPtr))
{
if (readFile >= 0)
@@ -12014,6 +12005,7 @@ retry:
goto next_record_is_invalid;
}
+ Assert(xlogreader->readPagePtr == targetPagePtr);
xlogreader->readLen = readLen;
return true;
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index 3d599325ee..9281b57379 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -40,7 +40,7 @@ static bool XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr,
int reqLen, bool header_inclusive);
static void XLogReaderInvalReadState(XLogReaderState *state);
static bool ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
- XLogRecPtr PrevRecPtr, XLogRecord *record, bool randAccess);
+ XLogRecPtr PrevRecPtr, XLogRecord *record);
static bool ValidXLogRecord(XLogReaderState *state, XLogRecord *record,
XLogRecPtr recptr);
static void ResetDecoder(XLogReaderState *state);
@@ -73,7 +73,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...)
*/
XLogReaderState *
XLogReaderAllocate(int wal_segment_size, const char *waldir,
- XLogReaderRoutine *routine, void *private_data)
+ WALSegmentCleanupCB cleanup_cb)
{
XLogReaderState *state;
@@ -84,7 +84,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
return NULL;
/* initialize caller-provided support functions */
- state->routine = *routine;
+ state->cleanup_cb = cleanup_cb;
state->max_block_id = -1;
@@ -107,8 +107,6 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
WALOpenSegmentInit(&state->seg, &state->segcxt, wal_segment_size,
waldir);
- /* system_identifier initialized to zeroes above */
- state->private_data = private_data;
/* ReadRecPtr, EndRecPtr and readLen initialized to zeroes above */
state->errormsg_buf = palloc_extended(MAX_ERRORMSG_LEN + 1,
MCXT_ALLOC_NO_OOM);
@@ -140,8 +138,8 @@ XLogReaderFree(XLogReaderState *state)
{
int block_id;
- if (state->seg.ws_file != -1)
- state->routine.segment_close(state);
+ if (state->seg.ws_file >= 0)
+ state->cleanup_cb(state);
for (block_id = 0; block_id <= XLR_MAX_BLOCK_ID; block_id++)
{
@@ -246,6 +244,7 @@ XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
/* Begin at the passed-in record pointer. */
state->EndRecPtr = RecPtr;
state->ReadRecPtr = InvalidXLogRecPtr;
+ state->readRecordState = XLREAD_NEXT_RECORD;
}
/*
@@ -254,317 +253,452 @@ XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
* XLogBeginRead() or XLogFindNextRecord() must be called before the first call
* to XLogReadRecord().
*
- * If the page_read callback fails to read the requested data, NULL is
- * returned. The callback is expected to have reported the error; errormsg
- * is set to NULL.
+ * This function may return XLREAD_NEED_DATA several times before returning a
+ * result record. The caller shall read in some new data then call this
+ * function again with the same parameters.
*
- * If the reading fails for some other reason, NULL is also returned, and
- * *errormsg is set to a string with details of the failure.
+ * When a record is successfully read, returns XLREAD_SUCCESS with result
+ * record being stored in *record. Otherwise *record is NULL.
*
- * The returned pointer (or *errormsg) points to an internal buffer that's
- * valid until the next call to XLogReadRecord.
+ * Returns XLREAD_NEED_DATA if more data is needed to finish reading the
+ * current record. In that case, state->readPagePtr and state->readLen inform
+ * the desired position and minimum length of data needed. The caller shall
+ * read in the requested data and set state->readBuf to point to a buffer
+ * containing it. The caller must also set state->readPageTLI and
+ * state->readLen to indicate the timeline that it was read from, and the
+ * length of data that is now available (which must be >= given readLen),
+ * respectively.
+ *
+ * If invalid data is encountered, returns XLREAD_FAIL with *record being set to
+ * NULL. *errormsg is set to a string with details of the failure.
+ * The returned pointer (or *errormsg) points to an internal buffer that's valid
+ * until the next call to XLogReadRecord.
+ *
+ *
+ * This function runs a state machine consists of the following states.
+ *
+ * XLREAD_NEXT_RECORD :
+ * The initial state, if called with valid RecPtr, try to read a record at
+ * that position. If invalid RecPtr is given try to read a record just after
+ * the last one previously read.
+ * This state ens after setting ReadRecPtr. Then goes to XLREAD_TOT_LEN.
+ *
+ * XLREAD_TOT_LEN:
+ * Examining record header. Ends after reading record total
+ * length. recordRemainLen and recordGotLen are initialized.
+ *
+ * XLREAD_FIRST_FRAGMENT:
+ * Reading the first fragment. Ends with finishing reading a single
+ * record. Goes to XLREAD_NEXT_RECORD if that's all or
+ * XLREAD_CONTINUATION if we have continuation.
+
+ * XLREAD_CONTINUATION:
+ * Reading continuation of record. Ends with finishing the whole record then
+ * goes to XLREAD_NEXT_RECORD. During this state, recordRemainLen indicates
+ * how much is left and readRecordBuf holds the partially assert
+ * record.recordContRecPtr points to the beginning of the next page where to
+ * continue.
+ *
+ * If wrong data found in any state, the state machine stays at the current
+ * state. This behavior allows to continue reading a reacord switching among
+ * different souces, while streaming replication.
*/
-XLogRecord *
-XLogReadRecord(XLogReaderState *state, char **errormsg)
+XLogReadRecordResult
+XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
{
- XLogRecPtr RecPtr;
- XLogRecord *record;
- XLogRecPtr targetPagePtr;
- bool randAccess;
- uint32 len,
- total_len;
- uint32 targetRecOff;
- uint32 pageHeaderSize;
- bool gotheader;
+ XLogRecord *prec;
- /*
- * randAccess indicates whether to verify the previous-record pointer of
- * the record we're reading. We only do this if we're reading
- * sequentially, which is what we initially assume.
- */
- randAccess = false;
+ *record = NULL;
/* reset error state */
*errormsg = NULL;
state->errormsg_buf[0] = '\0';
- ResetDecoder(state);
-
- RecPtr = state->EndRecPtr;
-
- if (state->ReadRecPtr != InvalidXLogRecPtr)
- {
- /* read the record after the one we just read */
-
- /*
- * EndRecPtr is pointing to end+1 of the previous WAL record. If
- * we're at a page boundary, no more records can fit on the current
- * page. We must skip over the page header, but we can't do that until
- * we've read in the page, since the header size is variable.
- */
- }
- else
- {
- /*
- * Caller supplied a position to start at.
- *
- * In this case, EndRecPtr should already be pointing to a valid
- * record starting position.
- */
- Assert(XRecOffIsValid(RecPtr));
- randAccess = true;
- }
-
- state->currRecPtr = RecPtr;
-
- targetPagePtr = RecPtr - (RecPtr % XLOG_BLCKSZ);
- targetRecOff = RecPtr % XLOG_BLCKSZ;
-
- /*
- * Read the page containing the record into state->readBuf. Request enough
- * byte to cover the whole record header, or at least the part of it that
- * fits on the same page.
- */
- while (XLogNeedData(state, targetPagePtr,
- Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
- targetRecOff != 0))
- {
- if (!state->routine.page_read(state, state->readPagePtr, state->readLen,
- RecPtr, state->readBuf))
- break;
- }
-
- if (!state->page_verified)
- goto err;
-
- /*
- * We have at least the page header, so we can examine it now.
- */
- pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
- if (targetRecOff == 0)
- {
- /*
- * At page start, so skip over page header.
- */
- RecPtr += pageHeaderSize;
- targetRecOff = pageHeaderSize;
- }
- else if (targetRecOff < pageHeaderSize)
- {
- report_invalid_record(state, "invalid record offset at %X/%X",
- (uint32) (RecPtr >> 32), (uint32) RecPtr);
- goto err;
- }
-
- if ((((XLogPageHeader) state->readBuf)->xlp_info & XLP_FIRST_IS_CONTRECORD) &&
- targetRecOff == pageHeaderSize)
- {
- report_invalid_record(state, "contrecord is requested by %X/%X",
- (uint32) (RecPtr >> 32), (uint32) RecPtr);
- goto err;
- }
-
- /* XLogNeedData has verified the page header */
- Assert(pageHeaderSize <= state->readLen);
-
- /*
- * Read the record length.
- *
- * NB: Even though we use an XLogRecord pointer here, the whole record
- * header might not fit on this page. xl_tot_len is the first field of the
- * struct, so it must be on this page (the records are MAXALIGNed), but we
- * cannot access any other fields until we've verified that we got the
- * whole header.
- */
- record = (XLogRecord *) (state->readBuf + RecPtr % XLOG_BLCKSZ);
- total_len = record->xl_tot_len;
-
- /*
- * If the whole record header is on this page, validate it immediately.
- * Otherwise do just a basic sanity check on xl_tot_len, and validate the
- * rest of the header after reading it from the next page. The xl_tot_len
- * check is necessary here to ensure that we enter the "Need to reassemble
- * record" code path below; otherwise we might fail to apply
- * ValidXLogRecordHeader at all.
- */
- if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
- {
- if (!ValidXLogRecordHeader(state, RecPtr, state->ReadRecPtr, record,
- randAccess))
- goto err;
- gotheader = true;
- }
- else
- {
- /* XXX: more validation should be done here */
- if (total_len < SizeOfXLogRecord)
- {
- report_invalid_record(state,
- "invalid record length at %X/%X: wanted %u, got %u",
- (uint32) (RecPtr >> 32), (uint32) RecPtr,
- (uint32) SizeOfXLogRecord, total_len);
- goto err;
- }
- gotheader = false;
- }
-
- len = XLOG_BLCKSZ - RecPtr % XLOG_BLCKSZ;
- if (total_len > len)
+ switch (state->readRecordState)
{
- /* Need to reassemble record */
- char *contdata;
- XLogPageHeader pageHeader;
- char *buffer;
- uint32 gotlen;
-
- /*
- * Enlarge readRecordBuf as needed.
- */
- if (total_len > state->readRecordBufSize &&
- !allocate_recordbuf(state, total_len))
- {
- /* We treat this as a "bogus data" condition */
- report_invalid_record(state, "record length %u at %X/%X too long",
- total_len,
- (uint32) (RecPtr >> 32), (uint32) RecPtr);
- goto err;
- }
+ case XLREAD_NEXT_RECORD:
+ ResetDecoder(state);
- /* Copy the first fragment of the record from the first page. */
- memcpy(state->readRecordBuf,
- state->readBuf + RecPtr % XLOG_BLCKSZ, len);
- buffer = state->readRecordBuf + len;
- gotlen = len;
-
- do
- {
- int rest_len = total_len - gotlen;
-
- /* Calculate pointer to beginning of next page */
- targetPagePtr += XLOG_BLCKSZ;
-
- /* Wait for the next page to become available */
- while (XLogNeedData(state, targetPagePtr,
- Min(rest_len, XLOG_BLCKSZ),
- false))
+ if (state->ReadRecPtr != InvalidXLogRecPtr)
{
- if (!state->routine.page_read(state, state->readPagePtr,
- state->readLen,
- state->ReadRecPtr,
- state->readBuf))
- break;
+ /* read the record after the one we just read */
+
+ /*
+ * EndRecPtr is pointing to end+1 of the previous WAL record.
+ * If we're at a page boundary, no more records can fit on the
+ * current page. We must skip over the page header, but we
+ * can't do that until we've read in the page, since the header
+ * size is variable.
+ */
+ state->PrevRecPtr = state->ReadRecPtr;
+ state->ReadRecPtr = state->EndRecPtr;
+ }
+ else
+ {
+ /*
+ * Caller supplied a position to start at.
+ *
+ * In this case, EndRecPtr should already be pointing to a
+ * valid record starting position.
+ */
+ Assert(XRecOffIsValid(state->EndRecPtr));
+ state->ReadRecPtr = state->EndRecPtr;
+
+ /*
+ * We cannot verify the previous-record pointer when we're
+ * seeking to a particular record. Reset PrevRecPtr so that we
+ * won't try doing that.
+ */
+ state->PrevRecPtr = InvalidXLogRecPtr;
+ state->EndRecPtr = InvalidXLogRecPtr; /* to be tidy */
}
+ state->record_verified = false;
+ state->readRecordState = XLREAD_TOT_LEN;
+ /* fall through */
+
+ case XLREAD_TOT_LEN:
+ {
+ uint32 total_len;
+ uint32 pageHeaderSize;
+ XLogRecPtr targetPagePtr;
+ uint32 targetRecOff;
+ XLogPageHeader pageHeader;
+
+ targetPagePtr =
+ state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
+ targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+
+ /*
+ * Check if we have enough data. For the first record in the page,
+ * the requesting length doesn't contain page header.
+ */
+ if (XLogNeedData(state, targetPagePtr,
+ Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+ targetRecOff != 0))
+ return XLREAD_NEED_DATA;
+
+ /* error out if caller supplied bogus page */
if (!state->page_verified)
goto err;
- Assert(SizeOfXLogShortPHD <= state->readLen);
+ /* examine page header now. */
+ pageHeaderSize =
+ XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+ if (targetRecOff == 0)
+ {
+ /* At page start, so skip over page header. */
+ state->ReadRecPtr += pageHeaderSize;
+ targetRecOff = pageHeaderSize;
+ }
+ else if (targetRecOff < pageHeaderSize)
+ {
+ report_invalid_record(state, "invalid record offset at %X/%X",
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr);
+ goto err;
+ }
- /* Check that the continuation on next page looks valid */
pageHeader = (XLogPageHeader) state->readBuf;
- if (!(pageHeader->xlp_info & XLP_FIRST_IS_CONTRECORD))
+ if ((pageHeader->xlp_info & XLP_FIRST_IS_CONTRECORD) &&
+ targetRecOff == pageHeaderSize)
{
- report_invalid_record(state,
- "there is no contrecord flag at %X/%X",
- (uint32) (RecPtr >> 32), (uint32) RecPtr);
+ report_invalid_record(state, "contrecord is requested by %X/%X",
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr);
goto err;
}
- /*
- * Cross-check that xlp_rem_len agrees with how much of the record
- * we expect there to be left.
- */
- if (pageHeader->xlp_rem_len == 0 ||
- total_len != (pageHeader->xlp_rem_len + gotlen))
- {
- report_invalid_record(state,
- "invalid contrecord length %u at %X/%X",
- pageHeader->xlp_rem_len,
- (uint32) (RecPtr >> 32), (uint32) RecPtr);
- goto err;
- }
-
- /* Append the continuation from this page to the buffer */
- pageHeaderSize = XLogPageHeaderSize(pageHeader);
-
+ /* XLogNeedData has verified the page header */
Assert(pageHeaderSize <= state->readLen);
- contdata = (char *) state->readBuf + pageHeaderSize;
- len = XLOG_BLCKSZ - pageHeaderSize;
- if (pageHeader->xlp_rem_len < len)
- len = pageHeader->xlp_rem_len;
+ /*
+ * Read the record length.
+ *
+ * NB: Even though we use an XLogRecord pointer here, the whole
+ * record header might not fit on this page. xl_tot_len is the first
+ * field of the struct, so it must be on this page (the records are
+ * MAXALIGNed), but we cannot access any other fields until we've
+ * verified that we got the whole header.
+ */
+ prec = (XLogRecord *) (state->readBuf +
+ state->ReadRecPtr % XLOG_BLCKSZ);
+ total_len = prec->xl_tot_len;
- Assert (pageHeaderSize + len <= state->readLen);
- memcpy(buffer, (char *) contdata, len);
- buffer += len;
- gotlen += len;
+ /*
+ * If the whole record header is on this page, validate it
+ * immediately. Otherwise do just a basic sanity check on
+ * xl_tot_len, and validate the rest of the header after reading it
+ * from the next page. The xl_tot_len check is necessary here to
+ * ensure that we enter the XLREAD_CONTINUATION state below;
+ * otherwise we might fail to apply ValidXLogRecordHeader at all.
+ */
+ if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
+ {
+ if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
+ state->PrevRecPtr, prec))
+ goto err;
- /* If we just reassembled the record header, validate it. */
- if (!gotheader)
+ state->record_verified = true;
+ }
+ else
{
- record = (XLogRecord *) state->readRecordBuf;
- if (!ValidXLogRecordHeader(state, RecPtr, state->ReadRecPtr,
- record, randAccess))
+ /* XXX: more validation should be done here */
+ if (total_len < SizeOfXLogRecord)
+ {
+ report_invalid_record(state,
+ "invalid record length at %X/%X: wanted %u, got %u",
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr,
+ (uint32) SizeOfXLogRecord, total_len);
goto err;
- gotheader = true;
+ }
}
- } while (gotlen < total_len);
-
- Assert(gotheader);
- record = (XLogRecord *) state->readRecordBuf;
- if (!ValidXLogRecord(state, record, RecPtr))
- goto err;
+ /*
+ * Wait for the rest of the record, or the part of the record that
+ * fit on the first page if crossed a page boundary, to become
+ * available.
+ */
+ state->recordGotLen = 0;
+ state->recordRemainLen = total_len;
+ state->readRecordState = XLREAD_FIRST_FRAGMENT;
+ }
+ /* fall through */
- pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
- state->ReadRecPtr = RecPtr;
- state->EndRecPtr = targetPagePtr + pageHeaderSize
- + MAXALIGN(pageHeader->xlp_rem_len);
- }
- else
- {
- /* Wait for the record data to become available */
- while (XLogNeedData(state, targetPagePtr,
- Min(targetRecOff + total_len, XLOG_BLCKSZ), true))
+ case XLREAD_FIRST_FRAGMENT:
{
- if (!state->routine.page_read(state, state->readPagePtr,
- state->readLen,
- state->ReadRecPtr, state->readBuf))
+ uint32 total_len = state->recordRemainLen;
+ uint32 request_len;
+ uint32 record_len;
+ XLogRecPtr targetPagePtr;
+ uint32 targetRecOff;
+
+ /*
+ * Wait for the rest of the record on the first page to become
+ * available
+ */
+ targetPagePtr =
+ state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
+ targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+
+ request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ);
+ record_len = request_len - targetRecOff;
+
+ /* ReadRecPtr contains page header */
+ Assert (targetRecOff != 0);
+ if (XLogNeedData(state, targetPagePtr, request_len, true))
+ return XLREAD_NEED_DATA;
+
+ /* error out if caller supplied bogus page */
+ if (!state->page_verified)
+ goto err;
+
+ prec = (XLogRecord *) (state->readBuf + targetRecOff);
+
+ /* validate record header if not yet */
+ if (!state->record_verified && record_len >= SizeOfXLogRecord)
+ {
+ if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
+ state->PrevRecPtr, prec))
+ goto err;
+
+ state->record_verified = true;
+ }
+
+
+ if (total_len == record_len)
+ {
+ /* Record does not cross a page boundary */
+ Assert(state->record_verified);
+
+ if (!ValidXLogRecord(state, prec, state->ReadRecPtr))
+ goto err;
+
+ state->record_verified = true; /* to be tidy */
+
+ /* We already checked the header earlier */
+ state->EndRecPtr = state->ReadRecPtr + MAXALIGN(record_len);
+
+ *record = prec;
+ state->readRecordState = XLREAD_NEXT_RECORD;
break;
+ }
+
+ /*
+ * The record continues on the next page. Need to reassemble
+ * record
+ */
+ Assert(total_len > record_len);
+
+ /* Enlarge readRecordBuf as needed. */
+ if (total_len > state->readRecordBufSize &&
+ !allocate_recordbuf(state, total_len))
+ {
+ /* We treat this as a "bogus data" condition */
+ report_invalid_record(state,
+ "record length %u at %X/%X too long",
+ total_len,
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr);
+ goto err;
+ }
+
+ /* Copy the first fragment of the record from the first page. */
+ memcpy(state->readRecordBuf, state->readBuf + targetRecOff,
+ record_len);
+ state->recordGotLen += record_len;
+ state->recordRemainLen -= record_len;
+
+ /* Calculate pointer to beginning of next page */
+ state->recordContRecPtr = state->ReadRecPtr + record_len;
+ Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0);
+
+ state->readRecordState = XLREAD_CONTINUATION;
}
+ /* fall through */
+
+ case XLREAD_CONTINUATION:
+ {
+ XLogPageHeader pageHeader;
+ uint32 pageHeaderSize;
+ XLogRecPtr targetPagePtr;
+
+ /* we enter this state only if we haven't read the whole record. */
+ Assert (state->recordRemainLen > 0);
+
+ while(state->recordRemainLen > 0)
+ {
+ char *contdata;
+ uint32 request_len;
+ uint32 record_len;
+
+ /* Wait for the next page to become available */
+ targetPagePtr = state->recordContRecPtr;
+
+ /* this request contains page header */
+ Assert (targetPagePtr != 0);
+ if (XLogNeedData(state, targetPagePtr,
+ Min(state->recordRemainLen, XLOG_BLCKSZ),
+ false))
+ return XLREAD_NEED_DATA;
+
+ if (!state->page_verified)
+ goto err;
+
+ Assert(SizeOfXLogShortPHD <= state->readLen);
- if (!state->page_verified)
- goto err;
+ /* Check that the continuation on next page looks valid */
+ pageHeader = (XLogPageHeader) state->readBuf;
+ if (!(pageHeader->xlp_info & XLP_FIRST_IS_CONTRECORD))
+ {
+ report_invalid_record(
+ state,
+ "there is no contrecord flag at %X/%X reading %X/%X",
+ (uint32) (state->recordContRecPtr >> 32),
+ (uint32) state->recordContRecPtr,
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr);
+ goto err;
+ }
- /* Record does not cross a page boundary */
- if (!ValidXLogRecord(state, record, RecPtr))
- goto err;
+ /*
+ * Cross-check that xlp_rem_len agrees with how much of the
+ * record we expect there to be left.
+ */
+ if (pageHeader->xlp_rem_len == 0 ||
+ pageHeader->xlp_rem_len != state->recordRemainLen)
+ {
+ report_invalid_record(
+ state,
+ "invalid contrecord length %u at %X/%X reading %X/%X, expected %u",
+ pageHeader->xlp_rem_len,
+ (uint32) (state->recordContRecPtr >> 32),
+ (uint32) state->recordContRecPtr,
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr,
+ state->recordRemainLen);
+ goto err;
+ }
- state->EndRecPtr = RecPtr + MAXALIGN(total_len);
+ /* Append the continuation from this page to the buffer */
+ pageHeaderSize = XLogPageHeaderSize(pageHeader);
- state->ReadRecPtr = RecPtr;
+ /*
+ * XLogNeedData should have ensured that the whole page header
+ * was read
+ */
+ Assert(state->readLen >= pageHeaderSize);
+
+ contdata = (char *) state->readBuf + pageHeaderSize;
+ record_len = XLOG_BLCKSZ - pageHeaderSize;
+ if (pageHeader->xlp_rem_len < record_len)
+ record_len = pageHeader->xlp_rem_len;
+
+ request_len = record_len + pageHeaderSize;
+
+ /* XLogNeedData should have ensured all needed data was read */
+ Assert (state->readLen >= request_len);
+
+ memcpy(state->readRecordBuf + state->recordGotLen,
+ (char *) contdata, record_len);
+ state->recordGotLen += record_len;
+ state->recordRemainLen -= record_len;
+
+ /* If we just reassembled the record header, validate it. */
+ if (!state->record_verified)
+ {
+ Assert(state->recordGotLen >= SizeOfXLogRecord);
+ if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
+ state->PrevRecPtr,
+ (XLogRecord *) state->readRecordBuf))
+ goto err;
+
+ state->record_verified = true;
+ }
+
+ /* Calculate pointer to beginning of next page, and continue */
+ state->recordContRecPtr += XLOG_BLCKSZ;
+ }
+
+ /* targetPagePtr is pointing the last-read page here */
+ prec = (XLogRecord *) state->readRecordBuf;
+ if (!ValidXLogRecord(state, prec, state->ReadRecPtr))
+ goto err;
+
+ pageHeaderSize =
+ XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+ state->EndRecPtr = targetPagePtr + pageHeaderSize
+ + MAXALIGN(pageHeader->xlp_rem_len);
+
+ *record = prec;
+ state->readRecordState = XLREAD_NEXT_RECORD;
+ break;
+ }
}
/*
* Special processing if it's an XLOG SWITCH record
*/
- if (record->xl_rmid == RM_XLOG_ID &&
- (record->xl_info & ~XLR_INFO_MASK) == XLOG_SWITCH)
+ if ((*record)->xl_rmid == RM_XLOG_ID &&
+ ((*record)->xl_info & ~XLR_INFO_MASK) == XLOG_SWITCH)
{
/* Pretend it extends to end of segment */
state->EndRecPtr += state->segcxt.ws_segsize - 1;
state->EndRecPtr -= XLogSegmentOffset(state->EndRecPtr, state->segcxt.ws_segsize);
}
- if (DecodeXLogRecord(state, record, errormsg))
- return record;
- else
- return NULL;
+ Assert (!*record || state->readLen >= 0);
+ if (DecodeXLogRecord(state, *record, errormsg))
+ return XLREAD_SUCCESS;
+
+ *record = NULL;
+ return XLREAD_FAIL;
err:
/*
- * Invalidate the read state. We might read from a different source after
+ * Invalidate the read page. We might read from a different source after
* failure.
*/
XLogReaderInvalReadState(state);
@@ -572,7 +706,8 @@ err:
if (state->errormsg_buf[0] != '\0')
*errormsg = state->errormsg_buf;
- return NULL;
+ *record = NULL;
+ return XLREAD_FAIL;
}
/*
@@ -724,11 +859,12 @@ XLogReaderInvalReadState(XLogReaderState *state)
*
* This is just a convenience subroutine to avoid duplicated code in
* XLogReadRecord. It's not intended for use from anywhere else.
+ *
+ * If PrevRecPtr is valid, the xl_prev is is cross-checked with it.
*/
static bool
ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
- XLogRecPtr PrevRecPtr, XLogRecord *record,
- bool randAccess)
+ XLogRecPtr PrevRecPtr, XLogRecord *record)
{
if (record->xl_tot_len < SizeOfXLogRecord)
{
@@ -746,7 +882,7 @@ ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
(uint32) RecPtr);
return false;
}
- if (randAccess)
+ if (PrevRecPtr == InvalidXLogRecPtr)
{
/*
* We can't exactly verify the prev-link, but surely it should be less
@@ -978,11 +1114,14 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
* XLogReadRecord() will read the next valid record.
*/
XLogRecPtr
-XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
+XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
+ XLogFindNextRecordCB read_page, void *private)
{
XLogRecPtr tmpRecPtr;
XLogRecPtr found = InvalidXLogRecPtr;
XLogPageHeader header;
+ XLogRecord *record;
+ XLogReadRecordResult result;
char *errormsg;
Assert(!XLogRecPtrIsInvalid(RecPtr));
@@ -1015,9 +1154,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
while(XLogNeedData(state, targetPagePtr, targetRecOff,
targetRecOff != 0))
{
- if (!state->routine.page_read(state, state->readPagePtr,
- state->readLen,
- state->ReadRecPtr, state->readBuf))
+ if (!read_page(state, private))
break;
}
@@ -1069,8 +1206,16 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
* or we just jumped over the remaining data of a continuation.
*/
XLogBeginRead(state, tmpRecPtr);
- while (XLogReadRecord(state, &errormsg) != NULL)
+ while ((result = XLogReadRecord(state, &record, &errormsg)) !=
+ XLREAD_FAIL)
{
+ if (result == XLREAD_NEED_DATA)
+ {
+ if (!read_page(state, private))
+ goto err;
+ continue;
+ }
+
/* past the record we've found, break out */
if (RecPtr <= state->ReadRecPtr)
{
@@ -1090,9 +1235,9 @@ err:
#endif /* FRONTEND */
/*
- * Helper function to ease writing of XLogRoutine->page_read callbacks.
- * If this function is used, caller must supply a segment_open callback in
- * 'state', as that is used here.
+ * Helper function to ease writing of page_read callback.
+ * If this function is used, caller must supply a segment_open callback and
+ * segment_close callback as that is used here.
*
* Read 'count' bytes into 'buf', starting at location 'startptr', from WAL
* fetched from timeline 'tli'.
@@ -1105,6 +1250,7 @@ err:
*/
bool
WALRead(XLogReaderState *state,
+ WALSegmentOpenCB segopenfn, WALSegmentCloseCB segclosefn,
char *buf, XLogRecPtr startptr, Size count, TimeLineID tli,
WALReadError *errinfo)
{
@@ -1136,10 +1282,10 @@ WALRead(XLogReaderState *state,
XLogSegNo nextSegNo;
if (state->seg.ws_file >= 0)
- state->routine.segment_close(state);
+ segclosefn(state);
XLByteToSeg(recptr, nextSegNo, state->segcxt.ws_segsize);
- state->routine.segment_open(state, nextSegNo, &tli);
+ segopenfn(state, nextSegNo, &tli);
/* This shouldn't happen -- indicates a bug in segment_open */
Assert(state->seg.ws_file >= 0);
diff --git a/src/backend/access/transam/xlogutils.c b/src/backend/access/transam/xlogutils.c
index 1b10b6df20..303333571f 100644
--- a/src/backend/access/transam/xlogutils.c
+++ b/src/backend/access/transam/xlogutils.c
@@ -689,8 +689,7 @@ XLogTruncateRelation(RelFileNode rnode, ForkNumber forkNum,
void
XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wantLength)
{
- const XLogRecPtr lastReadPage = state->seg.ws_segno *
- state->segcxt.ws_segsize + state->readLen;
+ const XLogRecPtr lastReadPage = state->readPagePtr;
Assert(wantPage != InvalidXLogRecPtr && wantPage % XLOG_BLCKSZ == 0);
Assert(wantLength <= XLOG_BLCKSZ);
@@ -705,7 +704,7 @@ XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wa
* current TLI has since become historical.
*/
if (lastReadPage == wantPage &&
- state->readLen != 0 &&
+ state->page_verified &&
lastReadPage + state->readLen >= wantPage + Min(wantLength, XLOG_BLCKSZ - 1))
return;
@@ -792,6 +791,7 @@ wal_segment_open(XLogReaderState *state, XLogSegNo nextSegNo,
char path[MAXPGPATH];
XLogFilePath(path, tli, nextSegNo, state->segcxt.ws_segsize);
+ elog(LOG, "HOGE: %lu, %d => %s", nextSegNo, tli, path);
state->seg.ws_file = BasicOpenFile(path, O_RDONLY | PG_BINARY);
if (state->seg.ws_file >= 0)
return;
@@ -829,9 +829,11 @@ wal_segment_close(XLogReaderState *state)
* loop for now.
*/
bool
-read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *cur_page)
+read_local_xlog_page(XLogReaderState *state)
{
+ XLogRecPtr targetPagePtr = state->readPagePtr;
+ int reqLen = state->readLen;
+ char *cur_page = state->readBuf;
XLogRecPtr read_upto,
loc;
TimeLineID tli;
@@ -944,11 +946,12 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
* as 'count', read the whole page anyway. It's guaranteed to be
* zero-padded up to the page boundary if it's incomplete.
*/
- if (!WALRead(state, cur_page, targetPagePtr, XLOG_BLCKSZ, tli,
- &errinfo))
+ if (!WALRead(state, wal_segment_open, wal_segment_close,
+ cur_page, targetPagePtr, XLOG_BLCKSZ, tli, &errinfo))
WALReadRaiseError(&errinfo);
/* number of valid bytes in the buffer */
+ state->readPagePtr = targetPagePtr;
state->readLen = count;
return true;
}
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index 61902be3b0..a15b0b3355 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -120,7 +120,8 @@ StartupDecodingContext(List *output_plugin_options,
TransactionId xmin_horizon,
bool need_full_snapshot,
bool fast_forward,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress)
@@ -169,11 +170,12 @@ StartupDecodingContext(List *output_plugin_options,
ctx->slot = slot;
- ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, xl_routine, ctx);
+ ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, cleanup_cb);
if (!ctx->reader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
errmsg("out of memory")));
+ ctx->page_read = page_read;
ctx->reorder = ReorderBufferAllocate();
ctx->snapshot_builder =
@@ -231,7 +233,8 @@ CreateInitDecodingContext(char *plugin,
List *output_plugin_options,
bool need_full_snapshot,
XLogRecPtr restart_lsn,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress)
@@ -328,7 +331,7 @@ CreateInitDecodingContext(char *plugin,
ctx = StartupDecodingContext(NIL, restart_lsn, xmin_horizon,
need_full_snapshot, false,
- xl_routine, prepare_write, do_write,
+ page_read, cleanup_cb, prepare_write, do_write,
update_progress);
/* call output plugin initialization callback */
@@ -376,7 +379,8 @@ LogicalDecodingContext *
CreateDecodingContext(XLogRecPtr start_lsn,
List *output_plugin_options,
bool fast_forward,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress)
@@ -429,8 +433,8 @@ CreateDecodingContext(XLogRecPtr start_lsn,
ctx = StartupDecodingContext(output_plugin_options,
start_lsn, InvalidTransactionId, false,
- fast_forward, xl_routine, prepare_write,
- do_write, update_progress);
+ fast_forward, page_read, cleanup_cb,
+ prepare_write, do_write, update_progress);
/* call output plugin initialization callback */
old_context = MemoryContextSwitchTo(ctx->context);
@@ -483,7 +487,13 @@ DecodingContextFindStartpoint(LogicalDecodingContext *ctx)
char *err = NULL;
/* the read_page callback waits for new WAL */
- record = XLogReadRecord(ctx->reader, &err);
+ while (XLogReadRecord(ctx->reader, &record, &err) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!ctx->page_read(ctx->reader))
+ break;
+ }
+
if (err)
elog(ERROR, "%s", err);
if (!record)
diff --git a/src/backend/replication/logical/logicalfuncs.c b/src/backend/replication/logical/logicalfuncs.c
index b99c94e848..fcc81b358e 100644
--- a/src/backend/replication/logical/logicalfuncs.c
+++ b/src/backend/replication/logical/logicalfuncs.c
@@ -233,9 +233,8 @@ pg_logical_slot_get_changes_guts(FunctionCallInfo fcinfo, bool confirm, bool bin
ctx = CreateDecodingContext(InvalidXLogRecPtr,
options,
false,
- XL_ROUTINE(.page_read = read_local_xlog_page,
- .segment_open = wal_segment_open,
- .segment_close = wal_segment_close),
+ read_local_xlog_page,
+ wal_segment_close,
LogicalOutputPrepareWrite,
LogicalOutputWrite, NULL);
@@ -284,7 +283,13 @@ pg_logical_slot_get_changes_guts(FunctionCallInfo fcinfo, bool confirm, bool bin
XLogRecord *record;
char *errm = NULL;
- record = XLogReadRecord(ctx->reader, &errm);
+ while (XLogReadRecord(ctx->reader, &record, &errm) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!ctx->page_read(ctx->reader))
+ break;
+ }
+
if (errm)
elog(ERROR, "%s", errm);
diff --git a/src/backend/replication/slotfuncs.c b/src/backend/replication/slotfuncs.c
index 88033a79b2..8a678074d5 100644
--- a/src/backend/replication/slotfuncs.c
+++ b/src/backend/replication/slotfuncs.c
@@ -152,9 +152,8 @@ create_logical_replication_slot(char *name, char *plugin,
ctx = CreateInitDecodingContext(plugin, NIL,
false, /* just catalogs is OK */
restart_lsn,
- XL_ROUTINE(.page_read = read_local_xlog_page,
- .segment_open = wal_segment_open,
- .segment_close = wal_segment_close),
+ read_local_xlog_page,
+ wal_segment_close,
NULL, NULL, NULL);
/*
@@ -486,9 +485,8 @@ pg_logical_replication_slot_advance(XLogRecPtr moveto)
ctx = CreateDecodingContext(InvalidXLogRecPtr,
NIL,
true, /* fast_forward */
- XL_ROUTINE(.page_read = read_local_xlog_page,
- .segment_open = wal_segment_open,
- .segment_close = wal_segment_close),
+ read_local_xlog_page,
+ wal_segment_close,
NULL, NULL, NULL);
/*
@@ -513,7 +511,13 @@ pg_logical_replication_slot_advance(XLogRecPtr moveto)
* Read records. No changes are generated in fast_forward mode,
* but snapbuilder/slot statuses are updated properly.
*/
- record = XLogReadRecord(ctx->reader, &errm);
+ while (XLogReadRecord(ctx->reader, &record, &errm) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!ctx->page_read(ctx->reader))
+ break;
+ }
+
if (errm)
elog(ERROR, "%s", errm);
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index bf2711bddd..2fd24a9b14 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -580,10 +580,7 @@ StartReplication(StartReplicationCmd *cmd)
/* create xlogreader for physical replication */
xlogreader =
- XLogReaderAllocate(wal_segment_size, NULL,
- XL_ROUTINE(.segment_open = WalSndSegmentOpen,
- .segment_close = wal_segment_close),
- NULL);
+ XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
if (!xlogreader)
ereport(ERROR,
@@ -812,9 +809,11 @@ StartReplication(StartReplicationCmd *cmd)
* set every time WAL is flushed.
*/
static bool
-logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetRecPtr, char *cur_page)
+logical_read_xlog_page(XLogReaderState *state)
{
+ XLogRecPtr targetPagePtr = state->readPagePtr;
+ int reqLen = state->readLen;
+ char *cur_page = state->readBuf;
XLogRecPtr flushptr;
int count;
WALReadError errinfo;
@@ -842,7 +841,7 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
count = flushptr - targetPagePtr; /* part of the page available */
/* now actually read the data, we know it's there */
- if (!WALRead(state,
+ if (!WALRead(state, WalSndSegmentOpen, wal_segment_close,
cur_page,
targetPagePtr,
XLOG_BLCKSZ,
@@ -1014,9 +1013,8 @@ CreateReplicationSlot(CreateReplicationSlotCmd *cmd)
ctx = CreateInitDecodingContext(cmd->plugin, NIL, need_full_snapshot,
InvalidXLogRecPtr,
- XL_ROUTINE(.page_read = logical_read_xlog_page,
- .segment_open = WalSndSegmentOpen,
- .segment_close = wal_segment_close),
+ logical_read_xlog_page,
+ wal_segment_close,
WalSndPrepareWrite, WalSndWriteData,
WalSndUpdateProgress);
@@ -1179,9 +1177,8 @@ StartLogicalReplication(StartReplicationCmd *cmd)
*/
logical_decoding_ctx =
CreateDecodingContext(cmd->startpoint, cmd->options, false,
- XL_ROUTINE(.page_read = logical_read_xlog_page,
- .segment_open = WalSndSegmentOpen,
- .segment_close = wal_segment_close),
+ logical_read_xlog_page,
+ wal_segment_close,
WalSndPrepareWrite, WalSndWriteData,
WalSndUpdateProgress);
xlogreader = logical_decoding_ctx->reader;
@@ -2761,7 +2758,7 @@ XLogSendPhysical(void)
enlargeStringInfo(&output_message, nbytes);
retry:
- if (!WALRead(xlogreader,
+ if (!WALRead(xlogreader, WalSndSegmentOpen, wal_segment_close,
&output_message.data[output_message.len],
startptr,
nbytes,
@@ -2859,7 +2856,12 @@ XLogSendLogical(void)
*/
WalSndCaughtUp = false;
- record = XLogReadRecord(logical_decoding_ctx->reader, &errm);
+ while (XLogReadRecord(logical_decoding_ctx->reader, &record, &errm) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!logical_decoding_ctx->page_read(logical_decoding_ctx->reader))
+ break;
+ }
/* xlog record was invalid */
if (errm != NULL)
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index 0b7e73ae79..3dd6df4be6 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -40,15 +40,9 @@ static int xlogreadfd = -1;
static XLogSegNo xlogreadsegno = -1;
static char xlogfpath[MAXPGPATH];
-typedef struct XLogPageReadPrivate
-{
- const char *restoreCommand;
- int tliIndex;
-} XLogPageReadPrivate;
-
-static bool SimpleXLogPageRead(XLogReaderState *xlogreader,
- XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
+static bool SimpleXLogPageRead(XLogReaderState *xlogreader,
+ const char *datadir, int *tliIndex,
+ const char *restoreCommand);
/*
* Read WAL from the datadir/pg_wal, starting from 'startpoint' on timeline
@@ -62,20 +56,22 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
XLogRecord *record;
XLogReaderState *xlogreader;
char *errormsg;
- XLogPageReadPrivate private;
- private.tliIndex = tliIndex;
- private.restoreCommand = restoreCommand;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir,
- XL_ROUTINE(.page_read = &SimpleXLogPageRead),
- &private);
+ xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+
if (xlogreader == NULL)
pg_fatal("out of memory");
XLogBeginRead(xlogreader, startpoint);
do
{
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!SimpleXLogPageRead(xlogreader, datadir,
+ &tliIndex, restoreCommand))
+ break;
+ }
if (record == NULL)
{
@@ -113,19 +109,19 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex,
XLogRecord *record;
XLogReaderState *xlogreader;
char *errormsg;
- XLogPageReadPrivate private;
XLogRecPtr endptr;
- private.tliIndex = tliIndex;
- private.restoreCommand = restoreCommand;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir,
- XL_ROUTINE(.page_read = &SimpleXLogPageRead),
- &private);
+ xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
XLogBeginRead(xlogreader, ptr);
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!SimpleXLogPageRead(xlogreader, datadir, &tliIndex, restoreCommand))
+ break;
+ }
if (record == NULL)
{
if (errormsg)
@@ -160,7 +156,6 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
XLogRecPtr searchptr;
XLogReaderState *xlogreader;
char *errormsg;
- XLogPageReadPrivate private;
/*
* The given fork pointer points to the end of the last common record,
@@ -176,11 +171,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
forkptr += SizeOfXLogShortPHD;
}
- private.tliIndex = tliIndex;
- private.restoreCommand = restoreCommand;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir,
- XL_ROUTINE(.page_read = &SimpleXLogPageRead),
- &private);
+ xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
@@ -190,7 +181,13 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
uint8 info;
XLogBeginRead(xlogreader, searchptr);
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!SimpleXLogPageRead(xlogreader, datadir,
+ &tliIndex, restoreCommand))
+ break;
+ }
if (record == NULL)
{
@@ -237,10 +234,11 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
/* XLogReader callback function, to read a WAL page */
static bool
-SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *readBuf)
+SimpleXLogPageRead(XLogReaderState *xlogreader, const char *datadir,
+ int *tliIndex, const char *restoreCommand)
{
- XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
+ XLogRecPtr targetPagePtr = xlogreader->readPagePtr;
+ char *readBuf = xlogreader->readBuf;
uint32 targetPageOff;
XLogRecPtr targetSegEnd;
XLogSegNo targetSegNo;
@@ -273,14 +271,14 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
* be done both forward and backward, consider also switching timeline
* accordingly.
*/
- while (private->tliIndex < targetNentries - 1 &&
- targetHistory[private->tliIndex].end < targetSegEnd)
- private->tliIndex++;
- while (private->tliIndex > 0 &&
- targetHistory[private->tliIndex].begin >= targetSegEnd)
- private->tliIndex--;
+ while (*tliIndex < targetNentries - 1 &&
+ targetHistory[*tliIndex].end < targetSegEnd)
+ (*tliIndex)++;
+ while (*tliIndex > 0 &&
+ targetHistory[*tliIndex].begin >= targetSegEnd)
+ (*tliIndex)--;
- XLogFileName(xlogfname, targetHistory[private->tliIndex].tli,
+ XLogFileName(xlogfname, targetHistory[*tliIndex].tli,
xlogreadsegno, WalSegSz);
snprintf(xlogfpath, MAXPGPATH, "%s/" XLOGDIR "/%s",
@@ -293,7 +291,7 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
/*
* If we have no restore_command to execute, then exit.
*/
- if (private->restoreCommand == NULL)
+ if (restoreCommand == NULL)
{
pg_log_error("could not open file \"%s\": %m", xlogfpath);
xlogreader->readLen = -1;
@@ -307,7 +305,7 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
xlogreadfd = RestoreArchivedFile(xlogreader->segcxt.ws_dir,
xlogfname,
WalSegSz,
- private->restoreCommand);
+ restoreCommand);
if (xlogreadfd < 0)
{
@@ -349,7 +347,7 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
Assert(targetSegNo == xlogreadsegno);
- xlogreader->seg.ws_tli = targetHistory[private->tliIndex].tli;
+ xlogreader->seg.ws_tli = targetHistory[*tliIndex].tli;
xlogreader->readLen = XLOG_BLCKSZ;
return true;
}
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index ab0c28d9b7..02e71138cc 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -330,12 +330,17 @@ WALDumpCloseSegment(XLogReaderState *state)
state->seg.ws_file = -1;
}
-/* pg_waldump's XLogReaderRoutine->page_read callback */
+/*
+ * pg_waldump's WAL page rader, also used as page_read callback for
+ * XLogFindNextRecord
+ */
static bool
-WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetPtr, char *readBuff)
+WALDumpReadPage(XLogReaderState *state, void *priv)
{
- XLogDumpPrivate *private = state->private_data;
+ XLogRecPtr targetPagePtr = state->readPagePtr;
+ int reqLen = state->readLen;
+ char *readBuff = state->readBuf;
+ XLogDumpPrivate *private = (XLogDumpPrivate *) priv;
int count = XLOG_BLCKSZ;
WALReadError errinfo;
@@ -353,8 +358,8 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
}
}
- if (!WALRead(state, readBuff, targetPagePtr, count, private->timeline,
- &errinfo))
+ if (!WALRead(state, WALDumpOpenSegment, WALDumpCloseSegment,
+ readBuff, targetPagePtr, count, private->timeline, &errinfo))
{
WALOpenSegment *seg = &errinfo.wre_seg;
char fname[MAXPGPATH];
@@ -374,6 +379,7 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
(Size) errinfo.wre_req);
}
+ Assert(count >= state->readLen);
state->readLen = count;
return true;
}
@@ -1042,16 +1048,14 @@ main(int argc, char **argv)
/* we have everything we need, start reading */
xlogreader_state =
- XLogReaderAllocate(WalSegSz, waldir,
- XL_ROUTINE(.page_read = WALDumpReadPage,
- .segment_open = WALDumpOpenSegment,
- .segment_close = WALDumpCloseSegment),
- &private);
+ XLogReaderAllocate(WalSegSz, waldir, WALDumpCloseSegment);
+
if (!xlogreader_state)
fatal_error("out of memory");
/* first find a valid recptr to start from */
- first_record = XLogFindNextRecord(xlogreader_state, private.startptr);
+ first_record = XLogFindNextRecord(xlogreader_state, private.startptr,
+ &WALDumpReadPage, (void*) &private);
if (first_record == InvalidXLogRecPtr)
fatal_error("could not find a valid record after %X/%X",
@@ -1075,7 +1079,13 @@ main(int argc, char **argv)
for (;;)
{
/* try to read the next record */
- record = XLogReadRecord(xlogreader_state, &errormsg);
+ while (XLogReadRecord(xlogreader_state, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!WALDumpReadPage(xlogreader_state, (void *) &private))
+ break;
+ }
+
if (!record)
{
if (!config.follow || private.endptr_reached)
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index 6c8848e14f..6a7fe140cb 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -57,64 +57,15 @@ typedef struct WALSegmentContext
typedef struct XLogReaderState XLogReaderState;
-/* Function type definition for the read_page callback */
-typedef bool (*XLogPageReadCB) (XLogReaderState *xlogreader,
- XLogRecPtr targetPagePtr,
- int reqLen,
- XLogRecPtr targetRecPtr,
- char *readBuf);
+/* Function type definition for the segment cleanup callback */
+typedef void (*WALSegmentCleanupCB) (XLogReaderState *xlogreader);
+
+/* Function type definition for the open/close callbacks for WALRead() */
typedef void (*WALSegmentOpenCB) (XLogReaderState *xlogreader,
XLogSegNo nextSegNo,
TimeLineID *tli_p);
typedef void (*WALSegmentCloseCB) (XLogReaderState *xlogreader);
-typedef struct XLogReaderRoutine
-{
- /*
- * Data input callback
- *
- * This callback shall read at least reqLen valid bytes of the xlog page
- * starting at targetPagePtr, and store them in readBuf. The callback
- * shall return the number of bytes read (never more than XLOG_BLCKSZ), or
- * -1 on failure. The callback shall sleep, if necessary, to wait for the
- * requested bytes to become available. The callback will not be invoked
- * again for the same page unless more than the returned number of bytes
- * are needed.
- *
- * targetRecPtr is the position of the WAL record we're reading. Usually
- * it is equal to targetPagePtr + reqLen, but sometimes xlogreader needs
- * to read and verify the page or segment header, before it reads the
- * actual WAL record it's interested in. In that case, targetRecPtr can
- * be used to determine which timeline to read the page from.
- *
- * The callback shall set ->seg.ws_tli to the TLI of the file the page was
- * read from.
- */
- XLogPageReadCB page_read;
-
- /*
- * Callback to open the specified WAL segment for reading. ->seg.ws_file
- * shall be set to the file descriptor of the opened segment. In case of
- * failure, an error shall be raised by the callback and it shall not
- * return.
- *
- * "nextSegNo" is the number of the segment to be opened.
- *
- * "tli_p" is an input/output argument. WALRead() uses it to pass the
- * timeline in which the new segment should be found, but the callback can
- * use it to return the TLI that it actually opened.
- */
- WALSegmentOpenCB segment_open;
-
- /*
- * WAL segment close callback. ->seg.ws_file shall be set to a negative
- * number.
- */
- WALSegmentCloseCB segment_close;
-} XLogReaderRoutine;
-
-#define XL_ROUTINE(...) &(XLogReaderRoutine){__VA_ARGS__}
-
typedef struct
{
/* Is this block ref in use? */
@@ -144,12 +95,35 @@ typedef struct
uint16 data_bufsz;
} DecodedBkpBlock;
+/* Return code from XLogReadRecord */
+typedef enum XLogReadRecordResult
+{
+ XLREAD_SUCCESS, /* record is successfully read */
+ XLREAD_NEED_DATA, /* need more data. see XLogReadRecord. */
+ XLREAD_FAIL /* failed during reading a record */
+} XLogReadRecordResult;
+
+/*
+ * internal state of XLogReadRecord
+ *
+ * XLogReadState runs a state machine while reading a record. Theses states
+ * are not seen outside the function. Each state may repeat several times
+ * exiting requesting caller for new data. See the comment of XLogReadRecrod
+ * for details.
+ */
+typedef enum XLogReadRecordState {
+ XLREAD_NEXT_RECORD,
+ XLREAD_TOT_LEN,
+ XLREAD_FIRST_FRAGMENT,
+ XLREAD_CONTINUATION
+} XLogReadRecordState;
+
struct XLogReaderState
{
/*
* Operational callbacks
*/
- XLogReaderRoutine routine;
+ WALSegmentCleanupCB cleanup_cb;
/* ----------------------------------------
* Public parameters
@@ -162,18 +136,14 @@ struct XLogReaderState
*/
uint64 system_identifier;
- /*
- * Opaque data for callbacks to use. Not used by XLogReader.
- */
- void *private_data;
-
/*
* Start and end point of last record read. EndRecPtr is also used as the
* position to read next. Calling XLogBeginRead() sets EndRecPtr to the
* starting position and ReadRecPtr to invalid.
*/
- XLogRecPtr ReadRecPtr; /* start of last record read */
+ XLogRecPtr ReadRecPtr; /* start of last record read or being read */
XLogRecPtr EndRecPtr; /* end+1 of last record read */
+ XLogRecPtr PrevRecPtr; /* start of previous record read */
/* ----------------------------------------
* Communication with page reader
@@ -187,7 +157,9 @@ struct XLogReaderState
* the request, or -1 on error */
TimeLineID readPageTLI; /* TLI for data currently in readBuf */
char *readBuf; /* buffer to store data */
- bool page_verified; /* is the page on the buffer verified? */
+ bool page_verified; /* is the page header on the buffer verified? */
+ bool record_verified;/* is the current record header verified? */
+
/* ----------------------------------------
@@ -227,8 +199,6 @@ struct XLogReaderState
XLogRecPtr latestPagePtr;
TimeLineID latestPageTLI;
- /* beginning of the WAL record being read. */
- XLogRecPtr currRecPtr;
/* timeline to read it from, 0 if a lookup is required */
TimeLineID currTLI;
@@ -255,6 +225,15 @@ struct XLogReaderState
char *readRecordBuf;
uint32 readRecordBufSize;
+ /*
+ * XLogReadRecord() state
+ */
+ XLogReadRecordState readRecordState;/* state machine state */
+ int recordGotLen; /* amount of current record that has
+ * already been read */
+ int recordRemainLen; /* length of current record that remains */
+ XLogRecPtr recordContRecPtr; /* where the current record continues */
+
/* Buffer to hold error message */
char *errormsg_buf;
};
@@ -262,9 +241,7 @@ struct XLogReaderState
/* Get a new XLogReader */
extern XLogReaderState *XLogReaderAllocate(int wal_segment_size,
const char *waldir,
- XLogReaderRoutine *routine,
- void *private_data);
-extern XLogReaderRoutine *LocalXLogReaderRoutine(void);
+ WALSegmentCleanupCB cleanup_cb);
/* Free an XLogReader */
extern void XLogReaderFree(XLogReaderState *state);
@@ -272,12 +249,17 @@ extern void XLogReaderFree(XLogReaderState *state);
/* Position the XLogReader to given record */
extern void XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr);
#ifdef FRONTEND
-extern XLogRecPtr XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr);
+/* Function type definition for the read_page callback */
+typedef bool (*XLogFindNextRecordCB) (XLogReaderState *xlogreader,
+ void *private);
+extern XLogRecPtr XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
+ XLogFindNextRecordCB read_page, void *private);
#endif /* FRONTEND */
/* Read the next XLog record. Returns NULL on end-of-WAL or failure */
-extern struct XLogRecord *XLogReadRecord(XLogReaderState *state,
- char **errormsg);
+extern XLogReadRecordResult XLogReadRecord(XLogReaderState *state,
+ XLogRecord **record,
+ char **errormsg);
/* Validate a page */
extern bool XLogReaderValidatePageHeader(XLogReaderState *state,
@@ -297,6 +279,7 @@ typedef struct WALReadError
} WALReadError;
extern bool WALRead(XLogReaderState *state,
+ WALSegmentOpenCB segopenfn, WALSegmentCloseCB sgclosefn,
char *buf, XLogRecPtr startptr, Size count,
TimeLineID tli, WALReadError *errinfo);
diff --git a/src/include/access/xlogutils.h b/src/include/access/xlogutils.h
index c287edf206..482e4ced69 100644
--- a/src/include/access/xlogutils.h
+++ b/src/include/access/xlogutils.h
@@ -47,9 +47,7 @@ extern Buffer XLogReadBufferExtended(RelFileNode rnode, ForkNumber forknum,
extern Relation CreateFakeRelcacheEntry(RelFileNode rnode);
extern void FreeFakeRelcacheEntry(Relation fakerel);
-extern bool read_local_xlog_page(XLogReaderState *state,
- XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetRecPtr, char *cur_page);
+extern bool read_local_xlog_page(XLogReaderState *state);
extern void wal_segment_open(XLogReaderState *state,
XLogSegNo nextSegNo,
TimeLineID *tli_p);
diff --git a/src/include/replication/logical.h b/src/include/replication/logical.h
index c2f2475e5d..a743db49cf 100644
--- a/src/include/replication/logical.h
+++ b/src/include/replication/logical.h
@@ -29,6 +29,10 @@ typedef void (*LogicalOutputPluginWriterUpdateProgress) (struct LogicalDecodingC
TransactionId xid
);
+typedef struct LogicalDecodingContext LogicalDecodingContext;
+
+typedef bool (*LogicalDecodingXLogPageReadCB)(XLogReaderState *ctx);
+
typedef struct LogicalDecodingContext
{
/* memory context this is all allocated in */
@@ -39,6 +43,7 @@ typedef struct LogicalDecodingContext
/* infrastructure pieces for decoding */
XLogReaderState *reader;
+ LogicalDecodingXLogPageReadCB page_read;
struct ReorderBuffer *reorder;
struct SnapBuild *snapshot_builder;
@@ -95,14 +100,16 @@ extern LogicalDecodingContext *CreateInitDecodingContext(char *plugin,
List *output_plugin_options,
bool need_full_snapshot,
XLogRecPtr restart_lsn,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress);
extern LogicalDecodingContext *CreateDecodingContext(XLogRecPtr start_lsn,
List *output_plugin_options,
bool fast_forward,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress);
--
2.18.4
----Next_Part(Thu_Jul__2_13_53_30_2020_072)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
filename="v15-0003-Remove-globals-readOff-readLen-and-readSegNo.patch"
^ permalink raw reply [nested|flat] 70+ messages in thread
* [PATCH v17 2/4] Move page-reader out of XLogReadRecord
@ 2019-09-10 03:58 Kyotaro Horiguchi <[email protected]>
0 siblings, 0 replies; 70+ messages in thread
From: Kyotaro Horiguchi @ 2019-09-10 03:58 UTC (permalink / raw)
This is the second step of removing callbacks from WAL record reader.
Since it is essential to take in additional data while reading a
record, the function have to ask caller for new data while keeping
working state. Thus the function is turned into a state machine.
---
src/backend/access/transam/twophase.c | 15 +-
src/backend/access/transam/xlog.c | 58 +-
src/backend/access/transam/xlogreader.c | 697 +++++++++++-------
src/backend/access/transam/xlogutils.c | 17 +-
src/backend/replication/logical/logical.c | 26 +-
.../replication/logical/logicalfuncs.c | 13 +-
src/backend/replication/slotfuncs.c | 18 +-
src/backend/replication/walsender.c | 32 +-
src/bin/pg_rewind/parsexlog.c | 84 ++-
src/bin/pg_waldump/pg_waldump.c | 36 +-
src/include/access/xlogreader.h | 121 ++-
src/include/access/xlogutils.h | 4 +-
src/include/pg_config_manual.h | 2 +-
src/include/replication/logical.h | 11 +-
14 files changed, 646 insertions(+), 488 deletions(-)
diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index 80d2d20d6c..7d5a96a5f5 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -1325,11 +1325,8 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
XLogReaderState *xlogreader;
char *errormsg;
- xlogreader = XLogReaderAllocate(wal_segment_size, NULL,
- XL_ROUTINE(.page_read = &read_local_xlog_page,
- .segment_open = &wal_segment_open,
- .segment_close = &wal_segment_close),
- NULL);
+ xlogreader = XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+
if (!xlogreader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
@@ -1337,7 +1334,13 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
errdetail("Failed while allocating a WAL reading processor.")));
XLogBeginRead(xlogreader, lsn);
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!read_local_xlog_page(xlogreader))
+ break;
+ }
+
if (record == NULL)
ereport(ERROR,
(errcode_for_file_access(),
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 606328a65a..670996ae67 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -836,13 +836,6 @@ static XLogSource currentSource = XLOG_FROM_ANY;
static bool lastSourceFailed = false;
static bool pendingWalRcvRestart = false;
-typedef struct XLogPageReadPrivate
-{
- int emode;
- bool fetching_ckpt; /* are we fetching a checkpoint record? */
- bool randAccess;
-} XLogPageReadPrivate;
-
/*
* These variables track when we last obtained some WAL data to process,
* and where we got it from. (XLogReceiptSource is initially the same as
@@ -917,8 +910,8 @@ static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath,
static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli,
XLogSource source, bool notfoundOk);
static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, XLogSource source);
-static bool XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
+static bool XLogPageRead(XLogReaderState *xlogreader,
+ bool fetching_ckpt, int emode, bool randAccess);
static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
bool fetching_ckpt, XLogRecPtr tliRecPtr);
static int emode_for_corrupt_record(int emode, XLogRecPtr RecPtr);
@@ -1231,8 +1224,7 @@ XLogInsertRecord(XLogRecData *rdata,
appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len);
if (!debug_reader)
- debug_reader = XLogReaderAllocate(wal_segment_size, NULL,
- XL_ROUTINE(), NULL);
+ debug_reader = XLogReaderAllocate(wal_segment_size, NULL, NULL);
if (!debug_reader)
{
@@ -4344,15 +4336,10 @@ CleanupBackupHistory(void)
* record is available.
*/
static XLogRecord *
-ReadRecord(XLogReaderState *xlogreader, int emode,
- bool fetching_ckpt)
+ReadRecord(XLogReaderState *xlogreader, int emode, bool fetching_ckpt)
{
XLogRecord *record;
- XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
-
- private->fetching_ckpt = fetching_ckpt;
- private->emode = emode;
- private->randAccess = (xlogreader->ReadRecPtr == InvalidXLogRecPtr);
+ bool randAccess = (xlogreader->ReadRecPtr == InvalidXLogRecPtr);
/* This is the first attempt to read this page. */
lastSourceFailed = false;
@@ -4360,8 +4347,16 @@ ReadRecord(XLogReaderState *xlogreader, int emode,
for (;;)
{
char *errormsg;
+ XLogReadRecordResult result;
+
+ while ((result = XLogReadRecord(xlogreader, &record, &errormsg))
+ == XLREAD_NEED_DATA)
+ {
+ if (!XLogPageRead(xlogreader, fetching_ckpt, emode, randAccess))
+ break;
+
+ }
- record = XLogReadRecord(xlogreader, &errormsg);
ReadRecPtr = xlogreader->ReadRecPtr;
EndRecPtr = xlogreader->EndRecPtr;
if (record == NULL)
@@ -6380,7 +6375,6 @@ StartupXLOG(void)
bool backupFromStandby = false;
DBState dbstate_at_startup;
XLogReaderState *xlogreader;
- XLogPageReadPrivate private;
bool promoted = false;
struct stat st;
@@ -6539,13 +6533,9 @@ StartupXLOG(void)
OwnLatch(&XLogCtl->recoveryWakeupLatch);
/* Set up XLOG reader facility */
- MemSet(&private, 0, sizeof(XLogPageReadPrivate));
xlogreader =
- XLogReaderAllocate(wal_segment_size, NULL,
- XL_ROUTINE(.page_read = &XLogPageRead,
- .segment_open = NULL,
- .segment_close = wal_segment_close),
- &private);
+ XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+
if (!xlogreader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
@@ -11987,12 +11977,13 @@ CancelBackup(void)
* sleep and retry.
*/
static bool
-XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetRecPtr, char *readBuf)
+XLogPageRead(XLogReaderState *xlogreader,
+ bool fetching_ckpt, int emode, bool randAccess)
{
- XLogPageReadPrivate *private =
- (XLogPageReadPrivate *) xlogreader->private_data;
- int emode = private->emode;
+ char *readBuf = xlogreader->readBuf;
+ XLogRecPtr targetPagePtr = xlogreader->readPagePtr;
+ int reqLen = xlogreader->readLen;
+ XLogRecPtr targetRecPtr = xlogreader->ReadRecPtr;
uint32 targetPageOff;
XLogSegNo targetSegNo PG_USED_FOR_ASSERTS_ONLY;
int r;
@@ -12035,8 +12026,8 @@ retry:
flushedUpto < targetPagePtr + reqLen))
{
if (!WaitForWALToBecomeAvailable(targetPagePtr + reqLen,
- private->randAccess,
- private->fetching_ckpt,
+ randAccess,
+ fetching_ckpt,
targetRecPtr))
{
if (readFile >= 0)
@@ -12141,6 +12132,7 @@ retry:
goto next_record_is_invalid;
}
+ Assert(xlogreader->readPagePtr == targetPagePtr);
xlogreader->readLen = readLen;
return true;
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index 7d04995b7b..ba3e4868f7 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -40,7 +40,7 @@ static bool XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr,
int reqLen, bool header_inclusive);
static void XLogReaderInvalReadState(XLogReaderState *state);
static bool ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
- XLogRecPtr PrevRecPtr, XLogRecord *record, bool randAccess);
+ XLogRecPtr PrevRecPtr, XLogRecord *record);
static bool ValidXLogRecord(XLogReaderState *state, XLogRecord *record,
XLogRecPtr recptr);
static void ResetDecoder(XLogReaderState *state);
@@ -73,7 +73,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...)
*/
XLogReaderState *
XLogReaderAllocate(int wal_segment_size, const char *waldir,
- XLogReaderRoutine *routine, void *private_data)
+ WALSegmentCleanupCB cleanup_cb)
{
XLogReaderState *state;
@@ -84,7 +84,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
return NULL;
/* initialize caller-provided support functions */
- state->routine = *routine;
+ state->cleanup_cb = cleanup_cb;
state->max_block_id = -1;
@@ -107,8 +107,6 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
WALOpenSegmentInit(&state->seg, &state->segcxt, wal_segment_size,
waldir);
- /* system_identifier initialized to zeroes above */
- state->private_data = private_data;
/* ReadRecPtr, EndRecPtr and readLen initialized to zeroes above */
state->errormsg_buf = palloc_extended(MAX_ERRORMSG_LEN + 1,
MCXT_ALLOC_NO_OOM);
@@ -140,8 +138,8 @@ XLogReaderFree(XLogReaderState *state)
{
int block_id;
- if (state->seg.ws_file != -1)
- state->routine.segment_close(state);
+ if (state->seg.ws_file >= 0)
+ state->cleanup_cb(state);
for (block_id = 0; block_id <= XLR_MAX_BLOCK_ID; block_id++)
{
@@ -246,6 +244,7 @@ XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
/* Begin at the passed-in record pointer. */
state->EndRecPtr = RecPtr;
state->ReadRecPtr = InvalidXLogRecPtr;
+ state->readRecordState = XLREAD_NEXT_RECORD;
}
/*
@@ -254,317 +253,449 @@ XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
* XLogBeginRead() or XLogFindNextRecord() must be called before the first call
* to XLogReadRecord().
*
- * If the page_read callback fails to read the requested data, NULL is
- * returned. The callback is expected to have reported the error; errormsg
- * is set to NULL.
+ * This function may return XLREAD_NEED_DATA several times before returning a
+ * result record. The caller shall read in some new data then call this
+ * function again with the same parameters.
*
- * If the reading fails for some other reason, NULL is also returned, and
- * *errormsg is set to a string with details of the failure.
+ * When a record is successfully read, returns XLREAD_SUCCESS with result
+ * record being stored in *record. Otherwise *record is NULL.
*
- * The returned pointer (or *errormsg) points to an internal buffer that's
- * valid until the next call to XLogReadRecord.
+ * Returns XLREAD_NEED_DATA if more data is needed to finish reading the
+ * current record. In that case, state->readPagePtr and state->readLen inform
+ * the desired position and minimum length of data needed. The caller shall
+ * read in the requested data and set state->readBuf to point to a buffer
+ * containing it. The caller must also set state->readPageTLI and
+ * state->readLen to indicate the timeline that it was read from, and the
+ * length of data that is now available (which must be >= given readLen),
+ * respectively.
+ *
+ * If invalid data is encountered, returns XLREAD_FAIL with *record being set to
+ * NULL. *errormsg is set to a string with details of the failure.
+ * The returned pointer (or *errormsg) points to an internal buffer that's valid
+ * until the next call to XLogReadRecord.
+ *
+ *
+ * This function runs a state machine consists of the following states.
+ *
+ * XLREAD_NEXT_RECORD :
+ * The initial state, if called with valid RecPtr, try to read a record at
+ * that position. If invalid RecPtr is given try to read a record just after
+ * the last one previously read.
+ * This state ens after setting ReadRecPtr. Then goes to XLREAD_TOT_LEN.
+ *
+ * XLREAD_TOT_LEN:
+ * Examining record header. Ends after reading record total
+ * length. recordRemainLen and recordGotLen are initialized.
+ *
+ * XLREAD_FIRST_FRAGMENT:
+ * Reading the first fragment. Ends with finishing reading a single
+ * record. Goes to XLREAD_NEXT_RECORD if that's all or
+ * XLREAD_CONTINUATION if we have continuation.
+
+ * XLREAD_CONTINUATION:
+ * Reading continuation of record. Ends with finishing the whole record then
+ * goes to XLREAD_NEXT_RECORD. During this state, recordRemainLen indicates
+ * how much is left and readRecordBuf holds the partially assert
+ * record.recordContRecPtr points to the beginning of the next page where to
+ * continue.
+ *
+ * If wrong data found in any state, the state machine stays at the current
+ * state. This behavior allows to continue reading a reacord switching among
+ * different souces, while streaming replication.
*/
-XLogRecord *
-XLogReadRecord(XLogReaderState *state, char **errormsg)
+XLogReadRecordResult
+XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
{
- XLogRecPtr RecPtr;
- XLogRecord *record;
- XLogRecPtr targetPagePtr;
- bool randAccess;
- uint32 len,
- total_len;
- uint32 targetRecOff;
- uint32 pageHeaderSize;
- bool gotheader;
+ XLogRecord *prec;
- /*
- * randAccess indicates whether to verify the previous-record pointer of
- * the record we're reading. We only do this if we're reading
- * sequentially, which is what we initially assume.
- */
- randAccess = false;
+ *record = NULL;
/* reset error state */
*errormsg = NULL;
state->errormsg_buf[0] = '\0';
- ResetDecoder(state);
-
- RecPtr = state->EndRecPtr;
-
- if (state->ReadRecPtr != InvalidXLogRecPtr)
- {
- /* read the record after the one we just read */
-
- /*
- * EndRecPtr is pointing to end+1 of the previous WAL record. If
- * we're at a page boundary, no more records can fit on the current
- * page. We must skip over the page header, but we can't do that until
- * we've read in the page, since the header size is variable.
- */
- }
- else
- {
- /*
- * Caller supplied a position to start at.
- *
- * In this case, EndRecPtr should already be pointing to a valid
- * record starting position.
- */
- Assert(XRecOffIsValid(RecPtr));
- randAccess = true;
- }
-
- state->currRecPtr = RecPtr;
-
- targetPagePtr = RecPtr - (RecPtr % XLOG_BLCKSZ);
- targetRecOff = RecPtr % XLOG_BLCKSZ;
-
- /*
- * Read the page containing the record into state->readBuf. Request enough
- * byte to cover the whole record header, or at least the part of it that
- * fits on the same page.
- */
- while (XLogNeedData(state, targetPagePtr,
- Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
- targetRecOff != 0))
- {
- if (!state->routine.page_read(state, state->readPagePtr, state->readLen,
- RecPtr, state->readBuf))
- break;
- }
-
- if (!state->page_verified)
- goto err;
-
- /*
- * We have at least the page header, so we can examine it now.
- */
- pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
- if (targetRecOff == 0)
- {
- /*
- * At page start, so skip over page header.
- */
- RecPtr += pageHeaderSize;
- targetRecOff = pageHeaderSize;
- }
- else if (targetRecOff < pageHeaderSize)
- {
- report_invalid_record(state, "invalid record offset at %X/%X",
- LSN_FORMAT_ARGS(RecPtr));
- goto err;
- }
-
- if ((((XLogPageHeader) state->readBuf)->xlp_info & XLP_FIRST_IS_CONTRECORD) &&
- targetRecOff == pageHeaderSize)
- {
- report_invalid_record(state, "contrecord is requested by %X/%X",
- LSN_FORMAT_ARGS(RecPtr));
- goto err;
- }
-
- /* XLogNeedData has verified the page header */
- Assert(pageHeaderSize <= state->readLen);
-
- /*
- * Read the record length.
- *
- * NB: Even though we use an XLogRecord pointer here, the whole record
- * header might not fit on this page. xl_tot_len is the first field of the
- * struct, so it must be on this page (the records are MAXALIGNed), but we
- * cannot access any other fields until we've verified that we got the
- * whole header.
- */
- record = (XLogRecord *) (state->readBuf + RecPtr % XLOG_BLCKSZ);
- total_len = record->xl_tot_len;
-
- /*
- * If the whole record header is on this page, validate it immediately.
- * Otherwise do just a basic sanity check on xl_tot_len, and validate the
- * rest of the header after reading it from the next page. The xl_tot_len
- * check is necessary here to ensure that we enter the "Need to reassemble
- * record" code path below; otherwise we might fail to apply
- * ValidXLogRecordHeader at all.
- */
- if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
- {
- if (!ValidXLogRecordHeader(state, RecPtr, state->ReadRecPtr, record,
- randAccess))
- goto err;
- gotheader = true;
- }
- else
- {
- /* XXX: more validation should be done here */
- if (total_len < SizeOfXLogRecord)
- {
- report_invalid_record(state,
- "invalid record length at %X/%X: wanted %u, got %u",
- LSN_FORMAT_ARGS(RecPtr),
- (uint32) SizeOfXLogRecord, total_len);
- goto err;
- }
- gotheader = false;
- }
-
- len = XLOG_BLCKSZ - RecPtr % XLOG_BLCKSZ;
- if (total_len > len)
+ switch (state->readRecordState)
{
- /* Need to reassemble record */
- char *contdata;
- XLogPageHeader pageHeader;
- char *buffer;
- uint32 gotlen;
-
- /*
- * Enlarge readRecordBuf as needed.
- */
- if (total_len > state->readRecordBufSize &&
- !allocate_recordbuf(state, total_len))
- {
- /* We treat this as a "bogus data" condition */
- report_invalid_record(state, "record length %u at %X/%X too long",
- total_len, LSN_FORMAT_ARGS(RecPtr));
- goto err;
- }
+ case XLREAD_NEXT_RECORD:
+ ResetDecoder(state);
- /* Copy the first fragment of the record from the first page. */
- memcpy(state->readRecordBuf,
- state->readBuf + RecPtr % XLOG_BLCKSZ, len);
- buffer = state->readRecordBuf + len;
- gotlen = len;
-
- do
- {
- int rest_len = total_len - gotlen;
-
- /* Calculate pointer to beginning of next page */
- targetPagePtr += XLOG_BLCKSZ;
-
- /* Wait for the next page to become available */
- while (XLogNeedData(state, targetPagePtr,
- Min(rest_len, XLOG_BLCKSZ),
- false))
+ if (state->ReadRecPtr != InvalidXLogRecPtr)
{
- if (!state->routine.page_read(state, state->readPagePtr,
- state->readLen,
- state->ReadRecPtr,
- state->readBuf))
- break;
+ /* read the record after the one we just read */
+
+ /*
+ * EndRecPtr is pointing to end+1 of the previous WAL record.
+ * If we're at a page boundary, no more records can fit on the
+ * current page. We must skip over the page header, but we
+ * can't do that until we've read in the page, since the header
+ * size is variable.
+ */
+ state->PrevRecPtr = state->ReadRecPtr;
+ state->ReadRecPtr = state->EndRecPtr;
+ }
+ else
+ {
+ /*
+ * Caller supplied a position to start at.
+ *
+ * In this case, EndRecPtr should already be pointing to a
+ * valid record starting position.
+ */
+ Assert(XRecOffIsValid(state->EndRecPtr));
+ state->ReadRecPtr = state->EndRecPtr;
+
+ /*
+ * We cannot verify the previous-record pointer when we're
+ * seeking to a particular record. Reset PrevRecPtr so that we
+ * won't try doing that.
+ */
+ state->PrevRecPtr = InvalidXLogRecPtr;
+ state->EndRecPtr = InvalidXLogRecPtr; /* to be tidy */
}
+ state->record_verified = false;
+ state->readRecordState = XLREAD_TOT_LEN;
+ /* fall through */
+
+ case XLREAD_TOT_LEN:
+ {
+ uint32 total_len;
+ uint32 pageHeaderSize;
+ XLogRecPtr targetPagePtr;
+ uint32 targetRecOff;
+ XLogPageHeader pageHeader;
+
+ targetPagePtr =
+ state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
+ targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+
+ /*
+ * Check if we have enough data. For the first record in the page,
+ * the requesting length doesn't contain page header.
+ */
+ if (XLogNeedData(state, targetPagePtr,
+ Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+ targetRecOff != 0))
+ return XLREAD_NEED_DATA;
+
+ /* error out if caller supplied bogus page */
if (!state->page_verified)
goto err;
- Assert(SizeOfXLogShortPHD <= state->readLen);
+ /* examine page header now. */
+ pageHeaderSize =
+ XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+ if (targetRecOff == 0)
+ {
+ /* At page start, so skip over page header. */
+ state->ReadRecPtr += pageHeaderSize;
+ targetRecOff = pageHeaderSize;
+ }
+ else if (targetRecOff < pageHeaderSize)
+ {
+ report_invalid_record(state, "invalid record offset at %X/%X",
+ LSN_FORMAT_ARGS(state->ReadRecPtr));
+ goto err;
+ }
- /* Check that the continuation on next page looks valid */
pageHeader = (XLogPageHeader) state->readBuf;
- if (!(pageHeader->xlp_info & XLP_FIRST_IS_CONTRECORD))
+ if ((pageHeader->xlp_info & XLP_FIRST_IS_CONTRECORD) &&
+ targetRecOff == pageHeaderSize)
{
- report_invalid_record(state,
- "there is no contrecord flag at %X/%X",
- LSN_FORMAT_ARGS(RecPtr));
+ report_invalid_record(state, "contrecord is requested by %X/%X",
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr);
goto err;
}
- /*
- * Cross-check that xlp_rem_len agrees with how much of the record
- * we expect there to be left.
- */
- if (pageHeader->xlp_rem_len == 0 ||
- total_len != (pageHeader->xlp_rem_len + gotlen))
- {
- report_invalid_record(state,
- "invalid contrecord length %u (expected %lld) at %X/%X",
- pageHeader->xlp_rem_len,
- ((long long) total_len) - gotlen,
- LSN_FORMAT_ARGS(RecPtr));
- goto err;
- }
-
- /* Append the continuation from this page to the buffer */
- pageHeaderSize = XLogPageHeaderSize(pageHeader);
-
+ /* XLogNeedData has verified the page header */
Assert(pageHeaderSize <= state->readLen);
- contdata = (char *) state->readBuf + pageHeaderSize;
- len = XLOG_BLCKSZ - pageHeaderSize;
- if (pageHeader->xlp_rem_len < len)
- len = pageHeader->xlp_rem_len;
+ /*
+ * Read the record length.
+ *
+ * NB: Even though we use an XLogRecord pointer here, the whole
+ * record header might not fit on this page. xl_tot_len is the first
+ * field of the struct, so it must be on this page (the records are
+ * MAXALIGNed), but we cannot access any other fields until we've
+ * verified that we got the whole header.
+ */
+ prec = (XLogRecord *) (state->readBuf +
+ state->ReadRecPtr % XLOG_BLCKSZ);
+ total_len = prec->xl_tot_len;
- Assert (pageHeaderSize + len <= state->readLen);
- memcpy(buffer, (char *) contdata, len);
- buffer += len;
- gotlen += len;
+ /*
+ * If the whole record header is on this page, validate it
+ * immediately. Otherwise do just a basic sanity check on
+ * xl_tot_len, and validate the rest of the header after reading it
+ * from the next page. The xl_tot_len check is necessary here to
+ * ensure that we enter the XLREAD_CONTINUATION state below;
+ * otherwise we might fail to apply ValidXLogRecordHeader at all.
+ */
+ if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
+ {
+ if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
+ state->PrevRecPtr, prec))
+ goto err;
- /* If we just reassembled the record header, validate it. */
- if (!gotheader)
+ state->record_verified = true;
+ }
+ else
{
- record = (XLogRecord *) state->readRecordBuf;
- if (!ValidXLogRecordHeader(state, RecPtr, state->ReadRecPtr,
- record, randAccess))
+ /* XXX: more validation should be done here */
+ if (total_len < SizeOfXLogRecord)
+ {
+ report_invalid_record(state,
+ "invalid record length at %X/%X: wanted %u, got %u",
+ LSN_FORMAT_ARGS(state->ReadRecPtr),
+ (uint32) SizeOfXLogRecord, total_len);
goto err;
- gotheader = true;
+ }
}
- } while (gotlen < total_len);
-
- Assert(gotheader);
- record = (XLogRecord *) state->readRecordBuf;
- if (!ValidXLogRecord(state, record, RecPtr))
- goto err;
+ /*
+ * Wait for the rest of the record, or the part of the record that
+ * fit on the first page if crossed a page boundary, to become
+ * available.
+ */
+ state->recordGotLen = 0;
+ state->recordRemainLen = total_len;
+ state->readRecordState = XLREAD_FIRST_FRAGMENT;
+ }
+ /* fall through */
- pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
- state->ReadRecPtr = RecPtr;
- state->EndRecPtr = targetPagePtr + pageHeaderSize
- + MAXALIGN(pageHeader->xlp_rem_len);
- }
- else
- {
- /* Wait for the record data to become available */
- while (XLogNeedData(state, targetPagePtr,
- Min(targetRecOff + total_len, XLOG_BLCKSZ), true))
+ case XLREAD_FIRST_FRAGMENT:
{
- if (!state->routine.page_read(state, state->readPagePtr,
- state->readLen,
- state->ReadRecPtr, state->readBuf))
+ uint32 total_len = state->recordRemainLen;
+ uint32 request_len;
+ uint32 record_len;
+ XLogRecPtr targetPagePtr;
+ uint32 targetRecOff;
+
+ /*
+ * Wait for the rest of the record on the first page to become
+ * available
+ */
+ targetPagePtr =
+ state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
+ targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+
+ request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ);
+ record_len = request_len - targetRecOff;
+
+ /* ReadRecPtr contains page header */
+ Assert (targetRecOff != 0);
+ if (XLogNeedData(state, targetPagePtr, request_len, true))
+ return XLREAD_NEED_DATA;
+
+ /* error out if caller supplied bogus page */
+ if (!state->page_verified)
+ goto err;
+
+ prec = (XLogRecord *) (state->readBuf + targetRecOff);
+
+ /* validate record header if not yet */
+ if (!state->record_verified && record_len >= SizeOfXLogRecord)
+ {
+ if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
+ state->PrevRecPtr, prec))
+ goto err;
+
+ state->record_verified = true;
+ }
+
+
+ if (total_len == record_len)
+ {
+ /* Record does not cross a page boundary */
+ Assert(state->record_verified);
+
+ if (!ValidXLogRecord(state, prec, state->ReadRecPtr))
+ goto err;
+
+ state->record_verified = true; /* to be tidy */
+
+ /* We already checked the header earlier */
+ state->EndRecPtr = state->ReadRecPtr + MAXALIGN(record_len);
+
+ *record = prec;
+ state->readRecordState = XLREAD_NEXT_RECORD;
break;
+ }
+
+ /*
+ * The record continues on the next page. Need to reassemble
+ * record
+ */
+ Assert(total_len > record_len);
+
+ /* Enlarge readRecordBuf as needed. */
+ if (total_len > state->readRecordBufSize &&
+ !allocate_recordbuf(state, total_len))
+ {
+ /* We treat this as a "bogus data" condition */
+ report_invalid_record(state,
+ "record length %u at %X/%X too long",
+ total_len,
+ LSN_FORMAT_ARGS(state->ReadRecPtr));
+ goto err;
+ }
+
+ /* Copy the first fragment of the record from the first page. */
+ memcpy(state->readRecordBuf, state->readBuf + targetRecOff,
+ record_len);
+ state->recordGotLen += record_len;
+ state->recordRemainLen -= record_len;
+
+ /* Calculate pointer to beginning of next page */
+ state->recordContRecPtr = state->ReadRecPtr + record_len;
+ Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0);
+
+ state->readRecordState = XLREAD_CONTINUATION;
}
+ /* fall through */
+
+ case XLREAD_CONTINUATION:
+ {
+ XLogPageHeader pageHeader;
+ uint32 pageHeaderSize;
+ XLogRecPtr targetPagePtr;
+
+ /* we enter this state only if we haven't read the whole record. */
+ Assert (state->recordRemainLen > 0);
+
+ while(state->recordRemainLen > 0)
+ {
+ char *contdata;
+ uint32 request_len;
+ uint32 record_len;
+
+ /* Wait for the next page to become available */
+ targetPagePtr = state->recordContRecPtr;
+
+ /* this request contains page header */
+ Assert (targetPagePtr != 0);
+ if (XLogNeedData(state, targetPagePtr,
+ Min(state->recordRemainLen, XLOG_BLCKSZ),
+ false))
+ return XLREAD_NEED_DATA;
+
+ if (!state->page_verified)
+ goto err;
+
+ Assert(SizeOfXLogShortPHD <= state->readLen);
- if (!state->page_verified)
- goto err;
+ /* Check that the continuation on next page looks valid */
+ pageHeader = (XLogPageHeader) state->readBuf;
+ if (!(pageHeader->xlp_info & XLP_FIRST_IS_CONTRECORD))
+ {
+ report_invalid_record(
+ state,
+ "there is no contrecord flag at %X/%X reading %X/%X",
+ (uint32) (state->recordContRecPtr >> 32),
+ (uint32) state->recordContRecPtr,
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr);
+ goto err;
+ }
- /* Record does not cross a page boundary */
- if (!ValidXLogRecord(state, record, RecPtr))
- goto err;
+ /*
+ * Cross-check that xlp_rem_len agrees with how much of the
+ * record we expect there to be left.
+ */
+ if (pageHeader->xlp_rem_len == 0 ||
+ pageHeader->xlp_rem_len != state->recordRemainLen)
+ {
+ report_invalid_record(
+ state,
+ "invalid contrecord length %u at %X/%X reading %X/%X, expected %u",
+ pageHeader->xlp_rem_len,
+ (uint32) (state->recordContRecPtr >> 32),
+ (uint32) state->recordContRecPtr,
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr,
+ state->recordRemainLen);
+ goto err;
+ }
- state->EndRecPtr = RecPtr + MAXALIGN(total_len);
+ /* Append the continuation from this page to the buffer */
+ pageHeaderSize = XLogPageHeaderSize(pageHeader);
- state->ReadRecPtr = RecPtr;
+ /*
+ * XLogNeedData should have ensured that the whole page header
+ * was read
+ */
+ Assert(state->readLen >= pageHeaderSize);
+
+ contdata = (char *) state->readBuf + pageHeaderSize;
+ record_len = XLOG_BLCKSZ - pageHeaderSize;
+ if (pageHeader->xlp_rem_len < record_len)
+ record_len = pageHeader->xlp_rem_len;
+
+ request_len = record_len + pageHeaderSize;
+
+ /* XLogNeedData should have ensured all needed data was read */
+ Assert (state->readLen >= request_len);
+
+ memcpy(state->readRecordBuf + state->recordGotLen,
+ (char *) contdata, record_len);
+ state->recordGotLen += record_len;
+ state->recordRemainLen -= record_len;
+
+ /* If we just reassembled the record header, validate it. */
+ if (!state->record_verified)
+ {
+ Assert(state->recordGotLen >= SizeOfXLogRecord);
+ if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
+ state->PrevRecPtr,
+ (XLogRecord *) state->readRecordBuf))
+ goto err;
+
+ state->record_verified = true;
+ }
+
+ /* Calculate pointer to beginning of next page, and continue */
+ state->recordContRecPtr += XLOG_BLCKSZ;
+ }
+
+ /* targetPagePtr is pointing the last-read page here */
+ prec = (XLogRecord *) state->readRecordBuf;
+ if (!ValidXLogRecord(state, prec, state->ReadRecPtr))
+ goto err;
+
+ pageHeaderSize =
+ XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+ state->EndRecPtr = targetPagePtr + pageHeaderSize
+ + MAXALIGN(pageHeader->xlp_rem_len);
+
+ *record = prec;
+ state->readRecordState = XLREAD_NEXT_RECORD;
+ break;
+ }
}
/*
* Special processing if it's an XLOG SWITCH record
*/
- if (record->xl_rmid == RM_XLOG_ID &&
- (record->xl_info & ~XLR_INFO_MASK) == XLOG_SWITCH)
+ if ((*record)->xl_rmid == RM_XLOG_ID &&
+ ((*record)->xl_info & ~XLR_INFO_MASK) == XLOG_SWITCH)
{
/* Pretend it extends to end of segment */
state->EndRecPtr += state->segcxt.ws_segsize - 1;
state->EndRecPtr -= XLogSegmentOffset(state->EndRecPtr, state->segcxt.ws_segsize);
}
- if (DecodeXLogRecord(state, record, errormsg))
- return record;
- else
- return NULL;
+ Assert (!*record || state->readLen >= 0);
+ if (DecodeXLogRecord(state, *record, errormsg))
+ return XLREAD_SUCCESS;
+
+ *record = NULL;
+ return XLREAD_FAIL;
err:
/*
- * Invalidate the read state. We might read from a different source after
+ * Invalidate the read page. We might read from a different source after
* failure.
*/
XLogReaderInvalReadState(state);
@@ -572,7 +703,8 @@ err:
if (state->errormsg_buf[0] != '\0')
*errormsg = state->errormsg_buf;
- return NULL;
+ *record = NULL;
+ return XLREAD_FAIL;
}
/*
@@ -724,11 +856,12 @@ XLogReaderInvalReadState(XLogReaderState *state)
*
* This is just a convenience subroutine to avoid duplicated code in
* XLogReadRecord. It's not intended for use from anywhere else.
+ *
+ * If PrevRecPtr is valid, the xl_prev is is cross-checked with it.
*/
static bool
ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
- XLogRecPtr PrevRecPtr, XLogRecord *record,
- bool randAccess)
+ XLogRecPtr PrevRecPtr, XLogRecord *record)
{
if (record->xl_tot_len < SizeOfXLogRecord)
{
@@ -745,7 +878,7 @@ ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
record->xl_rmid, LSN_FORMAT_ARGS(RecPtr));
return false;
}
- if (randAccess)
+ if (PrevRecPtr == InvalidXLogRecPtr)
{
/*
* We can't exactly verify the prev-link, but surely it should be less
@@ -975,11 +1108,14 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
* XLogReadRecord() will read the next valid record.
*/
XLogRecPtr
-XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
+XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
+ XLogFindNextRecordCB read_page, void *private)
{
XLogRecPtr tmpRecPtr;
XLogRecPtr found = InvalidXLogRecPtr;
XLogPageHeader header;
+ XLogRecord *record;
+ XLogReadRecordResult result;
char *errormsg;
Assert(!XLogRecPtrIsInvalid(RecPtr));
@@ -1012,9 +1148,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
while(XLogNeedData(state, targetPagePtr, targetRecOff,
targetRecOff != 0))
{
- if (!state->routine.page_read(state, state->readPagePtr,
- state->readLen,
- state->ReadRecPtr, state->readBuf))
+ if (!read_page(state, private))
break;
}
@@ -1066,8 +1200,16 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
* or we just jumped over the remaining data of a continuation.
*/
XLogBeginRead(state, tmpRecPtr);
- while (XLogReadRecord(state, &errormsg) != NULL)
+ while ((result = XLogReadRecord(state, &record, &errormsg)) !=
+ XLREAD_FAIL)
{
+ if (result == XLREAD_NEED_DATA)
+ {
+ if (!read_page(state, private))
+ goto err;
+ continue;
+ }
+
/* past the record we've found, break out */
if (RecPtr <= state->ReadRecPtr)
{
@@ -1087,9 +1229,9 @@ err:
#endif /* FRONTEND */
/*
- * Helper function to ease writing of XLogRoutine->page_read callbacks.
- * If this function is used, caller must supply a segment_open callback in
- * 'state', as that is used here.
+ * Helper function to ease writing of page_read callback.
+ * If this function is used, caller must supply a segment_open callback and
+ * segment_close callback as that is used here.
*
* Read 'count' bytes into 'buf', starting at location 'startptr', from WAL
* fetched from timeline 'tli'.
@@ -1102,6 +1244,7 @@ err:
*/
bool
WALRead(XLogReaderState *state,
+ WALSegmentOpenCB segopenfn, WALSegmentCloseCB segclosefn,
char *buf, XLogRecPtr startptr, Size count, TimeLineID tli,
WALReadError *errinfo)
{
@@ -1133,10 +1276,10 @@ WALRead(XLogReaderState *state,
XLogSegNo nextSegNo;
if (state->seg.ws_file >= 0)
- state->routine.segment_close(state);
+ segclosefn(state);
XLByteToSeg(recptr, nextSegNo, state->segcxt.ws_segsize);
- state->routine.segment_open(state, nextSegNo, &tli);
+ segopenfn(state, nextSegNo, &tli);
/* This shouldn't happen -- indicates a bug in segment_open */
Assert(state->seg.ws_file >= 0);
diff --git a/src/backend/access/transam/xlogutils.c b/src/backend/access/transam/xlogutils.c
index 46eda33f25..b003990745 100644
--- a/src/backend/access/transam/xlogutils.c
+++ b/src/backend/access/transam/xlogutils.c
@@ -686,8 +686,7 @@ XLogTruncateRelation(RelFileNode rnode, ForkNumber forkNum,
void
XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wantLength)
{
- const XLogRecPtr lastReadPage = state->seg.ws_segno *
- state->segcxt.ws_segsize + state->readLen;
+ const XLogRecPtr lastReadPage = state->readPagePtr;
Assert(wantPage != InvalidXLogRecPtr && wantPage % XLOG_BLCKSZ == 0);
Assert(wantLength <= XLOG_BLCKSZ);
@@ -702,7 +701,7 @@ XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wa
* current TLI has since become historical.
*/
if (lastReadPage == wantPage &&
- state->readLen != 0 &&
+ state->page_verified &&
lastReadPage + state->readLen >= wantPage + Min(wantLength, XLOG_BLCKSZ - 1))
return;
@@ -788,6 +787,7 @@ wal_segment_open(XLogReaderState *state, XLogSegNo nextSegNo,
char path[MAXPGPATH];
XLogFilePath(path, tli, nextSegNo, state->segcxt.ws_segsize);
+ elog(LOG, "HOGE: %lu, %d => %s", nextSegNo, tli, path);
state->seg.ws_file = BasicOpenFile(path, O_RDONLY | PG_BINARY);
if (state->seg.ws_file >= 0)
return;
@@ -825,9 +825,11 @@ wal_segment_close(XLogReaderState *state)
* loop for now.
*/
bool
-read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *cur_page)
+read_local_xlog_page(XLogReaderState *state)
{
+ XLogRecPtr targetPagePtr = state->readPagePtr;
+ int reqLen = state->readLen;
+ char *cur_page = state->readBuf;
XLogRecPtr read_upto,
loc;
TimeLineID tli;
@@ -940,11 +942,12 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
* as 'count', read the whole page anyway. It's guaranteed to be
* zero-padded up to the page boundary if it's incomplete.
*/
- if (!WALRead(state, cur_page, targetPagePtr, XLOG_BLCKSZ, tli,
- &errinfo))
+ if (!WALRead(state, wal_segment_open, wal_segment_close,
+ cur_page, targetPagePtr, XLOG_BLCKSZ, tli, &errinfo))
WALReadRaiseError(&errinfo);
/* number of valid bytes in the buffer */
+ state->readPagePtr = targetPagePtr;
state->readLen = count;
return true;
}
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index 37b75deb72..9cbd8afa49 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -148,7 +148,8 @@ StartupDecodingContext(List *output_plugin_options,
TransactionId xmin_horizon,
bool need_full_snapshot,
bool fast_forward,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress)
@@ -198,11 +199,12 @@ StartupDecodingContext(List *output_plugin_options,
ctx->slot = slot;
- ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, xl_routine, ctx);
+ ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, cleanup_cb);
if (!ctx->reader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
errmsg("out of memory")));
+ ctx->page_read = page_read;
ctx->reorder = ReorderBufferAllocate();
ctx->snapshot_builder =
@@ -319,7 +321,8 @@ CreateInitDecodingContext(const char *plugin,
List *output_plugin_options,
bool need_full_snapshot,
XLogRecPtr restart_lsn,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress)
@@ -422,7 +425,7 @@ CreateInitDecodingContext(const char *plugin,
ctx = StartupDecodingContext(NIL, restart_lsn, xmin_horizon,
need_full_snapshot, false,
- xl_routine, prepare_write, do_write,
+ page_read, cleanup_cb, prepare_write, do_write,
update_progress);
/* call output plugin initialization callback */
@@ -476,7 +479,8 @@ LogicalDecodingContext *
CreateDecodingContext(XLogRecPtr start_lsn,
List *output_plugin_options,
bool fast_forward,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress)
@@ -528,8 +532,8 @@ CreateDecodingContext(XLogRecPtr start_lsn,
ctx = StartupDecodingContext(output_plugin_options,
start_lsn, InvalidTransactionId, false,
- fast_forward, xl_routine, prepare_write,
- do_write, update_progress);
+ fast_forward, page_read, cleanup_cb,
+ prepare_write, do_write, update_progress);
/* call output plugin initialization callback */
old_context = MemoryContextSwitchTo(ctx->context);
@@ -585,7 +589,13 @@ DecodingContextFindStartpoint(LogicalDecodingContext *ctx)
char *err = NULL;
/* the read_page callback waits for new WAL */
- record = XLogReadRecord(ctx->reader, &err);
+ while (XLogReadRecord(ctx->reader, &record, &err) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!ctx->page_read(ctx->reader))
+ break;
+ }
+
if (err)
elog(ERROR, "%s", err);
if (!record)
diff --git a/src/backend/replication/logical/logicalfuncs.c b/src/backend/replication/logical/logicalfuncs.c
index f7e055874e..b69f6911c5 100644
--- a/src/backend/replication/logical/logicalfuncs.c
+++ b/src/backend/replication/logical/logicalfuncs.c
@@ -233,9 +233,8 @@ pg_logical_slot_get_changes_guts(FunctionCallInfo fcinfo, bool confirm, bool bin
ctx = CreateDecodingContext(InvalidXLogRecPtr,
options,
false,
- XL_ROUTINE(.page_read = read_local_xlog_page,
- .segment_open = wal_segment_open,
- .segment_close = wal_segment_close),
+ read_local_xlog_page,
+ wal_segment_close,
LogicalOutputPrepareWrite,
LogicalOutputWrite, NULL);
@@ -284,7 +283,13 @@ pg_logical_slot_get_changes_guts(FunctionCallInfo fcinfo, bool confirm, bool bin
XLogRecord *record;
char *errm = NULL;
- record = XLogReadRecord(ctx->reader, &errm);
+ while (XLogReadRecord(ctx->reader, &record, &errm) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!ctx->page_read(ctx->reader))
+ break;
+ }
+
if (errm)
elog(ERROR, "%s", errm);
diff --git a/src/backend/replication/slotfuncs.c b/src/backend/replication/slotfuncs.c
index 9817b44113..deb7cf15eb 100644
--- a/src/backend/replication/slotfuncs.c
+++ b/src/backend/replication/slotfuncs.c
@@ -153,9 +153,8 @@ create_logical_replication_slot(char *name, char *plugin,
ctx = CreateInitDecodingContext(plugin, NIL,
false, /* just catalogs is OK */
restart_lsn,
- XL_ROUTINE(.page_read = read_local_xlog_page,
- .segment_open = wal_segment_open,
- .segment_close = wal_segment_close),
+ read_local_xlog_page,
+ wal_segment_close,
NULL, NULL, NULL);
/*
@@ -512,9 +511,8 @@ pg_logical_replication_slot_advance(XLogRecPtr moveto)
ctx = CreateDecodingContext(InvalidXLogRecPtr,
NIL,
true, /* fast_forward */
- XL_ROUTINE(.page_read = read_local_xlog_page,
- .segment_open = wal_segment_open,
- .segment_close = wal_segment_close),
+ read_local_xlog_page,
+ wal_segment_close,
NULL, NULL, NULL);
/*
@@ -536,7 +534,13 @@ pg_logical_replication_slot_advance(XLogRecPtr moveto)
* Read records. No changes are generated in fast_forward mode,
* but snapbuilder/slot statuses are updated properly.
*/
- record = XLogReadRecord(ctx->reader, &errm);
+ while (XLogReadRecord(ctx->reader, &record, &errm) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!ctx->page_read(ctx->reader))
+ break;
+ }
+
if (errm)
elog(ERROR, "%s", errm);
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index c36b91ae5e..062093727d 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -580,10 +580,7 @@ StartReplication(StartReplicationCmd *cmd)
/* create xlogreader for physical replication */
xlogreader =
- XLogReaderAllocate(wal_segment_size, NULL,
- XL_ROUTINE(.segment_open = WalSndSegmentOpen,
- .segment_close = wal_segment_close),
- NULL);
+ XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
if (!xlogreader)
ereport(ERROR,
@@ -807,9 +804,11 @@ StartReplication(StartReplicationCmd *cmd)
* set every time WAL is flushed.
*/
static bool
-logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetRecPtr, char *cur_page)
+logical_read_xlog_page(XLogReaderState *state)
{
+ XLogRecPtr targetPagePtr = state->readPagePtr;
+ int reqLen = state->readLen;
+ char *cur_page = state->readBuf;
XLogRecPtr flushptr;
int count;
WALReadError errinfo;
@@ -837,7 +836,7 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
count = flushptr - targetPagePtr; /* part of the page available */
/* now actually read the data, we know it's there */
- if (!WALRead(state,
+ if (!WALRead(state, WalSndSegmentOpen, wal_segment_close,
cur_page,
targetPagePtr,
XLOG_BLCKSZ,
@@ -1011,9 +1010,8 @@ CreateReplicationSlot(CreateReplicationSlotCmd *cmd)
ctx = CreateInitDecodingContext(cmd->plugin, NIL, need_full_snapshot,
InvalidXLogRecPtr,
- XL_ROUTINE(.page_read = logical_read_xlog_page,
- .segment_open = WalSndSegmentOpen,
- .segment_close = wal_segment_close),
+ logical_read_xlog_page,
+ wal_segment_close,
WalSndPrepareWrite, WalSndWriteData,
WalSndUpdateProgress);
@@ -1171,9 +1169,8 @@ StartLogicalReplication(StartReplicationCmd *cmd)
*/
logical_decoding_ctx =
CreateDecodingContext(cmd->startpoint, cmd->options, false,
- XL_ROUTINE(.page_read = logical_read_xlog_page,
- .segment_open = WalSndSegmentOpen,
- .segment_close = wal_segment_close),
+ logical_read_xlog_page,
+ wal_segment_close,
WalSndPrepareWrite, WalSndWriteData,
WalSndUpdateProgress);
xlogreader = logical_decoding_ctx->reader;
@@ -2749,7 +2746,7 @@ XLogSendPhysical(void)
enlargeStringInfo(&output_message, nbytes);
retry:
- if (!WALRead(xlogreader,
+ if (!WALRead(xlogreader, WalSndSegmentOpen, wal_segment_close,
&output_message.data[output_message.len],
startptr,
nbytes,
@@ -2847,7 +2844,12 @@ XLogSendLogical(void)
*/
WalSndCaughtUp = false;
- record = XLogReadRecord(logical_decoding_ctx->reader, &errm);
+ while (XLogReadRecord(logical_decoding_ctx->reader, &record, &errm) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!logical_decoding_ctx->page_read(logical_decoding_ctx->reader))
+ break;
+ }
/* xlog record was invalid */
if (errm != NULL)
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index cf119848b0..712c85281c 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -41,15 +41,9 @@ static int xlogreadfd = -1;
static XLogSegNo xlogreadsegno = -1;
static char xlogfpath[MAXPGPATH];
-typedef struct XLogPageReadPrivate
-{
- const char *restoreCommand;
- int tliIndex;
-} XLogPageReadPrivate;
-
-static bool SimpleXLogPageRead(XLogReaderState *xlogreader,
- XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
+static bool SimpleXLogPageRead(XLogReaderState *xlogreader,
+ const char *datadir, int *tliIndex,
+ const char *restoreCommand);
/*
* Read WAL from the datadir/pg_wal, starting from 'startpoint' on timeline
@@ -66,20 +60,22 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
XLogRecord *record;
XLogReaderState *xlogreader;
char *errormsg;
- XLogPageReadPrivate private;
- private.tliIndex = tliIndex;
- private.restoreCommand = restoreCommand;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir,
- XL_ROUTINE(.page_read = &SimpleXLogPageRead),
- &private);
+ xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+
if (xlogreader == NULL)
pg_fatal("out of memory");
XLogBeginRead(xlogreader, startpoint);
do
{
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!SimpleXLogPageRead(xlogreader, datadir,
+ &tliIndex, restoreCommand))
+ break;
+ }
if (record == NULL)
{
@@ -123,19 +119,19 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex,
XLogRecord *record;
XLogReaderState *xlogreader;
char *errormsg;
- XLogPageReadPrivate private;
XLogRecPtr endptr;
- private.tliIndex = tliIndex;
- private.restoreCommand = restoreCommand;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir,
- XL_ROUTINE(.page_read = &SimpleXLogPageRead),
- &private);
+ xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
XLogBeginRead(xlogreader, ptr);
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!SimpleXLogPageRead(xlogreader, datadir, &tliIndex, restoreCommand))
+ break;
+ }
if (record == NULL)
{
if (errormsg)
@@ -170,7 +166,6 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
XLogRecPtr searchptr;
XLogReaderState *xlogreader;
char *errormsg;
- XLogPageReadPrivate private;
/*
* The given fork pointer points to the end of the last common record,
@@ -186,11 +181,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
forkptr += SizeOfXLogShortPHD;
}
- private.tliIndex = tliIndex;
- private.restoreCommand = restoreCommand;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir,
- XL_ROUTINE(.page_read = &SimpleXLogPageRead),
- &private);
+ xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
@@ -200,7 +191,13 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
uint8 info;
XLogBeginRead(xlogreader, searchptr);
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!SimpleXLogPageRead(xlogreader, datadir,
+ &tliIndex, restoreCommand))
+ break;
+ }
if (record == NULL)
{
@@ -247,10 +244,11 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
/* XLogReader callback function, to read a WAL page */
static bool
-SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *readBuf)
+SimpleXLogPageRead(XLogReaderState *xlogreader, const char *datadir,
+ int *tliIndex, const char *restoreCommand)
{
- XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
+ XLogRecPtr targetPagePtr = xlogreader->readPagePtr;
+ char *readBuf = xlogreader->readBuf;
uint32 targetPageOff;
XLogRecPtr targetSegEnd;
XLogSegNo targetSegNo;
@@ -283,14 +281,14 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
* be done both forward and backward, consider also switching timeline
* accordingly.
*/
- while (private->tliIndex < targetNentries - 1 &&
- targetHistory[private->tliIndex].end < targetSegEnd)
- private->tliIndex++;
- while (private->tliIndex > 0 &&
- targetHistory[private->tliIndex].begin >= targetSegEnd)
- private->tliIndex--;
+ while (*tliIndex < targetNentries - 1 &&
+ targetHistory[*tliIndex].end < targetSegEnd)
+ (*tliIndex)++;
+ while (*tliIndex > 0 &&
+ targetHistory[*tliIndex].begin >= targetSegEnd)
+ (*tliIndex)--;
- XLogFileName(xlogfname, targetHistory[private->tliIndex].tli,
+ XLogFileName(xlogfname, targetHistory[*tliIndex].tli,
xlogreadsegno, WalSegSz);
snprintf(xlogfpath, MAXPGPATH, "%s/" XLOGDIR "/%s",
@@ -303,7 +301,7 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
/*
* If we have no restore_command to execute, then exit.
*/
- if (private->restoreCommand == NULL)
+ if (restoreCommand == NULL)
{
pg_log_error("could not open file \"%s\": %m", xlogfpath);
xlogreader->readLen = -1;
@@ -317,7 +315,7 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
xlogreadfd = RestoreArchivedFile(xlogreader->segcxt.ws_dir,
xlogfname,
WalSegSz,
- private->restoreCommand);
+ restoreCommand);
if (xlogreadfd < 0)
{
@@ -359,7 +357,7 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
Assert(targetSegNo == xlogreadsegno);
- xlogreader->seg.ws_tli = targetHistory[private->tliIndex].tli;
+ xlogreader->seg.ws_tli = targetHistory[*tliIndex].tli;
xlogreader->readLen = XLOG_BLCKSZ;
return true;
}
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index 0f0e0723a3..f514de6b13 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -330,12 +330,17 @@ WALDumpCloseSegment(XLogReaderState *state)
state->seg.ws_file = -1;
}
-/* pg_waldump's XLogReaderRoutine->page_read callback */
+/*
+ * pg_waldump's WAL page rader, also used as page_read callback for
+ * XLogFindNextRecord
+ */
static bool
-WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetPtr, char *readBuff)
+WALDumpReadPage(XLogReaderState *state, void *priv)
{
- XLogDumpPrivate *private = state->private_data;
+ XLogRecPtr targetPagePtr = state->readPagePtr;
+ int reqLen = state->readLen;
+ char *readBuff = state->readBuf;
+ XLogDumpPrivate *private = (XLogDumpPrivate *) priv;
int count = XLOG_BLCKSZ;
WALReadError errinfo;
@@ -353,8 +358,8 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
}
}
- if (!WALRead(state, readBuff, targetPagePtr, count, private->timeline,
- &errinfo))
+ if (!WALRead(state, WALDumpOpenSegment, WALDumpCloseSegment,
+ readBuff, targetPagePtr, count, private->timeline, &errinfo))
{
WALOpenSegment *seg = &errinfo.wre_seg;
char fname[MAXPGPATH];
@@ -374,6 +379,7 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
(Size) errinfo.wre_req);
}
+ Assert(count >= state->readLen);
state->readLen = count;
return true;
}
@@ -1035,16 +1041,14 @@ main(int argc, char **argv)
/* we have everything we need, start reading */
xlogreader_state =
- XLogReaderAllocate(WalSegSz, waldir,
- XL_ROUTINE(.page_read = WALDumpReadPage,
- .segment_open = WALDumpOpenSegment,
- .segment_close = WALDumpCloseSegment),
- &private);
+ XLogReaderAllocate(WalSegSz, waldir, WALDumpCloseSegment);
+
if (!xlogreader_state)
fatal_error("out of memory");
/* first find a valid recptr to start from */
- first_record = XLogFindNextRecord(xlogreader_state, private.startptr);
+ first_record = XLogFindNextRecord(xlogreader_state, private.startptr,
+ &WALDumpReadPage, (void*) &private);
if (first_record == InvalidXLogRecPtr)
fatal_error("could not find a valid record after %X/%X",
@@ -1067,7 +1071,13 @@ main(int argc, char **argv)
for (;;)
{
/* try to read the next record */
- record = XLogReadRecord(xlogreader_state, &errormsg);
+ while (XLogReadRecord(xlogreader_state, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!WALDumpReadPage(xlogreader_state, (void *) &private))
+ break;
+ }
+
if (!record)
{
if (!config.follow || private.endptr_reached)
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index 23d9e70a59..4e9268b553 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -57,64 +57,15 @@ typedef struct WALSegmentContext
typedef struct XLogReaderState XLogReaderState;
-/* Function type definition for the read_page callback */
-typedef bool (*XLogPageReadCB) (XLogReaderState *xlogreader,
- XLogRecPtr targetPagePtr,
- int reqLen,
- XLogRecPtr targetRecPtr,
- char *readBuf);
+/* Function type definition for the segment cleanup callback */
+typedef void (*WALSegmentCleanupCB) (XLogReaderState *xlogreader);
+
+/* Function type definition for the open/close callbacks for WALRead() */
typedef void (*WALSegmentOpenCB) (XLogReaderState *xlogreader,
XLogSegNo nextSegNo,
TimeLineID *tli_p);
typedef void (*WALSegmentCloseCB) (XLogReaderState *xlogreader);
-typedef struct XLogReaderRoutine
-{
- /*
- * Data input callback
- *
- * This callback shall read at least reqLen valid bytes of the xlog page
- * starting at targetPagePtr, and store them in readBuf. The callback
- * shall return the number of bytes read (never more than XLOG_BLCKSZ), or
- * -1 on failure. The callback shall sleep, if necessary, to wait for the
- * requested bytes to become available. The callback will not be invoked
- * again for the same page unless more than the returned number of bytes
- * are needed.
- *
- * targetRecPtr is the position of the WAL record we're reading. Usually
- * it is equal to targetPagePtr + reqLen, but sometimes xlogreader needs
- * to read and verify the page or segment header, before it reads the
- * actual WAL record it's interested in. In that case, targetRecPtr can
- * be used to determine which timeline to read the page from.
- *
- * The callback shall set ->seg.ws_tli to the TLI of the file the page was
- * read from.
- */
- XLogPageReadCB page_read;
-
- /*
- * Callback to open the specified WAL segment for reading. ->seg.ws_file
- * shall be set to the file descriptor of the opened segment. In case of
- * failure, an error shall be raised by the callback and it shall not
- * return.
- *
- * "nextSegNo" is the number of the segment to be opened.
- *
- * "tli_p" is an input/output argument. WALRead() uses it to pass the
- * timeline in which the new segment should be found, but the callback can
- * use it to return the TLI that it actually opened.
- */
- WALSegmentOpenCB segment_open;
-
- /*
- * WAL segment close callback. ->seg.ws_file shall be set to a negative
- * number.
- */
- WALSegmentCloseCB segment_close;
-} XLogReaderRoutine;
-
-#define XL_ROUTINE(...) &(XLogReaderRoutine){__VA_ARGS__}
-
typedef struct
{
/* Is this block ref in use? */
@@ -144,12 +95,35 @@ typedef struct
uint16 data_bufsz;
} DecodedBkpBlock;
+/* Return code from XLogReadRecord */
+typedef enum XLogReadRecordResult
+{
+ XLREAD_SUCCESS, /* record is successfully read */
+ XLREAD_NEED_DATA, /* need more data. see XLogReadRecord. */
+ XLREAD_FAIL /* failed during reading a record */
+} XLogReadRecordResult;
+
+/*
+ * internal state of XLogReadRecord
+ *
+ * XLogReadState runs a state machine while reading a record. Theses states
+ * are not seen outside the function. Each state may repeat several times
+ * exiting requesting caller for new data. See the comment of XLogReadRecrod
+ * for details.
+ */
+typedef enum XLogReadRecordState {
+ XLREAD_NEXT_RECORD,
+ XLREAD_TOT_LEN,
+ XLREAD_FIRST_FRAGMENT,
+ XLREAD_CONTINUATION
+} XLogReadRecordState;
+
struct XLogReaderState
{
/*
* Operational callbacks
*/
- XLogReaderRoutine routine;
+ WALSegmentCleanupCB cleanup_cb;
/* ----------------------------------------
* Public parameters
@@ -162,18 +136,14 @@ struct XLogReaderState
*/
uint64 system_identifier;
- /*
- * Opaque data for callbacks to use. Not used by XLogReader.
- */
- void *private_data;
-
/*
* Start and end point of last record read. EndRecPtr is also used as the
* position to read next. Calling XLogBeginRead() sets EndRecPtr to the
* starting position and ReadRecPtr to invalid.
*/
- XLogRecPtr ReadRecPtr; /* start of last record read */
+ XLogRecPtr ReadRecPtr; /* start of last record read or being read */
XLogRecPtr EndRecPtr; /* end+1 of last record read */
+ XLogRecPtr PrevRecPtr; /* start of previous record read */
/* ----------------------------------------
* Communication with page reader
@@ -187,7 +157,9 @@ struct XLogReaderState
* the request, or -1 on error */
TimeLineID readPageTLI; /* TLI for data currently in readBuf */
char *readBuf; /* buffer to store data */
- bool page_verified; /* is the page on the buffer verified? */
+ bool page_verified; /* is the page header on the buffer verified? */
+ bool record_verified;/* is the current record header verified? */
+
/* ----------------------------------------
@@ -229,8 +201,6 @@ struct XLogReaderState
XLogRecPtr latestPagePtr;
TimeLineID latestPageTLI;
- /* beginning of the WAL record being read. */
- XLogRecPtr currRecPtr;
/* timeline to read it from, 0 if a lookup is required */
TimeLineID currTLI;
@@ -257,6 +227,15 @@ struct XLogReaderState
char *readRecordBuf;
uint32 readRecordBufSize;
+ /*
+ * XLogReadRecord() state
+ */
+ XLogReadRecordState readRecordState;/* state machine state */
+ int recordGotLen; /* amount of current record that has
+ * already been read */
+ int recordRemainLen; /* length of current record that remains */
+ XLogRecPtr recordContRecPtr; /* where the current record continues */
+
/* Buffer to hold error message */
char *errormsg_buf;
};
@@ -264,9 +243,7 @@ struct XLogReaderState
/* Get a new XLogReader */
extern XLogReaderState *XLogReaderAllocate(int wal_segment_size,
const char *waldir,
- XLogReaderRoutine *routine,
- void *private_data);
-extern XLogReaderRoutine *LocalXLogReaderRoutine(void);
+ WALSegmentCleanupCB cleanup_cb);
/* Free an XLogReader */
extern void XLogReaderFree(XLogReaderState *state);
@@ -274,12 +251,17 @@ extern void XLogReaderFree(XLogReaderState *state);
/* Position the XLogReader to given record */
extern void XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr);
#ifdef FRONTEND
-extern XLogRecPtr XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr);
+/* Function type definition for the read_page callback */
+typedef bool (*XLogFindNextRecordCB) (XLogReaderState *xlogreader,
+ void *private);
+extern XLogRecPtr XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
+ XLogFindNextRecordCB read_page, void *private);
#endif /* FRONTEND */
/* Read the next XLog record. Returns NULL on end-of-WAL or failure */
-extern struct XLogRecord *XLogReadRecord(XLogReaderState *state,
- char **errormsg);
+extern XLogReadRecordResult XLogReadRecord(XLogReaderState *state,
+ XLogRecord **record,
+ char **errormsg);
/* Validate a page */
extern bool XLogReaderValidatePageHeader(XLogReaderState *state,
@@ -299,6 +281,7 @@ typedef struct WALReadError
} WALReadError;
extern bool WALRead(XLogReaderState *state,
+ WALSegmentOpenCB segopenfn, WALSegmentCloseCB sgclosefn,
char *buf, XLogRecPtr startptr, Size count,
TimeLineID tli, WALReadError *errinfo);
diff --git a/src/include/access/xlogutils.h b/src/include/access/xlogutils.h
index 364a21c4ea..397fb27fc2 100644
--- a/src/include/access/xlogutils.h
+++ b/src/include/access/xlogutils.h
@@ -47,9 +47,7 @@ extern Buffer XLogReadBufferExtended(RelFileNode rnode, ForkNumber forknum,
extern Relation CreateFakeRelcacheEntry(RelFileNode rnode);
extern void FreeFakeRelcacheEntry(Relation fakerel);
-extern bool read_local_xlog_page(XLogReaderState *state,
- XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetRecPtr, char *cur_page);
+extern bool read_local_xlog_page(XLogReaderState *state);
extern void wal_segment_open(XLogReaderState *state,
XLogSegNo nextSegNo,
TimeLineID *tli_p);
diff --git a/src/include/pg_config_manual.h b/src/include/pg_config_manual.h
index f10ad0acd6..94749609b4 100644
--- a/src/include/pg_config_manual.h
+++ b/src/include/pg_config_manual.h
@@ -377,7 +377,7 @@
* Enable debugging print statements for WAL-related operations; see
* also the wal_debug GUC var.
*/
-/* #define WAL_DEBUG */
+#define WAL_DEBUG
/*
* Enable tracing of resource consumption during sort operations;
diff --git a/src/include/replication/logical.h b/src/include/replication/logical.h
index c253403372..b936f31393 100644
--- a/src/include/replication/logical.h
+++ b/src/include/replication/logical.h
@@ -29,6 +29,10 @@ typedef void (*LogicalOutputPluginWriterUpdateProgress) (struct LogicalDecodingC
TransactionId xid
);
+typedef struct LogicalDecodingContext LogicalDecodingContext;
+
+typedef bool (*LogicalDecodingXLogPageReadCB)(XLogReaderState *ctx);
+
typedef struct LogicalDecodingContext
{
/* memory context this is all allocated in */
@@ -39,6 +43,7 @@ typedef struct LogicalDecodingContext
/* infrastructure pieces for decoding */
XLogReaderState *reader;
+ LogicalDecodingXLogPageReadCB page_read;
struct ReorderBuffer *reorder;
struct SnapBuild *snapshot_builder;
@@ -105,14 +110,16 @@ extern LogicalDecodingContext *CreateInitDecodingContext(const char *plugin,
List *output_plugin_options,
bool need_full_snapshot,
XLogRecPtr restart_lsn,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress);
extern LogicalDecodingContext *CreateDecodingContext(XLogRecPtr start_lsn,
List *output_plugin_options,
bool fast_forward,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress);
--
2.27.0
----Next_Part(Thu_Mar__4_11_28_53_2021_014)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
filename="v17-0003-Remove-globals-readOff-readLen-and-readSegNo.patch"
^ permalink raw reply [nested|flat] 70+ messages in thread
* [PATCH v17 2/4] Move page-reader out of XLogReadRecord
@ 2019-09-10 03:58 Kyotaro Horiguchi <[email protected]>
0 siblings, 0 replies; 70+ messages in thread
From: Kyotaro Horiguchi @ 2019-09-10 03:58 UTC (permalink / raw)
This is the second step of removing callbacks from WAL record reader.
Since it is essential to take in additional data while reading a
record, the function have to ask caller for new data while keeping
working state. Thus the function is turned into a state machine.
---
src/backend/access/transam/twophase.c | 15 +-
src/backend/access/transam/xlog.c | 58 +-
src/backend/access/transam/xlogreader.c | 697 +++++++++++-------
src/backend/access/transam/xlogutils.c | 17 +-
src/backend/replication/logical/logical.c | 26 +-
.../replication/logical/logicalfuncs.c | 13 +-
src/backend/replication/slotfuncs.c | 18 +-
src/backend/replication/walsender.c | 32 +-
src/bin/pg_rewind/parsexlog.c | 84 ++-
src/bin/pg_waldump/pg_waldump.c | 36 +-
src/include/access/xlogreader.h | 121 ++-
src/include/access/xlogutils.h | 4 +-
src/include/pg_config_manual.h | 2 +-
src/include/replication/logical.h | 11 +-
14 files changed, 646 insertions(+), 488 deletions(-)
diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index 80d2d20d6c..7d5a96a5f5 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -1325,11 +1325,8 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
XLogReaderState *xlogreader;
char *errormsg;
- xlogreader = XLogReaderAllocate(wal_segment_size, NULL,
- XL_ROUTINE(.page_read = &read_local_xlog_page,
- .segment_open = &wal_segment_open,
- .segment_close = &wal_segment_close),
- NULL);
+ xlogreader = XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+
if (!xlogreader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
@@ -1337,7 +1334,13 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
errdetail("Failed while allocating a WAL reading processor.")));
XLogBeginRead(xlogreader, lsn);
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!read_local_xlog_page(xlogreader))
+ break;
+ }
+
if (record == NULL)
ereport(ERROR,
(errcode_for_file_access(),
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 606328a65a..670996ae67 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -836,13 +836,6 @@ static XLogSource currentSource = XLOG_FROM_ANY;
static bool lastSourceFailed = false;
static bool pendingWalRcvRestart = false;
-typedef struct XLogPageReadPrivate
-{
- int emode;
- bool fetching_ckpt; /* are we fetching a checkpoint record? */
- bool randAccess;
-} XLogPageReadPrivate;
-
/*
* These variables track when we last obtained some WAL data to process,
* and where we got it from. (XLogReceiptSource is initially the same as
@@ -917,8 +910,8 @@ static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath,
static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli,
XLogSource source, bool notfoundOk);
static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, XLogSource source);
-static bool XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
+static bool XLogPageRead(XLogReaderState *xlogreader,
+ bool fetching_ckpt, int emode, bool randAccess);
static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
bool fetching_ckpt, XLogRecPtr tliRecPtr);
static int emode_for_corrupt_record(int emode, XLogRecPtr RecPtr);
@@ -1231,8 +1224,7 @@ XLogInsertRecord(XLogRecData *rdata,
appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len);
if (!debug_reader)
- debug_reader = XLogReaderAllocate(wal_segment_size, NULL,
- XL_ROUTINE(), NULL);
+ debug_reader = XLogReaderAllocate(wal_segment_size, NULL, NULL);
if (!debug_reader)
{
@@ -4344,15 +4336,10 @@ CleanupBackupHistory(void)
* record is available.
*/
static XLogRecord *
-ReadRecord(XLogReaderState *xlogreader, int emode,
- bool fetching_ckpt)
+ReadRecord(XLogReaderState *xlogreader, int emode, bool fetching_ckpt)
{
XLogRecord *record;
- XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
-
- private->fetching_ckpt = fetching_ckpt;
- private->emode = emode;
- private->randAccess = (xlogreader->ReadRecPtr == InvalidXLogRecPtr);
+ bool randAccess = (xlogreader->ReadRecPtr == InvalidXLogRecPtr);
/* This is the first attempt to read this page. */
lastSourceFailed = false;
@@ -4360,8 +4347,16 @@ ReadRecord(XLogReaderState *xlogreader, int emode,
for (;;)
{
char *errormsg;
+ XLogReadRecordResult result;
+
+ while ((result = XLogReadRecord(xlogreader, &record, &errormsg))
+ == XLREAD_NEED_DATA)
+ {
+ if (!XLogPageRead(xlogreader, fetching_ckpt, emode, randAccess))
+ break;
+
+ }
- record = XLogReadRecord(xlogreader, &errormsg);
ReadRecPtr = xlogreader->ReadRecPtr;
EndRecPtr = xlogreader->EndRecPtr;
if (record == NULL)
@@ -6380,7 +6375,6 @@ StartupXLOG(void)
bool backupFromStandby = false;
DBState dbstate_at_startup;
XLogReaderState *xlogreader;
- XLogPageReadPrivate private;
bool promoted = false;
struct stat st;
@@ -6539,13 +6533,9 @@ StartupXLOG(void)
OwnLatch(&XLogCtl->recoveryWakeupLatch);
/* Set up XLOG reader facility */
- MemSet(&private, 0, sizeof(XLogPageReadPrivate));
xlogreader =
- XLogReaderAllocate(wal_segment_size, NULL,
- XL_ROUTINE(.page_read = &XLogPageRead,
- .segment_open = NULL,
- .segment_close = wal_segment_close),
- &private);
+ XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+
if (!xlogreader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
@@ -11987,12 +11977,13 @@ CancelBackup(void)
* sleep and retry.
*/
static bool
-XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetRecPtr, char *readBuf)
+XLogPageRead(XLogReaderState *xlogreader,
+ bool fetching_ckpt, int emode, bool randAccess)
{
- XLogPageReadPrivate *private =
- (XLogPageReadPrivate *) xlogreader->private_data;
- int emode = private->emode;
+ char *readBuf = xlogreader->readBuf;
+ XLogRecPtr targetPagePtr = xlogreader->readPagePtr;
+ int reqLen = xlogreader->readLen;
+ XLogRecPtr targetRecPtr = xlogreader->ReadRecPtr;
uint32 targetPageOff;
XLogSegNo targetSegNo PG_USED_FOR_ASSERTS_ONLY;
int r;
@@ -12035,8 +12026,8 @@ retry:
flushedUpto < targetPagePtr + reqLen))
{
if (!WaitForWALToBecomeAvailable(targetPagePtr + reqLen,
- private->randAccess,
- private->fetching_ckpt,
+ randAccess,
+ fetching_ckpt,
targetRecPtr))
{
if (readFile >= 0)
@@ -12141,6 +12132,7 @@ retry:
goto next_record_is_invalid;
}
+ Assert(xlogreader->readPagePtr == targetPagePtr);
xlogreader->readLen = readLen;
return true;
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index 7d04995b7b..ba3e4868f7 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -40,7 +40,7 @@ static bool XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr,
int reqLen, bool header_inclusive);
static void XLogReaderInvalReadState(XLogReaderState *state);
static bool ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
- XLogRecPtr PrevRecPtr, XLogRecord *record, bool randAccess);
+ XLogRecPtr PrevRecPtr, XLogRecord *record);
static bool ValidXLogRecord(XLogReaderState *state, XLogRecord *record,
XLogRecPtr recptr);
static void ResetDecoder(XLogReaderState *state);
@@ -73,7 +73,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...)
*/
XLogReaderState *
XLogReaderAllocate(int wal_segment_size, const char *waldir,
- XLogReaderRoutine *routine, void *private_data)
+ WALSegmentCleanupCB cleanup_cb)
{
XLogReaderState *state;
@@ -84,7 +84,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
return NULL;
/* initialize caller-provided support functions */
- state->routine = *routine;
+ state->cleanup_cb = cleanup_cb;
state->max_block_id = -1;
@@ -107,8 +107,6 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
WALOpenSegmentInit(&state->seg, &state->segcxt, wal_segment_size,
waldir);
- /* system_identifier initialized to zeroes above */
- state->private_data = private_data;
/* ReadRecPtr, EndRecPtr and readLen initialized to zeroes above */
state->errormsg_buf = palloc_extended(MAX_ERRORMSG_LEN + 1,
MCXT_ALLOC_NO_OOM);
@@ -140,8 +138,8 @@ XLogReaderFree(XLogReaderState *state)
{
int block_id;
- if (state->seg.ws_file != -1)
- state->routine.segment_close(state);
+ if (state->seg.ws_file >= 0)
+ state->cleanup_cb(state);
for (block_id = 0; block_id <= XLR_MAX_BLOCK_ID; block_id++)
{
@@ -246,6 +244,7 @@ XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
/* Begin at the passed-in record pointer. */
state->EndRecPtr = RecPtr;
state->ReadRecPtr = InvalidXLogRecPtr;
+ state->readRecordState = XLREAD_NEXT_RECORD;
}
/*
@@ -254,317 +253,449 @@ XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
* XLogBeginRead() or XLogFindNextRecord() must be called before the first call
* to XLogReadRecord().
*
- * If the page_read callback fails to read the requested data, NULL is
- * returned. The callback is expected to have reported the error; errormsg
- * is set to NULL.
+ * This function may return XLREAD_NEED_DATA several times before returning a
+ * result record. The caller shall read in some new data then call this
+ * function again with the same parameters.
*
- * If the reading fails for some other reason, NULL is also returned, and
- * *errormsg is set to a string with details of the failure.
+ * When a record is successfully read, returns XLREAD_SUCCESS with result
+ * record being stored in *record. Otherwise *record is NULL.
*
- * The returned pointer (or *errormsg) points to an internal buffer that's
- * valid until the next call to XLogReadRecord.
+ * Returns XLREAD_NEED_DATA if more data is needed to finish reading the
+ * current record. In that case, state->readPagePtr and state->readLen inform
+ * the desired position and minimum length of data needed. The caller shall
+ * read in the requested data and set state->readBuf to point to a buffer
+ * containing it. The caller must also set state->readPageTLI and
+ * state->readLen to indicate the timeline that it was read from, and the
+ * length of data that is now available (which must be >= given readLen),
+ * respectively.
+ *
+ * If invalid data is encountered, returns XLREAD_FAIL with *record being set to
+ * NULL. *errormsg is set to a string with details of the failure.
+ * The returned pointer (or *errormsg) points to an internal buffer that's valid
+ * until the next call to XLogReadRecord.
+ *
+ *
+ * This function runs a state machine consists of the following states.
+ *
+ * XLREAD_NEXT_RECORD :
+ * The initial state, if called with valid RecPtr, try to read a record at
+ * that position. If invalid RecPtr is given try to read a record just after
+ * the last one previously read.
+ * This state ens after setting ReadRecPtr. Then goes to XLREAD_TOT_LEN.
+ *
+ * XLREAD_TOT_LEN:
+ * Examining record header. Ends after reading record total
+ * length. recordRemainLen and recordGotLen are initialized.
+ *
+ * XLREAD_FIRST_FRAGMENT:
+ * Reading the first fragment. Ends with finishing reading a single
+ * record. Goes to XLREAD_NEXT_RECORD if that's all or
+ * XLREAD_CONTINUATION if we have continuation.
+
+ * XLREAD_CONTINUATION:
+ * Reading continuation of record. Ends with finishing the whole record then
+ * goes to XLREAD_NEXT_RECORD. During this state, recordRemainLen indicates
+ * how much is left and readRecordBuf holds the partially assert
+ * record.recordContRecPtr points to the beginning of the next page where to
+ * continue.
+ *
+ * If wrong data found in any state, the state machine stays at the current
+ * state. This behavior allows to continue reading a reacord switching among
+ * different souces, while streaming replication.
*/
-XLogRecord *
-XLogReadRecord(XLogReaderState *state, char **errormsg)
+XLogReadRecordResult
+XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
{
- XLogRecPtr RecPtr;
- XLogRecord *record;
- XLogRecPtr targetPagePtr;
- bool randAccess;
- uint32 len,
- total_len;
- uint32 targetRecOff;
- uint32 pageHeaderSize;
- bool gotheader;
+ XLogRecord *prec;
- /*
- * randAccess indicates whether to verify the previous-record pointer of
- * the record we're reading. We only do this if we're reading
- * sequentially, which is what we initially assume.
- */
- randAccess = false;
+ *record = NULL;
/* reset error state */
*errormsg = NULL;
state->errormsg_buf[0] = '\0';
- ResetDecoder(state);
-
- RecPtr = state->EndRecPtr;
-
- if (state->ReadRecPtr != InvalidXLogRecPtr)
- {
- /* read the record after the one we just read */
-
- /*
- * EndRecPtr is pointing to end+1 of the previous WAL record. If
- * we're at a page boundary, no more records can fit on the current
- * page. We must skip over the page header, but we can't do that until
- * we've read in the page, since the header size is variable.
- */
- }
- else
- {
- /*
- * Caller supplied a position to start at.
- *
- * In this case, EndRecPtr should already be pointing to a valid
- * record starting position.
- */
- Assert(XRecOffIsValid(RecPtr));
- randAccess = true;
- }
-
- state->currRecPtr = RecPtr;
-
- targetPagePtr = RecPtr - (RecPtr % XLOG_BLCKSZ);
- targetRecOff = RecPtr % XLOG_BLCKSZ;
-
- /*
- * Read the page containing the record into state->readBuf. Request enough
- * byte to cover the whole record header, or at least the part of it that
- * fits on the same page.
- */
- while (XLogNeedData(state, targetPagePtr,
- Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
- targetRecOff != 0))
- {
- if (!state->routine.page_read(state, state->readPagePtr, state->readLen,
- RecPtr, state->readBuf))
- break;
- }
-
- if (!state->page_verified)
- goto err;
-
- /*
- * We have at least the page header, so we can examine it now.
- */
- pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
- if (targetRecOff == 0)
- {
- /*
- * At page start, so skip over page header.
- */
- RecPtr += pageHeaderSize;
- targetRecOff = pageHeaderSize;
- }
- else if (targetRecOff < pageHeaderSize)
- {
- report_invalid_record(state, "invalid record offset at %X/%X",
- LSN_FORMAT_ARGS(RecPtr));
- goto err;
- }
-
- if ((((XLogPageHeader) state->readBuf)->xlp_info & XLP_FIRST_IS_CONTRECORD) &&
- targetRecOff == pageHeaderSize)
- {
- report_invalid_record(state, "contrecord is requested by %X/%X",
- LSN_FORMAT_ARGS(RecPtr));
- goto err;
- }
-
- /* XLogNeedData has verified the page header */
- Assert(pageHeaderSize <= state->readLen);
-
- /*
- * Read the record length.
- *
- * NB: Even though we use an XLogRecord pointer here, the whole record
- * header might not fit on this page. xl_tot_len is the first field of the
- * struct, so it must be on this page (the records are MAXALIGNed), but we
- * cannot access any other fields until we've verified that we got the
- * whole header.
- */
- record = (XLogRecord *) (state->readBuf + RecPtr % XLOG_BLCKSZ);
- total_len = record->xl_tot_len;
-
- /*
- * If the whole record header is on this page, validate it immediately.
- * Otherwise do just a basic sanity check on xl_tot_len, and validate the
- * rest of the header after reading it from the next page. The xl_tot_len
- * check is necessary here to ensure that we enter the "Need to reassemble
- * record" code path below; otherwise we might fail to apply
- * ValidXLogRecordHeader at all.
- */
- if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
- {
- if (!ValidXLogRecordHeader(state, RecPtr, state->ReadRecPtr, record,
- randAccess))
- goto err;
- gotheader = true;
- }
- else
- {
- /* XXX: more validation should be done here */
- if (total_len < SizeOfXLogRecord)
- {
- report_invalid_record(state,
- "invalid record length at %X/%X: wanted %u, got %u",
- LSN_FORMAT_ARGS(RecPtr),
- (uint32) SizeOfXLogRecord, total_len);
- goto err;
- }
- gotheader = false;
- }
-
- len = XLOG_BLCKSZ - RecPtr % XLOG_BLCKSZ;
- if (total_len > len)
+ switch (state->readRecordState)
{
- /* Need to reassemble record */
- char *contdata;
- XLogPageHeader pageHeader;
- char *buffer;
- uint32 gotlen;
-
- /*
- * Enlarge readRecordBuf as needed.
- */
- if (total_len > state->readRecordBufSize &&
- !allocate_recordbuf(state, total_len))
- {
- /* We treat this as a "bogus data" condition */
- report_invalid_record(state, "record length %u at %X/%X too long",
- total_len, LSN_FORMAT_ARGS(RecPtr));
- goto err;
- }
+ case XLREAD_NEXT_RECORD:
+ ResetDecoder(state);
- /* Copy the first fragment of the record from the first page. */
- memcpy(state->readRecordBuf,
- state->readBuf + RecPtr % XLOG_BLCKSZ, len);
- buffer = state->readRecordBuf + len;
- gotlen = len;
-
- do
- {
- int rest_len = total_len - gotlen;
-
- /* Calculate pointer to beginning of next page */
- targetPagePtr += XLOG_BLCKSZ;
-
- /* Wait for the next page to become available */
- while (XLogNeedData(state, targetPagePtr,
- Min(rest_len, XLOG_BLCKSZ),
- false))
+ if (state->ReadRecPtr != InvalidXLogRecPtr)
{
- if (!state->routine.page_read(state, state->readPagePtr,
- state->readLen,
- state->ReadRecPtr,
- state->readBuf))
- break;
+ /* read the record after the one we just read */
+
+ /*
+ * EndRecPtr is pointing to end+1 of the previous WAL record.
+ * If we're at a page boundary, no more records can fit on the
+ * current page. We must skip over the page header, but we
+ * can't do that until we've read in the page, since the header
+ * size is variable.
+ */
+ state->PrevRecPtr = state->ReadRecPtr;
+ state->ReadRecPtr = state->EndRecPtr;
+ }
+ else
+ {
+ /*
+ * Caller supplied a position to start at.
+ *
+ * In this case, EndRecPtr should already be pointing to a
+ * valid record starting position.
+ */
+ Assert(XRecOffIsValid(state->EndRecPtr));
+ state->ReadRecPtr = state->EndRecPtr;
+
+ /*
+ * We cannot verify the previous-record pointer when we're
+ * seeking to a particular record. Reset PrevRecPtr so that we
+ * won't try doing that.
+ */
+ state->PrevRecPtr = InvalidXLogRecPtr;
+ state->EndRecPtr = InvalidXLogRecPtr; /* to be tidy */
}
+ state->record_verified = false;
+ state->readRecordState = XLREAD_TOT_LEN;
+ /* fall through */
+
+ case XLREAD_TOT_LEN:
+ {
+ uint32 total_len;
+ uint32 pageHeaderSize;
+ XLogRecPtr targetPagePtr;
+ uint32 targetRecOff;
+ XLogPageHeader pageHeader;
+
+ targetPagePtr =
+ state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
+ targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+
+ /*
+ * Check if we have enough data. For the first record in the page,
+ * the requesting length doesn't contain page header.
+ */
+ if (XLogNeedData(state, targetPagePtr,
+ Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+ targetRecOff != 0))
+ return XLREAD_NEED_DATA;
+
+ /* error out if caller supplied bogus page */
if (!state->page_verified)
goto err;
- Assert(SizeOfXLogShortPHD <= state->readLen);
+ /* examine page header now. */
+ pageHeaderSize =
+ XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+ if (targetRecOff == 0)
+ {
+ /* At page start, so skip over page header. */
+ state->ReadRecPtr += pageHeaderSize;
+ targetRecOff = pageHeaderSize;
+ }
+ else if (targetRecOff < pageHeaderSize)
+ {
+ report_invalid_record(state, "invalid record offset at %X/%X",
+ LSN_FORMAT_ARGS(state->ReadRecPtr));
+ goto err;
+ }
- /* Check that the continuation on next page looks valid */
pageHeader = (XLogPageHeader) state->readBuf;
- if (!(pageHeader->xlp_info & XLP_FIRST_IS_CONTRECORD))
+ if ((pageHeader->xlp_info & XLP_FIRST_IS_CONTRECORD) &&
+ targetRecOff == pageHeaderSize)
{
- report_invalid_record(state,
- "there is no contrecord flag at %X/%X",
- LSN_FORMAT_ARGS(RecPtr));
+ report_invalid_record(state, "contrecord is requested by %X/%X",
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr);
goto err;
}
- /*
- * Cross-check that xlp_rem_len agrees with how much of the record
- * we expect there to be left.
- */
- if (pageHeader->xlp_rem_len == 0 ||
- total_len != (pageHeader->xlp_rem_len + gotlen))
- {
- report_invalid_record(state,
- "invalid contrecord length %u (expected %lld) at %X/%X",
- pageHeader->xlp_rem_len,
- ((long long) total_len) - gotlen,
- LSN_FORMAT_ARGS(RecPtr));
- goto err;
- }
-
- /* Append the continuation from this page to the buffer */
- pageHeaderSize = XLogPageHeaderSize(pageHeader);
-
+ /* XLogNeedData has verified the page header */
Assert(pageHeaderSize <= state->readLen);
- contdata = (char *) state->readBuf + pageHeaderSize;
- len = XLOG_BLCKSZ - pageHeaderSize;
- if (pageHeader->xlp_rem_len < len)
- len = pageHeader->xlp_rem_len;
+ /*
+ * Read the record length.
+ *
+ * NB: Even though we use an XLogRecord pointer here, the whole
+ * record header might not fit on this page. xl_tot_len is the first
+ * field of the struct, so it must be on this page (the records are
+ * MAXALIGNed), but we cannot access any other fields until we've
+ * verified that we got the whole header.
+ */
+ prec = (XLogRecord *) (state->readBuf +
+ state->ReadRecPtr % XLOG_BLCKSZ);
+ total_len = prec->xl_tot_len;
- Assert (pageHeaderSize + len <= state->readLen);
- memcpy(buffer, (char *) contdata, len);
- buffer += len;
- gotlen += len;
+ /*
+ * If the whole record header is on this page, validate it
+ * immediately. Otherwise do just a basic sanity check on
+ * xl_tot_len, and validate the rest of the header after reading it
+ * from the next page. The xl_tot_len check is necessary here to
+ * ensure that we enter the XLREAD_CONTINUATION state below;
+ * otherwise we might fail to apply ValidXLogRecordHeader at all.
+ */
+ if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
+ {
+ if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
+ state->PrevRecPtr, prec))
+ goto err;
- /* If we just reassembled the record header, validate it. */
- if (!gotheader)
+ state->record_verified = true;
+ }
+ else
{
- record = (XLogRecord *) state->readRecordBuf;
- if (!ValidXLogRecordHeader(state, RecPtr, state->ReadRecPtr,
- record, randAccess))
+ /* XXX: more validation should be done here */
+ if (total_len < SizeOfXLogRecord)
+ {
+ report_invalid_record(state,
+ "invalid record length at %X/%X: wanted %u, got %u",
+ LSN_FORMAT_ARGS(state->ReadRecPtr),
+ (uint32) SizeOfXLogRecord, total_len);
goto err;
- gotheader = true;
+ }
}
- } while (gotlen < total_len);
-
- Assert(gotheader);
- record = (XLogRecord *) state->readRecordBuf;
- if (!ValidXLogRecord(state, record, RecPtr))
- goto err;
+ /*
+ * Wait for the rest of the record, or the part of the record that
+ * fit on the first page if crossed a page boundary, to become
+ * available.
+ */
+ state->recordGotLen = 0;
+ state->recordRemainLen = total_len;
+ state->readRecordState = XLREAD_FIRST_FRAGMENT;
+ }
+ /* fall through */
- pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
- state->ReadRecPtr = RecPtr;
- state->EndRecPtr = targetPagePtr + pageHeaderSize
- + MAXALIGN(pageHeader->xlp_rem_len);
- }
- else
- {
- /* Wait for the record data to become available */
- while (XLogNeedData(state, targetPagePtr,
- Min(targetRecOff + total_len, XLOG_BLCKSZ), true))
+ case XLREAD_FIRST_FRAGMENT:
{
- if (!state->routine.page_read(state, state->readPagePtr,
- state->readLen,
- state->ReadRecPtr, state->readBuf))
+ uint32 total_len = state->recordRemainLen;
+ uint32 request_len;
+ uint32 record_len;
+ XLogRecPtr targetPagePtr;
+ uint32 targetRecOff;
+
+ /*
+ * Wait for the rest of the record on the first page to become
+ * available
+ */
+ targetPagePtr =
+ state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
+ targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+
+ request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ);
+ record_len = request_len - targetRecOff;
+
+ /* ReadRecPtr contains page header */
+ Assert (targetRecOff != 0);
+ if (XLogNeedData(state, targetPagePtr, request_len, true))
+ return XLREAD_NEED_DATA;
+
+ /* error out if caller supplied bogus page */
+ if (!state->page_verified)
+ goto err;
+
+ prec = (XLogRecord *) (state->readBuf + targetRecOff);
+
+ /* validate record header if not yet */
+ if (!state->record_verified && record_len >= SizeOfXLogRecord)
+ {
+ if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
+ state->PrevRecPtr, prec))
+ goto err;
+
+ state->record_verified = true;
+ }
+
+
+ if (total_len == record_len)
+ {
+ /* Record does not cross a page boundary */
+ Assert(state->record_verified);
+
+ if (!ValidXLogRecord(state, prec, state->ReadRecPtr))
+ goto err;
+
+ state->record_verified = true; /* to be tidy */
+
+ /* We already checked the header earlier */
+ state->EndRecPtr = state->ReadRecPtr + MAXALIGN(record_len);
+
+ *record = prec;
+ state->readRecordState = XLREAD_NEXT_RECORD;
break;
+ }
+
+ /*
+ * The record continues on the next page. Need to reassemble
+ * record
+ */
+ Assert(total_len > record_len);
+
+ /* Enlarge readRecordBuf as needed. */
+ if (total_len > state->readRecordBufSize &&
+ !allocate_recordbuf(state, total_len))
+ {
+ /* We treat this as a "bogus data" condition */
+ report_invalid_record(state,
+ "record length %u at %X/%X too long",
+ total_len,
+ LSN_FORMAT_ARGS(state->ReadRecPtr));
+ goto err;
+ }
+
+ /* Copy the first fragment of the record from the first page. */
+ memcpy(state->readRecordBuf, state->readBuf + targetRecOff,
+ record_len);
+ state->recordGotLen += record_len;
+ state->recordRemainLen -= record_len;
+
+ /* Calculate pointer to beginning of next page */
+ state->recordContRecPtr = state->ReadRecPtr + record_len;
+ Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0);
+
+ state->readRecordState = XLREAD_CONTINUATION;
}
+ /* fall through */
+
+ case XLREAD_CONTINUATION:
+ {
+ XLogPageHeader pageHeader;
+ uint32 pageHeaderSize;
+ XLogRecPtr targetPagePtr;
+
+ /* we enter this state only if we haven't read the whole record. */
+ Assert (state->recordRemainLen > 0);
+
+ while(state->recordRemainLen > 0)
+ {
+ char *contdata;
+ uint32 request_len;
+ uint32 record_len;
+
+ /* Wait for the next page to become available */
+ targetPagePtr = state->recordContRecPtr;
+
+ /* this request contains page header */
+ Assert (targetPagePtr != 0);
+ if (XLogNeedData(state, targetPagePtr,
+ Min(state->recordRemainLen, XLOG_BLCKSZ),
+ false))
+ return XLREAD_NEED_DATA;
+
+ if (!state->page_verified)
+ goto err;
+
+ Assert(SizeOfXLogShortPHD <= state->readLen);
- if (!state->page_verified)
- goto err;
+ /* Check that the continuation on next page looks valid */
+ pageHeader = (XLogPageHeader) state->readBuf;
+ if (!(pageHeader->xlp_info & XLP_FIRST_IS_CONTRECORD))
+ {
+ report_invalid_record(
+ state,
+ "there is no contrecord flag at %X/%X reading %X/%X",
+ (uint32) (state->recordContRecPtr >> 32),
+ (uint32) state->recordContRecPtr,
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr);
+ goto err;
+ }
- /* Record does not cross a page boundary */
- if (!ValidXLogRecord(state, record, RecPtr))
- goto err;
+ /*
+ * Cross-check that xlp_rem_len agrees with how much of the
+ * record we expect there to be left.
+ */
+ if (pageHeader->xlp_rem_len == 0 ||
+ pageHeader->xlp_rem_len != state->recordRemainLen)
+ {
+ report_invalid_record(
+ state,
+ "invalid contrecord length %u at %X/%X reading %X/%X, expected %u",
+ pageHeader->xlp_rem_len,
+ (uint32) (state->recordContRecPtr >> 32),
+ (uint32) state->recordContRecPtr,
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr,
+ state->recordRemainLen);
+ goto err;
+ }
- state->EndRecPtr = RecPtr + MAXALIGN(total_len);
+ /* Append the continuation from this page to the buffer */
+ pageHeaderSize = XLogPageHeaderSize(pageHeader);
- state->ReadRecPtr = RecPtr;
+ /*
+ * XLogNeedData should have ensured that the whole page header
+ * was read
+ */
+ Assert(state->readLen >= pageHeaderSize);
+
+ contdata = (char *) state->readBuf + pageHeaderSize;
+ record_len = XLOG_BLCKSZ - pageHeaderSize;
+ if (pageHeader->xlp_rem_len < record_len)
+ record_len = pageHeader->xlp_rem_len;
+
+ request_len = record_len + pageHeaderSize;
+
+ /* XLogNeedData should have ensured all needed data was read */
+ Assert (state->readLen >= request_len);
+
+ memcpy(state->readRecordBuf + state->recordGotLen,
+ (char *) contdata, record_len);
+ state->recordGotLen += record_len;
+ state->recordRemainLen -= record_len;
+
+ /* If we just reassembled the record header, validate it. */
+ if (!state->record_verified)
+ {
+ Assert(state->recordGotLen >= SizeOfXLogRecord);
+ if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
+ state->PrevRecPtr,
+ (XLogRecord *) state->readRecordBuf))
+ goto err;
+
+ state->record_verified = true;
+ }
+
+ /* Calculate pointer to beginning of next page, and continue */
+ state->recordContRecPtr += XLOG_BLCKSZ;
+ }
+
+ /* targetPagePtr is pointing the last-read page here */
+ prec = (XLogRecord *) state->readRecordBuf;
+ if (!ValidXLogRecord(state, prec, state->ReadRecPtr))
+ goto err;
+
+ pageHeaderSize =
+ XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+ state->EndRecPtr = targetPagePtr + pageHeaderSize
+ + MAXALIGN(pageHeader->xlp_rem_len);
+
+ *record = prec;
+ state->readRecordState = XLREAD_NEXT_RECORD;
+ break;
+ }
}
/*
* Special processing if it's an XLOG SWITCH record
*/
- if (record->xl_rmid == RM_XLOG_ID &&
- (record->xl_info & ~XLR_INFO_MASK) == XLOG_SWITCH)
+ if ((*record)->xl_rmid == RM_XLOG_ID &&
+ ((*record)->xl_info & ~XLR_INFO_MASK) == XLOG_SWITCH)
{
/* Pretend it extends to end of segment */
state->EndRecPtr += state->segcxt.ws_segsize - 1;
state->EndRecPtr -= XLogSegmentOffset(state->EndRecPtr, state->segcxt.ws_segsize);
}
- if (DecodeXLogRecord(state, record, errormsg))
- return record;
- else
- return NULL;
+ Assert (!*record || state->readLen >= 0);
+ if (DecodeXLogRecord(state, *record, errormsg))
+ return XLREAD_SUCCESS;
+
+ *record = NULL;
+ return XLREAD_FAIL;
err:
/*
- * Invalidate the read state. We might read from a different source after
+ * Invalidate the read page. We might read from a different source after
* failure.
*/
XLogReaderInvalReadState(state);
@@ -572,7 +703,8 @@ err:
if (state->errormsg_buf[0] != '\0')
*errormsg = state->errormsg_buf;
- return NULL;
+ *record = NULL;
+ return XLREAD_FAIL;
}
/*
@@ -724,11 +856,12 @@ XLogReaderInvalReadState(XLogReaderState *state)
*
* This is just a convenience subroutine to avoid duplicated code in
* XLogReadRecord. It's not intended for use from anywhere else.
+ *
+ * If PrevRecPtr is valid, the xl_prev is is cross-checked with it.
*/
static bool
ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
- XLogRecPtr PrevRecPtr, XLogRecord *record,
- bool randAccess)
+ XLogRecPtr PrevRecPtr, XLogRecord *record)
{
if (record->xl_tot_len < SizeOfXLogRecord)
{
@@ -745,7 +878,7 @@ ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
record->xl_rmid, LSN_FORMAT_ARGS(RecPtr));
return false;
}
- if (randAccess)
+ if (PrevRecPtr == InvalidXLogRecPtr)
{
/*
* We can't exactly verify the prev-link, but surely it should be less
@@ -975,11 +1108,14 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
* XLogReadRecord() will read the next valid record.
*/
XLogRecPtr
-XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
+XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
+ XLogFindNextRecordCB read_page, void *private)
{
XLogRecPtr tmpRecPtr;
XLogRecPtr found = InvalidXLogRecPtr;
XLogPageHeader header;
+ XLogRecord *record;
+ XLogReadRecordResult result;
char *errormsg;
Assert(!XLogRecPtrIsInvalid(RecPtr));
@@ -1012,9 +1148,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
while(XLogNeedData(state, targetPagePtr, targetRecOff,
targetRecOff != 0))
{
- if (!state->routine.page_read(state, state->readPagePtr,
- state->readLen,
- state->ReadRecPtr, state->readBuf))
+ if (!read_page(state, private))
break;
}
@@ -1066,8 +1200,16 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
* or we just jumped over the remaining data of a continuation.
*/
XLogBeginRead(state, tmpRecPtr);
- while (XLogReadRecord(state, &errormsg) != NULL)
+ while ((result = XLogReadRecord(state, &record, &errormsg)) !=
+ XLREAD_FAIL)
{
+ if (result == XLREAD_NEED_DATA)
+ {
+ if (!read_page(state, private))
+ goto err;
+ continue;
+ }
+
/* past the record we've found, break out */
if (RecPtr <= state->ReadRecPtr)
{
@@ -1087,9 +1229,9 @@ err:
#endif /* FRONTEND */
/*
- * Helper function to ease writing of XLogRoutine->page_read callbacks.
- * If this function is used, caller must supply a segment_open callback in
- * 'state', as that is used here.
+ * Helper function to ease writing of page_read callback.
+ * If this function is used, caller must supply a segment_open callback and
+ * segment_close callback as that is used here.
*
* Read 'count' bytes into 'buf', starting at location 'startptr', from WAL
* fetched from timeline 'tli'.
@@ -1102,6 +1244,7 @@ err:
*/
bool
WALRead(XLogReaderState *state,
+ WALSegmentOpenCB segopenfn, WALSegmentCloseCB segclosefn,
char *buf, XLogRecPtr startptr, Size count, TimeLineID tli,
WALReadError *errinfo)
{
@@ -1133,10 +1276,10 @@ WALRead(XLogReaderState *state,
XLogSegNo nextSegNo;
if (state->seg.ws_file >= 0)
- state->routine.segment_close(state);
+ segclosefn(state);
XLByteToSeg(recptr, nextSegNo, state->segcxt.ws_segsize);
- state->routine.segment_open(state, nextSegNo, &tli);
+ segopenfn(state, nextSegNo, &tli);
/* This shouldn't happen -- indicates a bug in segment_open */
Assert(state->seg.ws_file >= 0);
diff --git a/src/backend/access/transam/xlogutils.c b/src/backend/access/transam/xlogutils.c
index 46eda33f25..b003990745 100644
--- a/src/backend/access/transam/xlogutils.c
+++ b/src/backend/access/transam/xlogutils.c
@@ -686,8 +686,7 @@ XLogTruncateRelation(RelFileNode rnode, ForkNumber forkNum,
void
XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wantLength)
{
- const XLogRecPtr lastReadPage = state->seg.ws_segno *
- state->segcxt.ws_segsize + state->readLen;
+ const XLogRecPtr lastReadPage = state->readPagePtr;
Assert(wantPage != InvalidXLogRecPtr && wantPage % XLOG_BLCKSZ == 0);
Assert(wantLength <= XLOG_BLCKSZ);
@@ -702,7 +701,7 @@ XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wa
* current TLI has since become historical.
*/
if (lastReadPage == wantPage &&
- state->readLen != 0 &&
+ state->page_verified &&
lastReadPage + state->readLen >= wantPage + Min(wantLength, XLOG_BLCKSZ - 1))
return;
@@ -788,6 +787,7 @@ wal_segment_open(XLogReaderState *state, XLogSegNo nextSegNo,
char path[MAXPGPATH];
XLogFilePath(path, tli, nextSegNo, state->segcxt.ws_segsize);
+ elog(LOG, "HOGE: %lu, %d => %s", nextSegNo, tli, path);
state->seg.ws_file = BasicOpenFile(path, O_RDONLY | PG_BINARY);
if (state->seg.ws_file >= 0)
return;
@@ -825,9 +825,11 @@ wal_segment_close(XLogReaderState *state)
* loop for now.
*/
bool
-read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *cur_page)
+read_local_xlog_page(XLogReaderState *state)
{
+ XLogRecPtr targetPagePtr = state->readPagePtr;
+ int reqLen = state->readLen;
+ char *cur_page = state->readBuf;
XLogRecPtr read_upto,
loc;
TimeLineID tli;
@@ -940,11 +942,12 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
* as 'count', read the whole page anyway. It's guaranteed to be
* zero-padded up to the page boundary if it's incomplete.
*/
- if (!WALRead(state, cur_page, targetPagePtr, XLOG_BLCKSZ, tli,
- &errinfo))
+ if (!WALRead(state, wal_segment_open, wal_segment_close,
+ cur_page, targetPagePtr, XLOG_BLCKSZ, tli, &errinfo))
WALReadRaiseError(&errinfo);
/* number of valid bytes in the buffer */
+ state->readPagePtr = targetPagePtr;
state->readLen = count;
return true;
}
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index 37b75deb72..9cbd8afa49 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -148,7 +148,8 @@ StartupDecodingContext(List *output_plugin_options,
TransactionId xmin_horizon,
bool need_full_snapshot,
bool fast_forward,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress)
@@ -198,11 +199,12 @@ StartupDecodingContext(List *output_plugin_options,
ctx->slot = slot;
- ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, xl_routine, ctx);
+ ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, cleanup_cb);
if (!ctx->reader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
errmsg("out of memory")));
+ ctx->page_read = page_read;
ctx->reorder = ReorderBufferAllocate();
ctx->snapshot_builder =
@@ -319,7 +321,8 @@ CreateInitDecodingContext(const char *plugin,
List *output_plugin_options,
bool need_full_snapshot,
XLogRecPtr restart_lsn,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress)
@@ -422,7 +425,7 @@ CreateInitDecodingContext(const char *plugin,
ctx = StartupDecodingContext(NIL, restart_lsn, xmin_horizon,
need_full_snapshot, false,
- xl_routine, prepare_write, do_write,
+ page_read, cleanup_cb, prepare_write, do_write,
update_progress);
/* call output plugin initialization callback */
@@ -476,7 +479,8 @@ LogicalDecodingContext *
CreateDecodingContext(XLogRecPtr start_lsn,
List *output_plugin_options,
bool fast_forward,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress)
@@ -528,8 +532,8 @@ CreateDecodingContext(XLogRecPtr start_lsn,
ctx = StartupDecodingContext(output_plugin_options,
start_lsn, InvalidTransactionId, false,
- fast_forward, xl_routine, prepare_write,
- do_write, update_progress);
+ fast_forward, page_read, cleanup_cb,
+ prepare_write, do_write, update_progress);
/* call output plugin initialization callback */
old_context = MemoryContextSwitchTo(ctx->context);
@@ -585,7 +589,13 @@ DecodingContextFindStartpoint(LogicalDecodingContext *ctx)
char *err = NULL;
/* the read_page callback waits for new WAL */
- record = XLogReadRecord(ctx->reader, &err);
+ while (XLogReadRecord(ctx->reader, &record, &err) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!ctx->page_read(ctx->reader))
+ break;
+ }
+
if (err)
elog(ERROR, "%s", err);
if (!record)
diff --git a/src/backend/replication/logical/logicalfuncs.c b/src/backend/replication/logical/logicalfuncs.c
index f7e055874e..b69f6911c5 100644
--- a/src/backend/replication/logical/logicalfuncs.c
+++ b/src/backend/replication/logical/logicalfuncs.c
@@ -233,9 +233,8 @@ pg_logical_slot_get_changes_guts(FunctionCallInfo fcinfo, bool confirm, bool bin
ctx = CreateDecodingContext(InvalidXLogRecPtr,
options,
false,
- XL_ROUTINE(.page_read = read_local_xlog_page,
- .segment_open = wal_segment_open,
- .segment_close = wal_segment_close),
+ read_local_xlog_page,
+ wal_segment_close,
LogicalOutputPrepareWrite,
LogicalOutputWrite, NULL);
@@ -284,7 +283,13 @@ pg_logical_slot_get_changes_guts(FunctionCallInfo fcinfo, bool confirm, bool bin
XLogRecord *record;
char *errm = NULL;
- record = XLogReadRecord(ctx->reader, &errm);
+ while (XLogReadRecord(ctx->reader, &record, &errm) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!ctx->page_read(ctx->reader))
+ break;
+ }
+
if (errm)
elog(ERROR, "%s", errm);
diff --git a/src/backend/replication/slotfuncs.c b/src/backend/replication/slotfuncs.c
index 9817b44113..deb7cf15eb 100644
--- a/src/backend/replication/slotfuncs.c
+++ b/src/backend/replication/slotfuncs.c
@@ -153,9 +153,8 @@ create_logical_replication_slot(char *name, char *plugin,
ctx = CreateInitDecodingContext(plugin, NIL,
false, /* just catalogs is OK */
restart_lsn,
- XL_ROUTINE(.page_read = read_local_xlog_page,
- .segment_open = wal_segment_open,
- .segment_close = wal_segment_close),
+ read_local_xlog_page,
+ wal_segment_close,
NULL, NULL, NULL);
/*
@@ -512,9 +511,8 @@ pg_logical_replication_slot_advance(XLogRecPtr moveto)
ctx = CreateDecodingContext(InvalidXLogRecPtr,
NIL,
true, /* fast_forward */
- XL_ROUTINE(.page_read = read_local_xlog_page,
- .segment_open = wal_segment_open,
- .segment_close = wal_segment_close),
+ read_local_xlog_page,
+ wal_segment_close,
NULL, NULL, NULL);
/*
@@ -536,7 +534,13 @@ pg_logical_replication_slot_advance(XLogRecPtr moveto)
* Read records. No changes are generated in fast_forward mode,
* but snapbuilder/slot statuses are updated properly.
*/
- record = XLogReadRecord(ctx->reader, &errm);
+ while (XLogReadRecord(ctx->reader, &record, &errm) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!ctx->page_read(ctx->reader))
+ break;
+ }
+
if (errm)
elog(ERROR, "%s", errm);
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index c36b91ae5e..062093727d 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -580,10 +580,7 @@ StartReplication(StartReplicationCmd *cmd)
/* create xlogreader for physical replication */
xlogreader =
- XLogReaderAllocate(wal_segment_size, NULL,
- XL_ROUTINE(.segment_open = WalSndSegmentOpen,
- .segment_close = wal_segment_close),
- NULL);
+ XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
if (!xlogreader)
ereport(ERROR,
@@ -807,9 +804,11 @@ StartReplication(StartReplicationCmd *cmd)
* set every time WAL is flushed.
*/
static bool
-logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetRecPtr, char *cur_page)
+logical_read_xlog_page(XLogReaderState *state)
{
+ XLogRecPtr targetPagePtr = state->readPagePtr;
+ int reqLen = state->readLen;
+ char *cur_page = state->readBuf;
XLogRecPtr flushptr;
int count;
WALReadError errinfo;
@@ -837,7 +836,7 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
count = flushptr - targetPagePtr; /* part of the page available */
/* now actually read the data, we know it's there */
- if (!WALRead(state,
+ if (!WALRead(state, WalSndSegmentOpen, wal_segment_close,
cur_page,
targetPagePtr,
XLOG_BLCKSZ,
@@ -1011,9 +1010,8 @@ CreateReplicationSlot(CreateReplicationSlotCmd *cmd)
ctx = CreateInitDecodingContext(cmd->plugin, NIL, need_full_snapshot,
InvalidXLogRecPtr,
- XL_ROUTINE(.page_read = logical_read_xlog_page,
- .segment_open = WalSndSegmentOpen,
- .segment_close = wal_segment_close),
+ logical_read_xlog_page,
+ wal_segment_close,
WalSndPrepareWrite, WalSndWriteData,
WalSndUpdateProgress);
@@ -1171,9 +1169,8 @@ StartLogicalReplication(StartReplicationCmd *cmd)
*/
logical_decoding_ctx =
CreateDecodingContext(cmd->startpoint, cmd->options, false,
- XL_ROUTINE(.page_read = logical_read_xlog_page,
- .segment_open = WalSndSegmentOpen,
- .segment_close = wal_segment_close),
+ logical_read_xlog_page,
+ wal_segment_close,
WalSndPrepareWrite, WalSndWriteData,
WalSndUpdateProgress);
xlogreader = logical_decoding_ctx->reader;
@@ -2749,7 +2746,7 @@ XLogSendPhysical(void)
enlargeStringInfo(&output_message, nbytes);
retry:
- if (!WALRead(xlogreader,
+ if (!WALRead(xlogreader, WalSndSegmentOpen, wal_segment_close,
&output_message.data[output_message.len],
startptr,
nbytes,
@@ -2847,7 +2844,12 @@ XLogSendLogical(void)
*/
WalSndCaughtUp = false;
- record = XLogReadRecord(logical_decoding_ctx->reader, &errm);
+ while (XLogReadRecord(logical_decoding_ctx->reader, &record, &errm) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!logical_decoding_ctx->page_read(logical_decoding_ctx->reader))
+ break;
+ }
/* xlog record was invalid */
if (errm != NULL)
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index cf119848b0..712c85281c 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -41,15 +41,9 @@ static int xlogreadfd = -1;
static XLogSegNo xlogreadsegno = -1;
static char xlogfpath[MAXPGPATH];
-typedef struct XLogPageReadPrivate
-{
- const char *restoreCommand;
- int tliIndex;
-} XLogPageReadPrivate;
-
-static bool SimpleXLogPageRead(XLogReaderState *xlogreader,
- XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
+static bool SimpleXLogPageRead(XLogReaderState *xlogreader,
+ const char *datadir, int *tliIndex,
+ const char *restoreCommand);
/*
* Read WAL from the datadir/pg_wal, starting from 'startpoint' on timeline
@@ -66,20 +60,22 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
XLogRecord *record;
XLogReaderState *xlogreader;
char *errormsg;
- XLogPageReadPrivate private;
- private.tliIndex = tliIndex;
- private.restoreCommand = restoreCommand;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir,
- XL_ROUTINE(.page_read = &SimpleXLogPageRead),
- &private);
+ xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+
if (xlogreader == NULL)
pg_fatal("out of memory");
XLogBeginRead(xlogreader, startpoint);
do
{
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!SimpleXLogPageRead(xlogreader, datadir,
+ &tliIndex, restoreCommand))
+ break;
+ }
if (record == NULL)
{
@@ -123,19 +119,19 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex,
XLogRecord *record;
XLogReaderState *xlogreader;
char *errormsg;
- XLogPageReadPrivate private;
XLogRecPtr endptr;
- private.tliIndex = tliIndex;
- private.restoreCommand = restoreCommand;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir,
- XL_ROUTINE(.page_read = &SimpleXLogPageRead),
- &private);
+ xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
XLogBeginRead(xlogreader, ptr);
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!SimpleXLogPageRead(xlogreader, datadir, &tliIndex, restoreCommand))
+ break;
+ }
if (record == NULL)
{
if (errormsg)
@@ -170,7 +166,6 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
XLogRecPtr searchptr;
XLogReaderState *xlogreader;
char *errormsg;
- XLogPageReadPrivate private;
/*
* The given fork pointer points to the end of the last common record,
@@ -186,11 +181,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
forkptr += SizeOfXLogShortPHD;
}
- private.tliIndex = tliIndex;
- private.restoreCommand = restoreCommand;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir,
- XL_ROUTINE(.page_read = &SimpleXLogPageRead),
- &private);
+ xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
@@ -200,7 +191,13 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
uint8 info;
XLogBeginRead(xlogreader, searchptr);
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!SimpleXLogPageRead(xlogreader, datadir,
+ &tliIndex, restoreCommand))
+ break;
+ }
if (record == NULL)
{
@@ -247,10 +244,11 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
/* XLogReader callback function, to read a WAL page */
static bool
-SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *readBuf)
+SimpleXLogPageRead(XLogReaderState *xlogreader, const char *datadir,
+ int *tliIndex, const char *restoreCommand)
{
- XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
+ XLogRecPtr targetPagePtr = xlogreader->readPagePtr;
+ char *readBuf = xlogreader->readBuf;
uint32 targetPageOff;
XLogRecPtr targetSegEnd;
XLogSegNo targetSegNo;
@@ -283,14 +281,14 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
* be done both forward and backward, consider also switching timeline
* accordingly.
*/
- while (private->tliIndex < targetNentries - 1 &&
- targetHistory[private->tliIndex].end < targetSegEnd)
- private->tliIndex++;
- while (private->tliIndex > 0 &&
- targetHistory[private->tliIndex].begin >= targetSegEnd)
- private->tliIndex--;
+ while (*tliIndex < targetNentries - 1 &&
+ targetHistory[*tliIndex].end < targetSegEnd)
+ (*tliIndex)++;
+ while (*tliIndex > 0 &&
+ targetHistory[*tliIndex].begin >= targetSegEnd)
+ (*tliIndex)--;
- XLogFileName(xlogfname, targetHistory[private->tliIndex].tli,
+ XLogFileName(xlogfname, targetHistory[*tliIndex].tli,
xlogreadsegno, WalSegSz);
snprintf(xlogfpath, MAXPGPATH, "%s/" XLOGDIR "/%s",
@@ -303,7 +301,7 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
/*
* If we have no restore_command to execute, then exit.
*/
- if (private->restoreCommand == NULL)
+ if (restoreCommand == NULL)
{
pg_log_error("could not open file \"%s\": %m", xlogfpath);
xlogreader->readLen = -1;
@@ -317,7 +315,7 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
xlogreadfd = RestoreArchivedFile(xlogreader->segcxt.ws_dir,
xlogfname,
WalSegSz,
- private->restoreCommand);
+ restoreCommand);
if (xlogreadfd < 0)
{
@@ -359,7 +357,7 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
Assert(targetSegNo == xlogreadsegno);
- xlogreader->seg.ws_tli = targetHistory[private->tliIndex].tli;
+ xlogreader->seg.ws_tli = targetHistory[*tliIndex].tli;
xlogreader->readLen = XLOG_BLCKSZ;
return true;
}
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index 0f0e0723a3..f514de6b13 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -330,12 +330,17 @@ WALDumpCloseSegment(XLogReaderState *state)
state->seg.ws_file = -1;
}
-/* pg_waldump's XLogReaderRoutine->page_read callback */
+/*
+ * pg_waldump's WAL page rader, also used as page_read callback for
+ * XLogFindNextRecord
+ */
static bool
-WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetPtr, char *readBuff)
+WALDumpReadPage(XLogReaderState *state, void *priv)
{
- XLogDumpPrivate *private = state->private_data;
+ XLogRecPtr targetPagePtr = state->readPagePtr;
+ int reqLen = state->readLen;
+ char *readBuff = state->readBuf;
+ XLogDumpPrivate *private = (XLogDumpPrivate *) priv;
int count = XLOG_BLCKSZ;
WALReadError errinfo;
@@ -353,8 +358,8 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
}
}
- if (!WALRead(state, readBuff, targetPagePtr, count, private->timeline,
- &errinfo))
+ if (!WALRead(state, WALDumpOpenSegment, WALDumpCloseSegment,
+ readBuff, targetPagePtr, count, private->timeline, &errinfo))
{
WALOpenSegment *seg = &errinfo.wre_seg;
char fname[MAXPGPATH];
@@ -374,6 +379,7 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
(Size) errinfo.wre_req);
}
+ Assert(count >= state->readLen);
state->readLen = count;
return true;
}
@@ -1035,16 +1041,14 @@ main(int argc, char **argv)
/* we have everything we need, start reading */
xlogreader_state =
- XLogReaderAllocate(WalSegSz, waldir,
- XL_ROUTINE(.page_read = WALDumpReadPage,
- .segment_open = WALDumpOpenSegment,
- .segment_close = WALDumpCloseSegment),
- &private);
+ XLogReaderAllocate(WalSegSz, waldir, WALDumpCloseSegment);
+
if (!xlogreader_state)
fatal_error("out of memory");
/* first find a valid recptr to start from */
- first_record = XLogFindNextRecord(xlogreader_state, private.startptr);
+ first_record = XLogFindNextRecord(xlogreader_state, private.startptr,
+ &WALDumpReadPage, (void*) &private);
if (first_record == InvalidXLogRecPtr)
fatal_error("could not find a valid record after %X/%X",
@@ -1067,7 +1071,13 @@ main(int argc, char **argv)
for (;;)
{
/* try to read the next record */
- record = XLogReadRecord(xlogreader_state, &errormsg);
+ while (XLogReadRecord(xlogreader_state, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!WALDumpReadPage(xlogreader_state, (void *) &private))
+ break;
+ }
+
if (!record)
{
if (!config.follow || private.endptr_reached)
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index 23d9e70a59..4e9268b553 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -57,64 +57,15 @@ typedef struct WALSegmentContext
typedef struct XLogReaderState XLogReaderState;
-/* Function type definition for the read_page callback */
-typedef bool (*XLogPageReadCB) (XLogReaderState *xlogreader,
- XLogRecPtr targetPagePtr,
- int reqLen,
- XLogRecPtr targetRecPtr,
- char *readBuf);
+/* Function type definition for the segment cleanup callback */
+typedef void (*WALSegmentCleanupCB) (XLogReaderState *xlogreader);
+
+/* Function type definition for the open/close callbacks for WALRead() */
typedef void (*WALSegmentOpenCB) (XLogReaderState *xlogreader,
XLogSegNo nextSegNo,
TimeLineID *tli_p);
typedef void (*WALSegmentCloseCB) (XLogReaderState *xlogreader);
-typedef struct XLogReaderRoutine
-{
- /*
- * Data input callback
- *
- * This callback shall read at least reqLen valid bytes of the xlog page
- * starting at targetPagePtr, and store them in readBuf. The callback
- * shall return the number of bytes read (never more than XLOG_BLCKSZ), or
- * -1 on failure. The callback shall sleep, if necessary, to wait for the
- * requested bytes to become available. The callback will not be invoked
- * again for the same page unless more than the returned number of bytes
- * are needed.
- *
- * targetRecPtr is the position of the WAL record we're reading. Usually
- * it is equal to targetPagePtr + reqLen, but sometimes xlogreader needs
- * to read and verify the page or segment header, before it reads the
- * actual WAL record it's interested in. In that case, targetRecPtr can
- * be used to determine which timeline to read the page from.
- *
- * The callback shall set ->seg.ws_tli to the TLI of the file the page was
- * read from.
- */
- XLogPageReadCB page_read;
-
- /*
- * Callback to open the specified WAL segment for reading. ->seg.ws_file
- * shall be set to the file descriptor of the opened segment. In case of
- * failure, an error shall be raised by the callback and it shall not
- * return.
- *
- * "nextSegNo" is the number of the segment to be opened.
- *
- * "tli_p" is an input/output argument. WALRead() uses it to pass the
- * timeline in which the new segment should be found, but the callback can
- * use it to return the TLI that it actually opened.
- */
- WALSegmentOpenCB segment_open;
-
- /*
- * WAL segment close callback. ->seg.ws_file shall be set to a negative
- * number.
- */
- WALSegmentCloseCB segment_close;
-} XLogReaderRoutine;
-
-#define XL_ROUTINE(...) &(XLogReaderRoutine){__VA_ARGS__}
-
typedef struct
{
/* Is this block ref in use? */
@@ -144,12 +95,35 @@ typedef struct
uint16 data_bufsz;
} DecodedBkpBlock;
+/* Return code from XLogReadRecord */
+typedef enum XLogReadRecordResult
+{
+ XLREAD_SUCCESS, /* record is successfully read */
+ XLREAD_NEED_DATA, /* need more data. see XLogReadRecord. */
+ XLREAD_FAIL /* failed during reading a record */
+} XLogReadRecordResult;
+
+/*
+ * internal state of XLogReadRecord
+ *
+ * XLogReadState runs a state machine while reading a record. Theses states
+ * are not seen outside the function. Each state may repeat several times
+ * exiting requesting caller for new data. See the comment of XLogReadRecrod
+ * for details.
+ */
+typedef enum XLogReadRecordState {
+ XLREAD_NEXT_RECORD,
+ XLREAD_TOT_LEN,
+ XLREAD_FIRST_FRAGMENT,
+ XLREAD_CONTINUATION
+} XLogReadRecordState;
+
struct XLogReaderState
{
/*
* Operational callbacks
*/
- XLogReaderRoutine routine;
+ WALSegmentCleanupCB cleanup_cb;
/* ----------------------------------------
* Public parameters
@@ -162,18 +136,14 @@ struct XLogReaderState
*/
uint64 system_identifier;
- /*
- * Opaque data for callbacks to use. Not used by XLogReader.
- */
- void *private_data;
-
/*
* Start and end point of last record read. EndRecPtr is also used as the
* position to read next. Calling XLogBeginRead() sets EndRecPtr to the
* starting position and ReadRecPtr to invalid.
*/
- XLogRecPtr ReadRecPtr; /* start of last record read */
+ XLogRecPtr ReadRecPtr; /* start of last record read or being read */
XLogRecPtr EndRecPtr; /* end+1 of last record read */
+ XLogRecPtr PrevRecPtr; /* start of previous record read */
/* ----------------------------------------
* Communication with page reader
@@ -187,7 +157,9 @@ struct XLogReaderState
* the request, or -1 on error */
TimeLineID readPageTLI; /* TLI for data currently in readBuf */
char *readBuf; /* buffer to store data */
- bool page_verified; /* is the page on the buffer verified? */
+ bool page_verified; /* is the page header on the buffer verified? */
+ bool record_verified;/* is the current record header verified? */
+
/* ----------------------------------------
@@ -229,8 +201,6 @@ struct XLogReaderState
XLogRecPtr latestPagePtr;
TimeLineID latestPageTLI;
- /* beginning of the WAL record being read. */
- XLogRecPtr currRecPtr;
/* timeline to read it from, 0 if a lookup is required */
TimeLineID currTLI;
@@ -257,6 +227,15 @@ struct XLogReaderState
char *readRecordBuf;
uint32 readRecordBufSize;
+ /*
+ * XLogReadRecord() state
+ */
+ XLogReadRecordState readRecordState;/* state machine state */
+ int recordGotLen; /* amount of current record that has
+ * already been read */
+ int recordRemainLen; /* length of current record that remains */
+ XLogRecPtr recordContRecPtr; /* where the current record continues */
+
/* Buffer to hold error message */
char *errormsg_buf;
};
@@ -264,9 +243,7 @@ struct XLogReaderState
/* Get a new XLogReader */
extern XLogReaderState *XLogReaderAllocate(int wal_segment_size,
const char *waldir,
- XLogReaderRoutine *routine,
- void *private_data);
-extern XLogReaderRoutine *LocalXLogReaderRoutine(void);
+ WALSegmentCleanupCB cleanup_cb);
/* Free an XLogReader */
extern void XLogReaderFree(XLogReaderState *state);
@@ -274,12 +251,17 @@ extern void XLogReaderFree(XLogReaderState *state);
/* Position the XLogReader to given record */
extern void XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr);
#ifdef FRONTEND
-extern XLogRecPtr XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr);
+/* Function type definition for the read_page callback */
+typedef bool (*XLogFindNextRecordCB) (XLogReaderState *xlogreader,
+ void *private);
+extern XLogRecPtr XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
+ XLogFindNextRecordCB read_page, void *private);
#endif /* FRONTEND */
/* Read the next XLog record. Returns NULL on end-of-WAL or failure */
-extern struct XLogRecord *XLogReadRecord(XLogReaderState *state,
- char **errormsg);
+extern XLogReadRecordResult XLogReadRecord(XLogReaderState *state,
+ XLogRecord **record,
+ char **errormsg);
/* Validate a page */
extern bool XLogReaderValidatePageHeader(XLogReaderState *state,
@@ -299,6 +281,7 @@ typedef struct WALReadError
} WALReadError;
extern bool WALRead(XLogReaderState *state,
+ WALSegmentOpenCB segopenfn, WALSegmentCloseCB sgclosefn,
char *buf, XLogRecPtr startptr, Size count,
TimeLineID tli, WALReadError *errinfo);
diff --git a/src/include/access/xlogutils.h b/src/include/access/xlogutils.h
index 364a21c4ea..397fb27fc2 100644
--- a/src/include/access/xlogutils.h
+++ b/src/include/access/xlogutils.h
@@ -47,9 +47,7 @@ extern Buffer XLogReadBufferExtended(RelFileNode rnode, ForkNumber forknum,
extern Relation CreateFakeRelcacheEntry(RelFileNode rnode);
extern void FreeFakeRelcacheEntry(Relation fakerel);
-extern bool read_local_xlog_page(XLogReaderState *state,
- XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetRecPtr, char *cur_page);
+extern bool read_local_xlog_page(XLogReaderState *state);
extern void wal_segment_open(XLogReaderState *state,
XLogSegNo nextSegNo,
TimeLineID *tli_p);
diff --git a/src/include/pg_config_manual.h b/src/include/pg_config_manual.h
index f10ad0acd6..94749609b4 100644
--- a/src/include/pg_config_manual.h
+++ b/src/include/pg_config_manual.h
@@ -377,7 +377,7 @@
* Enable debugging print statements for WAL-related operations; see
* also the wal_debug GUC var.
*/
-/* #define WAL_DEBUG */
+#define WAL_DEBUG
/*
* Enable tracing of resource consumption during sort operations;
diff --git a/src/include/replication/logical.h b/src/include/replication/logical.h
index c253403372..b936f31393 100644
--- a/src/include/replication/logical.h
+++ b/src/include/replication/logical.h
@@ -29,6 +29,10 @@ typedef void (*LogicalOutputPluginWriterUpdateProgress) (struct LogicalDecodingC
TransactionId xid
);
+typedef struct LogicalDecodingContext LogicalDecodingContext;
+
+typedef bool (*LogicalDecodingXLogPageReadCB)(XLogReaderState *ctx);
+
typedef struct LogicalDecodingContext
{
/* memory context this is all allocated in */
@@ -39,6 +43,7 @@ typedef struct LogicalDecodingContext
/* infrastructure pieces for decoding */
XLogReaderState *reader;
+ LogicalDecodingXLogPageReadCB page_read;
struct ReorderBuffer *reorder;
struct SnapBuild *snapshot_builder;
@@ -105,14 +110,16 @@ extern LogicalDecodingContext *CreateInitDecodingContext(const char *plugin,
List *output_plugin_options,
bool need_full_snapshot,
XLogRecPtr restart_lsn,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress);
extern LogicalDecodingContext *CreateDecodingContext(XLogRecPtr start_lsn,
List *output_plugin_options,
bool fast_forward,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress);
--
2.27.0
----Next_Part(Thu_Mar__4_11_28_53_2021_014)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
filename="v17-0003-Remove-globals-readOff-readLen-and-readSegNo.patch"
^ permalink raw reply [nested|flat] 70+ messages in thread
* [PATCH v15 2/4] Move page-reader out of XLogReadRecord
@ 2019-09-10 03:58 Kyotaro Horiguchi <[email protected]>
0 siblings, 0 replies; 70+ messages in thread
From: Kyotaro Horiguchi @ 2019-09-10 03:58 UTC (permalink / raw)
This is the second step of removing callbacks from WAL record reader.
Since it is essential to take in additional data while reading a
record, the function have to ask caller for new data while keeping
working state. Thus the function is turned into a state machine.
---
src/backend/access/transam/twophase.c | 15 +-
src/backend/access/transam/xlog.c | 58 +-
src/backend/access/transam/xlogreader.c | 700 +++++++++++-------
src/backend/access/transam/xlogutils.c | 17 +-
src/backend/replication/logical/logical.c | 26 +-
.../replication/logical/logicalfuncs.c | 13 +-
src/backend/replication/slotfuncs.c | 18 +-
src/backend/replication/walsender.c | 32 +-
src/bin/pg_rewind/parsexlog.c | 84 +--
src/bin/pg_waldump/pg_waldump.c | 36 +-
src/include/access/xlogreader.h | 121 ++-
src/include/access/xlogutils.h | 4 +-
src/include/replication/logical.h | 11 +-
13 files changed, 648 insertions(+), 487 deletions(-)
diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index 9b2e59bf0e..b0d60a0d0f 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -1330,11 +1330,8 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
XLogReaderState *xlogreader;
char *errormsg;
- xlogreader = XLogReaderAllocate(wal_segment_size, NULL,
- XL_ROUTINE(.page_read = &read_local_xlog_page,
- .segment_open = &wal_segment_open,
- .segment_close = &wal_segment_close),
- NULL);
+ xlogreader = XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+
if (!xlogreader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
@@ -1342,7 +1339,13 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
errdetail("Failed while allocating a WAL reading processor.")));
XLogBeginRead(xlogreader, lsn);
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!read_local_xlog_page(xlogreader))
+ break;
+ }
+
if (record == NULL)
ereport(ERROR,
(errcode_for_file_access(),
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index e570e56a24..f9b0108602 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -831,13 +831,6 @@ static XLogSource currentSource = XLOG_FROM_ANY;
static bool lastSourceFailed = false;
static bool pendingWalRcvRestart = false;
-typedef struct XLogPageReadPrivate
-{
- int emode;
- bool fetching_ckpt; /* are we fetching a checkpoint record? */
- bool randAccess;
-} XLogPageReadPrivate;
-
/*
* These variables track when we last obtained some WAL data to process,
* and where we got it from. (XLogReceiptSource is initially the same as
@@ -912,8 +905,8 @@ static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath,
static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli,
XLogSource source, bool notfoundOk);
static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, XLogSource source);
-static bool XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
+static bool XLogPageRead(XLogReaderState *xlogreader,
+ bool fetching_ckpt, int emode, bool randAccess);
static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
bool fetching_ckpt, XLogRecPtr tliRecPtr);
static int emode_for_corrupt_record(int emode, XLogRecPtr RecPtr);
@@ -1225,8 +1218,7 @@ XLogInsertRecord(XLogRecData *rdata,
appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len);
if (!debug_reader)
- debug_reader = XLogReaderAllocate(wal_segment_size, NULL,
- XL_ROUTINE(), NULL);
+ debug_reader = XLogReaderAllocate(wal_segment_size, NULL NULL);
if (!debug_reader)
{
@@ -4326,15 +4318,10 @@ CleanupBackupHistory(void)
* record is available.
*/
static XLogRecord *
-ReadRecord(XLogReaderState *xlogreader, int emode,
- bool fetching_ckpt)
+ReadRecord(XLogReaderState *xlogreader, int emode, bool fetching_ckpt)
{
XLogRecord *record;
- XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
-
- private->fetching_ckpt = fetching_ckpt;
- private->emode = emode;
- private->randAccess = (xlogreader->ReadRecPtr == InvalidXLogRecPtr);
+ bool randAccess = (xlogreader->ReadRecPtr == InvalidXLogRecPtr);
/* This is the first attempt to read this page. */
lastSourceFailed = false;
@@ -4342,8 +4329,16 @@ ReadRecord(XLogReaderState *xlogreader, int emode,
for (;;)
{
char *errormsg;
+ XLogReadRecordResult result;
+
+ while ((result = XLogReadRecord(xlogreader, &record, &errormsg))
+ == XLREAD_NEED_DATA)
+ {
+ if (!XLogPageRead(xlogreader, fetching_ckpt, emode, randAccess))
+ break;
+
+ }
- record = XLogReadRecord(xlogreader, &errormsg);
ReadRecPtr = xlogreader->ReadRecPtr;
EndRecPtr = xlogreader->EndRecPtr;
if (record == NULL)
@@ -6320,7 +6315,6 @@ StartupXLOG(void)
bool backupFromStandby = false;
DBState dbstate_at_startup;
XLogReaderState *xlogreader;
- XLogPageReadPrivate private;
bool fast_promoted = false;
struct stat st;
@@ -6480,13 +6474,9 @@ StartupXLOG(void)
OwnLatch(&XLogCtl->recoveryWakeupLatch);
/* Set up XLOG reader facility */
- MemSet(&private, 0, sizeof(XLogPageReadPrivate));
xlogreader =
- XLogReaderAllocate(wal_segment_size, NULL,
- XL_ROUTINE(.page_read = &XLogPageRead,
- .segment_open = NULL,
- .segment_close = wal_segment_close),
- &private);
+ XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+
if (!xlogreader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
@@ -11860,12 +11850,13 @@ CancelBackup(void)
* sleep and retry.
*/
static bool
-XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetRecPtr, char *readBuf)
+XLogPageRead(XLogReaderState *xlogreader,
+ bool fetching_ckpt, int emode, bool randAccess)
{
- XLogPageReadPrivate *private =
- (XLogPageReadPrivate *) xlogreader->private_data;
- int emode = private->emode;
+ char *readBuf = xlogreader->readBuf;
+ XLogRecPtr targetPagePtr = xlogreader->readPagePtr;
+ int reqLen = xlogreader->readLen;
+ XLogRecPtr targetRecPtr = xlogreader->ReadRecPtr;
uint32 targetPageOff;
XLogSegNo targetSegNo PG_USED_FOR_ASSERTS_ONLY;
int r;
@@ -11908,8 +11899,8 @@ retry:
flushedUpto < targetPagePtr + reqLen))
{
if (!WaitForWALToBecomeAvailable(targetPagePtr + reqLen,
- private->randAccess,
- private->fetching_ckpt,
+ randAccess,
+ fetching_ckpt,
targetRecPtr))
{
if (readFile >= 0)
@@ -12014,6 +12005,7 @@ retry:
goto next_record_is_invalid;
}
+ Assert(xlogreader->readPagePtr == targetPagePtr);
xlogreader->readLen = readLen;
return true;
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index 3d599325ee..9281b57379 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -40,7 +40,7 @@ static bool XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr,
int reqLen, bool header_inclusive);
static void XLogReaderInvalReadState(XLogReaderState *state);
static bool ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
- XLogRecPtr PrevRecPtr, XLogRecord *record, bool randAccess);
+ XLogRecPtr PrevRecPtr, XLogRecord *record);
static bool ValidXLogRecord(XLogReaderState *state, XLogRecord *record,
XLogRecPtr recptr);
static void ResetDecoder(XLogReaderState *state);
@@ -73,7 +73,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...)
*/
XLogReaderState *
XLogReaderAllocate(int wal_segment_size, const char *waldir,
- XLogReaderRoutine *routine, void *private_data)
+ WALSegmentCleanupCB cleanup_cb)
{
XLogReaderState *state;
@@ -84,7 +84,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
return NULL;
/* initialize caller-provided support functions */
- state->routine = *routine;
+ state->cleanup_cb = cleanup_cb;
state->max_block_id = -1;
@@ -107,8 +107,6 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
WALOpenSegmentInit(&state->seg, &state->segcxt, wal_segment_size,
waldir);
- /* system_identifier initialized to zeroes above */
- state->private_data = private_data;
/* ReadRecPtr, EndRecPtr and readLen initialized to zeroes above */
state->errormsg_buf = palloc_extended(MAX_ERRORMSG_LEN + 1,
MCXT_ALLOC_NO_OOM);
@@ -140,8 +138,8 @@ XLogReaderFree(XLogReaderState *state)
{
int block_id;
- if (state->seg.ws_file != -1)
- state->routine.segment_close(state);
+ if (state->seg.ws_file >= 0)
+ state->cleanup_cb(state);
for (block_id = 0; block_id <= XLR_MAX_BLOCK_ID; block_id++)
{
@@ -246,6 +244,7 @@ XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
/* Begin at the passed-in record pointer. */
state->EndRecPtr = RecPtr;
state->ReadRecPtr = InvalidXLogRecPtr;
+ state->readRecordState = XLREAD_NEXT_RECORD;
}
/*
@@ -254,317 +253,452 @@ XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
* XLogBeginRead() or XLogFindNextRecord() must be called before the first call
* to XLogReadRecord().
*
- * If the page_read callback fails to read the requested data, NULL is
- * returned. The callback is expected to have reported the error; errormsg
- * is set to NULL.
+ * This function may return XLREAD_NEED_DATA several times before returning a
+ * result record. The caller shall read in some new data then call this
+ * function again with the same parameters.
*
- * If the reading fails for some other reason, NULL is also returned, and
- * *errormsg is set to a string with details of the failure.
+ * When a record is successfully read, returns XLREAD_SUCCESS with result
+ * record being stored in *record. Otherwise *record is NULL.
*
- * The returned pointer (or *errormsg) points to an internal buffer that's
- * valid until the next call to XLogReadRecord.
+ * Returns XLREAD_NEED_DATA if more data is needed to finish reading the
+ * current record. In that case, state->readPagePtr and state->readLen inform
+ * the desired position and minimum length of data needed. The caller shall
+ * read in the requested data and set state->readBuf to point to a buffer
+ * containing it. The caller must also set state->readPageTLI and
+ * state->readLen to indicate the timeline that it was read from, and the
+ * length of data that is now available (which must be >= given readLen),
+ * respectively.
+ *
+ * If invalid data is encountered, returns XLREAD_FAIL with *record being set to
+ * NULL. *errormsg is set to a string with details of the failure.
+ * The returned pointer (or *errormsg) points to an internal buffer that's valid
+ * until the next call to XLogReadRecord.
+ *
+ *
+ * This function runs a state machine consists of the following states.
+ *
+ * XLREAD_NEXT_RECORD :
+ * The initial state, if called with valid RecPtr, try to read a record at
+ * that position. If invalid RecPtr is given try to read a record just after
+ * the last one previously read.
+ * This state ens after setting ReadRecPtr. Then goes to XLREAD_TOT_LEN.
+ *
+ * XLREAD_TOT_LEN:
+ * Examining record header. Ends after reading record total
+ * length. recordRemainLen and recordGotLen are initialized.
+ *
+ * XLREAD_FIRST_FRAGMENT:
+ * Reading the first fragment. Ends with finishing reading a single
+ * record. Goes to XLREAD_NEXT_RECORD if that's all or
+ * XLREAD_CONTINUATION if we have continuation.
+
+ * XLREAD_CONTINUATION:
+ * Reading continuation of record. Ends with finishing the whole record then
+ * goes to XLREAD_NEXT_RECORD. During this state, recordRemainLen indicates
+ * how much is left and readRecordBuf holds the partially assert
+ * record.recordContRecPtr points to the beginning of the next page where to
+ * continue.
+ *
+ * If wrong data found in any state, the state machine stays at the current
+ * state. This behavior allows to continue reading a reacord switching among
+ * different souces, while streaming replication.
*/
-XLogRecord *
-XLogReadRecord(XLogReaderState *state, char **errormsg)
+XLogReadRecordResult
+XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
{
- XLogRecPtr RecPtr;
- XLogRecord *record;
- XLogRecPtr targetPagePtr;
- bool randAccess;
- uint32 len,
- total_len;
- uint32 targetRecOff;
- uint32 pageHeaderSize;
- bool gotheader;
+ XLogRecord *prec;
- /*
- * randAccess indicates whether to verify the previous-record pointer of
- * the record we're reading. We only do this if we're reading
- * sequentially, which is what we initially assume.
- */
- randAccess = false;
+ *record = NULL;
/* reset error state */
*errormsg = NULL;
state->errormsg_buf[0] = '\0';
- ResetDecoder(state);
-
- RecPtr = state->EndRecPtr;
-
- if (state->ReadRecPtr != InvalidXLogRecPtr)
- {
- /* read the record after the one we just read */
-
- /*
- * EndRecPtr is pointing to end+1 of the previous WAL record. If
- * we're at a page boundary, no more records can fit on the current
- * page. We must skip over the page header, but we can't do that until
- * we've read in the page, since the header size is variable.
- */
- }
- else
- {
- /*
- * Caller supplied a position to start at.
- *
- * In this case, EndRecPtr should already be pointing to a valid
- * record starting position.
- */
- Assert(XRecOffIsValid(RecPtr));
- randAccess = true;
- }
-
- state->currRecPtr = RecPtr;
-
- targetPagePtr = RecPtr - (RecPtr % XLOG_BLCKSZ);
- targetRecOff = RecPtr % XLOG_BLCKSZ;
-
- /*
- * Read the page containing the record into state->readBuf. Request enough
- * byte to cover the whole record header, or at least the part of it that
- * fits on the same page.
- */
- while (XLogNeedData(state, targetPagePtr,
- Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
- targetRecOff != 0))
- {
- if (!state->routine.page_read(state, state->readPagePtr, state->readLen,
- RecPtr, state->readBuf))
- break;
- }
-
- if (!state->page_verified)
- goto err;
-
- /*
- * We have at least the page header, so we can examine it now.
- */
- pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
- if (targetRecOff == 0)
- {
- /*
- * At page start, so skip over page header.
- */
- RecPtr += pageHeaderSize;
- targetRecOff = pageHeaderSize;
- }
- else if (targetRecOff < pageHeaderSize)
- {
- report_invalid_record(state, "invalid record offset at %X/%X",
- (uint32) (RecPtr >> 32), (uint32) RecPtr);
- goto err;
- }
-
- if ((((XLogPageHeader) state->readBuf)->xlp_info & XLP_FIRST_IS_CONTRECORD) &&
- targetRecOff == pageHeaderSize)
- {
- report_invalid_record(state, "contrecord is requested by %X/%X",
- (uint32) (RecPtr >> 32), (uint32) RecPtr);
- goto err;
- }
-
- /* XLogNeedData has verified the page header */
- Assert(pageHeaderSize <= state->readLen);
-
- /*
- * Read the record length.
- *
- * NB: Even though we use an XLogRecord pointer here, the whole record
- * header might not fit on this page. xl_tot_len is the first field of the
- * struct, so it must be on this page (the records are MAXALIGNed), but we
- * cannot access any other fields until we've verified that we got the
- * whole header.
- */
- record = (XLogRecord *) (state->readBuf + RecPtr % XLOG_BLCKSZ);
- total_len = record->xl_tot_len;
-
- /*
- * If the whole record header is on this page, validate it immediately.
- * Otherwise do just a basic sanity check on xl_tot_len, and validate the
- * rest of the header after reading it from the next page. The xl_tot_len
- * check is necessary here to ensure that we enter the "Need to reassemble
- * record" code path below; otherwise we might fail to apply
- * ValidXLogRecordHeader at all.
- */
- if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
- {
- if (!ValidXLogRecordHeader(state, RecPtr, state->ReadRecPtr, record,
- randAccess))
- goto err;
- gotheader = true;
- }
- else
- {
- /* XXX: more validation should be done here */
- if (total_len < SizeOfXLogRecord)
- {
- report_invalid_record(state,
- "invalid record length at %X/%X: wanted %u, got %u",
- (uint32) (RecPtr >> 32), (uint32) RecPtr,
- (uint32) SizeOfXLogRecord, total_len);
- goto err;
- }
- gotheader = false;
- }
-
- len = XLOG_BLCKSZ - RecPtr % XLOG_BLCKSZ;
- if (total_len > len)
+ switch (state->readRecordState)
{
- /* Need to reassemble record */
- char *contdata;
- XLogPageHeader pageHeader;
- char *buffer;
- uint32 gotlen;
-
- /*
- * Enlarge readRecordBuf as needed.
- */
- if (total_len > state->readRecordBufSize &&
- !allocate_recordbuf(state, total_len))
- {
- /* We treat this as a "bogus data" condition */
- report_invalid_record(state, "record length %u at %X/%X too long",
- total_len,
- (uint32) (RecPtr >> 32), (uint32) RecPtr);
- goto err;
- }
+ case XLREAD_NEXT_RECORD:
+ ResetDecoder(state);
- /* Copy the first fragment of the record from the first page. */
- memcpy(state->readRecordBuf,
- state->readBuf + RecPtr % XLOG_BLCKSZ, len);
- buffer = state->readRecordBuf + len;
- gotlen = len;
-
- do
- {
- int rest_len = total_len - gotlen;
-
- /* Calculate pointer to beginning of next page */
- targetPagePtr += XLOG_BLCKSZ;
-
- /* Wait for the next page to become available */
- while (XLogNeedData(state, targetPagePtr,
- Min(rest_len, XLOG_BLCKSZ),
- false))
+ if (state->ReadRecPtr != InvalidXLogRecPtr)
{
- if (!state->routine.page_read(state, state->readPagePtr,
- state->readLen,
- state->ReadRecPtr,
- state->readBuf))
- break;
+ /* read the record after the one we just read */
+
+ /*
+ * EndRecPtr is pointing to end+1 of the previous WAL record.
+ * If we're at a page boundary, no more records can fit on the
+ * current page. We must skip over the page header, but we
+ * can't do that until we've read in the page, since the header
+ * size is variable.
+ */
+ state->PrevRecPtr = state->ReadRecPtr;
+ state->ReadRecPtr = state->EndRecPtr;
+ }
+ else
+ {
+ /*
+ * Caller supplied a position to start at.
+ *
+ * In this case, EndRecPtr should already be pointing to a
+ * valid record starting position.
+ */
+ Assert(XRecOffIsValid(state->EndRecPtr));
+ state->ReadRecPtr = state->EndRecPtr;
+
+ /*
+ * We cannot verify the previous-record pointer when we're
+ * seeking to a particular record. Reset PrevRecPtr so that we
+ * won't try doing that.
+ */
+ state->PrevRecPtr = InvalidXLogRecPtr;
+ state->EndRecPtr = InvalidXLogRecPtr; /* to be tidy */
}
+ state->record_verified = false;
+ state->readRecordState = XLREAD_TOT_LEN;
+ /* fall through */
+
+ case XLREAD_TOT_LEN:
+ {
+ uint32 total_len;
+ uint32 pageHeaderSize;
+ XLogRecPtr targetPagePtr;
+ uint32 targetRecOff;
+ XLogPageHeader pageHeader;
+
+ targetPagePtr =
+ state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
+ targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+
+ /*
+ * Check if we have enough data. For the first record in the page,
+ * the requesting length doesn't contain page header.
+ */
+ if (XLogNeedData(state, targetPagePtr,
+ Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+ targetRecOff != 0))
+ return XLREAD_NEED_DATA;
+
+ /* error out if caller supplied bogus page */
if (!state->page_verified)
goto err;
- Assert(SizeOfXLogShortPHD <= state->readLen);
+ /* examine page header now. */
+ pageHeaderSize =
+ XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+ if (targetRecOff == 0)
+ {
+ /* At page start, so skip over page header. */
+ state->ReadRecPtr += pageHeaderSize;
+ targetRecOff = pageHeaderSize;
+ }
+ else if (targetRecOff < pageHeaderSize)
+ {
+ report_invalid_record(state, "invalid record offset at %X/%X",
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr);
+ goto err;
+ }
- /* Check that the continuation on next page looks valid */
pageHeader = (XLogPageHeader) state->readBuf;
- if (!(pageHeader->xlp_info & XLP_FIRST_IS_CONTRECORD))
+ if ((pageHeader->xlp_info & XLP_FIRST_IS_CONTRECORD) &&
+ targetRecOff == pageHeaderSize)
{
- report_invalid_record(state,
- "there is no contrecord flag at %X/%X",
- (uint32) (RecPtr >> 32), (uint32) RecPtr);
+ report_invalid_record(state, "contrecord is requested by %X/%X",
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr);
goto err;
}
- /*
- * Cross-check that xlp_rem_len agrees with how much of the record
- * we expect there to be left.
- */
- if (pageHeader->xlp_rem_len == 0 ||
- total_len != (pageHeader->xlp_rem_len + gotlen))
- {
- report_invalid_record(state,
- "invalid contrecord length %u at %X/%X",
- pageHeader->xlp_rem_len,
- (uint32) (RecPtr >> 32), (uint32) RecPtr);
- goto err;
- }
-
- /* Append the continuation from this page to the buffer */
- pageHeaderSize = XLogPageHeaderSize(pageHeader);
-
+ /* XLogNeedData has verified the page header */
Assert(pageHeaderSize <= state->readLen);
- contdata = (char *) state->readBuf + pageHeaderSize;
- len = XLOG_BLCKSZ - pageHeaderSize;
- if (pageHeader->xlp_rem_len < len)
- len = pageHeader->xlp_rem_len;
+ /*
+ * Read the record length.
+ *
+ * NB: Even though we use an XLogRecord pointer here, the whole
+ * record header might not fit on this page. xl_tot_len is the first
+ * field of the struct, so it must be on this page (the records are
+ * MAXALIGNed), but we cannot access any other fields until we've
+ * verified that we got the whole header.
+ */
+ prec = (XLogRecord *) (state->readBuf +
+ state->ReadRecPtr % XLOG_BLCKSZ);
+ total_len = prec->xl_tot_len;
- Assert (pageHeaderSize + len <= state->readLen);
- memcpy(buffer, (char *) contdata, len);
- buffer += len;
- gotlen += len;
+ /*
+ * If the whole record header is on this page, validate it
+ * immediately. Otherwise do just a basic sanity check on
+ * xl_tot_len, and validate the rest of the header after reading it
+ * from the next page. The xl_tot_len check is necessary here to
+ * ensure that we enter the XLREAD_CONTINUATION state below;
+ * otherwise we might fail to apply ValidXLogRecordHeader at all.
+ */
+ if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
+ {
+ if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
+ state->PrevRecPtr, prec))
+ goto err;
- /* If we just reassembled the record header, validate it. */
- if (!gotheader)
+ state->record_verified = true;
+ }
+ else
{
- record = (XLogRecord *) state->readRecordBuf;
- if (!ValidXLogRecordHeader(state, RecPtr, state->ReadRecPtr,
- record, randAccess))
+ /* XXX: more validation should be done here */
+ if (total_len < SizeOfXLogRecord)
+ {
+ report_invalid_record(state,
+ "invalid record length at %X/%X: wanted %u, got %u",
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr,
+ (uint32) SizeOfXLogRecord, total_len);
goto err;
- gotheader = true;
+ }
}
- } while (gotlen < total_len);
-
- Assert(gotheader);
- record = (XLogRecord *) state->readRecordBuf;
- if (!ValidXLogRecord(state, record, RecPtr))
- goto err;
+ /*
+ * Wait for the rest of the record, or the part of the record that
+ * fit on the first page if crossed a page boundary, to become
+ * available.
+ */
+ state->recordGotLen = 0;
+ state->recordRemainLen = total_len;
+ state->readRecordState = XLREAD_FIRST_FRAGMENT;
+ }
+ /* fall through */
- pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
- state->ReadRecPtr = RecPtr;
- state->EndRecPtr = targetPagePtr + pageHeaderSize
- + MAXALIGN(pageHeader->xlp_rem_len);
- }
- else
- {
- /* Wait for the record data to become available */
- while (XLogNeedData(state, targetPagePtr,
- Min(targetRecOff + total_len, XLOG_BLCKSZ), true))
+ case XLREAD_FIRST_FRAGMENT:
{
- if (!state->routine.page_read(state, state->readPagePtr,
- state->readLen,
- state->ReadRecPtr, state->readBuf))
+ uint32 total_len = state->recordRemainLen;
+ uint32 request_len;
+ uint32 record_len;
+ XLogRecPtr targetPagePtr;
+ uint32 targetRecOff;
+
+ /*
+ * Wait for the rest of the record on the first page to become
+ * available
+ */
+ targetPagePtr =
+ state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
+ targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+
+ request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ);
+ record_len = request_len - targetRecOff;
+
+ /* ReadRecPtr contains page header */
+ Assert (targetRecOff != 0);
+ if (XLogNeedData(state, targetPagePtr, request_len, true))
+ return XLREAD_NEED_DATA;
+
+ /* error out if caller supplied bogus page */
+ if (!state->page_verified)
+ goto err;
+
+ prec = (XLogRecord *) (state->readBuf + targetRecOff);
+
+ /* validate record header if not yet */
+ if (!state->record_verified && record_len >= SizeOfXLogRecord)
+ {
+ if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
+ state->PrevRecPtr, prec))
+ goto err;
+
+ state->record_verified = true;
+ }
+
+
+ if (total_len == record_len)
+ {
+ /* Record does not cross a page boundary */
+ Assert(state->record_verified);
+
+ if (!ValidXLogRecord(state, prec, state->ReadRecPtr))
+ goto err;
+
+ state->record_verified = true; /* to be tidy */
+
+ /* We already checked the header earlier */
+ state->EndRecPtr = state->ReadRecPtr + MAXALIGN(record_len);
+
+ *record = prec;
+ state->readRecordState = XLREAD_NEXT_RECORD;
break;
+ }
+
+ /*
+ * The record continues on the next page. Need to reassemble
+ * record
+ */
+ Assert(total_len > record_len);
+
+ /* Enlarge readRecordBuf as needed. */
+ if (total_len > state->readRecordBufSize &&
+ !allocate_recordbuf(state, total_len))
+ {
+ /* We treat this as a "bogus data" condition */
+ report_invalid_record(state,
+ "record length %u at %X/%X too long",
+ total_len,
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr);
+ goto err;
+ }
+
+ /* Copy the first fragment of the record from the first page. */
+ memcpy(state->readRecordBuf, state->readBuf + targetRecOff,
+ record_len);
+ state->recordGotLen += record_len;
+ state->recordRemainLen -= record_len;
+
+ /* Calculate pointer to beginning of next page */
+ state->recordContRecPtr = state->ReadRecPtr + record_len;
+ Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0);
+
+ state->readRecordState = XLREAD_CONTINUATION;
}
+ /* fall through */
+
+ case XLREAD_CONTINUATION:
+ {
+ XLogPageHeader pageHeader;
+ uint32 pageHeaderSize;
+ XLogRecPtr targetPagePtr;
+
+ /* we enter this state only if we haven't read the whole record. */
+ Assert (state->recordRemainLen > 0);
+
+ while(state->recordRemainLen > 0)
+ {
+ char *contdata;
+ uint32 request_len;
+ uint32 record_len;
+
+ /* Wait for the next page to become available */
+ targetPagePtr = state->recordContRecPtr;
+
+ /* this request contains page header */
+ Assert (targetPagePtr != 0);
+ if (XLogNeedData(state, targetPagePtr,
+ Min(state->recordRemainLen, XLOG_BLCKSZ),
+ false))
+ return XLREAD_NEED_DATA;
+
+ if (!state->page_verified)
+ goto err;
+
+ Assert(SizeOfXLogShortPHD <= state->readLen);
- if (!state->page_verified)
- goto err;
+ /* Check that the continuation on next page looks valid */
+ pageHeader = (XLogPageHeader) state->readBuf;
+ if (!(pageHeader->xlp_info & XLP_FIRST_IS_CONTRECORD))
+ {
+ report_invalid_record(
+ state,
+ "there is no contrecord flag at %X/%X reading %X/%X",
+ (uint32) (state->recordContRecPtr >> 32),
+ (uint32) state->recordContRecPtr,
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr);
+ goto err;
+ }
- /* Record does not cross a page boundary */
- if (!ValidXLogRecord(state, record, RecPtr))
- goto err;
+ /*
+ * Cross-check that xlp_rem_len agrees with how much of the
+ * record we expect there to be left.
+ */
+ if (pageHeader->xlp_rem_len == 0 ||
+ pageHeader->xlp_rem_len != state->recordRemainLen)
+ {
+ report_invalid_record(
+ state,
+ "invalid contrecord length %u at %X/%X reading %X/%X, expected %u",
+ pageHeader->xlp_rem_len,
+ (uint32) (state->recordContRecPtr >> 32),
+ (uint32) state->recordContRecPtr,
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr,
+ state->recordRemainLen);
+ goto err;
+ }
- state->EndRecPtr = RecPtr + MAXALIGN(total_len);
+ /* Append the continuation from this page to the buffer */
+ pageHeaderSize = XLogPageHeaderSize(pageHeader);
- state->ReadRecPtr = RecPtr;
+ /*
+ * XLogNeedData should have ensured that the whole page header
+ * was read
+ */
+ Assert(state->readLen >= pageHeaderSize);
+
+ contdata = (char *) state->readBuf + pageHeaderSize;
+ record_len = XLOG_BLCKSZ - pageHeaderSize;
+ if (pageHeader->xlp_rem_len < record_len)
+ record_len = pageHeader->xlp_rem_len;
+
+ request_len = record_len + pageHeaderSize;
+
+ /* XLogNeedData should have ensured all needed data was read */
+ Assert (state->readLen >= request_len);
+
+ memcpy(state->readRecordBuf + state->recordGotLen,
+ (char *) contdata, record_len);
+ state->recordGotLen += record_len;
+ state->recordRemainLen -= record_len;
+
+ /* If we just reassembled the record header, validate it. */
+ if (!state->record_verified)
+ {
+ Assert(state->recordGotLen >= SizeOfXLogRecord);
+ if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
+ state->PrevRecPtr,
+ (XLogRecord *) state->readRecordBuf))
+ goto err;
+
+ state->record_verified = true;
+ }
+
+ /* Calculate pointer to beginning of next page, and continue */
+ state->recordContRecPtr += XLOG_BLCKSZ;
+ }
+
+ /* targetPagePtr is pointing the last-read page here */
+ prec = (XLogRecord *) state->readRecordBuf;
+ if (!ValidXLogRecord(state, prec, state->ReadRecPtr))
+ goto err;
+
+ pageHeaderSize =
+ XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+ state->EndRecPtr = targetPagePtr + pageHeaderSize
+ + MAXALIGN(pageHeader->xlp_rem_len);
+
+ *record = prec;
+ state->readRecordState = XLREAD_NEXT_RECORD;
+ break;
+ }
}
/*
* Special processing if it's an XLOG SWITCH record
*/
- if (record->xl_rmid == RM_XLOG_ID &&
- (record->xl_info & ~XLR_INFO_MASK) == XLOG_SWITCH)
+ if ((*record)->xl_rmid == RM_XLOG_ID &&
+ ((*record)->xl_info & ~XLR_INFO_MASK) == XLOG_SWITCH)
{
/* Pretend it extends to end of segment */
state->EndRecPtr += state->segcxt.ws_segsize - 1;
state->EndRecPtr -= XLogSegmentOffset(state->EndRecPtr, state->segcxt.ws_segsize);
}
- if (DecodeXLogRecord(state, record, errormsg))
- return record;
- else
- return NULL;
+ Assert (!*record || state->readLen >= 0);
+ if (DecodeXLogRecord(state, *record, errormsg))
+ return XLREAD_SUCCESS;
+
+ *record = NULL;
+ return XLREAD_FAIL;
err:
/*
- * Invalidate the read state. We might read from a different source after
+ * Invalidate the read page. We might read from a different source after
* failure.
*/
XLogReaderInvalReadState(state);
@@ -572,7 +706,8 @@ err:
if (state->errormsg_buf[0] != '\0')
*errormsg = state->errormsg_buf;
- return NULL;
+ *record = NULL;
+ return XLREAD_FAIL;
}
/*
@@ -724,11 +859,12 @@ XLogReaderInvalReadState(XLogReaderState *state)
*
* This is just a convenience subroutine to avoid duplicated code in
* XLogReadRecord. It's not intended for use from anywhere else.
+ *
+ * If PrevRecPtr is valid, the xl_prev is is cross-checked with it.
*/
static bool
ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
- XLogRecPtr PrevRecPtr, XLogRecord *record,
- bool randAccess)
+ XLogRecPtr PrevRecPtr, XLogRecord *record)
{
if (record->xl_tot_len < SizeOfXLogRecord)
{
@@ -746,7 +882,7 @@ ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
(uint32) RecPtr);
return false;
}
- if (randAccess)
+ if (PrevRecPtr == InvalidXLogRecPtr)
{
/*
* We can't exactly verify the prev-link, but surely it should be less
@@ -978,11 +1114,14 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
* XLogReadRecord() will read the next valid record.
*/
XLogRecPtr
-XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
+XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
+ XLogFindNextRecordCB read_page, void *private)
{
XLogRecPtr tmpRecPtr;
XLogRecPtr found = InvalidXLogRecPtr;
XLogPageHeader header;
+ XLogRecord *record;
+ XLogReadRecordResult result;
char *errormsg;
Assert(!XLogRecPtrIsInvalid(RecPtr));
@@ -1015,9 +1154,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
while(XLogNeedData(state, targetPagePtr, targetRecOff,
targetRecOff != 0))
{
- if (!state->routine.page_read(state, state->readPagePtr,
- state->readLen,
- state->ReadRecPtr, state->readBuf))
+ if (!read_page(state, private))
break;
}
@@ -1069,8 +1206,16 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
* or we just jumped over the remaining data of a continuation.
*/
XLogBeginRead(state, tmpRecPtr);
- while (XLogReadRecord(state, &errormsg) != NULL)
+ while ((result = XLogReadRecord(state, &record, &errormsg)) !=
+ XLREAD_FAIL)
{
+ if (result == XLREAD_NEED_DATA)
+ {
+ if (!read_page(state, private))
+ goto err;
+ continue;
+ }
+
/* past the record we've found, break out */
if (RecPtr <= state->ReadRecPtr)
{
@@ -1090,9 +1235,9 @@ err:
#endif /* FRONTEND */
/*
- * Helper function to ease writing of XLogRoutine->page_read callbacks.
- * If this function is used, caller must supply a segment_open callback in
- * 'state', as that is used here.
+ * Helper function to ease writing of page_read callback.
+ * If this function is used, caller must supply a segment_open callback and
+ * segment_close callback as that is used here.
*
* Read 'count' bytes into 'buf', starting at location 'startptr', from WAL
* fetched from timeline 'tli'.
@@ -1105,6 +1250,7 @@ err:
*/
bool
WALRead(XLogReaderState *state,
+ WALSegmentOpenCB segopenfn, WALSegmentCloseCB segclosefn,
char *buf, XLogRecPtr startptr, Size count, TimeLineID tli,
WALReadError *errinfo)
{
@@ -1136,10 +1282,10 @@ WALRead(XLogReaderState *state,
XLogSegNo nextSegNo;
if (state->seg.ws_file >= 0)
- state->routine.segment_close(state);
+ segclosefn(state);
XLByteToSeg(recptr, nextSegNo, state->segcxt.ws_segsize);
- state->routine.segment_open(state, nextSegNo, &tli);
+ segopenfn(state, nextSegNo, &tli);
/* This shouldn't happen -- indicates a bug in segment_open */
Assert(state->seg.ws_file >= 0);
diff --git a/src/backend/access/transam/xlogutils.c b/src/backend/access/transam/xlogutils.c
index 1b10b6df20..303333571f 100644
--- a/src/backend/access/transam/xlogutils.c
+++ b/src/backend/access/transam/xlogutils.c
@@ -689,8 +689,7 @@ XLogTruncateRelation(RelFileNode rnode, ForkNumber forkNum,
void
XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wantLength)
{
- const XLogRecPtr lastReadPage = state->seg.ws_segno *
- state->segcxt.ws_segsize + state->readLen;
+ const XLogRecPtr lastReadPage = state->readPagePtr;
Assert(wantPage != InvalidXLogRecPtr && wantPage % XLOG_BLCKSZ == 0);
Assert(wantLength <= XLOG_BLCKSZ);
@@ -705,7 +704,7 @@ XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wa
* current TLI has since become historical.
*/
if (lastReadPage == wantPage &&
- state->readLen != 0 &&
+ state->page_verified &&
lastReadPage + state->readLen >= wantPage + Min(wantLength, XLOG_BLCKSZ - 1))
return;
@@ -792,6 +791,7 @@ wal_segment_open(XLogReaderState *state, XLogSegNo nextSegNo,
char path[MAXPGPATH];
XLogFilePath(path, tli, nextSegNo, state->segcxt.ws_segsize);
+ elog(LOG, "HOGE: %lu, %d => %s", nextSegNo, tli, path);
state->seg.ws_file = BasicOpenFile(path, O_RDONLY | PG_BINARY);
if (state->seg.ws_file >= 0)
return;
@@ -829,9 +829,11 @@ wal_segment_close(XLogReaderState *state)
* loop for now.
*/
bool
-read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *cur_page)
+read_local_xlog_page(XLogReaderState *state)
{
+ XLogRecPtr targetPagePtr = state->readPagePtr;
+ int reqLen = state->readLen;
+ char *cur_page = state->readBuf;
XLogRecPtr read_upto,
loc;
TimeLineID tli;
@@ -944,11 +946,12 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
* as 'count', read the whole page anyway. It's guaranteed to be
* zero-padded up to the page boundary if it's incomplete.
*/
- if (!WALRead(state, cur_page, targetPagePtr, XLOG_BLCKSZ, tli,
- &errinfo))
+ if (!WALRead(state, wal_segment_open, wal_segment_close,
+ cur_page, targetPagePtr, XLOG_BLCKSZ, tli, &errinfo))
WALReadRaiseError(&errinfo);
/* number of valid bytes in the buffer */
+ state->readPagePtr = targetPagePtr;
state->readLen = count;
return true;
}
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index 61902be3b0..a15b0b3355 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -120,7 +120,8 @@ StartupDecodingContext(List *output_plugin_options,
TransactionId xmin_horizon,
bool need_full_snapshot,
bool fast_forward,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress)
@@ -169,11 +170,12 @@ StartupDecodingContext(List *output_plugin_options,
ctx->slot = slot;
- ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, xl_routine, ctx);
+ ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, cleanup_cb);
if (!ctx->reader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
errmsg("out of memory")));
+ ctx->page_read = page_read;
ctx->reorder = ReorderBufferAllocate();
ctx->snapshot_builder =
@@ -231,7 +233,8 @@ CreateInitDecodingContext(char *plugin,
List *output_plugin_options,
bool need_full_snapshot,
XLogRecPtr restart_lsn,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress)
@@ -328,7 +331,7 @@ CreateInitDecodingContext(char *plugin,
ctx = StartupDecodingContext(NIL, restart_lsn, xmin_horizon,
need_full_snapshot, false,
- xl_routine, prepare_write, do_write,
+ page_read, cleanup_cb, prepare_write, do_write,
update_progress);
/* call output plugin initialization callback */
@@ -376,7 +379,8 @@ LogicalDecodingContext *
CreateDecodingContext(XLogRecPtr start_lsn,
List *output_plugin_options,
bool fast_forward,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress)
@@ -429,8 +433,8 @@ CreateDecodingContext(XLogRecPtr start_lsn,
ctx = StartupDecodingContext(output_plugin_options,
start_lsn, InvalidTransactionId, false,
- fast_forward, xl_routine, prepare_write,
- do_write, update_progress);
+ fast_forward, page_read, cleanup_cb,
+ prepare_write, do_write, update_progress);
/* call output plugin initialization callback */
old_context = MemoryContextSwitchTo(ctx->context);
@@ -483,7 +487,13 @@ DecodingContextFindStartpoint(LogicalDecodingContext *ctx)
char *err = NULL;
/* the read_page callback waits for new WAL */
- record = XLogReadRecord(ctx->reader, &err);
+ while (XLogReadRecord(ctx->reader, &record, &err) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!ctx->page_read(ctx->reader))
+ break;
+ }
+
if (err)
elog(ERROR, "%s", err);
if (!record)
diff --git a/src/backend/replication/logical/logicalfuncs.c b/src/backend/replication/logical/logicalfuncs.c
index b99c94e848..fcc81b358e 100644
--- a/src/backend/replication/logical/logicalfuncs.c
+++ b/src/backend/replication/logical/logicalfuncs.c
@@ -233,9 +233,8 @@ pg_logical_slot_get_changes_guts(FunctionCallInfo fcinfo, bool confirm, bool bin
ctx = CreateDecodingContext(InvalidXLogRecPtr,
options,
false,
- XL_ROUTINE(.page_read = read_local_xlog_page,
- .segment_open = wal_segment_open,
- .segment_close = wal_segment_close),
+ read_local_xlog_page,
+ wal_segment_close,
LogicalOutputPrepareWrite,
LogicalOutputWrite, NULL);
@@ -284,7 +283,13 @@ pg_logical_slot_get_changes_guts(FunctionCallInfo fcinfo, bool confirm, bool bin
XLogRecord *record;
char *errm = NULL;
- record = XLogReadRecord(ctx->reader, &errm);
+ while (XLogReadRecord(ctx->reader, &record, &errm) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!ctx->page_read(ctx->reader))
+ break;
+ }
+
if (errm)
elog(ERROR, "%s", errm);
diff --git a/src/backend/replication/slotfuncs.c b/src/backend/replication/slotfuncs.c
index 88033a79b2..8a678074d5 100644
--- a/src/backend/replication/slotfuncs.c
+++ b/src/backend/replication/slotfuncs.c
@@ -152,9 +152,8 @@ create_logical_replication_slot(char *name, char *plugin,
ctx = CreateInitDecodingContext(plugin, NIL,
false, /* just catalogs is OK */
restart_lsn,
- XL_ROUTINE(.page_read = read_local_xlog_page,
- .segment_open = wal_segment_open,
- .segment_close = wal_segment_close),
+ read_local_xlog_page,
+ wal_segment_close,
NULL, NULL, NULL);
/*
@@ -486,9 +485,8 @@ pg_logical_replication_slot_advance(XLogRecPtr moveto)
ctx = CreateDecodingContext(InvalidXLogRecPtr,
NIL,
true, /* fast_forward */
- XL_ROUTINE(.page_read = read_local_xlog_page,
- .segment_open = wal_segment_open,
- .segment_close = wal_segment_close),
+ read_local_xlog_page,
+ wal_segment_close,
NULL, NULL, NULL);
/*
@@ -513,7 +511,13 @@ pg_logical_replication_slot_advance(XLogRecPtr moveto)
* Read records. No changes are generated in fast_forward mode,
* but snapbuilder/slot statuses are updated properly.
*/
- record = XLogReadRecord(ctx->reader, &errm);
+ while (XLogReadRecord(ctx->reader, &record, &errm) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!ctx->page_read(ctx->reader))
+ break;
+ }
+
if (errm)
elog(ERROR, "%s", errm);
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index bf2711bddd..2fd24a9b14 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -580,10 +580,7 @@ StartReplication(StartReplicationCmd *cmd)
/* create xlogreader for physical replication */
xlogreader =
- XLogReaderAllocate(wal_segment_size, NULL,
- XL_ROUTINE(.segment_open = WalSndSegmentOpen,
- .segment_close = wal_segment_close),
- NULL);
+ XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
if (!xlogreader)
ereport(ERROR,
@@ -812,9 +809,11 @@ StartReplication(StartReplicationCmd *cmd)
* set every time WAL is flushed.
*/
static bool
-logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetRecPtr, char *cur_page)
+logical_read_xlog_page(XLogReaderState *state)
{
+ XLogRecPtr targetPagePtr = state->readPagePtr;
+ int reqLen = state->readLen;
+ char *cur_page = state->readBuf;
XLogRecPtr flushptr;
int count;
WALReadError errinfo;
@@ -842,7 +841,7 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
count = flushptr - targetPagePtr; /* part of the page available */
/* now actually read the data, we know it's there */
- if (!WALRead(state,
+ if (!WALRead(state, WalSndSegmentOpen, wal_segment_close,
cur_page,
targetPagePtr,
XLOG_BLCKSZ,
@@ -1014,9 +1013,8 @@ CreateReplicationSlot(CreateReplicationSlotCmd *cmd)
ctx = CreateInitDecodingContext(cmd->plugin, NIL, need_full_snapshot,
InvalidXLogRecPtr,
- XL_ROUTINE(.page_read = logical_read_xlog_page,
- .segment_open = WalSndSegmentOpen,
- .segment_close = wal_segment_close),
+ logical_read_xlog_page,
+ wal_segment_close,
WalSndPrepareWrite, WalSndWriteData,
WalSndUpdateProgress);
@@ -1179,9 +1177,8 @@ StartLogicalReplication(StartReplicationCmd *cmd)
*/
logical_decoding_ctx =
CreateDecodingContext(cmd->startpoint, cmd->options, false,
- XL_ROUTINE(.page_read = logical_read_xlog_page,
- .segment_open = WalSndSegmentOpen,
- .segment_close = wal_segment_close),
+ logical_read_xlog_page,
+ wal_segment_close,
WalSndPrepareWrite, WalSndWriteData,
WalSndUpdateProgress);
xlogreader = logical_decoding_ctx->reader;
@@ -2761,7 +2758,7 @@ XLogSendPhysical(void)
enlargeStringInfo(&output_message, nbytes);
retry:
- if (!WALRead(xlogreader,
+ if (!WALRead(xlogreader, WalSndSegmentOpen, wal_segment_close,
&output_message.data[output_message.len],
startptr,
nbytes,
@@ -2859,7 +2856,12 @@ XLogSendLogical(void)
*/
WalSndCaughtUp = false;
- record = XLogReadRecord(logical_decoding_ctx->reader, &errm);
+ while (XLogReadRecord(logical_decoding_ctx->reader, &record, &errm) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!logical_decoding_ctx->page_read(logical_decoding_ctx->reader))
+ break;
+ }
/* xlog record was invalid */
if (errm != NULL)
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index 0b7e73ae79..3dd6df4be6 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -40,15 +40,9 @@ static int xlogreadfd = -1;
static XLogSegNo xlogreadsegno = -1;
static char xlogfpath[MAXPGPATH];
-typedef struct XLogPageReadPrivate
-{
- const char *restoreCommand;
- int tliIndex;
-} XLogPageReadPrivate;
-
-static bool SimpleXLogPageRead(XLogReaderState *xlogreader,
- XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
+static bool SimpleXLogPageRead(XLogReaderState *xlogreader,
+ const char *datadir, int *tliIndex,
+ const char *restoreCommand);
/*
* Read WAL from the datadir/pg_wal, starting from 'startpoint' on timeline
@@ -62,20 +56,22 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
XLogRecord *record;
XLogReaderState *xlogreader;
char *errormsg;
- XLogPageReadPrivate private;
- private.tliIndex = tliIndex;
- private.restoreCommand = restoreCommand;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir,
- XL_ROUTINE(.page_read = &SimpleXLogPageRead),
- &private);
+ xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+
if (xlogreader == NULL)
pg_fatal("out of memory");
XLogBeginRead(xlogreader, startpoint);
do
{
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!SimpleXLogPageRead(xlogreader, datadir,
+ &tliIndex, restoreCommand))
+ break;
+ }
if (record == NULL)
{
@@ -113,19 +109,19 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex,
XLogRecord *record;
XLogReaderState *xlogreader;
char *errormsg;
- XLogPageReadPrivate private;
XLogRecPtr endptr;
- private.tliIndex = tliIndex;
- private.restoreCommand = restoreCommand;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir,
- XL_ROUTINE(.page_read = &SimpleXLogPageRead),
- &private);
+ xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
XLogBeginRead(xlogreader, ptr);
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!SimpleXLogPageRead(xlogreader, datadir, &tliIndex, restoreCommand))
+ break;
+ }
if (record == NULL)
{
if (errormsg)
@@ -160,7 +156,6 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
XLogRecPtr searchptr;
XLogReaderState *xlogreader;
char *errormsg;
- XLogPageReadPrivate private;
/*
* The given fork pointer points to the end of the last common record,
@@ -176,11 +171,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
forkptr += SizeOfXLogShortPHD;
}
- private.tliIndex = tliIndex;
- private.restoreCommand = restoreCommand;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir,
- XL_ROUTINE(.page_read = &SimpleXLogPageRead),
- &private);
+ xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
@@ -190,7 +181,13 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
uint8 info;
XLogBeginRead(xlogreader, searchptr);
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!SimpleXLogPageRead(xlogreader, datadir,
+ &tliIndex, restoreCommand))
+ break;
+ }
if (record == NULL)
{
@@ -237,10 +234,11 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
/* XLogReader callback function, to read a WAL page */
static bool
-SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *readBuf)
+SimpleXLogPageRead(XLogReaderState *xlogreader, const char *datadir,
+ int *tliIndex, const char *restoreCommand)
{
- XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
+ XLogRecPtr targetPagePtr = xlogreader->readPagePtr;
+ char *readBuf = xlogreader->readBuf;
uint32 targetPageOff;
XLogRecPtr targetSegEnd;
XLogSegNo targetSegNo;
@@ -273,14 +271,14 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
* be done both forward and backward, consider also switching timeline
* accordingly.
*/
- while (private->tliIndex < targetNentries - 1 &&
- targetHistory[private->tliIndex].end < targetSegEnd)
- private->tliIndex++;
- while (private->tliIndex > 0 &&
- targetHistory[private->tliIndex].begin >= targetSegEnd)
- private->tliIndex--;
+ while (*tliIndex < targetNentries - 1 &&
+ targetHistory[*tliIndex].end < targetSegEnd)
+ (*tliIndex)++;
+ while (*tliIndex > 0 &&
+ targetHistory[*tliIndex].begin >= targetSegEnd)
+ (*tliIndex)--;
- XLogFileName(xlogfname, targetHistory[private->tliIndex].tli,
+ XLogFileName(xlogfname, targetHistory[*tliIndex].tli,
xlogreadsegno, WalSegSz);
snprintf(xlogfpath, MAXPGPATH, "%s/" XLOGDIR "/%s",
@@ -293,7 +291,7 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
/*
* If we have no restore_command to execute, then exit.
*/
- if (private->restoreCommand == NULL)
+ if (restoreCommand == NULL)
{
pg_log_error("could not open file \"%s\": %m", xlogfpath);
xlogreader->readLen = -1;
@@ -307,7 +305,7 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
xlogreadfd = RestoreArchivedFile(xlogreader->segcxt.ws_dir,
xlogfname,
WalSegSz,
- private->restoreCommand);
+ restoreCommand);
if (xlogreadfd < 0)
{
@@ -349,7 +347,7 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
Assert(targetSegNo == xlogreadsegno);
- xlogreader->seg.ws_tli = targetHistory[private->tliIndex].tli;
+ xlogreader->seg.ws_tli = targetHistory[*tliIndex].tli;
xlogreader->readLen = XLOG_BLCKSZ;
return true;
}
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index ab0c28d9b7..02e71138cc 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -330,12 +330,17 @@ WALDumpCloseSegment(XLogReaderState *state)
state->seg.ws_file = -1;
}
-/* pg_waldump's XLogReaderRoutine->page_read callback */
+/*
+ * pg_waldump's WAL page rader, also used as page_read callback for
+ * XLogFindNextRecord
+ */
static bool
-WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetPtr, char *readBuff)
+WALDumpReadPage(XLogReaderState *state, void *priv)
{
- XLogDumpPrivate *private = state->private_data;
+ XLogRecPtr targetPagePtr = state->readPagePtr;
+ int reqLen = state->readLen;
+ char *readBuff = state->readBuf;
+ XLogDumpPrivate *private = (XLogDumpPrivate *) priv;
int count = XLOG_BLCKSZ;
WALReadError errinfo;
@@ -353,8 +358,8 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
}
}
- if (!WALRead(state, readBuff, targetPagePtr, count, private->timeline,
- &errinfo))
+ if (!WALRead(state, WALDumpOpenSegment, WALDumpCloseSegment,
+ readBuff, targetPagePtr, count, private->timeline, &errinfo))
{
WALOpenSegment *seg = &errinfo.wre_seg;
char fname[MAXPGPATH];
@@ -374,6 +379,7 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
(Size) errinfo.wre_req);
}
+ Assert(count >= state->readLen);
state->readLen = count;
return true;
}
@@ -1042,16 +1048,14 @@ main(int argc, char **argv)
/* we have everything we need, start reading */
xlogreader_state =
- XLogReaderAllocate(WalSegSz, waldir,
- XL_ROUTINE(.page_read = WALDumpReadPage,
- .segment_open = WALDumpOpenSegment,
- .segment_close = WALDumpCloseSegment),
- &private);
+ XLogReaderAllocate(WalSegSz, waldir, WALDumpCloseSegment);
+
if (!xlogreader_state)
fatal_error("out of memory");
/* first find a valid recptr to start from */
- first_record = XLogFindNextRecord(xlogreader_state, private.startptr);
+ first_record = XLogFindNextRecord(xlogreader_state, private.startptr,
+ &WALDumpReadPage, (void*) &private);
if (first_record == InvalidXLogRecPtr)
fatal_error("could not find a valid record after %X/%X",
@@ -1075,7 +1079,13 @@ main(int argc, char **argv)
for (;;)
{
/* try to read the next record */
- record = XLogReadRecord(xlogreader_state, &errormsg);
+ while (XLogReadRecord(xlogreader_state, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!WALDumpReadPage(xlogreader_state, (void *) &private))
+ break;
+ }
+
if (!record)
{
if (!config.follow || private.endptr_reached)
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index 6c8848e14f..6a7fe140cb 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -57,64 +57,15 @@ typedef struct WALSegmentContext
typedef struct XLogReaderState XLogReaderState;
-/* Function type definition for the read_page callback */
-typedef bool (*XLogPageReadCB) (XLogReaderState *xlogreader,
- XLogRecPtr targetPagePtr,
- int reqLen,
- XLogRecPtr targetRecPtr,
- char *readBuf);
+/* Function type definition for the segment cleanup callback */
+typedef void (*WALSegmentCleanupCB) (XLogReaderState *xlogreader);
+
+/* Function type definition for the open/close callbacks for WALRead() */
typedef void (*WALSegmentOpenCB) (XLogReaderState *xlogreader,
XLogSegNo nextSegNo,
TimeLineID *tli_p);
typedef void (*WALSegmentCloseCB) (XLogReaderState *xlogreader);
-typedef struct XLogReaderRoutine
-{
- /*
- * Data input callback
- *
- * This callback shall read at least reqLen valid bytes of the xlog page
- * starting at targetPagePtr, and store them in readBuf. The callback
- * shall return the number of bytes read (never more than XLOG_BLCKSZ), or
- * -1 on failure. The callback shall sleep, if necessary, to wait for the
- * requested bytes to become available. The callback will not be invoked
- * again for the same page unless more than the returned number of bytes
- * are needed.
- *
- * targetRecPtr is the position of the WAL record we're reading. Usually
- * it is equal to targetPagePtr + reqLen, but sometimes xlogreader needs
- * to read and verify the page or segment header, before it reads the
- * actual WAL record it's interested in. In that case, targetRecPtr can
- * be used to determine which timeline to read the page from.
- *
- * The callback shall set ->seg.ws_tli to the TLI of the file the page was
- * read from.
- */
- XLogPageReadCB page_read;
-
- /*
- * Callback to open the specified WAL segment for reading. ->seg.ws_file
- * shall be set to the file descriptor of the opened segment. In case of
- * failure, an error shall be raised by the callback and it shall not
- * return.
- *
- * "nextSegNo" is the number of the segment to be opened.
- *
- * "tli_p" is an input/output argument. WALRead() uses it to pass the
- * timeline in which the new segment should be found, but the callback can
- * use it to return the TLI that it actually opened.
- */
- WALSegmentOpenCB segment_open;
-
- /*
- * WAL segment close callback. ->seg.ws_file shall be set to a negative
- * number.
- */
- WALSegmentCloseCB segment_close;
-} XLogReaderRoutine;
-
-#define XL_ROUTINE(...) &(XLogReaderRoutine){__VA_ARGS__}
-
typedef struct
{
/* Is this block ref in use? */
@@ -144,12 +95,35 @@ typedef struct
uint16 data_bufsz;
} DecodedBkpBlock;
+/* Return code from XLogReadRecord */
+typedef enum XLogReadRecordResult
+{
+ XLREAD_SUCCESS, /* record is successfully read */
+ XLREAD_NEED_DATA, /* need more data. see XLogReadRecord. */
+ XLREAD_FAIL /* failed during reading a record */
+} XLogReadRecordResult;
+
+/*
+ * internal state of XLogReadRecord
+ *
+ * XLogReadState runs a state machine while reading a record. Theses states
+ * are not seen outside the function. Each state may repeat several times
+ * exiting requesting caller for new data. See the comment of XLogReadRecrod
+ * for details.
+ */
+typedef enum XLogReadRecordState {
+ XLREAD_NEXT_RECORD,
+ XLREAD_TOT_LEN,
+ XLREAD_FIRST_FRAGMENT,
+ XLREAD_CONTINUATION
+} XLogReadRecordState;
+
struct XLogReaderState
{
/*
* Operational callbacks
*/
- XLogReaderRoutine routine;
+ WALSegmentCleanupCB cleanup_cb;
/* ----------------------------------------
* Public parameters
@@ -162,18 +136,14 @@ struct XLogReaderState
*/
uint64 system_identifier;
- /*
- * Opaque data for callbacks to use. Not used by XLogReader.
- */
- void *private_data;
-
/*
* Start and end point of last record read. EndRecPtr is also used as the
* position to read next. Calling XLogBeginRead() sets EndRecPtr to the
* starting position and ReadRecPtr to invalid.
*/
- XLogRecPtr ReadRecPtr; /* start of last record read */
+ XLogRecPtr ReadRecPtr; /* start of last record read or being read */
XLogRecPtr EndRecPtr; /* end+1 of last record read */
+ XLogRecPtr PrevRecPtr; /* start of previous record read */
/* ----------------------------------------
* Communication with page reader
@@ -187,7 +157,9 @@ struct XLogReaderState
* the request, or -1 on error */
TimeLineID readPageTLI; /* TLI for data currently in readBuf */
char *readBuf; /* buffer to store data */
- bool page_verified; /* is the page on the buffer verified? */
+ bool page_verified; /* is the page header on the buffer verified? */
+ bool record_verified;/* is the current record header verified? */
+
/* ----------------------------------------
@@ -227,8 +199,6 @@ struct XLogReaderState
XLogRecPtr latestPagePtr;
TimeLineID latestPageTLI;
- /* beginning of the WAL record being read. */
- XLogRecPtr currRecPtr;
/* timeline to read it from, 0 if a lookup is required */
TimeLineID currTLI;
@@ -255,6 +225,15 @@ struct XLogReaderState
char *readRecordBuf;
uint32 readRecordBufSize;
+ /*
+ * XLogReadRecord() state
+ */
+ XLogReadRecordState readRecordState;/* state machine state */
+ int recordGotLen; /* amount of current record that has
+ * already been read */
+ int recordRemainLen; /* length of current record that remains */
+ XLogRecPtr recordContRecPtr; /* where the current record continues */
+
/* Buffer to hold error message */
char *errormsg_buf;
};
@@ -262,9 +241,7 @@ struct XLogReaderState
/* Get a new XLogReader */
extern XLogReaderState *XLogReaderAllocate(int wal_segment_size,
const char *waldir,
- XLogReaderRoutine *routine,
- void *private_data);
-extern XLogReaderRoutine *LocalXLogReaderRoutine(void);
+ WALSegmentCleanupCB cleanup_cb);
/* Free an XLogReader */
extern void XLogReaderFree(XLogReaderState *state);
@@ -272,12 +249,17 @@ extern void XLogReaderFree(XLogReaderState *state);
/* Position the XLogReader to given record */
extern void XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr);
#ifdef FRONTEND
-extern XLogRecPtr XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr);
+/* Function type definition for the read_page callback */
+typedef bool (*XLogFindNextRecordCB) (XLogReaderState *xlogreader,
+ void *private);
+extern XLogRecPtr XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
+ XLogFindNextRecordCB read_page, void *private);
#endif /* FRONTEND */
/* Read the next XLog record. Returns NULL on end-of-WAL or failure */
-extern struct XLogRecord *XLogReadRecord(XLogReaderState *state,
- char **errormsg);
+extern XLogReadRecordResult XLogReadRecord(XLogReaderState *state,
+ XLogRecord **record,
+ char **errormsg);
/* Validate a page */
extern bool XLogReaderValidatePageHeader(XLogReaderState *state,
@@ -297,6 +279,7 @@ typedef struct WALReadError
} WALReadError;
extern bool WALRead(XLogReaderState *state,
+ WALSegmentOpenCB segopenfn, WALSegmentCloseCB sgclosefn,
char *buf, XLogRecPtr startptr, Size count,
TimeLineID tli, WALReadError *errinfo);
diff --git a/src/include/access/xlogutils.h b/src/include/access/xlogutils.h
index c287edf206..482e4ced69 100644
--- a/src/include/access/xlogutils.h
+++ b/src/include/access/xlogutils.h
@@ -47,9 +47,7 @@ extern Buffer XLogReadBufferExtended(RelFileNode rnode, ForkNumber forknum,
extern Relation CreateFakeRelcacheEntry(RelFileNode rnode);
extern void FreeFakeRelcacheEntry(Relation fakerel);
-extern bool read_local_xlog_page(XLogReaderState *state,
- XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetRecPtr, char *cur_page);
+extern bool read_local_xlog_page(XLogReaderState *state);
extern void wal_segment_open(XLogReaderState *state,
XLogSegNo nextSegNo,
TimeLineID *tli_p);
diff --git a/src/include/replication/logical.h b/src/include/replication/logical.h
index c2f2475e5d..a743db49cf 100644
--- a/src/include/replication/logical.h
+++ b/src/include/replication/logical.h
@@ -29,6 +29,10 @@ typedef void (*LogicalOutputPluginWriterUpdateProgress) (struct LogicalDecodingC
TransactionId xid
);
+typedef struct LogicalDecodingContext LogicalDecodingContext;
+
+typedef bool (*LogicalDecodingXLogPageReadCB)(XLogReaderState *ctx);
+
typedef struct LogicalDecodingContext
{
/* memory context this is all allocated in */
@@ -39,6 +43,7 @@ typedef struct LogicalDecodingContext
/* infrastructure pieces for decoding */
XLogReaderState *reader;
+ LogicalDecodingXLogPageReadCB page_read;
struct ReorderBuffer *reorder;
struct SnapBuild *snapshot_builder;
@@ -95,14 +100,16 @@ extern LogicalDecodingContext *CreateInitDecodingContext(char *plugin,
List *output_plugin_options,
bool need_full_snapshot,
XLogRecPtr restart_lsn,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress);
extern LogicalDecodingContext *CreateDecodingContext(XLogRecPtr start_lsn,
List *output_plugin_options,
bool fast_forward,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress);
--
2.18.4
----Next_Part(Thu_Jul__2_13_53_30_2020_072)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
filename="v15-0003-Remove-globals-readOff-readLen-and-readSegNo.patch"
^ permalink raw reply [nested|flat] 70+ messages in thread
* [PATCH v16 2/4] Move page-reader out of XLogReadRecord
@ 2019-09-10 03:58 Kyotaro Horiguchi <[email protected]>
0 siblings, 0 replies; 70+ messages in thread
From: Kyotaro Horiguchi @ 2019-09-10 03:58 UTC (permalink / raw)
This is the second step of removing callbacks from WAL record reader.
Since it is essential to take in additional data while reading a
record, the function have to ask caller for new data while keeping
working state. Thus the function is turned into a state machine.
---
src/backend/access/transam/twophase.c | 15 +-
src/backend/access/transam/xlog.c | 58 +-
src/backend/access/transam/xlogreader.c | 701 +++++++++++-------
src/backend/access/transam/xlogutils.c | 17 +-
src/backend/replication/logical/logical.c | 26 +-
.../replication/logical/logicalfuncs.c | 13 +-
src/backend/replication/slotfuncs.c | 18 +-
src/backend/replication/walsender.c | 32 +-
src/bin/pg_rewind/parsexlog.c | 84 +--
src/bin/pg_waldump/pg_waldump.c | 36 +-
src/include/access/xlogreader.h | 121 ++-
src/include/access/xlogutils.h | 4 +-
src/include/pg_config_manual.h | 2 +-
src/include/replication/logical.h | 11 +-
14 files changed, 649 insertions(+), 489 deletions(-)
diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index ef4f9981e3..9a56b3487e 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -1324,11 +1324,8 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
XLogReaderState *xlogreader;
char *errormsg;
- xlogreader = XLogReaderAllocate(wal_segment_size, NULL,
- XL_ROUTINE(.page_read = &read_local_xlog_page,
- .segment_open = &wal_segment_open,
- .segment_close = &wal_segment_close),
- NULL);
+ xlogreader = XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+
if (!xlogreader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
@@ -1336,7 +1333,13 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
errdetail("Failed while allocating a WAL reading processor.")));
XLogBeginRead(xlogreader, lsn);
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!read_local_xlog_page(xlogreader))
+ break;
+ }
+
if (record == NULL)
ereport(ERROR,
(errcode_for_file_access(),
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index a91e86b290..6ef1f817ad 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -828,13 +828,6 @@ static XLogSource currentSource = XLOG_FROM_ANY;
static bool lastSourceFailed = false;
static bool pendingWalRcvRestart = false;
-typedef struct XLogPageReadPrivate
-{
- int emode;
- bool fetching_ckpt; /* are we fetching a checkpoint record? */
- bool randAccess;
-} XLogPageReadPrivate;
-
/*
* These variables track when we last obtained some WAL data to process,
* and where we got it from. (XLogReceiptSource is initially the same as
@@ -909,8 +902,8 @@ static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath,
static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli,
XLogSource source, bool notfoundOk);
static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, XLogSource source);
-static bool XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
+static bool XLogPageRead(XLogReaderState *xlogreader,
+ bool fetching_ckpt, int emode, bool randAccess);
static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
bool fetching_ckpt, XLogRecPtr tliRecPtr);
static int emode_for_corrupt_record(int emode, XLogRecPtr RecPtr);
@@ -1222,8 +1215,7 @@ XLogInsertRecord(XLogRecData *rdata,
appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len);
if (!debug_reader)
- debug_reader = XLogReaderAllocate(wal_segment_size, NULL,
- XL_ROUTINE(), NULL);
+ debug_reader = XLogReaderAllocate(wal_segment_size, NULL, NULL);
if (!debug_reader)
{
@@ -4323,15 +4315,10 @@ CleanupBackupHistory(void)
* record is available.
*/
static XLogRecord *
-ReadRecord(XLogReaderState *xlogreader, int emode,
- bool fetching_ckpt)
+ReadRecord(XLogReaderState *xlogreader, int emode, bool fetching_ckpt)
{
XLogRecord *record;
- XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
-
- private->fetching_ckpt = fetching_ckpt;
- private->emode = emode;
- private->randAccess = (xlogreader->ReadRecPtr == InvalidXLogRecPtr);
+ bool randAccess = (xlogreader->ReadRecPtr == InvalidXLogRecPtr);
/* This is the first attempt to read this page. */
lastSourceFailed = false;
@@ -4339,8 +4326,16 @@ ReadRecord(XLogReaderState *xlogreader, int emode,
for (;;)
{
char *errormsg;
+ XLogReadRecordResult result;
+
+ while ((result = XLogReadRecord(xlogreader, &record, &errormsg))
+ == XLREAD_NEED_DATA)
+ {
+ if (!XLogPageRead(xlogreader, fetching_ckpt, emode, randAccess))
+ break;
+
+ }
- record = XLogReadRecord(xlogreader, &errormsg);
ReadRecPtr = xlogreader->ReadRecPtr;
EndRecPtr = xlogreader->EndRecPtr;
if (record == NULL)
@@ -6317,7 +6312,6 @@ StartupXLOG(void)
bool backupFromStandby = false;
DBState dbstate_at_startup;
XLogReaderState *xlogreader;
- XLogPageReadPrivate private;
bool promoted = false;
struct stat st;
@@ -6477,13 +6471,9 @@ StartupXLOG(void)
OwnLatch(&XLogCtl->recoveryWakeupLatch);
/* Set up XLOG reader facility */
- MemSet(&private, 0, sizeof(XLogPageReadPrivate));
xlogreader =
- XLogReaderAllocate(wal_segment_size, NULL,
- XL_ROUTINE(.page_read = &XLogPageRead,
- .segment_open = NULL,
- .segment_close = wal_segment_close),
- &private);
+ XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+
if (!xlogreader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
@@ -11866,12 +11856,13 @@ CancelBackup(void)
* sleep and retry.
*/
static bool
-XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetRecPtr, char *readBuf)
+XLogPageRead(XLogReaderState *xlogreader,
+ bool fetching_ckpt, int emode, bool randAccess)
{
- XLogPageReadPrivate *private =
- (XLogPageReadPrivate *) xlogreader->private_data;
- int emode = private->emode;
+ char *readBuf = xlogreader->readBuf;
+ XLogRecPtr targetPagePtr = xlogreader->readPagePtr;
+ int reqLen = xlogreader->readLen;
+ XLogRecPtr targetRecPtr = xlogreader->ReadRecPtr;
uint32 targetPageOff;
XLogSegNo targetSegNo PG_USED_FOR_ASSERTS_ONLY;
int r;
@@ -11914,8 +11905,8 @@ retry:
flushedUpto < targetPagePtr + reqLen))
{
if (!WaitForWALToBecomeAvailable(targetPagePtr + reqLen,
- private->randAccess,
- private->fetching_ckpt,
+ randAccess,
+ fetching_ckpt,
targetRecPtr))
{
if (readFile >= 0)
@@ -12020,6 +12011,7 @@ retry:
goto next_record_is_invalid;
}
+ Assert(xlogreader->readPagePtr == targetPagePtr);
xlogreader->readLen = readLen;
return true;
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index f403261626..d829e57baa 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -40,7 +40,7 @@ static bool XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr,
int reqLen, bool header_inclusive);
static void XLogReaderInvalReadState(XLogReaderState *state);
static bool ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
- XLogRecPtr PrevRecPtr, XLogRecord *record, bool randAccess);
+ XLogRecPtr PrevRecPtr, XLogRecord *record);
static bool ValidXLogRecord(XLogReaderState *state, XLogRecord *record,
XLogRecPtr recptr);
static void ResetDecoder(XLogReaderState *state);
@@ -73,7 +73,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...)
*/
XLogReaderState *
XLogReaderAllocate(int wal_segment_size, const char *waldir,
- XLogReaderRoutine *routine, void *private_data)
+ WALSegmentCleanupCB cleanup_cb)
{
XLogReaderState *state;
@@ -84,7 +84,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
return NULL;
/* initialize caller-provided support functions */
- state->routine = *routine;
+ state->cleanup_cb = cleanup_cb;
state->max_block_id = -1;
@@ -107,8 +107,6 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
WALOpenSegmentInit(&state->seg, &state->segcxt, wal_segment_size,
waldir);
- /* system_identifier initialized to zeroes above */
- state->private_data = private_data;
/* ReadRecPtr, EndRecPtr and readLen initialized to zeroes above */
state->errormsg_buf = palloc_extended(MAX_ERRORMSG_LEN + 1,
MCXT_ALLOC_NO_OOM);
@@ -140,8 +138,8 @@ XLogReaderFree(XLogReaderState *state)
{
int block_id;
- if (state->seg.ws_file != -1)
- state->routine.segment_close(state);
+ if (state->seg.ws_file >= 0)
+ state->cleanup_cb(state);
for (block_id = 0; block_id <= XLR_MAX_BLOCK_ID; block_id++)
{
@@ -246,6 +244,7 @@ XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
/* Begin at the passed-in record pointer. */
state->EndRecPtr = RecPtr;
state->ReadRecPtr = InvalidXLogRecPtr;
+ state->readRecordState = XLREAD_NEXT_RECORD;
}
/*
@@ -254,318 +253,452 @@ XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
* XLogBeginRead() or XLogFindNextRecord() must be called before the first call
* to XLogReadRecord().
*
- * If the page_read callback fails to read the requested data, NULL is
- * returned. The callback is expected to have reported the error; errormsg
- * is set to NULL.
+ * This function may return XLREAD_NEED_DATA several times before returning a
+ * result record. The caller shall read in some new data then call this
+ * function again with the same parameters.
*
- * If the reading fails for some other reason, NULL is also returned, and
- * *errormsg is set to a string with details of the failure.
+ * When a record is successfully read, returns XLREAD_SUCCESS with result
+ * record being stored in *record. Otherwise *record is NULL.
*
- * The returned pointer (or *errormsg) points to an internal buffer that's
- * valid until the next call to XLogReadRecord.
+ * Returns XLREAD_NEED_DATA if more data is needed to finish reading the
+ * current record. In that case, state->readPagePtr and state->readLen inform
+ * the desired position and minimum length of data needed. The caller shall
+ * read in the requested data and set state->readBuf to point to a buffer
+ * containing it. The caller must also set state->readPageTLI and
+ * state->readLen to indicate the timeline that it was read from, and the
+ * length of data that is now available (which must be >= given readLen),
+ * respectively.
+ *
+ * If invalid data is encountered, returns XLREAD_FAIL with *record being set to
+ * NULL. *errormsg is set to a string with details of the failure.
+ * The returned pointer (or *errormsg) points to an internal buffer that's valid
+ * until the next call to XLogReadRecord.
+ *
+ *
+ * This function runs a state machine consists of the following states.
+ *
+ * XLREAD_NEXT_RECORD :
+ * The initial state, if called with valid RecPtr, try to read a record at
+ * that position. If invalid RecPtr is given try to read a record just after
+ * the last one previously read.
+ * This state ens after setting ReadRecPtr. Then goes to XLREAD_TOT_LEN.
+ *
+ * XLREAD_TOT_LEN:
+ * Examining record header. Ends after reading record total
+ * length. recordRemainLen and recordGotLen are initialized.
+ *
+ * XLREAD_FIRST_FRAGMENT:
+ * Reading the first fragment. Ends with finishing reading a single
+ * record. Goes to XLREAD_NEXT_RECORD if that's all or
+ * XLREAD_CONTINUATION if we have continuation.
+
+ * XLREAD_CONTINUATION:
+ * Reading continuation of record. Ends with finishing the whole record then
+ * goes to XLREAD_NEXT_RECORD. During this state, recordRemainLen indicates
+ * how much is left and readRecordBuf holds the partially assert
+ * record.recordContRecPtr points to the beginning of the next page where to
+ * continue.
+ *
+ * If wrong data found in any state, the state machine stays at the current
+ * state. This behavior allows to continue reading a reacord switching among
+ * different souces, while streaming replication.
*/
-XLogRecord *
-XLogReadRecord(XLogReaderState *state, char **errormsg)
+XLogReadRecordResult
+XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
{
- XLogRecPtr RecPtr;
- XLogRecord *record;
- XLogRecPtr targetPagePtr;
- bool randAccess;
- uint32 len,
- total_len;
- uint32 targetRecOff;
- uint32 pageHeaderSize;
- bool gotheader;
+ XLogRecord *prec;
- /*
- * randAccess indicates whether to verify the previous-record pointer of
- * the record we're reading. We only do this if we're reading
- * sequentially, which is what we initially assume.
- */
- randAccess = false;
+ *record = NULL;
/* reset error state */
*errormsg = NULL;
state->errormsg_buf[0] = '\0';
- ResetDecoder(state);
-
- RecPtr = state->EndRecPtr;
-
- if (state->ReadRecPtr != InvalidXLogRecPtr)
- {
- /* read the record after the one we just read */
-
- /*
- * EndRecPtr is pointing to end+1 of the previous WAL record. If
- * we're at a page boundary, no more records can fit on the current
- * page. We must skip over the page header, but we can't do that until
- * we've read in the page, since the header size is variable.
- */
- }
- else
- {
- /*
- * Caller supplied a position to start at.
- *
- * In this case, EndRecPtr should already be pointing to a valid
- * record starting position.
- */
- Assert(XRecOffIsValid(RecPtr));
- randAccess = true;
- }
-
- state->currRecPtr = RecPtr;
-
- targetPagePtr = RecPtr - (RecPtr % XLOG_BLCKSZ);
- targetRecOff = RecPtr % XLOG_BLCKSZ;
-
- /*
- * Read the page containing the record into state->readBuf. Request enough
- * byte to cover the whole record header, or at least the part of it that
- * fits on the same page.
- */
- while (XLogNeedData(state, targetPagePtr,
- Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
- targetRecOff != 0))
- {
- if (!state->routine.page_read(state, state->readPagePtr, state->readLen,
- RecPtr, state->readBuf))
- break;
- }
-
- if (!state->page_verified)
- goto err;
-
- /*
- * We have at least the page header, so we can examine it now.
- */
- pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
- if (targetRecOff == 0)
- {
- /*
- * At page start, so skip over page header.
- */
- RecPtr += pageHeaderSize;
- targetRecOff = pageHeaderSize;
- }
- else if (targetRecOff < pageHeaderSize)
- {
- report_invalid_record(state, "invalid record offset at %X/%X",
- (uint32) (RecPtr >> 32), (uint32) RecPtr);
- goto err;
- }
-
- if ((((XLogPageHeader) state->readBuf)->xlp_info & XLP_FIRST_IS_CONTRECORD) &&
- targetRecOff == pageHeaderSize)
- {
- report_invalid_record(state, "contrecord is requested by %X/%X",
- (uint32) (RecPtr >> 32), (uint32) RecPtr);
- goto err;
- }
-
- /* XLogNeedData has verified the page header */
- Assert(pageHeaderSize <= state->readLen);
-
- /*
- * Read the record length.
- *
- * NB: Even though we use an XLogRecord pointer here, the whole record
- * header might not fit on this page. xl_tot_len is the first field of the
- * struct, so it must be on this page (the records are MAXALIGNed), but we
- * cannot access any other fields until we've verified that we got the
- * whole header.
- */
- record = (XLogRecord *) (state->readBuf + RecPtr % XLOG_BLCKSZ);
- total_len = record->xl_tot_len;
-
- /*
- * If the whole record header is on this page, validate it immediately.
- * Otherwise do just a basic sanity check on xl_tot_len, and validate the
- * rest of the header after reading it from the next page. The xl_tot_len
- * check is necessary here to ensure that we enter the "Need to reassemble
- * record" code path below; otherwise we might fail to apply
- * ValidXLogRecordHeader at all.
- */
- if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
- {
- if (!ValidXLogRecordHeader(state, RecPtr, state->ReadRecPtr, record,
- randAccess))
- goto err;
- gotheader = true;
- }
- else
- {
- /* XXX: more validation should be done here */
- if (total_len < SizeOfXLogRecord)
- {
- report_invalid_record(state,
- "invalid record length at %X/%X: wanted %u, got %u",
- (uint32) (RecPtr >> 32), (uint32) RecPtr,
- (uint32) SizeOfXLogRecord, total_len);
- goto err;
- }
- gotheader = false;
- }
-
- len = XLOG_BLCKSZ - RecPtr % XLOG_BLCKSZ;
- if (total_len > len)
+ switch (state->readRecordState)
{
- /* Need to reassemble record */
- char *contdata;
- XLogPageHeader pageHeader;
- char *buffer;
- uint32 gotlen;
-
- /*
- * Enlarge readRecordBuf as needed.
- */
- if (total_len > state->readRecordBufSize &&
- !allocate_recordbuf(state, total_len))
- {
- /* We treat this as a "bogus data" condition */
- report_invalid_record(state, "record length %u at %X/%X too long",
- total_len,
- (uint32) (RecPtr >> 32), (uint32) RecPtr);
- goto err;
- }
+ case XLREAD_NEXT_RECORD:
+ ResetDecoder(state);
- /* Copy the first fragment of the record from the first page. */
- memcpy(state->readRecordBuf,
- state->readBuf + RecPtr % XLOG_BLCKSZ, len);
- buffer = state->readRecordBuf + len;
- gotlen = len;
-
- do
- {
- int rest_len = total_len - gotlen;
-
- /* Calculate pointer to beginning of next page */
- targetPagePtr += XLOG_BLCKSZ;
-
- /* Wait for the next page to become available */
- while (XLogNeedData(state, targetPagePtr,
- Min(rest_len, XLOG_BLCKSZ),
- false))
+ if (state->ReadRecPtr != InvalidXLogRecPtr)
{
- if (!state->routine.page_read(state, state->readPagePtr,
- state->readLen,
- state->ReadRecPtr,
- state->readBuf))
- break;
+ /* read the record after the one we just read */
+
+ /*
+ * EndRecPtr is pointing to end+1 of the previous WAL record.
+ * If we're at a page boundary, no more records can fit on the
+ * current page. We must skip over the page header, but we
+ * can't do that until we've read in the page, since the header
+ * size is variable.
+ */
+ state->PrevRecPtr = state->ReadRecPtr;
+ state->ReadRecPtr = state->EndRecPtr;
+ }
+ else
+ {
+ /*
+ * Caller supplied a position to start at.
+ *
+ * In this case, EndRecPtr should already be pointing to a
+ * valid record starting position.
+ */
+ Assert(XRecOffIsValid(state->EndRecPtr));
+ state->ReadRecPtr = state->EndRecPtr;
+
+ /*
+ * We cannot verify the previous-record pointer when we're
+ * seeking to a particular record. Reset PrevRecPtr so that we
+ * won't try doing that.
+ */
+ state->PrevRecPtr = InvalidXLogRecPtr;
+ state->EndRecPtr = InvalidXLogRecPtr; /* to be tidy */
}
+ state->record_verified = false;
+ state->readRecordState = XLREAD_TOT_LEN;
+ /* fall through */
+
+ case XLREAD_TOT_LEN:
+ {
+ uint32 total_len;
+ uint32 pageHeaderSize;
+ XLogRecPtr targetPagePtr;
+ uint32 targetRecOff;
+ XLogPageHeader pageHeader;
+
+ targetPagePtr =
+ state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
+ targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+
+ /*
+ * Check if we have enough data. For the first record in the page,
+ * the requesting length doesn't contain page header.
+ */
+ if (XLogNeedData(state, targetPagePtr,
+ Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+ targetRecOff != 0))
+ return XLREAD_NEED_DATA;
+
+ /* error out if caller supplied bogus page */
if (!state->page_verified)
goto err;
- Assert(SizeOfXLogShortPHD <= state->readLen);
+ /* examine page header now. */
+ pageHeaderSize =
+ XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+ if (targetRecOff == 0)
+ {
+ /* At page start, so skip over page header. */
+ state->ReadRecPtr += pageHeaderSize;
+ targetRecOff = pageHeaderSize;
+ }
+ else if (targetRecOff < pageHeaderSize)
+ {
+ report_invalid_record(state, "invalid record offset at %X/%X",
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr);
+ goto err;
+ }
- /* Check that the continuation on next page looks valid */
pageHeader = (XLogPageHeader) state->readBuf;
- if (!(pageHeader->xlp_info & XLP_FIRST_IS_CONTRECORD))
+ if ((pageHeader->xlp_info & XLP_FIRST_IS_CONTRECORD) &&
+ targetRecOff == pageHeaderSize)
{
- report_invalid_record(state,
- "there is no contrecord flag at %X/%X",
- (uint32) (RecPtr >> 32), (uint32) RecPtr);
+ report_invalid_record(state, "contrecord is requested by %X/%X",
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr);
goto err;
}
- /*
- * Cross-check that xlp_rem_len agrees with how much of the record
- * we expect there to be left.
- */
- if (pageHeader->xlp_rem_len == 0 ||
- total_len != (pageHeader->xlp_rem_len + gotlen))
- {
- report_invalid_record(state,
- "invalid contrecord length %u (expected %lld) at %X/%X",
- pageHeader->xlp_rem_len,
- ((long long) total_len) - gotlen,
- (uint32) (RecPtr >> 32), (uint32) RecPtr);
- goto err;
- }
-
- /* Append the continuation from this page to the buffer */
- pageHeaderSize = XLogPageHeaderSize(pageHeader);
-
+ /* XLogNeedData has verified the page header */
Assert(pageHeaderSize <= state->readLen);
- contdata = (char *) state->readBuf + pageHeaderSize;
- len = XLOG_BLCKSZ - pageHeaderSize;
- if (pageHeader->xlp_rem_len < len)
- len = pageHeader->xlp_rem_len;
+ /*
+ * Read the record length.
+ *
+ * NB: Even though we use an XLogRecord pointer here, the whole
+ * record header might not fit on this page. xl_tot_len is the first
+ * field of the struct, so it must be on this page (the records are
+ * MAXALIGNed), but we cannot access any other fields until we've
+ * verified that we got the whole header.
+ */
+ prec = (XLogRecord *) (state->readBuf +
+ state->ReadRecPtr % XLOG_BLCKSZ);
+ total_len = prec->xl_tot_len;
- Assert (pageHeaderSize + len <= state->readLen);
- memcpy(buffer, (char *) contdata, len);
- buffer += len;
- gotlen += len;
+ /*
+ * If the whole record header is on this page, validate it
+ * immediately. Otherwise do just a basic sanity check on
+ * xl_tot_len, and validate the rest of the header after reading it
+ * from the next page. The xl_tot_len check is necessary here to
+ * ensure that we enter the XLREAD_CONTINUATION state below;
+ * otherwise we might fail to apply ValidXLogRecordHeader at all.
+ */
+ if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
+ {
+ if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
+ state->PrevRecPtr, prec))
+ goto err;
- /* If we just reassembled the record header, validate it. */
- if (!gotheader)
+ state->record_verified = true;
+ }
+ else
{
- record = (XLogRecord *) state->readRecordBuf;
- if (!ValidXLogRecordHeader(state, RecPtr, state->ReadRecPtr,
- record, randAccess))
+ /* XXX: more validation should be done here */
+ if (total_len < SizeOfXLogRecord)
+ {
+ report_invalid_record(state,
+ "invalid record length at %X/%X: wanted %u, got %u",
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr,
+ (uint32) SizeOfXLogRecord, total_len);
goto err;
- gotheader = true;
+ }
}
- } while (gotlen < total_len);
-
- Assert(gotheader);
- record = (XLogRecord *) state->readRecordBuf;
- if (!ValidXLogRecord(state, record, RecPtr))
- goto err;
+ /*
+ * Wait for the rest of the record, or the part of the record that
+ * fit on the first page if crossed a page boundary, to become
+ * available.
+ */
+ state->recordGotLen = 0;
+ state->recordRemainLen = total_len;
+ state->readRecordState = XLREAD_FIRST_FRAGMENT;
+ }
+ /* fall through */
- pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
- state->ReadRecPtr = RecPtr;
- state->EndRecPtr = targetPagePtr + pageHeaderSize
- + MAXALIGN(pageHeader->xlp_rem_len);
- }
- else
- {
- /* Wait for the record data to become available */
- while (XLogNeedData(state, targetPagePtr,
- Min(targetRecOff + total_len, XLOG_BLCKSZ), true))
+ case XLREAD_FIRST_FRAGMENT:
{
- if (!state->routine.page_read(state, state->readPagePtr,
- state->readLen,
- state->ReadRecPtr, state->readBuf))
+ uint32 total_len = state->recordRemainLen;
+ uint32 request_len;
+ uint32 record_len;
+ XLogRecPtr targetPagePtr;
+ uint32 targetRecOff;
+
+ /*
+ * Wait for the rest of the record on the first page to become
+ * available
+ */
+ targetPagePtr =
+ state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
+ targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+
+ request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ);
+ record_len = request_len - targetRecOff;
+
+ /* ReadRecPtr contains page header */
+ Assert (targetRecOff != 0);
+ if (XLogNeedData(state, targetPagePtr, request_len, true))
+ return XLREAD_NEED_DATA;
+
+ /* error out if caller supplied bogus page */
+ if (!state->page_verified)
+ goto err;
+
+ prec = (XLogRecord *) (state->readBuf + targetRecOff);
+
+ /* validate record header if not yet */
+ if (!state->record_verified && record_len >= SizeOfXLogRecord)
+ {
+ if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
+ state->PrevRecPtr, prec))
+ goto err;
+
+ state->record_verified = true;
+ }
+
+
+ if (total_len == record_len)
+ {
+ /* Record does not cross a page boundary */
+ Assert(state->record_verified);
+
+ if (!ValidXLogRecord(state, prec, state->ReadRecPtr))
+ goto err;
+
+ state->record_verified = true; /* to be tidy */
+
+ /* We already checked the header earlier */
+ state->EndRecPtr = state->ReadRecPtr + MAXALIGN(record_len);
+
+ *record = prec;
+ state->readRecordState = XLREAD_NEXT_RECORD;
break;
+ }
+
+ /*
+ * The record continues on the next page. Need to reassemble
+ * record
+ */
+ Assert(total_len > record_len);
+
+ /* Enlarge readRecordBuf as needed. */
+ if (total_len > state->readRecordBufSize &&
+ !allocate_recordbuf(state, total_len))
+ {
+ /* We treat this as a "bogus data" condition */
+ report_invalid_record(state,
+ "record length %u at %X/%X too long",
+ total_len,
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr);
+ goto err;
+ }
+
+ /* Copy the first fragment of the record from the first page. */
+ memcpy(state->readRecordBuf, state->readBuf + targetRecOff,
+ record_len);
+ state->recordGotLen += record_len;
+ state->recordRemainLen -= record_len;
+
+ /* Calculate pointer to beginning of next page */
+ state->recordContRecPtr = state->ReadRecPtr + record_len;
+ Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0);
+
+ state->readRecordState = XLREAD_CONTINUATION;
}
+ /* fall through */
+
+ case XLREAD_CONTINUATION:
+ {
+ XLogPageHeader pageHeader;
+ uint32 pageHeaderSize;
+ XLogRecPtr targetPagePtr;
+
+ /* we enter this state only if we haven't read the whole record. */
+ Assert (state->recordRemainLen > 0);
+
+ while(state->recordRemainLen > 0)
+ {
+ char *contdata;
+ uint32 request_len;
+ uint32 record_len;
+
+ /* Wait for the next page to become available */
+ targetPagePtr = state->recordContRecPtr;
+
+ /* this request contains page header */
+ Assert (targetPagePtr != 0);
+ if (XLogNeedData(state, targetPagePtr,
+ Min(state->recordRemainLen, XLOG_BLCKSZ),
+ false))
+ return XLREAD_NEED_DATA;
+
+ if (!state->page_verified)
+ goto err;
+
+ Assert(SizeOfXLogShortPHD <= state->readLen);
- if (!state->page_verified)
- goto err;
+ /* Check that the continuation on next page looks valid */
+ pageHeader = (XLogPageHeader) state->readBuf;
+ if (!(pageHeader->xlp_info & XLP_FIRST_IS_CONTRECORD))
+ {
+ report_invalid_record(
+ state,
+ "there is no contrecord flag at %X/%X reading %X/%X",
+ (uint32) (state->recordContRecPtr >> 32),
+ (uint32) state->recordContRecPtr,
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr);
+ goto err;
+ }
- /* Record does not cross a page boundary */
- if (!ValidXLogRecord(state, record, RecPtr))
- goto err;
+ /*
+ * Cross-check that xlp_rem_len agrees with how much of the
+ * record we expect there to be left.
+ */
+ if (pageHeader->xlp_rem_len == 0 ||
+ pageHeader->xlp_rem_len != state->recordRemainLen)
+ {
+ report_invalid_record(
+ state,
+ "invalid contrecord length %u at %X/%X reading %X/%X, expected %u",
+ pageHeader->xlp_rem_len,
+ (uint32) (state->recordContRecPtr >> 32),
+ (uint32) state->recordContRecPtr,
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr,
+ state->recordRemainLen);
+ goto err;
+ }
- state->EndRecPtr = RecPtr + MAXALIGN(total_len);
+ /* Append the continuation from this page to the buffer */
+ pageHeaderSize = XLogPageHeaderSize(pageHeader);
- state->ReadRecPtr = RecPtr;
+ /*
+ * XLogNeedData should have ensured that the whole page header
+ * was read
+ */
+ Assert(state->readLen >= pageHeaderSize);
+
+ contdata = (char *) state->readBuf + pageHeaderSize;
+ record_len = XLOG_BLCKSZ - pageHeaderSize;
+ if (pageHeader->xlp_rem_len < record_len)
+ record_len = pageHeader->xlp_rem_len;
+
+ request_len = record_len + pageHeaderSize;
+
+ /* XLogNeedData should have ensured all needed data was read */
+ Assert (state->readLen >= request_len);
+
+ memcpy(state->readRecordBuf + state->recordGotLen,
+ (char *) contdata, record_len);
+ state->recordGotLen += record_len;
+ state->recordRemainLen -= record_len;
+
+ /* If we just reassembled the record header, validate it. */
+ if (!state->record_verified)
+ {
+ Assert(state->recordGotLen >= SizeOfXLogRecord);
+ if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
+ state->PrevRecPtr,
+ (XLogRecord *) state->readRecordBuf))
+ goto err;
+
+ state->record_verified = true;
+ }
+
+ /* Calculate pointer to beginning of next page, and continue */
+ state->recordContRecPtr += XLOG_BLCKSZ;
+ }
+
+ /* targetPagePtr is pointing the last-read page here */
+ prec = (XLogRecord *) state->readRecordBuf;
+ if (!ValidXLogRecord(state, prec, state->ReadRecPtr))
+ goto err;
+
+ pageHeaderSize =
+ XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+ state->EndRecPtr = targetPagePtr + pageHeaderSize
+ + MAXALIGN(pageHeader->xlp_rem_len);
+
+ *record = prec;
+ state->readRecordState = XLREAD_NEXT_RECORD;
+ break;
+ }
}
/*
* Special processing if it's an XLOG SWITCH record
*/
- if (record->xl_rmid == RM_XLOG_ID &&
- (record->xl_info & ~XLR_INFO_MASK) == XLOG_SWITCH)
+ if ((*record)->xl_rmid == RM_XLOG_ID &&
+ ((*record)->xl_info & ~XLR_INFO_MASK) == XLOG_SWITCH)
{
/* Pretend it extends to end of segment */
state->EndRecPtr += state->segcxt.ws_segsize - 1;
state->EndRecPtr -= XLogSegmentOffset(state->EndRecPtr, state->segcxt.ws_segsize);
}
- if (DecodeXLogRecord(state, record, errormsg))
- return record;
- else
- return NULL;
+ Assert (!*record || state->readLen >= 0);
+ if (DecodeXLogRecord(state, *record, errormsg))
+ return XLREAD_SUCCESS;
+
+ *record = NULL;
+ return XLREAD_FAIL;
err:
/*
- * Invalidate the read state. We might read from a different source after
+ * Invalidate the read page. We might read from a different source after
* failure.
*/
XLogReaderInvalReadState(state);
@@ -573,7 +706,8 @@ err:
if (state->errormsg_buf[0] != '\0')
*errormsg = state->errormsg_buf;
- return NULL;
+ *record = NULL;
+ return XLREAD_FAIL;
}
/*
@@ -725,11 +859,12 @@ XLogReaderInvalReadState(XLogReaderState *state)
*
* This is just a convenience subroutine to avoid duplicated code in
* XLogReadRecord. It's not intended for use from anywhere else.
+ *
+ * If PrevRecPtr is valid, the xl_prev is is cross-checked with it.
*/
static bool
ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
- XLogRecPtr PrevRecPtr, XLogRecord *record,
- bool randAccess)
+ XLogRecPtr PrevRecPtr, XLogRecord *record)
{
if (record->xl_tot_len < SizeOfXLogRecord)
{
@@ -747,7 +882,7 @@ ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
(uint32) RecPtr);
return false;
}
- if (randAccess)
+ if (PrevRecPtr == InvalidXLogRecPtr)
{
/*
* We can't exactly verify the prev-link, but surely it should be less
@@ -979,11 +1114,14 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
* XLogReadRecord() will read the next valid record.
*/
XLogRecPtr
-XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
+XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
+ XLogFindNextRecordCB read_page, void *private)
{
XLogRecPtr tmpRecPtr;
XLogRecPtr found = InvalidXLogRecPtr;
XLogPageHeader header;
+ XLogRecord *record;
+ XLogReadRecordResult result;
char *errormsg;
Assert(!XLogRecPtrIsInvalid(RecPtr));
@@ -1016,9 +1154,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
while(XLogNeedData(state, targetPagePtr, targetRecOff,
targetRecOff != 0))
{
- if (!state->routine.page_read(state, state->readPagePtr,
- state->readLen,
- state->ReadRecPtr, state->readBuf))
+ if (!read_page(state, private))
break;
}
@@ -1070,8 +1206,16 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
* or we just jumped over the remaining data of a continuation.
*/
XLogBeginRead(state, tmpRecPtr);
- while (XLogReadRecord(state, &errormsg) != NULL)
+ while ((result = XLogReadRecord(state, &record, &errormsg)) !=
+ XLREAD_FAIL)
{
+ if (result == XLREAD_NEED_DATA)
+ {
+ if (!read_page(state, private))
+ goto err;
+ continue;
+ }
+
/* past the record we've found, break out */
if (RecPtr <= state->ReadRecPtr)
{
@@ -1091,9 +1235,9 @@ err:
#endif /* FRONTEND */
/*
- * Helper function to ease writing of XLogRoutine->page_read callbacks.
- * If this function is used, caller must supply a segment_open callback in
- * 'state', as that is used here.
+ * Helper function to ease writing of page_read callback.
+ * If this function is used, caller must supply a segment_open callback and
+ * segment_close callback as that is used here.
*
* Read 'count' bytes into 'buf', starting at location 'startptr', from WAL
* fetched from timeline 'tli'.
@@ -1106,6 +1250,7 @@ err:
*/
bool
WALRead(XLogReaderState *state,
+ WALSegmentOpenCB segopenfn, WALSegmentCloseCB segclosefn,
char *buf, XLogRecPtr startptr, Size count, TimeLineID tli,
WALReadError *errinfo)
{
@@ -1137,10 +1282,10 @@ WALRead(XLogReaderState *state,
XLogSegNo nextSegNo;
if (state->seg.ws_file >= 0)
- state->routine.segment_close(state);
+ segclosefn(state);
XLByteToSeg(recptr, nextSegNo, state->segcxt.ws_segsize);
- state->routine.segment_open(state, nextSegNo, &tli);
+ segopenfn(state, nextSegNo, &tli);
/* This shouldn't happen -- indicates a bug in segment_open */
Assert(state->seg.ws_file >= 0);
diff --git a/src/backend/access/transam/xlogutils.c b/src/backend/access/transam/xlogutils.c
index 4c6d072720..c43812ee94 100644
--- a/src/backend/access/transam/xlogutils.c
+++ b/src/backend/access/transam/xlogutils.c
@@ -688,8 +688,7 @@ XLogTruncateRelation(RelFileNode rnode, ForkNumber forkNum,
void
XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wantLength)
{
- const XLogRecPtr lastReadPage = state->seg.ws_segno *
- state->segcxt.ws_segsize + state->readLen;
+ const XLogRecPtr lastReadPage = state->readPagePtr;
Assert(wantPage != InvalidXLogRecPtr && wantPage % XLOG_BLCKSZ == 0);
Assert(wantLength <= XLOG_BLCKSZ);
@@ -704,7 +703,7 @@ XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wa
* current TLI has since become historical.
*/
if (lastReadPage == wantPage &&
- state->readLen != 0 &&
+ state->page_verified &&
lastReadPage + state->readLen >= wantPage + Min(wantLength, XLOG_BLCKSZ - 1))
return;
@@ -791,6 +790,7 @@ wal_segment_open(XLogReaderState *state, XLogSegNo nextSegNo,
char path[MAXPGPATH];
XLogFilePath(path, tli, nextSegNo, state->segcxt.ws_segsize);
+ elog(LOG, "HOGE: %lu, %d => %s", nextSegNo, tli, path);
state->seg.ws_file = BasicOpenFile(path, O_RDONLY | PG_BINARY);
if (state->seg.ws_file >= 0)
return;
@@ -828,9 +828,11 @@ wal_segment_close(XLogReaderState *state)
* loop for now.
*/
bool
-read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *cur_page)
+read_local_xlog_page(XLogReaderState *state)
{
+ XLogRecPtr targetPagePtr = state->readPagePtr;
+ int reqLen = state->readLen;
+ char *cur_page = state->readBuf;
XLogRecPtr read_upto,
loc;
TimeLineID tli;
@@ -943,11 +945,12 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
* as 'count', read the whole page anyway. It's guaranteed to be
* zero-padded up to the page boundary if it's incomplete.
*/
- if (!WALRead(state, cur_page, targetPagePtr, XLOG_BLCKSZ, tli,
- &errinfo))
+ if (!WALRead(state, wal_segment_open, wal_segment_close,
+ cur_page, targetPagePtr, XLOG_BLCKSZ, tli, &errinfo))
WALReadRaiseError(&errinfo);
/* number of valid bytes in the buffer */
+ state->readPagePtr = targetPagePtr;
state->readLen = count;
return true;
}
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index 0f6af952f9..97def6f781 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -138,7 +138,8 @@ StartupDecodingContext(List *output_plugin_options,
TransactionId xmin_horizon,
bool need_full_snapshot,
bool fast_forward,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress)
@@ -188,11 +189,12 @@ StartupDecodingContext(List *output_plugin_options,
ctx->slot = slot;
- ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, xl_routine, ctx);
+ ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, cleanup_cb);
if (!ctx->reader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
errmsg("out of memory")));
+ ctx->page_read = page_read;
ctx->reorder = ReorderBufferAllocate();
ctx->snapshot_builder =
@@ -283,7 +285,8 @@ CreateInitDecodingContext(const char *plugin,
List *output_plugin_options,
bool need_full_snapshot,
XLogRecPtr restart_lsn,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress)
@@ -386,7 +389,7 @@ CreateInitDecodingContext(const char *plugin,
ctx = StartupDecodingContext(NIL, restart_lsn, xmin_horizon,
need_full_snapshot, false,
- xl_routine, prepare_write, do_write,
+ page_read, cleanup_cb, prepare_write, do_write,
update_progress);
/* call output plugin initialization callback */
@@ -434,7 +437,8 @@ LogicalDecodingContext *
CreateDecodingContext(XLogRecPtr start_lsn,
List *output_plugin_options,
bool fast_forward,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress)
@@ -487,8 +491,8 @@ CreateDecodingContext(XLogRecPtr start_lsn,
ctx = StartupDecodingContext(output_plugin_options,
start_lsn, InvalidTransactionId, false,
- fast_forward, xl_routine, prepare_write,
- do_write, update_progress);
+ fast_forward, page_read, cleanup_cb,
+ prepare_write, do_write, update_progress);
/* call output plugin initialization callback */
old_context = MemoryContextSwitchTo(ctx->context);
@@ -541,7 +545,13 @@ DecodingContextFindStartpoint(LogicalDecodingContext *ctx)
char *err = NULL;
/* the read_page callback waits for new WAL */
- record = XLogReadRecord(ctx->reader, &err);
+ while (XLogReadRecord(ctx->reader, &record, &err) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!ctx->page_read(ctx->reader))
+ break;
+ }
+
if (err)
elog(ERROR, "%s", err);
if (!record)
diff --git a/src/backend/replication/logical/logicalfuncs.c b/src/backend/replication/logical/logicalfuncs.c
index b99c94e848..fcc81b358e 100644
--- a/src/backend/replication/logical/logicalfuncs.c
+++ b/src/backend/replication/logical/logicalfuncs.c
@@ -233,9 +233,8 @@ pg_logical_slot_get_changes_guts(FunctionCallInfo fcinfo, bool confirm, bool bin
ctx = CreateDecodingContext(InvalidXLogRecPtr,
options,
false,
- XL_ROUTINE(.page_read = read_local_xlog_page,
- .segment_open = wal_segment_open,
- .segment_close = wal_segment_close),
+ read_local_xlog_page,
+ wal_segment_close,
LogicalOutputPrepareWrite,
LogicalOutputWrite, NULL);
@@ -284,7 +283,13 @@ pg_logical_slot_get_changes_guts(FunctionCallInfo fcinfo, bool confirm, bool bin
XLogRecord *record;
char *errm = NULL;
- record = XLogReadRecord(ctx->reader, &errm);
+ while (XLogReadRecord(ctx->reader, &record, &errm) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!ctx->page_read(ctx->reader))
+ break;
+ }
+
if (errm)
elog(ERROR, "%s", errm);
diff --git a/src/backend/replication/slotfuncs.c b/src/backend/replication/slotfuncs.c
index 1725ad0736..b013da9631 100644
--- a/src/backend/replication/slotfuncs.c
+++ b/src/backend/replication/slotfuncs.c
@@ -152,9 +152,8 @@ create_logical_replication_slot(char *name, char *plugin,
ctx = CreateInitDecodingContext(plugin, NIL,
false, /* just catalogs is OK */
restart_lsn,
- XL_ROUTINE(.page_read = read_local_xlog_page,
- .segment_open = wal_segment_open,
- .segment_close = wal_segment_close),
+ read_local_xlog_page,
+ wal_segment_close,
NULL, NULL, NULL);
/*
@@ -507,9 +506,8 @@ pg_logical_replication_slot_advance(XLogRecPtr moveto)
ctx = CreateDecodingContext(InvalidXLogRecPtr,
NIL,
true, /* fast_forward */
- XL_ROUTINE(.page_read = read_local_xlog_page,
- .segment_open = wal_segment_open,
- .segment_close = wal_segment_close),
+ read_local_xlog_page,
+ wal_segment_close,
NULL, NULL, NULL);
/*
@@ -531,7 +529,13 @@ pg_logical_replication_slot_advance(XLogRecPtr moveto)
* Read records. No changes are generated in fast_forward mode,
* but snapbuilder/slot statuses are updated properly.
*/
- record = XLogReadRecord(ctx->reader, &errm);
+ while (XLogReadRecord(ctx->reader, &record, &errm) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!ctx->page_read(ctx->reader))
+ break;
+ }
+
if (errm)
elog(ERROR, "%s", errm);
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index 76f94fa61f..33d952f9fd 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -579,10 +579,7 @@ StartReplication(StartReplicationCmd *cmd)
/* create xlogreader for physical replication */
xlogreader =
- XLogReaderAllocate(wal_segment_size, NULL,
- XL_ROUTINE(.segment_open = WalSndSegmentOpen,
- .segment_close = wal_segment_close),
- NULL);
+ XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
if (!xlogreader)
ereport(ERROR,
@@ -811,9 +808,11 @@ StartReplication(StartReplicationCmd *cmd)
* set every time WAL is flushed.
*/
static bool
-logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetRecPtr, char *cur_page)
+logical_read_xlog_page(XLogReaderState *state)
{
+ XLogRecPtr targetPagePtr = state->readPagePtr;
+ int reqLen = state->readLen;
+ char *cur_page = state->readBuf;
XLogRecPtr flushptr;
int count;
WALReadError errinfo;
@@ -841,7 +840,7 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
count = flushptr - targetPagePtr; /* part of the page available */
/* now actually read the data, we know it's there */
- if (!WALRead(state,
+ if (!WALRead(state, WalSndSegmentOpen, wal_segment_close,
cur_page,
targetPagePtr,
XLOG_BLCKSZ,
@@ -1013,9 +1012,8 @@ CreateReplicationSlot(CreateReplicationSlotCmd *cmd)
ctx = CreateInitDecodingContext(cmd->plugin, NIL, need_full_snapshot,
InvalidXLogRecPtr,
- XL_ROUTINE(.page_read = logical_read_xlog_page,
- .segment_open = WalSndSegmentOpen,
- .segment_close = wal_segment_close),
+ logical_read_xlog_page,
+ wal_segment_close,
WalSndPrepareWrite, WalSndWriteData,
WalSndUpdateProgress);
@@ -1178,9 +1176,8 @@ StartLogicalReplication(StartReplicationCmd *cmd)
*/
logical_decoding_ctx =
CreateDecodingContext(cmd->startpoint, cmd->options, false,
- XL_ROUTINE(.page_read = logical_read_xlog_page,
- .segment_open = WalSndSegmentOpen,
- .segment_close = wal_segment_close),
+ logical_read_xlog_page,
+ wal_segment_close,
WalSndPrepareWrite, WalSndWriteData,
WalSndUpdateProgress);
xlogreader = logical_decoding_ctx->reader;
@@ -2749,7 +2746,7 @@ XLogSendPhysical(void)
enlargeStringInfo(&output_message, nbytes);
retry:
- if (!WALRead(xlogreader,
+ if (!WALRead(xlogreader, WalSndSegmentOpen, wal_segment_close,
&output_message.data[output_message.len],
startptr,
nbytes,
@@ -2847,7 +2844,12 @@ XLogSendLogical(void)
*/
WalSndCaughtUp = false;
- record = XLogReadRecord(logical_decoding_ctx->reader, &errm);
+ while (XLogReadRecord(logical_decoding_ctx->reader, &record, &errm) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!logical_decoding_ctx->page_read(logical_decoding_ctx->reader))
+ break;
+ }
/* xlog record was invalid */
if (errm != NULL)
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index 4be2eede92..7344a7c692 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -41,15 +41,9 @@ static int xlogreadfd = -1;
static XLogSegNo xlogreadsegno = -1;
static char xlogfpath[MAXPGPATH];
-typedef struct XLogPageReadPrivate
-{
- const char *restoreCommand;
- int tliIndex;
-} XLogPageReadPrivate;
-
-static bool SimpleXLogPageRead(XLogReaderState *xlogreader,
- XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
+static bool SimpleXLogPageRead(XLogReaderState *xlogreader,
+ const char *datadir, int *tliIndex,
+ const char *restoreCommand);
/*
* Read WAL from the datadir/pg_wal, starting from 'startpoint' on timeline
@@ -63,20 +57,22 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
XLogRecord *record;
XLogReaderState *xlogreader;
char *errormsg;
- XLogPageReadPrivate private;
- private.tliIndex = tliIndex;
- private.restoreCommand = restoreCommand;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir,
- XL_ROUTINE(.page_read = &SimpleXLogPageRead),
- &private);
+ xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+
if (xlogreader == NULL)
pg_fatal("out of memory");
XLogBeginRead(xlogreader, startpoint);
do
{
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!SimpleXLogPageRead(xlogreader, datadir,
+ &tliIndex, restoreCommand))
+ break;
+ }
if (record == NULL)
{
@@ -114,19 +110,19 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex,
XLogRecord *record;
XLogReaderState *xlogreader;
char *errormsg;
- XLogPageReadPrivate private;
XLogRecPtr endptr;
- private.tliIndex = tliIndex;
- private.restoreCommand = restoreCommand;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir,
- XL_ROUTINE(.page_read = &SimpleXLogPageRead),
- &private);
+ xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
XLogBeginRead(xlogreader, ptr);
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!SimpleXLogPageRead(xlogreader, datadir, &tliIndex, restoreCommand))
+ break;
+ }
if (record == NULL)
{
if (errormsg)
@@ -161,7 +157,6 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
XLogRecPtr searchptr;
XLogReaderState *xlogreader;
char *errormsg;
- XLogPageReadPrivate private;
/*
* The given fork pointer points to the end of the last common record,
@@ -177,11 +172,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
forkptr += SizeOfXLogShortPHD;
}
- private.tliIndex = tliIndex;
- private.restoreCommand = restoreCommand;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir,
- XL_ROUTINE(.page_read = &SimpleXLogPageRead),
- &private);
+ xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
@@ -191,7 +182,13 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
uint8 info;
XLogBeginRead(xlogreader, searchptr);
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!SimpleXLogPageRead(xlogreader, datadir,
+ &tliIndex, restoreCommand))
+ break;
+ }
if (record == NULL)
{
@@ -238,10 +235,11 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
/* XLogReader callback function, to read a WAL page */
static bool
-SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *readBuf)
+SimpleXLogPageRead(XLogReaderState *xlogreader, const char *datadir,
+ int *tliIndex, const char *restoreCommand)
{
- XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
+ XLogRecPtr targetPagePtr = xlogreader->readPagePtr;
+ char *readBuf = xlogreader->readBuf;
uint32 targetPageOff;
XLogRecPtr targetSegEnd;
XLogSegNo targetSegNo;
@@ -274,14 +272,14 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
* be done both forward and backward, consider also switching timeline
* accordingly.
*/
- while (private->tliIndex < targetNentries - 1 &&
- targetHistory[private->tliIndex].end < targetSegEnd)
- private->tliIndex++;
- while (private->tliIndex > 0 &&
- targetHistory[private->tliIndex].begin >= targetSegEnd)
- private->tliIndex--;
+ while (*tliIndex < targetNentries - 1 &&
+ targetHistory[*tliIndex].end < targetSegEnd)
+ (*tliIndex)++;
+ while (*tliIndex > 0 &&
+ targetHistory[*tliIndex].begin >= targetSegEnd)
+ (*tliIndex)--;
- XLogFileName(xlogfname, targetHistory[private->tliIndex].tli,
+ XLogFileName(xlogfname, targetHistory[*tliIndex].tli,
xlogreadsegno, WalSegSz);
snprintf(xlogfpath, MAXPGPATH, "%s/" XLOGDIR "/%s",
@@ -294,7 +292,7 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
/*
* If we have no restore_command to execute, then exit.
*/
- if (private->restoreCommand == NULL)
+ if (restoreCommand == NULL)
{
pg_log_error("could not open file \"%s\": %m", xlogfpath);
xlogreader->readLen = -1;
@@ -308,7 +306,7 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
xlogreadfd = RestoreArchivedFile(xlogreader->segcxt.ws_dir,
xlogfname,
WalSegSz,
- private->restoreCommand);
+ restoreCommand);
if (xlogreadfd < 0)
{
@@ -350,7 +348,7 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
Assert(targetSegNo == xlogreadsegno);
- xlogreader->seg.ws_tli = targetHistory[private->tliIndex].tli;
+ xlogreader->seg.ws_tli = targetHistory[*tliIndex].tli;
xlogreader->readLen = XLOG_BLCKSZ;
return true;
}
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index da929bf4cd..9329a6a6d6 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -330,12 +330,17 @@ WALDumpCloseSegment(XLogReaderState *state)
state->seg.ws_file = -1;
}
-/* pg_waldump's XLogReaderRoutine->page_read callback */
+/*
+ * pg_waldump's WAL page rader, also used as page_read callback for
+ * XLogFindNextRecord
+ */
static bool
-WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetPtr, char *readBuff)
+WALDumpReadPage(XLogReaderState *state, void *priv)
{
- XLogDumpPrivate *private = state->private_data;
+ XLogRecPtr targetPagePtr = state->readPagePtr;
+ int reqLen = state->readLen;
+ char *readBuff = state->readBuf;
+ XLogDumpPrivate *private = (XLogDumpPrivate *) priv;
int count = XLOG_BLCKSZ;
WALReadError errinfo;
@@ -353,8 +358,8 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
}
}
- if (!WALRead(state, readBuff, targetPagePtr, count, private->timeline,
- &errinfo))
+ if (!WALRead(state, WALDumpOpenSegment, WALDumpCloseSegment,
+ readBuff, targetPagePtr, count, private->timeline, &errinfo))
{
WALOpenSegment *seg = &errinfo.wre_seg;
char fname[MAXPGPATH];
@@ -374,6 +379,7 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
(Size) errinfo.wre_req);
}
+ Assert(count >= state->readLen);
state->readLen = count;
return true;
}
@@ -1037,16 +1043,14 @@ main(int argc, char **argv)
/* we have everything we need, start reading */
xlogreader_state =
- XLogReaderAllocate(WalSegSz, waldir,
- XL_ROUTINE(.page_read = WALDumpReadPage,
- .segment_open = WALDumpOpenSegment,
- .segment_close = WALDumpCloseSegment),
- &private);
+ XLogReaderAllocate(WalSegSz, waldir, WALDumpCloseSegment);
+
if (!xlogreader_state)
fatal_error("out of memory");
/* first find a valid recptr to start from */
- first_record = XLogFindNextRecord(xlogreader_state, private.startptr);
+ first_record = XLogFindNextRecord(xlogreader_state, private.startptr,
+ &WALDumpReadPage, (void*) &private);
if (first_record == InvalidXLogRecPtr)
fatal_error("could not find a valid record after %X/%X",
@@ -1070,7 +1074,13 @@ main(int argc, char **argv)
for (;;)
{
/* try to read the next record */
- record = XLogReadRecord(xlogreader_state, &errormsg);
+ while (XLogReadRecord(xlogreader_state, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!WALDumpReadPage(xlogreader_state, (void *) &private))
+ break;
+ }
+
if (!record)
{
if (!config.follow || private.endptr_reached)
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index f8f6a6264c..38f9e59f99 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -57,64 +57,15 @@ typedef struct WALSegmentContext
typedef struct XLogReaderState XLogReaderState;
-/* Function type definition for the read_page callback */
-typedef bool (*XLogPageReadCB) (XLogReaderState *xlogreader,
- XLogRecPtr targetPagePtr,
- int reqLen,
- XLogRecPtr targetRecPtr,
- char *readBuf);
+/* Function type definition for the segment cleanup callback */
+typedef void (*WALSegmentCleanupCB) (XLogReaderState *xlogreader);
+
+/* Function type definition for the open/close callbacks for WALRead() */
typedef void (*WALSegmentOpenCB) (XLogReaderState *xlogreader,
XLogSegNo nextSegNo,
TimeLineID *tli_p);
typedef void (*WALSegmentCloseCB) (XLogReaderState *xlogreader);
-typedef struct XLogReaderRoutine
-{
- /*
- * Data input callback
- *
- * This callback shall read at least reqLen valid bytes of the xlog page
- * starting at targetPagePtr, and store them in readBuf. The callback
- * shall return the number of bytes read (never more than XLOG_BLCKSZ), or
- * -1 on failure. The callback shall sleep, if necessary, to wait for the
- * requested bytes to become available. The callback will not be invoked
- * again for the same page unless more than the returned number of bytes
- * are needed.
- *
- * targetRecPtr is the position of the WAL record we're reading. Usually
- * it is equal to targetPagePtr + reqLen, but sometimes xlogreader needs
- * to read and verify the page or segment header, before it reads the
- * actual WAL record it's interested in. In that case, targetRecPtr can
- * be used to determine which timeline to read the page from.
- *
- * The callback shall set ->seg.ws_tli to the TLI of the file the page was
- * read from.
- */
- XLogPageReadCB page_read;
-
- /*
- * Callback to open the specified WAL segment for reading. ->seg.ws_file
- * shall be set to the file descriptor of the opened segment. In case of
- * failure, an error shall be raised by the callback and it shall not
- * return.
- *
- * "nextSegNo" is the number of the segment to be opened.
- *
- * "tli_p" is an input/output argument. WALRead() uses it to pass the
- * timeline in which the new segment should be found, but the callback can
- * use it to return the TLI that it actually opened.
- */
- WALSegmentOpenCB segment_open;
-
- /*
- * WAL segment close callback. ->seg.ws_file shall be set to a negative
- * number.
- */
- WALSegmentCloseCB segment_close;
-} XLogReaderRoutine;
-
-#define XL_ROUTINE(...) &(XLogReaderRoutine){__VA_ARGS__}
-
typedef struct
{
/* Is this block ref in use? */
@@ -144,12 +95,35 @@ typedef struct
uint16 data_bufsz;
} DecodedBkpBlock;
+/* Return code from XLogReadRecord */
+typedef enum XLogReadRecordResult
+{
+ XLREAD_SUCCESS, /* record is successfully read */
+ XLREAD_NEED_DATA, /* need more data. see XLogReadRecord. */
+ XLREAD_FAIL /* failed during reading a record */
+} XLogReadRecordResult;
+
+/*
+ * internal state of XLogReadRecord
+ *
+ * XLogReadState runs a state machine while reading a record. Theses states
+ * are not seen outside the function. Each state may repeat several times
+ * exiting requesting caller for new data. See the comment of XLogReadRecrod
+ * for details.
+ */
+typedef enum XLogReadRecordState {
+ XLREAD_NEXT_RECORD,
+ XLREAD_TOT_LEN,
+ XLREAD_FIRST_FRAGMENT,
+ XLREAD_CONTINUATION
+} XLogReadRecordState;
+
struct XLogReaderState
{
/*
* Operational callbacks
*/
- XLogReaderRoutine routine;
+ WALSegmentCleanupCB cleanup_cb;
/* ----------------------------------------
* Public parameters
@@ -162,18 +136,14 @@ struct XLogReaderState
*/
uint64 system_identifier;
- /*
- * Opaque data for callbacks to use. Not used by XLogReader.
- */
- void *private_data;
-
/*
* Start and end point of last record read. EndRecPtr is also used as the
* position to read next. Calling XLogBeginRead() sets EndRecPtr to the
* starting position and ReadRecPtr to invalid.
*/
- XLogRecPtr ReadRecPtr; /* start of last record read */
+ XLogRecPtr ReadRecPtr; /* start of last record read or being read */
XLogRecPtr EndRecPtr; /* end+1 of last record read */
+ XLogRecPtr PrevRecPtr; /* start of previous record read */
/* ----------------------------------------
* Communication with page reader
@@ -187,7 +157,9 @@ struct XLogReaderState
* the request, or -1 on error */
TimeLineID readPageTLI; /* TLI for data currently in readBuf */
char *readBuf; /* buffer to store data */
- bool page_verified; /* is the page on the buffer verified? */
+ bool page_verified; /* is the page header on the buffer verified? */
+ bool record_verified;/* is the current record header verified? */
+
/* ----------------------------------------
@@ -229,8 +201,6 @@ struct XLogReaderState
XLogRecPtr latestPagePtr;
TimeLineID latestPageTLI;
- /* beginning of the WAL record being read. */
- XLogRecPtr currRecPtr;
/* timeline to read it from, 0 if a lookup is required */
TimeLineID currTLI;
@@ -257,6 +227,15 @@ struct XLogReaderState
char *readRecordBuf;
uint32 readRecordBufSize;
+ /*
+ * XLogReadRecord() state
+ */
+ XLogReadRecordState readRecordState;/* state machine state */
+ int recordGotLen; /* amount of current record that has
+ * already been read */
+ int recordRemainLen; /* length of current record that remains */
+ XLogRecPtr recordContRecPtr; /* where the current record continues */
+
/* Buffer to hold error message */
char *errormsg_buf;
};
@@ -264,9 +243,7 @@ struct XLogReaderState
/* Get a new XLogReader */
extern XLogReaderState *XLogReaderAllocate(int wal_segment_size,
const char *waldir,
- XLogReaderRoutine *routine,
- void *private_data);
-extern XLogReaderRoutine *LocalXLogReaderRoutine(void);
+ WALSegmentCleanupCB cleanup_cb);
/* Free an XLogReader */
extern void XLogReaderFree(XLogReaderState *state);
@@ -274,12 +251,17 @@ extern void XLogReaderFree(XLogReaderState *state);
/* Position the XLogReader to given record */
extern void XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr);
#ifdef FRONTEND
-extern XLogRecPtr XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr);
+/* Function type definition for the read_page callback */
+typedef bool (*XLogFindNextRecordCB) (XLogReaderState *xlogreader,
+ void *private);
+extern XLogRecPtr XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
+ XLogFindNextRecordCB read_page, void *private);
#endif /* FRONTEND */
/* Read the next XLog record. Returns NULL on end-of-WAL or failure */
-extern struct XLogRecord *XLogReadRecord(XLogReaderState *state,
- char **errormsg);
+extern XLogReadRecordResult XLogReadRecord(XLogReaderState *state,
+ XLogRecord **record,
+ char **errormsg);
/* Validate a page */
extern bool XLogReaderValidatePageHeader(XLogReaderState *state,
@@ -299,6 +281,7 @@ typedef struct WALReadError
} WALReadError;
extern bool WALRead(XLogReaderState *state,
+ WALSegmentOpenCB segopenfn, WALSegmentCloseCB sgclosefn,
char *buf, XLogRecPtr startptr, Size count,
TimeLineID tli, WALReadError *errinfo);
diff --git a/src/include/access/xlogutils.h b/src/include/access/xlogutils.h
index c287edf206..482e4ced69 100644
--- a/src/include/access/xlogutils.h
+++ b/src/include/access/xlogutils.h
@@ -47,9 +47,7 @@ extern Buffer XLogReadBufferExtended(RelFileNode rnode, ForkNumber forknum,
extern Relation CreateFakeRelcacheEntry(RelFileNode rnode);
extern void FreeFakeRelcacheEntry(Relation fakerel);
-extern bool read_local_xlog_page(XLogReaderState *state,
- XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetRecPtr, char *cur_page);
+extern bool read_local_xlog_page(XLogReaderState *state);
extern void wal_segment_open(XLogReaderState *state,
XLogSegNo nextSegNo,
TimeLineID *tli_p);
diff --git a/src/include/pg_config_manual.h b/src/include/pg_config_manual.h
index 705dc69c06..fdc41fff91 100644
--- a/src/include/pg_config_manual.h
+++ b/src/include/pg_config_manual.h
@@ -339,7 +339,7 @@
* Enable debugging print statements for WAL-related operations; see
* also the wal_debug GUC var.
*/
-/* #define WAL_DEBUG */
+#define WAL_DEBUG
/*
* Enable tracing of resource consumption during sort operations;
diff --git a/src/include/replication/logical.h b/src/include/replication/logical.h
index 45abc444b7..93f9769ab4 100644
--- a/src/include/replication/logical.h
+++ b/src/include/replication/logical.h
@@ -29,6 +29,10 @@ typedef void (*LogicalOutputPluginWriterUpdateProgress) (struct LogicalDecodingC
TransactionId xid
);
+typedef struct LogicalDecodingContext LogicalDecodingContext;
+
+typedef bool (*LogicalDecodingXLogPageReadCB)(XLogReaderState *ctx);
+
typedef struct LogicalDecodingContext
{
/* memory context this is all allocated in */
@@ -39,6 +43,7 @@ typedef struct LogicalDecodingContext
/* infrastructure pieces for decoding */
XLogReaderState *reader;
+ LogicalDecodingXLogPageReadCB page_read;
struct ReorderBuffer *reorder;
struct SnapBuild *snapshot_builder;
@@ -100,14 +105,16 @@ extern LogicalDecodingContext *CreateInitDecodingContext(const char *plugin,
List *output_plugin_options,
bool need_full_snapshot,
XLogRecPtr restart_lsn,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress);
extern LogicalDecodingContext *CreateDecodingContext(XLogRecPtr start_lsn,
List *output_plugin_options,
bool fast_forward,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress);
--
2.18.4
----Next_Part(Tue_Sep__8_16_35_16_2020_701)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
filename="v16-0003-Remove-globals-readOff-readLen-and-readSegNo.patch"
^ permalink raw reply [nested|flat] 70+ messages in thread
* [PATCH v17 2/4] Move page-reader out of XLogReadRecord
@ 2019-09-10 03:58 Kyotaro Horiguchi <[email protected]>
0 siblings, 0 replies; 70+ messages in thread
From: Kyotaro Horiguchi @ 2019-09-10 03:58 UTC (permalink / raw)
This is the second step of removing callbacks from WAL record reader.
Since it is essential to take in additional data while reading a
record, the function have to ask caller for new data while keeping
working state. Thus the function is turned into a state machine.
---
src/backend/access/transam/twophase.c | 15 +-
src/backend/access/transam/xlog.c | 58 +-
src/backend/access/transam/xlogreader.c | 697 +++++++++++-------
src/backend/access/transam/xlogutils.c | 17 +-
src/backend/replication/logical/logical.c | 26 +-
.../replication/logical/logicalfuncs.c | 13 +-
src/backend/replication/slotfuncs.c | 18 +-
src/backend/replication/walsender.c | 32 +-
src/bin/pg_rewind/parsexlog.c | 84 ++-
src/bin/pg_waldump/pg_waldump.c | 36 +-
src/include/access/xlogreader.h | 121 ++-
src/include/access/xlogutils.h | 4 +-
src/include/pg_config_manual.h | 2 +-
src/include/replication/logical.h | 11 +-
14 files changed, 646 insertions(+), 488 deletions(-)
diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index 80d2d20d6c..7d5a96a5f5 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -1325,11 +1325,8 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
XLogReaderState *xlogreader;
char *errormsg;
- xlogreader = XLogReaderAllocate(wal_segment_size, NULL,
- XL_ROUTINE(.page_read = &read_local_xlog_page,
- .segment_open = &wal_segment_open,
- .segment_close = &wal_segment_close),
- NULL);
+ xlogreader = XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+
if (!xlogreader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
@@ -1337,7 +1334,13 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
errdetail("Failed while allocating a WAL reading processor.")));
XLogBeginRead(xlogreader, lsn);
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!read_local_xlog_page(xlogreader))
+ break;
+ }
+
if (record == NULL)
ereport(ERROR,
(errcode_for_file_access(),
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 606328a65a..670996ae67 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -836,13 +836,6 @@ static XLogSource currentSource = XLOG_FROM_ANY;
static bool lastSourceFailed = false;
static bool pendingWalRcvRestart = false;
-typedef struct XLogPageReadPrivate
-{
- int emode;
- bool fetching_ckpt; /* are we fetching a checkpoint record? */
- bool randAccess;
-} XLogPageReadPrivate;
-
/*
* These variables track when we last obtained some WAL data to process,
* and where we got it from. (XLogReceiptSource is initially the same as
@@ -917,8 +910,8 @@ static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath,
static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli,
XLogSource source, bool notfoundOk);
static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, XLogSource source);
-static bool XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
+static bool XLogPageRead(XLogReaderState *xlogreader,
+ bool fetching_ckpt, int emode, bool randAccess);
static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
bool fetching_ckpt, XLogRecPtr tliRecPtr);
static int emode_for_corrupt_record(int emode, XLogRecPtr RecPtr);
@@ -1231,8 +1224,7 @@ XLogInsertRecord(XLogRecData *rdata,
appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len);
if (!debug_reader)
- debug_reader = XLogReaderAllocate(wal_segment_size, NULL,
- XL_ROUTINE(), NULL);
+ debug_reader = XLogReaderAllocate(wal_segment_size, NULL, NULL);
if (!debug_reader)
{
@@ -4344,15 +4336,10 @@ CleanupBackupHistory(void)
* record is available.
*/
static XLogRecord *
-ReadRecord(XLogReaderState *xlogreader, int emode,
- bool fetching_ckpt)
+ReadRecord(XLogReaderState *xlogreader, int emode, bool fetching_ckpt)
{
XLogRecord *record;
- XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
-
- private->fetching_ckpt = fetching_ckpt;
- private->emode = emode;
- private->randAccess = (xlogreader->ReadRecPtr == InvalidXLogRecPtr);
+ bool randAccess = (xlogreader->ReadRecPtr == InvalidXLogRecPtr);
/* This is the first attempt to read this page. */
lastSourceFailed = false;
@@ -4360,8 +4347,16 @@ ReadRecord(XLogReaderState *xlogreader, int emode,
for (;;)
{
char *errormsg;
+ XLogReadRecordResult result;
+
+ while ((result = XLogReadRecord(xlogreader, &record, &errormsg))
+ == XLREAD_NEED_DATA)
+ {
+ if (!XLogPageRead(xlogreader, fetching_ckpt, emode, randAccess))
+ break;
+
+ }
- record = XLogReadRecord(xlogreader, &errormsg);
ReadRecPtr = xlogreader->ReadRecPtr;
EndRecPtr = xlogreader->EndRecPtr;
if (record == NULL)
@@ -6380,7 +6375,6 @@ StartupXLOG(void)
bool backupFromStandby = false;
DBState dbstate_at_startup;
XLogReaderState *xlogreader;
- XLogPageReadPrivate private;
bool promoted = false;
struct stat st;
@@ -6539,13 +6533,9 @@ StartupXLOG(void)
OwnLatch(&XLogCtl->recoveryWakeupLatch);
/* Set up XLOG reader facility */
- MemSet(&private, 0, sizeof(XLogPageReadPrivate));
xlogreader =
- XLogReaderAllocate(wal_segment_size, NULL,
- XL_ROUTINE(.page_read = &XLogPageRead,
- .segment_open = NULL,
- .segment_close = wal_segment_close),
- &private);
+ XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+
if (!xlogreader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
@@ -11987,12 +11977,13 @@ CancelBackup(void)
* sleep and retry.
*/
static bool
-XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetRecPtr, char *readBuf)
+XLogPageRead(XLogReaderState *xlogreader,
+ bool fetching_ckpt, int emode, bool randAccess)
{
- XLogPageReadPrivate *private =
- (XLogPageReadPrivate *) xlogreader->private_data;
- int emode = private->emode;
+ char *readBuf = xlogreader->readBuf;
+ XLogRecPtr targetPagePtr = xlogreader->readPagePtr;
+ int reqLen = xlogreader->readLen;
+ XLogRecPtr targetRecPtr = xlogreader->ReadRecPtr;
uint32 targetPageOff;
XLogSegNo targetSegNo PG_USED_FOR_ASSERTS_ONLY;
int r;
@@ -12035,8 +12026,8 @@ retry:
flushedUpto < targetPagePtr + reqLen))
{
if (!WaitForWALToBecomeAvailable(targetPagePtr + reqLen,
- private->randAccess,
- private->fetching_ckpt,
+ randAccess,
+ fetching_ckpt,
targetRecPtr))
{
if (readFile >= 0)
@@ -12141,6 +12132,7 @@ retry:
goto next_record_is_invalid;
}
+ Assert(xlogreader->readPagePtr == targetPagePtr);
xlogreader->readLen = readLen;
return true;
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index 7d04995b7b..ba3e4868f7 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -40,7 +40,7 @@ static bool XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr,
int reqLen, bool header_inclusive);
static void XLogReaderInvalReadState(XLogReaderState *state);
static bool ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
- XLogRecPtr PrevRecPtr, XLogRecord *record, bool randAccess);
+ XLogRecPtr PrevRecPtr, XLogRecord *record);
static bool ValidXLogRecord(XLogReaderState *state, XLogRecord *record,
XLogRecPtr recptr);
static void ResetDecoder(XLogReaderState *state);
@@ -73,7 +73,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...)
*/
XLogReaderState *
XLogReaderAllocate(int wal_segment_size, const char *waldir,
- XLogReaderRoutine *routine, void *private_data)
+ WALSegmentCleanupCB cleanup_cb)
{
XLogReaderState *state;
@@ -84,7 +84,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
return NULL;
/* initialize caller-provided support functions */
- state->routine = *routine;
+ state->cleanup_cb = cleanup_cb;
state->max_block_id = -1;
@@ -107,8 +107,6 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
WALOpenSegmentInit(&state->seg, &state->segcxt, wal_segment_size,
waldir);
- /* system_identifier initialized to zeroes above */
- state->private_data = private_data;
/* ReadRecPtr, EndRecPtr and readLen initialized to zeroes above */
state->errormsg_buf = palloc_extended(MAX_ERRORMSG_LEN + 1,
MCXT_ALLOC_NO_OOM);
@@ -140,8 +138,8 @@ XLogReaderFree(XLogReaderState *state)
{
int block_id;
- if (state->seg.ws_file != -1)
- state->routine.segment_close(state);
+ if (state->seg.ws_file >= 0)
+ state->cleanup_cb(state);
for (block_id = 0; block_id <= XLR_MAX_BLOCK_ID; block_id++)
{
@@ -246,6 +244,7 @@ XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
/* Begin at the passed-in record pointer. */
state->EndRecPtr = RecPtr;
state->ReadRecPtr = InvalidXLogRecPtr;
+ state->readRecordState = XLREAD_NEXT_RECORD;
}
/*
@@ -254,317 +253,449 @@ XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
* XLogBeginRead() or XLogFindNextRecord() must be called before the first call
* to XLogReadRecord().
*
- * If the page_read callback fails to read the requested data, NULL is
- * returned. The callback is expected to have reported the error; errormsg
- * is set to NULL.
+ * This function may return XLREAD_NEED_DATA several times before returning a
+ * result record. The caller shall read in some new data then call this
+ * function again with the same parameters.
*
- * If the reading fails for some other reason, NULL is also returned, and
- * *errormsg is set to a string with details of the failure.
+ * When a record is successfully read, returns XLREAD_SUCCESS with result
+ * record being stored in *record. Otherwise *record is NULL.
*
- * The returned pointer (or *errormsg) points to an internal buffer that's
- * valid until the next call to XLogReadRecord.
+ * Returns XLREAD_NEED_DATA if more data is needed to finish reading the
+ * current record. In that case, state->readPagePtr and state->readLen inform
+ * the desired position and minimum length of data needed. The caller shall
+ * read in the requested data and set state->readBuf to point to a buffer
+ * containing it. The caller must also set state->readPageTLI and
+ * state->readLen to indicate the timeline that it was read from, and the
+ * length of data that is now available (which must be >= given readLen),
+ * respectively.
+ *
+ * If invalid data is encountered, returns XLREAD_FAIL with *record being set to
+ * NULL. *errormsg is set to a string with details of the failure.
+ * The returned pointer (or *errormsg) points to an internal buffer that's valid
+ * until the next call to XLogReadRecord.
+ *
+ *
+ * This function runs a state machine consists of the following states.
+ *
+ * XLREAD_NEXT_RECORD :
+ * The initial state, if called with valid RecPtr, try to read a record at
+ * that position. If invalid RecPtr is given try to read a record just after
+ * the last one previously read.
+ * This state ens after setting ReadRecPtr. Then goes to XLREAD_TOT_LEN.
+ *
+ * XLREAD_TOT_LEN:
+ * Examining record header. Ends after reading record total
+ * length. recordRemainLen and recordGotLen are initialized.
+ *
+ * XLREAD_FIRST_FRAGMENT:
+ * Reading the first fragment. Ends with finishing reading a single
+ * record. Goes to XLREAD_NEXT_RECORD if that's all or
+ * XLREAD_CONTINUATION if we have continuation.
+
+ * XLREAD_CONTINUATION:
+ * Reading continuation of record. Ends with finishing the whole record then
+ * goes to XLREAD_NEXT_RECORD. During this state, recordRemainLen indicates
+ * how much is left and readRecordBuf holds the partially assert
+ * record.recordContRecPtr points to the beginning of the next page where to
+ * continue.
+ *
+ * If wrong data found in any state, the state machine stays at the current
+ * state. This behavior allows to continue reading a reacord switching among
+ * different souces, while streaming replication.
*/
-XLogRecord *
-XLogReadRecord(XLogReaderState *state, char **errormsg)
+XLogReadRecordResult
+XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
{
- XLogRecPtr RecPtr;
- XLogRecord *record;
- XLogRecPtr targetPagePtr;
- bool randAccess;
- uint32 len,
- total_len;
- uint32 targetRecOff;
- uint32 pageHeaderSize;
- bool gotheader;
+ XLogRecord *prec;
- /*
- * randAccess indicates whether to verify the previous-record pointer of
- * the record we're reading. We only do this if we're reading
- * sequentially, which is what we initially assume.
- */
- randAccess = false;
+ *record = NULL;
/* reset error state */
*errormsg = NULL;
state->errormsg_buf[0] = '\0';
- ResetDecoder(state);
-
- RecPtr = state->EndRecPtr;
-
- if (state->ReadRecPtr != InvalidXLogRecPtr)
- {
- /* read the record after the one we just read */
-
- /*
- * EndRecPtr is pointing to end+1 of the previous WAL record. If
- * we're at a page boundary, no more records can fit on the current
- * page. We must skip over the page header, but we can't do that until
- * we've read in the page, since the header size is variable.
- */
- }
- else
- {
- /*
- * Caller supplied a position to start at.
- *
- * In this case, EndRecPtr should already be pointing to a valid
- * record starting position.
- */
- Assert(XRecOffIsValid(RecPtr));
- randAccess = true;
- }
-
- state->currRecPtr = RecPtr;
-
- targetPagePtr = RecPtr - (RecPtr % XLOG_BLCKSZ);
- targetRecOff = RecPtr % XLOG_BLCKSZ;
-
- /*
- * Read the page containing the record into state->readBuf. Request enough
- * byte to cover the whole record header, or at least the part of it that
- * fits on the same page.
- */
- while (XLogNeedData(state, targetPagePtr,
- Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
- targetRecOff != 0))
- {
- if (!state->routine.page_read(state, state->readPagePtr, state->readLen,
- RecPtr, state->readBuf))
- break;
- }
-
- if (!state->page_verified)
- goto err;
-
- /*
- * We have at least the page header, so we can examine it now.
- */
- pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
- if (targetRecOff == 0)
- {
- /*
- * At page start, so skip over page header.
- */
- RecPtr += pageHeaderSize;
- targetRecOff = pageHeaderSize;
- }
- else if (targetRecOff < pageHeaderSize)
- {
- report_invalid_record(state, "invalid record offset at %X/%X",
- LSN_FORMAT_ARGS(RecPtr));
- goto err;
- }
-
- if ((((XLogPageHeader) state->readBuf)->xlp_info & XLP_FIRST_IS_CONTRECORD) &&
- targetRecOff == pageHeaderSize)
- {
- report_invalid_record(state, "contrecord is requested by %X/%X",
- LSN_FORMAT_ARGS(RecPtr));
- goto err;
- }
-
- /* XLogNeedData has verified the page header */
- Assert(pageHeaderSize <= state->readLen);
-
- /*
- * Read the record length.
- *
- * NB: Even though we use an XLogRecord pointer here, the whole record
- * header might not fit on this page. xl_tot_len is the first field of the
- * struct, so it must be on this page (the records are MAXALIGNed), but we
- * cannot access any other fields until we've verified that we got the
- * whole header.
- */
- record = (XLogRecord *) (state->readBuf + RecPtr % XLOG_BLCKSZ);
- total_len = record->xl_tot_len;
-
- /*
- * If the whole record header is on this page, validate it immediately.
- * Otherwise do just a basic sanity check on xl_tot_len, and validate the
- * rest of the header after reading it from the next page. The xl_tot_len
- * check is necessary here to ensure that we enter the "Need to reassemble
- * record" code path below; otherwise we might fail to apply
- * ValidXLogRecordHeader at all.
- */
- if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
- {
- if (!ValidXLogRecordHeader(state, RecPtr, state->ReadRecPtr, record,
- randAccess))
- goto err;
- gotheader = true;
- }
- else
- {
- /* XXX: more validation should be done here */
- if (total_len < SizeOfXLogRecord)
- {
- report_invalid_record(state,
- "invalid record length at %X/%X: wanted %u, got %u",
- LSN_FORMAT_ARGS(RecPtr),
- (uint32) SizeOfXLogRecord, total_len);
- goto err;
- }
- gotheader = false;
- }
-
- len = XLOG_BLCKSZ - RecPtr % XLOG_BLCKSZ;
- if (total_len > len)
+ switch (state->readRecordState)
{
- /* Need to reassemble record */
- char *contdata;
- XLogPageHeader pageHeader;
- char *buffer;
- uint32 gotlen;
-
- /*
- * Enlarge readRecordBuf as needed.
- */
- if (total_len > state->readRecordBufSize &&
- !allocate_recordbuf(state, total_len))
- {
- /* We treat this as a "bogus data" condition */
- report_invalid_record(state, "record length %u at %X/%X too long",
- total_len, LSN_FORMAT_ARGS(RecPtr));
- goto err;
- }
+ case XLREAD_NEXT_RECORD:
+ ResetDecoder(state);
- /* Copy the first fragment of the record from the first page. */
- memcpy(state->readRecordBuf,
- state->readBuf + RecPtr % XLOG_BLCKSZ, len);
- buffer = state->readRecordBuf + len;
- gotlen = len;
-
- do
- {
- int rest_len = total_len - gotlen;
-
- /* Calculate pointer to beginning of next page */
- targetPagePtr += XLOG_BLCKSZ;
-
- /* Wait for the next page to become available */
- while (XLogNeedData(state, targetPagePtr,
- Min(rest_len, XLOG_BLCKSZ),
- false))
+ if (state->ReadRecPtr != InvalidXLogRecPtr)
{
- if (!state->routine.page_read(state, state->readPagePtr,
- state->readLen,
- state->ReadRecPtr,
- state->readBuf))
- break;
+ /* read the record after the one we just read */
+
+ /*
+ * EndRecPtr is pointing to end+1 of the previous WAL record.
+ * If we're at a page boundary, no more records can fit on the
+ * current page. We must skip over the page header, but we
+ * can't do that until we've read in the page, since the header
+ * size is variable.
+ */
+ state->PrevRecPtr = state->ReadRecPtr;
+ state->ReadRecPtr = state->EndRecPtr;
+ }
+ else
+ {
+ /*
+ * Caller supplied a position to start at.
+ *
+ * In this case, EndRecPtr should already be pointing to a
+ * valid record starting position.
+ */
+ Assert(XRecOffIsValid(state->EndRecPtr));
+ state->ReadRecPtr = state->EndRecPtr;
+
+ /*
+ * We cannot verify the previous-record pointer when we're
+ * seeking to a particular record. Reset PrevRecPtr so that we
+ * won't try doing that.
+ */
+ state->PrevRecPtr = InvalidXLogRecPtr;
+ state->EndRecPtr = InvalidXLogRecPtr; /* to be tidy */
}
+ state->record_verified = false;
+ state->readRecordState = XLREAD_TOT_LEN;
+ /* fall through */
+
+ case XLREAD_TOT_LEN:
+ {
+ uint32 total_len;
+ uint32 pageHeaderSize;
+ XLogRecPtr targetPagePtr;
+ uint32 targetRecOff;
+ XLogPageHeader pageHeader;
+
+ targetPagePtr =
+ state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
+ targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+
+ /*
+ * Check if we have enough data. For the first record in the page,
+ * the requesting length doesn't contain page header.
+ */
+ if (XLogNeedData(state, targetPagePtr,
+ Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+ targetRecOff != 0))
+ return XLREAD_NEED_DATA;
+
+ /* error out if caller supplied bogus page */
if (!state->page_verified)
goto err;
- Assert(SizeOfXLogShortPHD <= state->readLen);
+ /* examine page header now. */
+ pageHeaderSize =
+ XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+ if (targetRecOff == 0)
+ {
+ /* At page start, so skip over page header. */
+ state->ReadRecPtr += pageHeaderSize;
+ targetRecOff = pageHeaderSize;
+ }
+ else if (targetRecOff < pageHeaderSize)
+ {
+ report_invalid_record(state, "invalid record offset at %X/%X",
+ LSN_FORMAT_ARGS(state->ReadRecPtr));
+ goto err;
+ }
- /* Check that the continuation on next page looks valid */
pageHeader = (XLogPageHeader) state->readBuf;
- if (!(pageHeader->xlp_info & XLP_FIRST_IS_CONTRECORD))
+ if ((pageHeader->xlp_info & XLP_FIRST_IS_CONTRECORD) &&
+ targetRecOff == pageHeaderSize)
{
- report_invalid_record(state,
- "there is no contrecord flag at %X/%X",
- LSN_FORMAT_ARGS(RecPtr));
+ report_invalid_record(state, "contrecord is requested by %X/%X",
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr);
goto err;
}
- /*
- * Cross-check that xlp_rem_len agrees with how much of the record
- * we expect there to be left.
- */
- if (pageHeader->xlp_rem_len == 0 ||
- total_len != (pageHeader->xlp_rem_len + gotlen))
- {
- report_invalid_record(state,
- "invalid contrecord length %u (expected %lld) at %X/%X",
- pageHeader->xlp_rem_len,
- ((long long) total_len) - gotlen,
- LSN_FORMAT_ARGS(RecPtr));
- goto err;
- }
-
- /* Append the continuation from this page to the buffer */
- pageHeaderSize = XLogPageHeaderSize(pageHeader);
-
+ /* XLogNeedData has verified the page header */
Assert(pageHeaderSize <= state->readLen);
- contdata = (char *) state->readBuf + pageHeaderSize;
- len = XLOG_BLCKSZ - pageHeaderSize;
- if (pageHeader->xlp_rem_len < len)
- len = pageHeader->xlp_rem_len;
+ /*
+ * Read the record length.
+ *
+ * NB: Even though we use an XLogRecord pointer here, the whole
+ * record header might not fit on this page. xl_tot_len is the first
+ * field of the struct, so it must be on this page (the records are
+ * MAXALIGNed), but we cannot access any other fields until we've
+ * verified that we got the whole header.
+ */
+ prec = (XLogRecord *) (state->readBuf +
+ state->ReadRecPtr % XLOG_BLCKSZ);
+ total_len = prec->xl_tot_len;
- Assert (pageHeaderSize + len <= state->readLen);
- memcpy(buffer, (char *) contdata, len);
- buffer += len;
- gotlen += len;
+ /*
+ * If the whole record header is on this page, validate it
+ * immediately. Otherwise do just a basic sanity check on
+ * xl_tot_len, and validate the rest of the header after reading it
+ * from the next page. The xl_tot_len check is necessary here to
+ * ensure that we enter the XLREAD_CONTINUATION state below;
+ * otherwise we might fail to apply ValidXLogRecordHeader at all.
+ */
+ if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
+ {
+ if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
+ state->PrevRecPtr, prec))
+ goto err;
- /* If we just reassembled the record header, validate it. */
- if (!gotheader)
+ state->record_verified = true;
+ }
+ else
{
- record = (XLogRecord *) state->readRecordBuf;
- if (!ValidXLogRecordHeader(state, RecPtr, state->ReadRecPtr,
- record, randAccess))
+ /* XXX: more validation should be done here */
+ if (total_len < SizeOfXLogRecord)
+ {
+ report_invalid_record(state,
+ "invalid record length at %X/%X: wanted %u, got %u",
+ LSN_FORMAT_ARGS(state->ReadRecPtr),
+ (uint32) SizeOfXLogRecord, total_len);
goto err;
- gotheader = true;
+ }
}
- } while (gotlen < total_len);
-
- Assert(gotheader);
- record = (XLogRecord *) state->readRecordBuf;
- if (!ValidXLogRecord(state, record, RecPtr))
- goto err;
+ /*
+ * Wait for the rest of the record, or the part of the record that
+ * fit on the first page if crossed a page boundary, to become
+ * available.
+ */
+ state->recordGotLen = 0;
+ state->recordRemainLen = total_len;
+ state->readRecordState = XLREAD_FIRST_FRAGMENT;
+ }
+ /* fall through */
- pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
- state->ReadRecPtr = RecPtr;
- state->EndRecPtr = targetPagePtr + pageHeaderSize
- + MAXALIGN(pageHeader->xlp_rem_len);
- }
- else
- {
- /* Wait for the record data to become available */
- while (XLogNeedData(state, targetPagePtr,
- Min(targetRecOff + total_len, XLOG_BLCKSZ), true))
+ case XLREAD_FIRST_FRAGMENT:
{
- if (!state->routine.page_read(state, state->readPagePtr,
- state->readLen,
- state->ReadRecPtr, state->readBuf))
+ uint32 total_len = state->recordRemainLen;
+ uint32 request_len;
+ uint32 record_len;
+ XLogRecPtr targetPagePtr;
+ uint32 targetRecOff;
+
+ /*
+ * Wait for the rest of the record on the first page to become
+ * available
+ */
+ targetPagePtr =
+ state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
+ targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+
+ request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ);
+ record_len = request_len - targetRecOff;
+
+ /* ReadRecPtr contains page header */
+ Assert (targetRecOff != 0);
+ if (XLogNeedData(state, targetPagePtr, request_len, true))
+ return XLREAD_NEED_DATA;
+
+ /* error out if caller supplied bogus page */
+ if (!state->page_verified)
+ goto err;
+
+ prec = (XLogRecord *) (state->readBuf + targetRecOff);
+
+ /* validate record header if not yet */
+ if (!state->record_verified && record_len >= SizeOfXLogRecord)
+ {
+ if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
+ state->PrevRecPtr, prec))
+ goto err;
+
+ state->record_verified = true;
+ }
+
+
+ if (total_len == record_len)
+ {
+ /* Record does not cross a page boundary */
+ Assert(state->record_verified);
+
+ if (!ValidXLogRecord(state, prec, state->ReadRecPtr))
+ goto err;
+
+ state->record_verified = true; /* to be tidy */
+
+ /* We already checked the header earlier */
+ state->EndRecPtr = state->ReadRecPtr + MAXALIGN(record_len);
+
+ *record = prec;
+ state->readRecordState = XLREAD_NEXT_RECORD;
break;
+ }
+
+ /*
+ * The record continues on the next page. Need to reassemble
+ * record
+ */
+ Assert(total_len > record_len);
+
+ /* Enlarge readRecordBuf as needed. */
+ if (total_len > state->readRecordBufSize &&
+ !allocate_recordbuf(state, total_len))
+ {
+ /* We treat this as a "bogus data" condition */
+ report_invalid_record(state,
+ "record length %u at %X/%X too long",
+ total_len,
+ LSN_FORMAT_ARGS(state->ReadRecPtr));
+ goto err;
+ }
+
+ /* Copy the first fragment of the record from the first page. */
+ memcpy(state->readRecordBuf, state->readBuf + targetRecOff,
+ record_len);
+ state->recordGotLen += record_len;
+ state->recordRemainLen -= record_len;
+
+ /* Calculate pointer to beginning of next page */
+ state->recordContRecPtr = state->ReadRecPtr + record_len;
+ Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0);
+
+ state->readRecordState = XLREAD_CONTINUATION;
}
+ /* fall through */
+
+ case XLREAD_CONTINUATION:
+ {
+ XLogPageHeader pageHeader;
+ uint32 pageHeaderSize;
+ XLogRecPtr targetPagePtr;
+
+ /* we enter this state only if we haven't read the whole record. */
+ Assert (state->recordRemainLen > 0);
+
+ while(state->recordRemainLen > 0)
+ {
+ char *contdata;
+ uint32 request_len;
+ uint32 record_len;
+
+ /* Wait for the next page to become available */
+ targetPagePtr = state->recordContRecPtr;
+
+ /* this request contains page header */
+ Assert (targetPagePtr != 0);
+ if (XLogNeedData(state, targetPagePtr,
+ Min(state->recordRemainLen, XLOG_BLCKSZ),
+ false))
+ return XLREAD_NEED_DATA;
+
+ if (!state->page_verified)
+ goto err;
+
+ Assert(SizeOfXLogShortPHD <= state->readLen);
- if (!state->page_verified)
- goto err;
+ /* Check that the continuation on next page looks valid */
+ pageHeader = (XLogPageHeader) state->readBuf;
+ if (!(pageHeader->xlp_info & XLP_FIRST_IS_CONTRECORD))
+ {
+ report_invalid_record(
+ state,
+ "there is no contrecord flag at %X/%X reading %X/%X",
+ (uint32) (state->recordContRecPtr >> 32),
+ (uint32) state->recordContRecPtr,
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr);
+ goto err;
+ }
- /* Record does not cross a page boundary */
- if (!ValidXLogRecord(state, record, RecPtr))
- goto err;
+ /*
+ * Cross-check that xlp_rem_len agrees with how much of the
+ * record we expect there to be left.
+ */
+ if (pageHeader->xlp_rem_len == 0 ||
+ pageHeader->xlp_rem_len != state->recordRemainLen)
+ {
+ report_invalid_record(
+ state,
+ "invalid contrecord length %u at %X/%X reading %X/%X, expected %u",
+ pageHeader->xlp_rem_len,
+ (uint32) (state->recordContRecPtr >> 32),
+ (uint32) state->recordContRecPtr,
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr,
+ state->recordRemainLen);
+ goto err;
+ }
- state->EndRecPtr = RecPtr + MAXALIGN(total_len);
+ /* Append the continuation from this page to the buffer */
+ pageHeaderSize = XLogPageHeaderSize(pageHeader);
- state->ReadRecPtr = RecPtr;
+ /*
+ * XLogNeedData should have ensured that the whole page header
+ * was read
+ */
+ Assert(state->readLen >= pageHeaderSize);
+
+ contdata = (char *) state->readBuf + pageHeaderSize;
+ record_len = XLOG_BLCKSZ - pageHeaderSize;
+ if (pageHeader->xlp_rem_len < record_len)
+ record_len = pageHeader->xlp_rem_len;
+
+ request_len = record_len + pageHeaderSize;
+
+ /* XLogNeedData should have ensured all needed data was read */
+ Assert (state->readLen >= request_len);
+
+ memcpy(state->readRecordBuf + state->recordGotLen,
+ (char *) contdata, record_len);
+ state->recordGotLen += record_len;
+ state->recordRemainLen -= record_len;
+
+ /* If we just reassembled the record header, validate it. */
+ if (!state->record_verified)
+ {
+ Assert(state->recordGotLen >= SizeOfXLogRecord);
+ if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
+ state->PrevRecPtr,
+ (XLogRecord *) state->readRecordBuf))
+ goto err;
+
+ state->record_verified = true;
+ }
+
+ /* Calculate pointer to beginning of next page, and continue */
+ state->recordContRecPtr += XLOG_BLCKSZ;
+ }
+
+ /* targetPagePtr is pointing the last-read page here */
+ prec = (XLogRecord *) state->readRecordBuf;
+ if (!ValidXLogRecord(state, prec, state->ReadRecPtr))
+ goto err;
+
+ pageHeaderSize =
+ XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+ state->EndRecPtr = targetPagePtr + pageHeaderSize
+ + MAXALIGN(pageHeader->xlp_rem_len);
+
+ *record = prec;
+ state->readRecordState = XLREAD_NEXT_RECORD;
+ break;
+ }
}
/*
* Special processing if it's an XLOG SWITCH record
*/
- if (record->xl_rmid == RM_XLOG_ID &&
- (record->xl_info & ~XLR_INFO_MASK) == XLOG_SWITCH)
+ if ((*record)->xl_rmid == RM_XLOG_ID &&
+ ((*record)->xl_info & ~XLR_INFO_MASK) == XLOG_SWITCH)
{
/* Pretend it extends to end of segment */
state->EndRecPtr += state->segcxt.ws_segsize - 1;
state->EndRecPtr -= XLogSegmentOffset(state->EndRecPtr, state->segcxt.ws_segsize);
}
- if (DecodeXLogRecord(state, record, errormsg))
- return record;
- else
- return NULL;
+ Assert (!*record || state->readLen >= 0);
+ if (DecodeXLogRecord(state, *record, errormsg))
+ return XLREAD_SUCCESS;
+
+ *record = NULL;
+ return XLREAD_FAIL;
err:
/*
- * Invalidate the read state. We might read from a different source after
+ * Invalidate the read page. We might read from a different source after
* failure.
*/
XLogReaderInvalReadState(state);
@@ -572,7 +703,8 @@ err:
if (state->errormsg_buf[0] != '\0')
*errormsg = state->errormsg_buf;
- return NULL;
+ *record = NULL;
+ return XLREAD_FAIL;
}
/*
@@ -724,11 +856,12 @@ XLogReaderInvalReadState(XLogReaderState *state)
*
* This is just a convenience subroutine to avoid duplicated code in
* XLogReadRecord. It's not intended for use from anywhere else.
+ *
+ * If PrevRecPtr is valid, the xl_prev is is cross-checked with it.
*/
static bool
ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
- XLogRecPtr PrevRecPtr, XLogRecord *record,
- bool randAccess)
+ XLogRecPtr PrevRecPtr, XLogRecord *record)
{
if (record->xl_tot_len < SizeOfXLogRecord)
{
@@ -745,7 +878,7 @@ ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
record->xl_rmid, LSN_FORMAT_ARGS(RecPtr));
return false;
}
- if (randAccess)
+ if (PrevRecPtr == InvalidXLogRecPtr)
{
/*
* We can't exactly verify the prev-link, but surely it should be less
@@ -975,11 +1108,14 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
* XLogReadRecord() will read the next valid record.
*/
XLogRecPtr
-XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
+XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
+ XLogFindNextRecordCB read_page, void *private)
{
XLogRecPtr tmpRecPtr;
XLogRecPtr found = InvalidXLogRecPtr;
XLogPageHeader header;
+ XLogRecord *record;
+ XLogReadRecordResult result;
char *errormsg;
Assert(!XLogRecPtrIsInvalid(RecPtr));
@@ -1012,9 +1148,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
while(XLogNeedData(state, targetPagePtr, targetRecOff,
targetRecOff != 0))
{
- if (!state->routine.page_read(state, state->readPagePtr,
- state->readLen,
- state->ReadRecPtr, state->readBuf))
+ if (!read_page(state, private))
break;
}
@@ -1066,8 +1200,16 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
* or we just jumped over the remaining data of a continuation.
*/
XLogBeginRead(state, tmpRecPtr);
- while (XLogReadRecord(state, &errormsg) != NULL)
+ while ((result = XLogReadRecord(state, &record, &errormsg)) !=
+ XLREAD_FAIL)
{
+ if (result == XLREAD_NEED_DATA)
+ {
+ if (!read_page(state, private))
+ goto err;
+ continue;
+ }
+
/* past the record we've found, break out */
if (RecPtr <= state->ReadRecPtr)
{
@@ -1087,9 +1229,9 @@ err:
#endif /* FRONTEND */
/*
- * Helper function to ease writing of XLogRoutine->page_read callbacks.
- * If this function is used, caller must supply a segment_open callback in
- * 'state', as that is used here.
+ * Helper function to ease writing of page_read callback.
+ * If this function is used, caller must supply a segment_open callback and
+ * segment_close callback as that is used here.
*
* Read 'count' bytes into 'buf', starting at location 'startptr', from WAL
* fetched from timeline 'tli'.
@@ -1102,6 +1244,7 @@ err:
*/
bool
WALRead(XLogReaderState *state,
+ WALSegmentOpenCB segopenfn, WALSegmentCloseCB segclosefn,
char *buf, XLogRecPtr startptr, Size count, TimeLineID tli,
WALReadError *errinfo)
{
@@ -1133,10 +1276,10 @@ WALRead(XLogReaderState *state,
XLogSegNo nextSegNo;
if (state->seg.ws_file >= 0)
- state->routine.segment_close(state);
+ segclosefn(state);
XLByteToSeg(recptr, nextSegNo, state->segcxt.ws_segsize);
- state->routine.segment_open(state, nextSegNo, &tli);
+ segopenfn(state, nextSegNo, &tli);
/* This shouldn't happen -- indicates a bug in segment_open */
Assert(state->seg.ws_file >= 0);
diff --git a/src/backend/access/transam/xlogutils.c b/src/backend/access/transam/xlogutils.c
index 46eda33f25..b003990745 100644
--- a/src/backend/access/transam/xlogutils.c
+++ b/src/backend/access/transam/xlogutils.c
@@ -686,8 +686,7 @@ XLogTruncateRelation(RelFileNode rnode, ForkNumber forkNum,
void
XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wantLength)
{
- const XLogRecPtr lastReadPage = state->seg.ws_segno *
- state->segcxt.ws_segsize + state->readLen;
+ const XLogRecPtr lastReadPage = state->readPagePtr;
Assert(wantPage != InvalidXLogRecPtr && wantPage % XLOG_BLCKSZ == 0);
Assert(wantLength <= XLOG_BLCKSZ);
@@ -702,7 +701,7 @@ XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wa
* current TLI has since become historical.
*/
if (lastReadPage == wantPage &&
- state->readLen != 0 &&
+ state->page_verified &&
lastReadPage + state->readLen >= wantPage + Min(wantLength, XLOG_BLCKSZ - 1))
return;
@@ -788,6 +787,7 @@ wal_segment_open(XLogReaderState *state, XLogSegNo nextSegNo,
char path[MAXPGPATH];
XLogFilePath(path, tli, nextSegNo, state->segcxt.ws_segsize);
+ elog(LOG, "HOGE: %lu, %d => %s", nextSegNo, tli, path);
state->seg.ws_file = BasicOpenFile(path, O_RDONLY | PG_BINARY);
if (state->seg.ws_file >= 0)
return;
@@ -825,9 +825,11 @@ wal_segment_close(XLogReaderState *state)
* loop for now.
*/
bool
-read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *cur_page)
+read_local_xlog_page(XLogReaderState *state)
{
+ XLogRecPtr targetPagePtr = state->readPagePtr;
+ int reqLen = state->readLen;
+ char *cur_page = state->readBuf;
XLogRecPtr read_upto,
loc;
TimeLineID tli;
@@ -940,11 +942,12 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
* as 'count', read the whole page anyway. It's guaranteed to be
* zero-padded up to the page boundary if it's incomplete.
*/
- if (!WALRead(state, cur_page, targetPagePtr, XLOG_BLCKSZ, tli,
- &errinfo))
+ if (!WALRead(state, wal_segment_open, wal_segment_close,
+ cur_page, targetPagePtr, XLOG_BLCKSZ, tli, &errinfo))
WALReadRaiseError(&errinfo);
/* number of valid bytes in the buffer */
+ state->readPagePtr = targetPagePtr;
state->readLen = count;
return true;
}
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index 37b75deb72..9cbd8afa49 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -148,7 +148,8 @@ StartupDecodingContext(List *output_plugin_options,
TransactionId xmin_horizon,
bool need_full_snapshot,
bool fast_forward,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress)
@@ -198,11 +199,12 @@ StartupDecodingContext(List *output_plugin_options,
ctx->slot = slot;
- ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, xl_routine, ctx);
+ ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, cleanup_cb);
if (!ctx->reader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
errmsg("out of memory")));
+ ctx->page_read = page_read;
ctx->reorder = ReorderBufferAllocate();
ctx->snapshot_builder =
@@ -319,7 +321,8 @@ CreateInitDecodingContext(const char *plugin,
List *output_plugin_options,
bool need_full_snapshot,
XLogRecPtr restart_lsn,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress)
@@ -422,7 +425,7 @@ CreateInitDecodingContext(const char *plugin,
ctx = StartupDecodingContext(NIL, restart_lsn, xmin_horizon,
need_full_snapshot, false,
- xl_routine, prepare_write, do_write,
+ page_read, cleanup_cb, prepare_write, do_write,
update_progress);
/* call output plugin initialization callback */
@@ -476,7 +479,8 @@ LogicalDecodingContext *
CreateDecodingContext(XLogRecPtr start_lsn,
List *output_plugin_options,
bool fast_forward,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress)
@@ -528,8 +532,8 @@ CreateDecodingContext(XLogRecPtr start_lsn,
ctx = StartupDecodingContext(output_plugin_options,
start_lsn, InvalidTransactionId, false,
- fast_forward, xl_routine, prepare_write,
- do_write, update_progress);
+ fast_forward, page_read, cleanup_cb,
+ prepare_write, do_write, update_progress);
/* call output plugin initialization callback */
old_context = MemoryContextSwitchTo(ctx->context);
@@ -585,7 +589,13 @@ DecodingContextFindStartpoint(LogicalDecodingContext *ctx)
char *err = NULL;
/* the read_page callback waits for new WAL */
- record = XLogReadRecord(ctx->reader, &err);
+ while (XLogReadRecord(ctx->reader, &record, &err) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!ctx->page_read(ctx->reader))
+ break;
+ }
+
if (err)
elog(ERROR, "%s", err);
if (!record)
diff --git a/src/backend/replication/logical/logicalfuncs.c b/src/backend/replication/logical/logicalfuncs.c
index f7e055874e..b69f6911c5 100644
--- a/src/backend/replication/logical/logicalfuncs.c
+++ b/src/backend/replication/logical/logicalfuncs.c
@@ -233,9 +233,8 @@ pg_logical_slot_get_changes_guts(FunctionCallInfo fcinfo, bool confirm, bool bin
ctx = CreateDecodingContext(InvalidXLogRecPtr,
options,
false,
- XL_ROUTINE(.page_read = read_local_xlog_page,
- .segment_open = wal_segment_open,
- .segment_close = wal_segment_close),
+ read_local_xlog_page,
+ wal_segment_close,
LogicalOutputPrepareWrite,
LogicalOutputWrite, NULL);
@@ -284,7 +283,13 @@ pg_logical_slot_get_changes_guts(FunctionCallInfo fcinfo, bool confirm, bool bin
XLogRecord *record;
char *errm = NULL;
- record = XLogReadRecord(ctx->reader, &errm);
+ while (XLogReadRecord(ctx->reader, &record, &errm) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!ctx->page_read(ctx->reader))
+ break;
+ }
+
if (errm)
elog(ERROR, "%s", errm);
diff --git a/src/backend/replication/slotfuncs.c b/src/backend/replication/slotfuncs.c
index 9817b44113..deb7cf15eb 100644
--- a/src/backend/replication/slotfuncs.c
+++ b/src/backend/replication/slotfuncs.c
@@ -153,9 +153,8 @@ create_logical_replication_slot(char *name, char *plugin,
ctx = CreateInitDecodingContext(plugin, NIL,
false, /* just catalogs is OK */
restart_lsn,
- XL_ROUTINE(.page_read = read_local_xlog_page,
- .segment_open = wal_segment_open,
- .segment_close = wal_segment_close),
+ read_local_xlog_page,
+ wal_segment_close,
NULL, NULL, NULL);
/*
@@ -512,9 +511,8 @@ pg_logical_replication_slot_advance(XLogRecPtr moveto)
ctx = CreateDecodingContext(InvalidXLogRecPtr,
NIL,
true, /* fast_forward */
- XL_ROUTINE(.page_read = read_local_xlog_page,
- .segment_open = wal_segment_open,
- .segment_close = wal_segment_close),
+ read_local_xlog_page,
+ wal_segment_close,
NULL, NULL, NULL);
/*
@@ -536,7 +534,13 @@ pg_logical_replication_slot_advance(XLogRecPtr moveto)
* Read records. No changes are generated in fast_forward mode,
* but snapbuilder/slot statuses are updated properly.
*/
- record = XLogReadRecord(ctx->reader, &errm);
+ while (XLogReadRecord(ctx->reader, &record, &errm) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!ctx->page_read(ctx->reader))
+ break;
+ }
+
if (errm)
elog(ERROR, "%s", errm);
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index c36b91ae5e..062093727d 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -580,10 +580,7 @@ StartReplication(StartReplicationCmd *cmd)
/* create xlogreader for physical replication */
xlogreader =
- XLogReaderAllocate(wal_segment_size, NULL,
- XL_ROUTINE(.segment_open = WalSndSegmentOpen,
- .segment_close = wal_segment_close),
- NULL);
+ XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
if (!xlogreader)
ereport(ERROR,
@@ -807,9 +804,11 @@ StartReplication(StartReplicationCmd *cmd)
* set every time WAL is flushed.
*/
static bool
-logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetRecPtr, char *cur_page)
+logical_read_xlog_page(XLogReaderState *state)
{
+ XLogRecPtr targetPagePtr = state->readPagePtr;
+ int reqLen = state->readLen;
+ char *cur_page = state->readBuf;
XLogRecPtr flushptr;
int count;
WALReadError errinfo;
@@ -837,7 +836,7 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
count = flushptr - targetPagePtr; /* part of the page available */
/* now actually read the data, we know it's there */
- if (!WALRead(state,
+ if (!WALRead(state, WalSndSegmentOpen, wal_segment_close,
cur_page,
targetPagePtr,
XLOG_BLCKSZ,
@@ -1011,9 +1010,8 @@ CreateReplicationSlot(CreateReplicationSlotCmd *cmd)
ctx = CreateInitDecodingContext(cmd->plugin, NIL, need_full_snapshot,
InvalidXLogRecPtr,
- XL_ROUTINE(.page_read = logical_read_xlog_page,
- .segment_open = WalSndSegmentOpen,
- .segment_close = wal_segment_close),
+ logical_read_xlog_page,
+ wal_segment_close,
WalSndPrepareWrite, WalSndWriteData,
WalSndUpdateProgress);
@@ -1171,9 +1169,8 @@ StartLogicalReplication(StartReplicationCmd *cmd)
*/
logical_decoding_ctx =
CreateDecodingContext(cmd->startpoint, cmd->options, false,
- XL_ROUTINE(.page_read = logical_read_xlog_page,
- .segment_open = WalSndSegmentOpen,
- .segment_close = wal_segment_close),
+ logical_read_xlog_page,
+ wal_segment_close,
WalSndPrepareWrite, WalSndWriteData,
WalSndUpdateProgress);
xlogreader = logical_decoding_ctx->reader;
@@ -2749,7 +2746,7 @@ XLogSendPhysical(void)
enlargeStringInfo(&output_message, nbytes);
retry:
- if (!WALRead(xlogreader,
+ if (!WALRead(xlogreader, WalSndSegmentOpen, wal_segment_close,
&output_message.data[output_message.len],
startptr,
nbytes,
@@ -2847,7 +2844,12 @@ XLogSendLogical(void)
*/
WalSndCaughtUp = false;
- record = XLogReadRecord(logical_decoding_ctx->reader, &errm);
+ while (XLogReadRecord(logical_decoding_ctx->reader, &record, &errm) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!logical_decoding_ctx->page_read(logical_decoding_ctx->reader))
+ break;
+ }
/* xlog record was invalid */
if (errm != NULL)
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index cf119848b0..712c85281c 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -41,15 +41,9 @@ static int xlogreadfd = -1;
static XLogSegNo xlogreadsegno = -1;
static char xlogfpath[MAXPGPATH];
-typedef struct XLogPageReadPrivate
-{
- const char *restoreCommand;
- int tliIndex;
-} XLogPageReadPrivate;
-
-static bool SimpleXLogPageRead(XLogReaderState *xlogreader,
- XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
+static bool SimpleXLogPageRead(XLogReaderState *xlogreader,
+ const char *datadir, int *tliIndex,
+ const char *restoreCommand);
/*
* Read WAL from the datadir/pg_wal, starting from 'startpoint' on timeline
@@ -66,20 +60,22 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
XLogRecord *record;
XLogReaderState *xlogreader;
char *errormsg;
- XLogPageReadPrivate private;
- private.tliIndex = tliIndex;
- private.restoreCommand = restoreCommand;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir,
- XL_ROUTINE(.page_read = &SimpleXLogPageRead),
- &private);
+ xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+
if (xlogreader == NULL)
pg_fatal("out of memory");
XLogBeginRead(xlogreader, startpoint);
do
{
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!SimpleXLogPageRead(xlogreader, datadir,
+ &tliIndex, restoreCommand))
+ break;
+ }
if (record == NULL)
{
@@ -123,19 +119,19 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex,
XLogRecord *record;
XLogReaderState *xlogreader;
char *errormsg;
- XLogPageReadPrivate private;
XLogRecPtr endptr;
- private.tliIndex = tliIndex;
- private.restoreCommand = restoreCommand;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir,
- XL_ROUTINE(.page_read = &SimpleXLogPageRead),
- &private);
+ xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
XLogBeginRead(xlogreader, ptr);
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!SimpleXLogPageRead(xlogreader, datadir, &tliIndex, restoreCommand))
+ break;
+ }
if (record == NULL)
{
if (errormsg)
@@ -170,7 +166,6 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
XLogRecPtr searchptr;
XLogReaderState *xlogreader;
char *errormsg;
- XLogPageReadPrivate private;
/*
* The given fork pointer points to the end of the last common record,
@@ -186,11 +181,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
forkptr += SizeOfXLogShortPHD;
}
- private.tliIndex = tliIndex;
- private.restoreCommand = restoreCommand;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir,
- XL_ROUTINE(.page_read = &SimpleXLogPageRead),
- &private);
+ xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
@@ -200,7 +191,13 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
uint8 info;
XLogBeginRead(xlogreader, searchptr);
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!SimpleXLogPageRead(xlogreader, datadir,
+ &tliIndex, restoreCommand))
+ break;
+ }
if (record == NULL)
{
@@ -247,10 +244,11 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
/* XLogReader callback function, to read a WAL page */
static bool
-SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *readBuf)
+SimpleXLogPageRead(XLogReaderState *xlogreader, const char *datadir,
+ int *tliIndex, const char *restoreCommand)
{
- XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
+ XLogRecPtr targetPagePtr = xlogreader->readPagePtr;
+ char *readBuf = xlogreader->readBuf;
uint32 targetPageOff;
XLogRecPtr targetSegEnd;
XLogSegNo targetSegNo;
@@ -283,14 +281,14 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
* be done both forward and backward, consider also switching timeline
* accordingly.
*/
- while (private->tliIndex < targetNentries - 1 &&
- targetHistory[private->tliIndex].end < targetSegEnd)
- private->tliIndex++;
- while (private->tliIndex > 0 &&
- targetHistory[private->tliIndex].begin >= targetSegEnd)
- private->tliIndex--;
+ while (*tliIndex < targetNentries - 1 &&
+ targetHistory[*tliIndex].end < targetSegEnd)
+ (*tliIndex)++;
+ while (*tliIndex > 0 &&
+ targetHistory[*tliIndex].begin >= targetSegEnd)
+ (*tliIndex)--;
- XLogFileName(xlogfname, targetHistory[private->tliIndex].tli,
+ XLogFileName(xlogfname, targetHistory[*tliIndex].tli,
xlogreadsegno, WalSegSz);
snprintf(xlogfpath, MAXPGPATH, "%s/" XLOGDIR "/%s",
@@ -303,7 +301,7 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
/*
* If we have no restore_command to execute, then exit.
*/
- if (private->restoreCommand == NULL)
+ if (restoreCommand == NULL)
{
pg_log_error("could not open file \"%s\": %m", xlogfpath);
xlogreader->readLen = -1;
@@ -317,7 +315,7 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
xlogreadfd = RestoreArchivedFile(xlogreader->segcxt.ws_dir,
xlogfname,
WalSegSz,
- private->restoreCommand);
+ restoreCommand);
if (xlogreadfd < 0)
{
@@ -359,7 +357,7 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
Assert(targetSegNo == xlogreadsegno);
- xlogreader->seg.ws_tli = targetHistory[private->tliIndex].tli;
+ xlogreader->seg.ws_tli = targetHistory[*tliIndex].tli;
xlogreader->readLen = XLOG_BLCKSZ;
return true;
}
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index 0f0e0723a3..f514de6b13 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -330,12 +330,17 @@ WALDumpCloseSegment(XLogReaderState *state)
state->seg.ws_file = -1;
}
-/* pg_waldump's XLogReaderRoutine->page_read callback */
+/*
+ * pg_waldump's WAL page rader, also used as page_read callback for
+ * XLogFindNextRecord
+ */
static bool
-WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetPtr, char *readBuff)
+WALDumpReadPage(XLogReaderState *state, void *priv)
{
- XLogDumpPrivate *private = state->private_data;
+ XLogRecPtr targetPagePtr = state->readPagePtr;
+ int reqLen = state->readLen;
+ char *readBuff = state->readBuf;
+ XLogDumpPrivate *private = (XLogDumpPrivate *) priv;
int count = XLOG_BLCKSZ;
WALReadError errinfo;
@@ -353,8 +358,8 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
}
}
- if (!WALRead(state, readBuff, targetPagePtr, count, private->timeline,
- &errinfo))
+ if (!WALRead(state, WALDumpOpenSegment, WALDumpCloseSegment,
+ readBuff, targetPagePtr, count, private->timeline, &errinfo))
{
WALOpenSegment *seg = &errinfo.wre_seg;
char fname[MAXPGPATH];
@@ -374,6 +379,7 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
(Size) errinfo.wre_req);
}
+ Assert(count >= state->readLen);
state->readLen = count;
return true;
}
@@ -1035,16 +1041,14 @@ main(int argc, char **argv)
/* we have everything we need, start reading */
xlogreader_state =
- XLogReaderAllocate(WalSegSz, waldir,
- XL_ROUTINE(.page_read = WALDumpReadPage,
- .segment_open = WALDumpOpenSegment,
- .segment_close = WALDumpCloseSegment),
- &private);
+ XLogReaderAllocate(WalSegSz, waldir, WALDumpCloseSegment);
+
if (!xlogreader_state)
fatal_error("out of memory");
/* first find a valid recptr to start from */
- first_record = XLogFindNextRecord(xlogreader_state, private.startptr);
+ first_record = XLogFindNextRecord(xlogreader_state, private.startptr,
+ &WALDumpReadPage, (void*) &private);
if (first_record == InvalidXLogRecPtr)
fatal_error("could not find a valid record after %X/%X",
@@ -1067,7 +1071,13 @@ main(int argc, char **argv)
for (;;)
{
/* try to read the next record */
- record = XLogReadRecord(xlogreader_state, &errormsg);
+ while (XLogReadRecord(xlogreader_state, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!WALDumpReadPage(xlogreader_state, (void *) &private))
+ break;
+ }
+
if (!record)
{
if (!config.follow || private.endptr_reached)
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index 23d9e70a59..4e9268b553 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -57,64 +57,15 @@ typedef struct WALSegmentContext
typedef struct XLogReaderState XLogReaderState;
-/* Function type definition for the read_page callback */
-typedef bool (*XLogPageReadCB) (XLogReaderState *xlogreader,
- XLogRecPtr targetPagePtr,
- int reqLen,
- XLogRecPtr targetRecPtr,
- char *readBuf);
+/* Function type definition for the segment cleanup callback */
+typedef void (*WALSegmentCleanupCB) (XLogReaderState *xlogreader);
+
+/* Function type definition for the open/close callbacks for WALRead() */
typedef void (*WALSegmentOpenCB) (XLogReaderState *xlogreader,
XLogSegNo nextSegNo,
TimeLineID *tli_p);
typedef void (*WALSegmentCloseCB) (XLogReaderState *xlogreader);
-typedef struct XLogReaderRoutine
-{
- /*
- * Data input callback
- *
- * This callback shall read at least reqLen valid bytes of the xlog page
- * starting at targetPagePtr, and store them in readBuf. The callback
- * shall return the number of bytes read (never more than XLOG_BLCKSZ), or
- * -1 on failure. The callback shall sleep, if necessary, to wait for the
- * requested bytes to become available. The callback will not be invoked
- * again for the same page unless more than the returned number of bytes
- * are needed.
- *
- * targetRecPtr is the position of the WAL record we're reading. Usually
- * it is equal to targetPagePtr + reqLen, but sometimes xlogreader needs
- * to read and verify the page or segment header, before it reads the
- * actual WAL record it's interested in. In that case, targetRecPtr can
- * be used to determine which timeline to read the page from.
- *
- * The callback shall set ->seg.ws_tli to the TLI of the file the page was
- * read from.
- */
- XLogPageReadCB page_read;
-
- /*
- * Callback to open the specified WAL segment for reading. ->seg.ws_file
- * shall be set to the file descriptor of the opened segment. In case of
- * failure, an error shall be raised by the callback and it shall not
- * return.
- *
- * "nextSegNo" is the number of the segment to be opened.
- *
- * "tli_p" is an input/output argument. WALRead() uses it to pass the
- * timeline in which the new segment should be found, but the callback can
- * use it to return the TLI that it actually opened.
- */
- WALSegmentOpenCB segment_open;
-
- /*
- * WAL segment close callback. ->seg.ws_file shall be set to a negative
- * number.
- */
- WALSegmentCloseCB segment_close;
-} XLogReaderRoutine;
-
-#define XL_ROUTINE(...) &(XLogReaderRoutine){__VA_ARGS__}
-
typedef struct
{
/* Is this block ref in use? */
@@ -144,12 +95,35 @@ typedef struct
uint16 data_bufsz;
} DecodedBkpBlock;
+/* Return code from XLogReadRecord */
+typedef enum XLogReadRecordResult
+{
+ XLREAD_SUCCESS, /* record is successfully read */
+ XLREAD_NEED_DATA, /* need more data. see XLogReadRecord. */
+ XLREAD_FAIL /* failed during reading a record */
+} XLogReadRecordResult;
+
+/*
+ * internal state of XLogReadRecord
+ *
+ * XLogReadState runs a state machine while reading a record. Theses states
+ * are not seen outside the function. Each state may repeat several times
+ * exiting requesting caller for new data. See the comment of XLogReadRecrod
+ * for details.
+ */
+typedef enum XLogReadRecordState {
+ XLREAD_NEXT_RECORD,
+ XLREAD_TOT_LEN,
+ XLREAD_FIRST_FRAGMENT,
+ XLREAD_CONTINUATION
+} XLogReadRecordState;
+
struct XLogReaderState
{
/*
* Operational callbacks
*/
- XLogReaderRoutine routine;
+ WALSegmentCleanupCB cleanup_cb;
/* ----------------------------------------
* Public parameters
@@ -162,18 +136,14 @@ struct XLogReaderState
*/
uint64 system_identifier;
- /*
- * Opaque data for callbacks to use. Not used by XLogReader.
- */
- void *private_data;
-
/*
* Start and end point of last record read. EndRecPtr is also used as the
* position to read next. Calling XLogBeginRead() sets EndRecPtr to the
* starting position and ReadRecPtr to invalid.
*/
- XLogRecPtr ReadRecPtr; /* start of last record read */
+ XLogRecPtr ReadRecPtr; /* start of last record read or being read */
XLogRecPtr EndRecPtr; /* end+1 of last record read */
+ XLogRecPtr PrevRecPtr; /* start of previous record read */
/* ----------------------------------------
* Communication with page reader
@@ -187,7 +157,9 @@ struct XLogReaderState
* the request, or -1 on error */
TimeLineID readPageTLI; /* TLI for data currently in readBuf */
char *readBuf; /* buffer to store data */
- bool page_verified; /* is the page on the buffer verified? */
+ bool page_verified; /* is the page header on the buffer verified? */
+ bool record_verified;/* is the current record header verified? */
+
/* ----------------------------------------
@@ -229,8 +201,6 @@ struct XLogReaderState
XLogRecPtr latestPagePtr;
TimeLineID latestPageTLI;
- /* beginning of the WAL record being read. */
- XLogRecPtr currRecPtr;
/* timeline to read it from, 0 if a lookup is required */
TimeLineID currTLI;
@@ -257,6 +227,15 @@ struct XLogReaderState
char *readRecordBuf;
uint32 readRecordBufSize;
+ /*
+ * XLogReadRecord() state
+ */
+ XLogReadRecordState readRecordState;/* state machine state */
+ int recordGotLen; /* amount of current record that has
+ * already been read */
+ int recordRemainLen; /* length of current record that remains */
+ XLogRecPtr recordContRecPtr; /* where the current record continues */
+
/* Buffer to hold error message */
char *errormsg_buf;
};
@@ -264,9 +243,7 @@ struct XLogReaderState
/* Get a new XLogReader */
extern XLogReaderState *XLogReaderAllocate(int wal_segment_size,
const char *waldir,
- XLogReaderRoutine *routine,
- void *private_data);
-extern XLogReaderRoutine *LocalXLogReaderRoutine(void);
+ WALSegmentCleanupCB cleanup_cb);
/* Free an XLogReader */
extern void XLogReaderFree(XLogReaderState *state);
@@ -274,12 +251,17 @@ extern void XLogReaderFree(XLogReaderState *state);
/* Position the XLogReader to given record */
extern void XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr);
#ifdef FRONTEND
-extern XLogRecPtr XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr);
+/* Function type definition for the read_page callback */
+typedef bool (*XLogFindNextRecordCB) (XLogReaderState *xlogreader,
+ void *private);
+extern XLogRecPtr XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
+ XLogFindNextRecordCB read_page, void *private);
#endif /* FRONTEND */
/* Read the next XLog record. Returns NULL on end-of-WAL or failure */
-extern struct XLogRecord *XLogReadRecord(XLogReaderState *state,
- char **errormsg);
+extern XLogReadRecordResult XLogReadRecord(XLogReaderState *state,
+ XLogRecord **record,
+ char **errormsg);
/* Validate a page */
extern bool XLogReaderValidatePageHeader(XLogReaderState *state,
@@ -299,6 +281,7 @@ typedef struct WALReadError
} WALReadError;
extern bool WALRead(XLogReaderState *state,
+ WALSegmentOpenCB segopenfn, WALSegmentCloseCB sgclosefn,
char *buf, XLogRecPtr startptr, Size count,
TimeLineID tli, WALReadError *errinfo);
diff --git a/src/include/access/xlogutils.h b/src/include/access/xlogutils.h
index 364a21c4ea..397fb27fc2 100644
--- a/src/include/access/xlogutils.h
+++ b/src/include/access/xlogutils.h
@@ -47,9 +47,7 @@ extern Buffer XLogReadBufferExtended(RelFileNode rnode, ForkNumber forknum,
extern Relation CreateFakeRelcacheEntry(RelFileNode rnode);
extern void FreeFakeRelcacheEntry(Relation fakerel);
-extern bool read_local_xlog_page(XLogReaderState *state,
- XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetRecPtr, char *cur_page);
+extern bool read_local_xlog_page(XLogReaderState *state);
extern void wal_segment_open(XLogReaderState *state,
XLogSegNo nextSegNo,
TimeLineID *tli_p);
diff --git a/src/include/pg_config_manual.h b/src/include/pg_config_manual.h
index f10ad0acd6..94749609b4 100644
--- a/src/include/pg_config_manual.h
+++ b/src/include/pg_config_manual.h
@@ -377,7 +377,7 @@
* Enable debugging print statements for WAL-related operations; see
* also the wal_debug GUC var.
*/
-/* #define WAL_DEBUG */
+#define WAL_DEBUG
/*
* Enable tracing of resource consumption during sort operations;
diff --git a/src/include/replication/logical.h b/src/include/replication/logical.h
index c253403372..b936f31393 100644
--- a/src/include/replication/logical.h
+++ b/src/include/replication/logical.h
@@ -29,6 +29,10 @@ typedef void (*LogicalOutputPluginWriterUpdateProgress) (struct LogicalDecodingC
TransactionId xid
);
+typedef struct LogicalDecodingContext LogicalDecodingContext;
+
+typedef bool (*LogicalDecodingXLogPageReadCB)(XLogReaderState *ctx);
+
typedef struct LogicalDecodingContext
{
/* memory context this is all allocated in */
@@ -39,6 +43,7 @@ typedef struct LogicalDecodingContext
/* infrastructure pieces for decoding */
XLogReaderState *reader;
+ LogicalDecodingXLogPageReadCB page_read;
struct ReorderBuffer *reorder;
struct SnapBuild *snapshot_builder;
@@ -105,14 +110,16 @@ extern LogicalDecodingContext *CreateInitDecodingContext(const char *plugin,
List *output_plugin_options,
bool need_full_snapshot,
XLogRecPtr restart_lsn,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress);
extern LogicalDecodingContext *CreateDecodingContext(XLogRecPtr start_lsn,
List *output_plugin_options,
bool fast_forward,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress);
--
2.27.0
----Next_Part(Thu_Mar__4_11_28_53_2021_014)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
filename="v17-0003-Remove-globals-readOff-readLen-and-readSegNo.patch"
^ permalink raw reply [nested|flat] 70+ messages in thread
* [PATCH v15 2/4] Move page-reader out of XLogReadRecord
@ 2019-09-10 03:58 Kyotaro Horiguchi <[email protected]>
0 siblings, 0 replies; 70+ messages in thread
From: Kyotaro Horiguchi @ 2019-09-10 03:58 UTC (permalink / raw)
This is the second step of removing callbacks from WAL record reader.
Since it is essential to take in additional data while reading a
record, the function have to ask caller for new data while keeping
working state. Thus the function is turned into a state machine.
---
src/backend/access/transam/twophase.c | 15 +-
src/backend/access/transam/xlog.c | 58 +-
src/backend/access/transam/xlogreader.c | 700 +++++++++++-------
src/backend/access/transam/xlogutils.c | 17 +-
src/backend/replication/logical/logical.c | 26 +-
.../replication/logical/logicalfuncs.c | 13 +-
src/backend/replication/slotfuncs.c | 18 +-
src/backend/replication/walsender.c | 32 +-
src/bin/pg_rewind/parsexlog.c | 84 +--
src/bin/pg_waldump/pg_waldump.c | 36 +-
src/include/access/xlogreader.h | 121 ++-
src/include/access/xlogutils.h | 4 +-
src/include/replication/logical.h | 11 +-
13 files changed, 648 insertions(+), 487 deletions(-)
diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index 9b2e59bf0e..b0d60a0d0f 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -1330,11 +1330,8 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
XLogReaderState *xlogreader;
char *errormsg;
- xlogreader = XLogReaderAllocate(wal_segment_size, NULL,
- XL_ROUTINE(.page_read = &read_local_xlog_page,
- .segment_open = &wal_segment_open,
- .segment_close = &wal_segment_close),
- NULL);
+ xlogreader = XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+
if (!xlogreader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
@@ -1342,7 +1339,13 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
errdetail("Failed while allocating a WAL reading processor.")));
XLogBeginRead(xlogreader, lsn);
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!read_local_xlog_page(xlogreader))
+ break;
+ }
+
if (record == NULL)
ereport(ERROR,
(errcode_for_file_access(),
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index e570e56a24..f9b0108602 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -831,13 +831,6 @@ static XLogSource currentSource = XLOG_FROM_ANY;
static bool lastSourceFailed = false;
static bool pendingWalRcvRestart = false;
-typedef struct XLogPageReadPrivate
-{
- int emode;
- bool fetching_ckpt; /* are we fetching a checkpoint record? */
- bool randAccess;
-} XLogPageReadPrivate;
-
/*
* These variables track when we last obtained some WAL data to process,
* and where we got it from. (XLogReceiptSource is initially the same as
@@ -912,8 +905,8 @@ static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath,
static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli,
XLogSource source, bool notfoundOk);
static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, XLogSource source);
-static bool XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
+static bool XLogPageRead(XLogReaderState *xlogreader,
+ bool fetching_ckpt, int emode, bool randAccess);
static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
bool fetching_ckpt, XLogRecPtr tliRecPtr);
static int emode_for_corrupt_record(int emode, XLogRecPtr RecPtr);
@@ -1225,8 +1218,7 @@ XLogInsertRecord(XLogRecData *rdata,
appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len);
if (!debug_reader)
- debug_reader = XLogReaderAllocate(wal_segment_size, NULL,
- XL_ROUTINE(), NULL);
+ debug_reader = XLogReaderAllocate(wal_segment_size, NULL NULL);
if (!debug_reader)
{
@@ -4326,15 +4318,10 @@ CleanupBackupHistory(void)
* record is available.
*/
static XLogRecord *
-ReadRecord(XLogReaderState *xlogreader, int emode,
- bool fetching_ckpt)
+ReadRecord(XLogReaderState *xlogreader, int emode, bool fetching_ckpt)
{
XLogRecord *record;
- XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
-
- private->fetching_ckpt = fetching_ckpt;
- private->emode = emode;
- private->randAccess = (xlogreader->ReadRecPtr == InvalidXLogRecPtr);
+ bool randAccess = (xlogreader->ReadRecPtr == InvalidXLogRecPtr);
/* This is the first attempt to read this page. */
lastSourceFailed = false;
@@ -4342,8 +4329,16 @@ ReadRecord(XLogReaderState *xlogreader, int emode,
for (;;)
{
char *errormsg;
+ XLogReadRecordResult result;
+
+ while ((result = XLogReadRecord(xlogreader, &record, &errormsg))
+ == XLREAD_NEED_DATA)
+ {
+ if (!XLogPageRead(xlogreader, fetching_ckpt, emode, randAccess))
+ break;
+
+ }
- record = XLogReadRecord(xlogreader, &errormsg);
ReadRecPtr = xlogreader->ReadRecPtr;
EndRecPtr = xlogreader->EndRecPtr;
if (record == NULL)
@@ -6320,7 +6315,6 @@ StartupXLOG(void)
bool backupFromStandby = false;
DBState dbstate_at_startup;
XLogReaderState *xlogreader;
- XLogPageReadPrivate private;
bool fast_promoted = false;
struct stat st;
@@ -6480,13 +6474,9 @@ StartupXLOG(void)
OwnLatch(&XLogCtl->recoveryWakeupLatch);
/* Set up XLOG reader facility */
- MemSet(&private, 0, sizeof(XLogPageReadPrivate));
xlogreader =
- XLogReaderAllocate(wal_segment_size, NULL,
- XL_ROUTINE(.page_read = &XLogPageRead,
- .segment_open = NULL,
- .segment_close = wal_segment_close),
- &private);
+ XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+
if (!xlogreader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
@@ -11860,12 +11850,13 @@ CancelBackup(void)
* sleep and retry.
*/
static bool
-XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetRecPtr, char *readBuf)
+XLogPageRead(XLogReaderState *xlogreader,
+ bool fetching_ckpt, int emode, bool randAccess)
{
- XLogPageReadPrivate *private =
- (XLogPageReadPrivate *) xlogreader->private_data;
- int emode = private->emode;
+ char *readBuf = xlogreader->readBuf;
+ XLogRecPtr targetPagePtr = xlogreader->readPagePtr;
+ int reqLen = xlogreader->readLen;
+ XLogRecPtr targetRecPtr = xlogreader->ReadRecPtr;
uint32 targetPageOff;
XLogSegNo targetSegNo PG_USED_FOR_ASSERTS_ONLY;
int r;
@@ -11908,8 +11899,8 @@ retry:
flushedUpto < targetPagePtr + reqLen))
{
if (!WaitForWALToBecomeAvailable(targetPagePtr + reqLen,
- private->randAccess,
- private->fetching_ckpt,
+ randAccess,
+ fetching_ckpt,
targetRecPtr))
{
if (readFile >= 0)
@@ -12014,6 +12005,7 @@ retry:
goto next_record_is_invalid;
}
+ Assert(xlogreader->readPagePtr == targetPagePtr);
xlogreader->readLen = readLen;
return true;
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index 3d599325ee..9281b57379 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -40,7 +40,7 @@ static bool XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr,
int reqLen, bool header_inclusive);
static void XLogReaderInvalReadState(XLogReaderState *state);
static bool ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
- XLogRecPtr PrevRecPtr, XLogRecord *record, bool randAccess);
+ XLogRecPtr PrevRecPtr, XLogRecord *record);
static bool ValidXLogRecord(XLogReaderState *state, XLogRecord *record,
XLogRecPtr recptr);
static void ResetDecoder(XLogReaderState *state);
@@ -73,7 +73,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...)
*/
XLogReaderState *
XLogReaderAllocate(int wal_segment_size, const char *waldir,
- XLogReaderRoutine *routine, void *private_data)
+ WALSegmentCleanupCB cleanup_cb)
{
XLogReaderState *state;
@@ -84,7 +84,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
return NULL;
/* initialize caller-provided support functions */
- state->routine = *routine;
+ state->cleanup_cb = cleanup_cb;
state->max_block_id = -1;
@@ -107,8 +107,6 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
WALOpenSegmentInit(&state->seg, &state->segcxt, wal_segment_size,
waldir);
- /* system_identifier initialized to zeroes above */
- state->private_data = private_data;
/* ReadRecPtr, EndRecPtr and readLen initialized to zeroes above */
state->errormsg_buf = palloc_extended(MAX_ERRORMSG_LEN + 1,
MCXT_ALLOC_NO_OOM);
@@ -140,8 +138,8 @@ XLogReaderFree(XLogReaderState *state)
{
int block_id;
- if (state->seg.ws_file != -1)
- state->routine.segment_close(state);
+ if (state->seg.ws_file >= 0)
+ state->cleanup_cb(state);
for (block_id = 0; block_id <= XLR_MAX_BLOCK_ID; block_id++)
{
@@ -246,6 +244,7 @@ XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
/* Begin at the passed-in record pointer. */
state->EndRecPtr = RecPtr;
state->ReadRecPtr = InvalidXLogRecPtr;
+ state->readRecordState = XLREAD_NEXT_RECORD;
}
/*
@@ -254,317 +253,452 @@ XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
* XLogBeginRead() or XLogFindNextRecord() must be called before the first call
* to XLogReadRecord().
*
- * If the page_read callback fails to read the requested data, NULL is
- * returned. The callback is expected to have reported the error; errormsg
- * is set to NULL.
+ * This function may return XLREAD_NEED_DATA several times before returning a
+ * result record. The caller shall read in some new data then call this
+ * function again with the same parameters.
*
- * If the reading fails for some other reason, NULL is also returned, and
- * *errormsg is set to a string with details of the failure.
+ * When a record is successfully read, returns XLREAD_SUCCESS with result
+ * record being stored in *record. Otherwise *record is NULL.
*
- * The returned pointer (or *errormsg) points to an internal buffer that's
- * valid until the next call to XLogReadRecord.
+ * Returns XLREAD_NEED_DATA if more data is needed to finish reading the
+ * current record. In that case, state->readPagePtr and state->readLen inform
+ * the desired position and minimum length of data needed. The caller shall
+ * read in the requested data and set state->readBuf to point to a buffer
+ * containing it. The caller must also set state->readPageTLI and
+ * state->readLen to indicate the timeline that it was read from, and the
+ * length of data that is now available (which must be >= given readLen),
+ * respectively.
+ *
+ * If invalid data is encountered, returns XLREAD_FAIL with *record being set to
+ * NULL. *errormsg is set to a string with details of the failure.
+ * The returned pointer (or *errormsg) points to an internal buffer that's valid
+ * until the next call to XLogReadRecord.
+ *
+ *
+ * This function runs a state machine consists of the following states.
+ *
+ * XLREAD_NEXT_RECORD :
+ * The initial state, if called with valid RecPtr, try to read a record at
+ * that position. If invalid RecPtr is given try to read a record just after
+ * the last one previously read.
+ * This state ens after setting ReadRecPtr. Then goes to XLREAD_TOT_LEN.
+ *
+ * XLREAD_TOT_LEN:
+ * Examining record header. Ends after reading record total
+ * length. recordRemainLen and recordGotLen are initialized.
+ *
+ * XLREAD_FIRST_FRAGMENT:
+ * Reading the first fragment. Ends with finishing reading a single
+ * record. Goes to XLREAD_NEXT_RECORD if that's all or
+ * XLREAD_CONTINUATION if we have continuation.
+
+ * XLREAD_CONTINUATION:
+ * Reading continuation of record. Ends with finishing the whole record then
+ * goes to XLREAD_NEXT_RECORD. During this state, recordRemainLen indicates
+ * how much is left and readRecordBuf holds the partially assert
+ * record.recordContRecPtr points to the beginning of the next page where to
+ * continue.
+ *
+ * If wrong data found in any state, the state machine stays at the current
+ * state. This behavior allows to continue reading a reacord switching among
+ * different souces, while streaming replication.
*/
-XLogRecord *
-XLogReadRecord(XLogReaderState *state, char **errormsg)
+XLogReadRecordResult
+XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
{
- XLogRecPtr RecPtr;
- XLogRecord *record;
- XLogRecPtr targetPagePtr;
- bool randAccess;
- uint32 len,
- total_len;
- uint32 targetRecOff;
- uint32 pageHeaderSize;
- bool gotheader;
+ XLogRecord *prec;
- /*
- * randAccess indicates whether to verify the previous-record pointer of
- * the record we're reading. We only do this if we're reading
- * sequentially, which is what we initially assume.
- */
- randAccess = false;
+ *record = NULL;
/* reset error state */
*errormsg = NULL;
state->errormsg_buf[0] = '\0';
- ResetDecoder(state);
-
- RecPtr = state->EndRecPtr;
-
- if (state->ReadRecPtr != InvalidXLogRecPtr)
- {
- /* read the record after the one we just read */
-
- /*
- * EndRecPtr is pointing to end+1 of the previous WAL record. If
- * we're at a page boundary, no more records can fit on the current
- * page. We must skip over the page header, but we can't do that until
- * we've read in the page, since the header size is variable.
- */
- }
- else
- {
- /*
- * Caller supplied a position to start at.
- *
- * In this case, EndRecPtr should already be pointing to a valid
- * record starting position.
- */
- Assert(XRecOffIsValid(RecPtr));
- randAccess = true;
- }
-
- state->currRecPtr = RecPtr;
-
- targetPagePtr = RecPtr - (RecPtr % XLOG_BLCKSZ);
- targetRecOff = RecPtr % XLOG_BLCKSZ;
-
- /*
- * Read the page containing the record into state->readBuf. Request enough
- * byte to cover the whole record header, or at least the part of it that
- * fits on the same page.
- */
- while (XLogNeedData(state, targetPagePtr,
- Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
- targetRecOff != 0))
- {
- if (!state->routine.page_read(state, state->readPagePtr, state->readLen,
- RecPtr, state->readBuf))
- break;
- }
-
- if (!state->page_verified)
- goto err;
-
- /*
- * We have at least the page header, so we can examine it now.
- */
- pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
- if (targetRecOff == 0)
- {
- /*
- * At page start, so skip over page header.
- */
- RecPtr += pageHeaderSize;
- targetRecOff = pageHeaderSize;
- }
- else if (targetRecOff < pageHeaderSize)
- {
- report_invalid_record(state, "invalid record offset at %X/%X",
- (uint32) (RecPtr >> 32), (uint32) RecPtr);
- goto err;
- }
-
- if ((((XLogPageHeader) state->readBuf)->xlp_info & XLP_FIRST_IS_CONTRECORD) &&
- targetRecOff == pageHeaderSize)
- {
- report_invalid_record(state, "contrecord is requested by %X/%X",
- (uint32) (RecPtr >> 32), (uint32) RecPtr);
- goto err;
- }
-
- /* XLogNeedData has verified the page header */
- Assert(pageHeaderSize <= state->readLen);
-
- /*
- * Read the record length.
- *
- * NB: Even though we use an XLogRecord pointer here, the whole record
- * header might not fit on this page. xl_tot_len is the first field of the
- * struct, so it must be on this page (the records are MAXALIGNed), but we
- * cannot access any other fields until we've verified that we got the
- * whole header.
- */
- record = (XLogRecord *) (state->readBuf + RecPtr % XLOG_BLCKSZ);
- total_len = record->xl_tot_len;
-
- /*
- * If the whole record header is on this page, validate it immediately.
- * Otherwise do just a basic sanity check on xl_tot_len, and validate the
- * rest of the header after reading it from the next page. The xl_tot_len
- * check is necessary here to ensure that we enter the "Need to reassemble
- * record" code path below; otherwise we might fail to apply
- * ValidXLogRecordHeader at all.
- */
- if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
- {
- if (!ValidXLogRecordHeader(state, RecPtr, state->ReadRecPtr, record,
- randAccess))
- goto err;
- gotheader = true;
- }
- else
- {
- /* XXX: more validation should be done here */
- if (total_len < SizeOfXLogRecord)
- {
- report_invalid_record(state,
- "invalid record length at %X/%X: wanted %u, got %u",
- (uint32) (RecPtr >> 32), (uint32) RecPtr,
- (uint32) SizeOfXLogRecord, total_len);
- goto err;
- }
- gotheader = false;
- }
-
- len = XLOG_BLCKSZ - RecPtr % XLOG_BLCKSZ;
- if (total_len > len)
+ switch (state->readRecordState)
{
- /* Need to reassemble record */
- char *contdata;
- XLogPageHeader pageHeader;
- char *buffer;
- uint32 gotlen;
-
- /*
- * Enlarge readRecordBuf as needed.
- */
- if (total_len > state->readRecordBufSize &&
- !allocate_recordbuf(state, total_len))
- {
- /* We treat this as a "bogus data" condition */
- report_invalid_record(state, "record length %u at %X/%X too long",
- total_len,
- (uint32) (RecPtr >> 32), (uint32) RecPtr);
- goto err;
- }
+ case XLREAD_NEXT_RECORD:
+ ResetDecoder(state);
- /* Copy the first fragment of the record from the first page. */
- memcpy(state->readRecordBuf,
- state->readBuf + RecPtr % XLOG_BLCKSZ, len);
- buffer = state->readRecordBuf + len;
- gotlen = len;
-
- do
- {
- int rest_len = total_len - gotlen;
-
- /* Calculate pointer to beginning of next page */
- targetPagePtr += XLOG_BLCKSZ;
-
- /* Wait for the next page to become available */
- while (XLogNeedData(state, targetPagePtr,
- Min(rest_len, XLOG_BLCKSZ),
- false))
+ if (state->ReadRecPtr != InvalidXLogRecPtr)
{
- if (!state->routine.page_read(state, state->readPagePtr,
- state->readLen,
- state->ReadRecPtr,
- state->readBuf))
- break;
+ /* read the record after the one we just read */
+
+ /*
+ * EndRecPtr is pointing to end+1 of the previous WAL record.
+ * If we're at a page boundary, no more records can fit on the
+ * current page. We must skip over the page header, but we
+ * can't do that until we've read in the page, since the header
+ * size is variable.
+ */
+ state->PrevRecPtr = state->ReadRecPtr;
+ state->ReadRecPtr = state->EndRecPtr;
+ }
+ else
+ {
+ /*
+ * Caller supplied a position to start at.
+ *
+ * In this case, EndRecPtr should already be pointing to a
+ * valid record starting position.
+ */
+ Assert(XRecOffIsValid(state->EndRecPtr));
+ state->ReadRecPtr = state->EndRecPtr;
+
+ /*
+ * We cannot verify the previous-record pointer when we're
+ * seeking to a particular record. Reset PrevRecPtr so that we
+ * won't try doing that.
+ */
+ state->PrevRecPtr = InvalidXLogRecPtr;
+ state->EndRecPtr = InvalidXLogRecPtr; /* to be tidy */
}
+ state->record_verified = false;
+ state->readRecordState = XLREAD_TOT_LEN;
+ /* fall through */
+
+ case XLREAD_TOT_LEN:
+ {
+ uint32 total_len;
+ uint32 pageHeaderSize;
+ XLogRecPtr targetPagePtr;
+ uint32 targetRecOff;
+ XLogPageHeader pageHeader;
+
+ targetPagePtr =
+ state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
+ targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+
+ /*
+ * Check if we have enough data. For the first record in the page,
+ * the requesting length doesn't contain page header.
+ */
+ if (XLogNeedData(state, targetPagePtr,
+ Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+ targetRecOff != 0))
+ return XLREAD_NEED_DATA;
+
+ /* error out if caller supplied bogus page */
if (!state->page_verified)
goto err;
- Assert(SizeOfXLogShortPHD <= state->readLen);
+ /* examine page header now. */
+ pageHeaderSize =
+ XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+ if (targetRecOff == 0)
+ {
+ /* At page start, so skip over page header. */
+ state->ReadRecPtr += pageHeaderSize;
+ targetRecOff = pageHeaderSize;
+ }
+ else if (targetRecOff < pageHeaderSize)
+ {
+ report_invalid_record(state, "invalid record offset at %X/%X",
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr);
+ goto err;
+ }
- /* Check that the continuation on next page looks valid */
pageHeader = (XLogPageHeader) state->readBuf;
- if (!(pageHeader->xlp_info & XLP_FIRST_IS_CONTRECORD))
+ if ((pageHeader->xlp_info & XLP_FIRST_IS_CONTRECORD) &&
+ targetRecOff == pageHeaderSize)
{
- report_invalid_record(state,
- "there is no contrecord flag at %X/%X",
- (uint32) (RecPtr >> 32), (uint32) RecPtr);
+ report_invalid_record(state, "contrecord is requested by %X/%X",
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr);
goto err;
}
- /*
- * Cross-check that xlp_rem_len agrees with how much of the record
- * we expect there to be left.
- */
- if (pageHeader->xlp_rem_len == 0 ||
- total_len != (pageHeader->xlp_rem_len + gotlen))
- {
- report_invalid_record(state,
- "invalid contrecord length %u at %X/%X",
- pageHeader->xlp_rem_len,
- (uint32) (RecPtr >> 32), (uint32) RecPtr);
- goto err;
- }
-
- /* Append the continuation from this page to the buffer */
- pageHeaderSize = XLogPageHeaderSize(pageHeader);
-
+ /* XLogNeedData has verified the page header */
Assert(pageHeaderSize <= state->readLen);
- contdata = (char *) state->readBuf + pageHeaderSize;
- len = XLOG_BLCKSZ - pageHeaderSize;
- if (pageHeader->xlp_rem_len < len)
- len = pageHeader->xlp_rem_len;
+ /*
+ * Read the record length.
+ *
+ * NB: Even though we use an XLogRecord pointer here, the whole
+ * record header might not fit on this page. xl_tot_len is the first
+ * field of the struct, so it must be on this page (the records are
+ * MAXALIGNed), but we cannot access any other fields until we've
+ * verified that we got the whole header.
+ */
+ prec = (XLogRecord *) (state->readBuf +
+ state->ReadRecPtr % XLOG_BLCKSZ);
+ total_len = prec->xl_tot_len;
- Assert (pageHeaderSize + len <= state->readLen);
- memcpy(buffer, (char *) contdata, len);
- buffer += len;
- gotlen += len;
+ /*
+ * If the whole record header is on this page, validate it
+ * immediately. Otherwise do just a basic sanity check on
+ * xl_tot_len, and validate the rest of the header after reading it
+ * from the next page. The xl_tot_len check is necessary here to
+ * ensure that we enter the XLREAD_CONTINUATION state below;
+ * otherwise we might fail to apply ValidXLogRecordHeader at all.
+ */
+ if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
+ {
+ if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
+ state->PrevRecPtr, prec))
+ goto err;
- /* If we just reassembled the record header, validate it. */
- if (!gotheader)
+ state->record_verified = true;
+ }
+ else
{
- record = (XLogRecord *) state->readRecordBuf;
- if (!ValidXLogRecordHeader(state, RecPtr, state->ReadRecPtr,
- record, randAccess))
+ /* XXX: more validation should be done here */
+ if (total_len < SizeOfXLogRecord)
+ {
+ report_invalid_record(state,
+ "invalid record length at %X/%X: wanted %u, got %u",
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr,
+ (uint32) SizeOfXLogRecord, total_len);
goto err;
- gotheader = true;
+ }
}
- } while (gotlen < total_len);
-
- Assert(gotheader);
- record = (XLogRecord *) state->readRecordBuf;
- if (!ValidXLogRecord(state, record, RecPtr))
- goto err;
+ /*
+ * Wait for the rest of the record, or the part of the record that
+ * fit on the first page if crossed a page boundary, to become
+ * available.
+ */
+ state->recordGotLen = 0;
+ state->recordRemainLen = total_len;
+ state->readRecordState = XLREAD_FIRST_FRAGMENT;
+ }
+ /* fall through */
- pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
- state->ReadRecPtr = RecPtr;
- state->EndRecPtr = targetPagePtr + pageHeaderSize
- + MAXALIGN(pageHeader->xlp_rem_len);
- }
- else
- {
- /* Wait for the record data to become available */
- while (XLogNeedData(state, targetPagePtr,
- Min(targetRecOff + total_len, XLOG_BLCKSZ), true))
+ case XLREAD_FIRST_FRAGMENT:
{
- if (!state->routine.page_read(state, state->readPagePtr,
- state->readLen,
- state->ReadRecPtr, state->readBuf))
+ uint32 total_len = state->recordRemainLen;
+ uint32 request_len;
+ uint32 record_len;
+ XLogRecPtr targetPagePtr;
+ uint32 targetRecOff;
+
+ /*
+ * Wait for the rest of the record on the first page to become
+ * available
+ */
+ targetPagePtr =
+ state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
+ targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+
+ request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ);
+ record_len = request_len - targetRecOff;
+
+ /* ReadRecPtr contains page header */
+ Assert (targetRecOff != 0);
+ if (XLogNeedData(state, targetPagePtr, request_len, true))
+ return XLREAD_NEED_DATA;
+
+ /* error out if caller supplied bogus page */
+ if (!state->page_verified)
+ goto err;
+
+ prec = (XLogRecord *) (state->readBuf + targetRecOff);
+
+ /* validate record header if not yet */
+ if (!state->record_verified && record_len >= SizeOfXLogRecord)
+ {
+ if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
+ state->PrevRecPtr, prec))
+ goto err;
+
+ state->record_verified = true;
+ }
+
+
+ if (total_len == record_len)
+ {
+ /* Record does not cross a page boundary */
+ Assert(state->record_verified);
+
+ if (!ValidXLogRecord(state, prec, state->ReadRecPtr))
+ goto err;
+
+ state->record_verified = true; /* to be tidy */
+
+ /* We already checked the header earlier */
+ state->EndRecPtr = state->ReadRecPtr + MAXALIGN(record_len);
+
+ *record = prec;
+ state->readRecordState = XLREAD_NEXT_RECORD;
break;
+ }
+
+ /*
+ * The record continues on the next page. Need to reassemble
+ * record
+ */
+ Assert(total_len > record_len);
+
+ /* Enlarge readRecordBuf as needed. */
+ if (total_len > state->readRecordBufSize &&
+ !allocate_recordbuf(state, total_len))
+ {
+ /* We treat this as a "bogus data" condition */
+ report_invalid_record(state,
+ "record length %u at %X/%X too long",
+ total_len,
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr);
+ goto err;
+ }
+
+ /* Copy the first fragment of the record from the first page. */
+ memcpy(state->readRecordBuf, state->readBuf + targetRecOff,
+ record_len);
+ state->recordGotLen += record_len;
+ state->recordRemainLen -= record_len;
+
+ /* Calculate pointer to beginning of next page */
+ state->recordContRecPtr = state->ReadRecPtr + record_len;
+ Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0);
+
+ state->readRecordState = XLREAD_CONTINUATION;
}
+ /* fall through */
+
+ case XLREAD_CONTINUATION:
+ {
+ XLogPageHeader pageHeader;
+ uint32 pageHeaderSize;
+ XLogRecPtr targetPagePtr;
+
+ /* we enter this state only if we haven't read the whole record. */
+ Assert (state->recordRemainLen > 0);
+
+ while(state->recordRemainLen > 0)
+ {
+ char *contdata;
+ uint32 request_len;
+ uint32 record_len;
+
+ /* Wait for the next page to become available */
+ targetPagePtr = state->recordContRecPtr;
+
+ /* this request contains page header */
+ Assert (targetPagePtr != 0);
+ if (XLogNeedData(state, targetPagePtr,
+ Min(state->recordRemainLen, XLOG_BLCKSZ),
+ false))
+ return XLREAD_NEED_DATA;
+
+ if (!state->page_verified)
+ goto err;
+
+ Assert(SizeOfXLogShortPHD <= state->readLen);
- if (!state->page_verified)
- goto err;
+ /* Check that the continuation on next page looks valid */
+ pageHeader = (XLogPageHeader) state->readBuf;
+ if (!(pageHeader->xlp_info & XLP_FIRST_IS_CONTRECORD))
+ {
+ report_invalid_record(
+ state,
+ "there is no contrecord flag at %X/%X reading %X/%X",
+ (uint32) (state->recordContRecPtr >> 32),
+ (uint32) state->recordContRecPtr,
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr);
+ goto err;
+ }
- /* Record does not cross a page boundary */
- if (!ValidXLogRecord(state, record, RecPtr))
- goto err;
+ /*
+ * Cross-check that xlp_rem_len agrees with how much of the
+ * record we expect there to be left.
+ */
+ if (pageHeader->xlp_rem_len == 0 ||
+ pageHeader->xlp_rem_len != state->recordRemainLen)
+ {
+ report_invalid_record(
+ state,
+ "invalid contrecord length %u at %X/%X reading %X/%X, expected %u",
+ pageHeader->xlp_rem_len,
+ (uint32) (state->recordContRecPtr >> 32),
+ (uint32) state->recordContRecPtr,
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr,
+ state->recordRemainLen);
+ goto err;
+ }
- state->EndRecPtr = RecPtr + MAXALIGN(total_len);
+ /* Append the continuation from this page to the buffer */
+ pageHeaderSize = XLogPageHeaderSize(pageHeader);
- state->ReadRecPtr = RecPtr;
+ /*
+ * XLogNeedData should have ensured that the whole page header
+ * was read
+ */
+ Assert(state->readLen >= pageHeaderSize);
+
+ contdata = (char *) state->readBuf + pageHeaderSize;
+ record_len = XLOG_BLCKSZ - pageHeaderSize;
+ if (pageHeader->xlp_rem_len < record_len)
+ record_len = pageHeader->xlp_rem_len;
+
+ request_len = record_len + pageHeaderSize;
+
+ /* XLogNeedData should have ensured all needed data was read */
+ Assert (state->readLen >= request_len);
+
+ memcpy(state->readRecordBuf + state->recordGotLen,
+ (char *) contdata, record_len);
+ state->recordGotLen += record_len;
+ state->recordRemainLen -= record_len;
+
+ /* If we just reassembled the record header, validate it. */
+ if (!state->record_verified)
+ {
+ Assert(state->recordGotLen >= SizeOfXLogRecord);
+ if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
+ state->PrevRecPtr,
+ (XLogRecord *) state->readRecordBuf))
+ goto err;
+
+ state->record_verified = true;
+ }
+
+ /* Calculate pointer to beginning of next page, and continue */
+ state->recordContRecPtr += XLOG_BLCKSZ;
+ }
+
+ /* targetPagePtr is pointing the last-read page here */
+ prec = (XLogRecord *) state->readRecordBuf;
+ if (!ValidXLogRecord(state, prec, state->ReadRecPtr))
+ goto err;
+
+ pageHeaderSize =
+ XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+ state->EndRecPtr = targetPagePtr + pageHeaderSize
+ + MAXALIGN(pageHeader->xlp_rem_len);
+
+ *record = prec;
+ state->readRecordState = XLREAD_NEXT_RECORD;
+ break;
+ }
}
/*
* Special processing if it's an XLOG SWITCH record
*/
- if (record->xl_rmid == RM_XLOG_ID &&
- (record->xl_info & ~XLR_INFO_MASK) == XLOG_SWITCH)
+ if ((*record)->xl_rmid == RM_XLOG_ID &&
+ ((*record)->xl_info & ~XLR_INFO_MASK) == XLOG_SWITCH)
{
/* Pretend it extends to end of segment */
state->EndRecPtr += state->segcxt.ws_segsize - 1;
state->EndRecPtr -= XLogSegmentOffset(state->EndRecPtr, state->segcxt.ws_segsize);
}
- if (DecodeXLogRecord(state, record, errormsg))
- return record;
- else
- return NULL;
+ Assert (!*record || state->readLen >= 0);
+ if (DecodeXLogRecord(state, *record, errormsg))
+ return XLREAD_SUCCESS;
+
+ *record = NULL;
+ return XLREAD_FAIL;
err:
/*
- * Invalidate the read state. We might read from a different source after
+ * Invalidate the read page. We might read from a different source after
* failure.
*/
XLogReaderInvalReadState(state);
@@ -572,7 +706,8 @@ err:
if (state->errormsg_buf[0] != '\0')
*errormsg = state->errormsg_buf;
- return NULL;
+ *record = NULL;
+ return XLREAD_FAIL;
}
/*
@@ -724,11 +859,12 @@ XLogReaderInvalReadState(XLogReaderState *state)
*
* This is just a convenience subroutine to avoid duplicated code in
* XLogReadRecord. It's not intended for use from anywhere else.
+ *
+ * If PrevRecPtr is valid, the xl_prev is is cross-checked with it.
*/
static bool
ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
- XLogRecPtr PrevRecPtr, XLogRecord *record,
- bool randAccess)
+ XLogRecPtr PrevRecPtr, XLogRecord *record)
{
if (record->xl_tot_len < SizeOfXLogRecord)
{
@@ -746,7 +882,7 @@ ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
(uint32) RecPtr);
return false;
}
- if (randAccess)
+ if (PrevRecPtr == InvalidXLogRecPtr)
{
/*
* We can't exactly verify the prev-link, but surely it should be less
@@ -978,11 +1114,14 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
* XLogReadRecord() will read the next valid record.
*/
XLogRecPtr
-XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
+XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
+ XLogFindNextRecordCB read_page, void *private)
{
XLogRecPtr tmpRecPtr;
XLogRecPtr found = InvalidXLogRecPtr;
XLogPageHeader header;
+ XLogRecord *record;
+ XLogReadRecordResult result;
char *errormsg;
Assert(!XLogRecPtrIsInvalid(RecPtr));
@@ -1015,9 +1154,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
while(XLogNeedData(state, targetPagePtr, targetRecOff,
targetRecOff != 0))
{
- if (!state->routine.page_read(state, state->readPagePtr,
- state->readLen,
- state->ReadRecPtr, state->readBuf))
+ if (!read_page(state, private))
break;
}
@@ -1069,8 +1206,16 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
* or we just jumped over the remaining data of a continuation.
*/
XLogBeginRead(state, tmpRecPtr);
- while (XLogReadRecord(state, &errormsg) != NULL)
+ while ((result = XLogReadRecord(state, &record, &errormsg)) !=
+ XLREAD_FAIL)
{
+ if (result == XLREAD_NEED_DATA)
+ {
+ if (!read_page(state, private))
+ goto err;
+ continue;
+ }
+
/* past the record we've found, break out */
if (RecPtr <= state->ReadRecPtr)
{
@@ -1090,9 +1235,9 @@ err:
#endif /* FRONTEND */
/*
- * Helper function to ease writing of XLogRoutine->page_read callbacks.
- * If this function is used, caller must supply a segment_open callback in
- * 'state', as that is used here.
+ * Helper function to ease writing of page_read callback.
+ * If this function is used, caller must supply a segment_open callback and
+ * segment_close callback as that is used here.
*
* Read 'count' bytes into 'buf', starting at location 'startptr', from WAL
* fetched from timeline 'tli'.
@@ -1105,6 +1250,7 @@ err:
*/
bool
WALRead(XLogReaderState *state,
+ WALSegmentOpenCB segopenfn, WALSegmentCloseCB segclosefn,
char *buf, XLogRecPtr startptr, Size count, TimeLineID tli,
WALReadError *errinfo)
{
@@ -1136,10 +1282,10 @@ WALRead(XLogReaderState *state,
XLogSegNo nextSegNo;
if (state->seg.ws_file >= 0)
- state->routine.segment_close(state);
+ segclosefn(state);
XLByteToSeg(recptr, nextSegNo, state->segcxt.ws_segsize);
- state->routine.segment_open(state, nextSegNo, &tli);
+ segopenfn(state, nextSegNo, &tli);
/* This shouldn't happen -- indicates a bug in segment_open */
Assert(state->seg.ws_file >= 0);
diff --git a/src/backend/access/transam/xlogutils.c b/src/backend/access/transam/xlogutils.c
index 1b10b6df20..303333571f 100644
--- a/src/backend/access/transam/xlogutils.c
+++ b/src/backend/access/transam/xlogutils.c
@@ -689,8 +689,7 @@ XLogTruncateRelation(RelFileNode rnode, ForkNumber forkNum,
void
XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wantLength)
{
- const XLogRecPtr lastReadPage = state->seg.ws_segno *
- state->segcxt.ws_segsize + state->readLen;
+ const XLogRecPtr lastReadPage = state->readPagePtr;
Assert(wantPage != InvalidXLogRecPtr && wantPage % XLOG_BLCKSZ == 0);
Assert(wantLength <= XLOG_BLCKSZ);
@@ -705,7 +704,7 @@ XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wa
* current TLI has since become historical.
*/
if (lastReadPage == wantPage &&
- state->readLen != 0 &&
+ state->page_verified &&
lastReadPage + state->readLen >= wantPage + Min(wantLength, XLOG_BLCKSZ - 1))
return;
@@ -792,6 +791,7 @@ wal_segment_open(XLogReaderState *state, XLogSegNo nextSegNo,
char path[MAXPGPATH];
XLogFilePath(path, tli, nextSegNo, state->segcxt.ws_segsize);
+ elog(LOG, "HOGE: %lu, %d => %s", nextSegNo, tli, path);
state->seg.ws_file = BasicOpenFile(path, O_RDONLY | PG_BINARY);
if (state->seg.ws_file >= 0)
return;
@@ -829,9 +829,11 @@ wal_segment_close(XLogReaderState *state)
* loop for now.
*/
bool
-read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *cur_page)
+read_local_xlog_page(XLogReaderState *state)
{
+ XLogRecPtr targetPagePtr = state->readPagePtr;
+ int reqLen = state->readLen;
+ char *cur_page = state->readBuf;
XLogRecPtr read_upto,
loc;
TimeLineID tli;
@@ -944,11 +946,12 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
* as 'count', read the whole page anyway. It's guaranteed to be
* zero-padded up to the page boundary if it's incomplete.
*/
- if (!WALRead(state, cur_page, targetPagePtr, XLOG_BLCKSZ, tli,
- &errinfo))
+ if (!WALRead(state, wal_segment_open, wal_segment_close,
+ cur_page, targetPagePtr, XLOG_BLCKSZ, tli, &errinfo))
WALReadRaiseError(&errinfo);
/* number of valid bytes in the buffer */
+ state->readPagePtr = targetPagePtr;
state->readLen = count;
return true;
}
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index 61902be3b0..a15b0b3355 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -120,7 +120,8 @@ StartupDecodingContext(List *output_plugin_options,
TransactionId xmin_horizon,
bool need_full_snapshot,
bool fast_forward,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress)
@@ -169,11 +170,12 @@ StartupDecodingContext(List *output_plugin_options,
ctx->slot = slot;
- ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, xl_routine, ctx);
+ ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, cleanup_cb);
if (!ctx->reader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
errmsg("out of memory")));
+ ctx->page_read = page_read;
ctx->reorder = ReorderBufferAllocate();
ctx->snapshot_builder =
@@ -231,7 +233,8 @@ CreateInitDecodingContext(char *plugin,
List *output_plugin_options,
bool need_full_snapshot,
XLogRecPtr restart_lsn,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress)
@@ -328,7 +331,7 @@ CreateInitDecodingContext(char *plugin,
ctx = StartupDecodingContext(NIL, restart_lsn, xmin_horizon,
need_full_snapshot, false,
- xl_routine, prepare_write, do_write,
+ page_read, cleanup_cb, prepare_write, do_write,
update_progress);
/* call output plugin initialization callback */
@@ -376,7 +379,8 @@ LogicalDecodingContext *
CreateDecodingContext(XLogRecPtr start_lsn,
List *output_plugin_options,
bool fast_forward,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress)
@@ -429,8 +433,8 @@ CreateDecodingContext(XLogRecPtr start_lsn,
ctx = StartupDecodingContext(output_plugin_options,
start_lsn, InvalidTransactionId, false,
- fast_forward, xl_routine, prepare_write,
- do_write, update_progress);
+ fast_forward, page_read, cleanup_cb,
+ prepare_write, do_write, update_progress);
/* call output plugin initialization callback */
old_context = MemoryContextSwitchTo(ctx->context);
@@ -483,7 +487,13 @@ DecodingContextFindStartpoint(LogicalDecodingContext *ctx)
char *err = NULL;
/* the read_page callback waits for new WAL */
- record = XLogReadRecord(ctx->reader, &err);
+ while (XLogReadRecord(ctx->reader, &record, &err) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!ctx->page_read(ctx->reader))
+ break;
+ }
+
if (err)
elog(ERROR, "%s", err);
if (!record)
diff --git a/src/backend/replication/logical/logicalfuncs.c b/src/backend/replication/logical/logicalfuncs.c
index b99c94e848..fcc81b358e 100644
--- a/src/backend/replication/logical/logicalfuncs.c
+++ b/src/backend/replication/logical/logicalfuncs.c
@@ -233,9 +233,8 @@ pg_logical_slot_get_changes_guts(FunctionCallInfo fcinfo, bool confirm, bool bin
ctx = CreateDecodingContext(InvalidXLogRecPtr,
options,
false,
- XL_ROUTINE(.page_read = read_local_xlog_page,
- .segment_open = wal_segment_open,
- .segment_close = wal_segment_close),
+ read_local_xlog_page,
+ wal_segment_close,
LogicalOutputPrepareWrite,
LogicalOutputWrite, NULL);
@@ -284,7 +283,13 @@ pg_logical_slot_get_changes_guts(FunctionCallInfo fcinfo, bool confirm, bool bin
XLogRecord *record;
char *errm = NULL;
- record = XLogReadRecord(ctx->reader, &errm);
+ while (XLogReadRecord(ctx->reader, &record, &errm) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!ctx->page_read(ctx->reader))
+ break;
+ }
+
if (errm)
elog(ERROR, "%s", errm);
diff --git a/src/backend/replication/slotfuncs.c b/src/backend/replication/slotfuncs.c
index 88033a79b2..8a678074d5 100644
--- a/src/backend/replication/slotfuncs.c
+++ b/src/backend/replication/slotfuncs.c
@@ -152,9 +152,8 @@ create_logical_replication_slot(char *name, char *plugin,
ctx = CreateInitDecodingContext(plugin, NIL,
false, /* just catalogs is OK */
restart_lsn,
- XL_ROUTINE(.page_read = read_local_xlog_page,
- .segment_open = wal_segment_open,
- .segment_close = wal_segment_close),
+ read_local_xlog_page,
+ wal_segment_close,
NULL, NULL, NULL);
/*
@@ -486,9 +485,8 @@ pg_logical_replication_slot_advance(XLogRecPtr moveto)
ctx = CreateDecodingContext(InvalidXLogRecPtr,
NIL,
true, /* fast_forward */
- XL_ROUTINE(.page_read = read_local_xlog_page,
- .segment_open = wal_segment_open,
- .segment_close = wal_segment_close),
+ read_local_xlog_page,
+ wal_segment_close,
NULL, NULL, NULL);
/*
@@ -513,7 +511,13 @@ pg_logical_replication_slot_advance(XLogRecPtr moveto)
* Read records. No changes are generated in fast_forward mode,
* but snapbuilder/slot statuses are updated properly.
*/
- record = XLogReadRecord(ctx->reader, &errm);
+ while (XLogReadRecord(ctx->reader, &record, &errm) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!ctx->page_read(ctx->reader))
+ break;
+ }
+
if (errm)
elog(ERROR, "%s", errm);
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index bf2711bddd..2fd24a9b14 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -580,10 +580,7 @@ StartReplication(StartReplicationCmd *cmd)
/* create xlogreader for physical replication */
xlogreader =
- XLogReaderAllocate(wal_segment_size, NULL,
- XL_ROUTINE(.segment_open = WalSndSegmentOpen,
- .segment_close = wal_segment_close),
- NULL);
+ XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
if (!xlogreader)
ereport(ERROR,
@@ -812,9 +809,11 @@ StartReplication(StartReplicationCmd *cmd)
* set every time WAL is flushed.
*/
static bool
-logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetRecPtr, char *cur_page)
+logical_read_xlog_page(XLogReaderState *state)
{
+ XLogRecPtr targetPagePtr = state->readPagePtr;
+ int reqLen = state->readLen;
+ char *cur_page = state->readBuf;
XLogRecPtr flushptr;
int count;
WALReadError errinfo;
@@ -842,7 +841,7 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
count = flushptr - targetPagePtr; /* part of the page available */
/* now actually read the data, we know it's there */
- if (!WALRead(state,
+ if (!WALRead(state, WalSndSegmentOpen, wal_segment_close,
cur_page,
targetPagePtr,
XLOG_BLCKSZ,
@@ -1014,9 +1013,8 @@ CreateReplicationSlot(CreateReplicationSlotCmd *cmd)
ctx = CreateInitDecodingContext(cmd->plugin, NIL, need_full_snapshot,
InvalidXLogRecPtr,
- XL_ROUTINE(.page_read = logical_read_xlog_page,
- .segment_open = WalSndSegmentOpen,
- .segment_close = wal_segment_close),
+ logical_read_xlog_page,
+ wal_segment_close,
WalSndPrepareWrite, WalSndWriteData,
WalSndUpdateProgress);
@@ -1179,9 +1177,8 @@ StartLogicalReplication(StartReplicationCmd *cmd)
*/
logical_decoding_ctx =
CreateDecodingContext(cmd->startpoint, cmd->options, false,
- XL_ROUTINE(.page_read = logical_read_xlog_page,
- .segment_open = WalSndSegmentOpen,
- .segment_close = wal_segment_close),
+ logical_read_xlog_page,
+ wal_segment_close,
WalSndPrepareWrite, WalSndWriteData,
WalSndUpdateProgress);
xlogreader = logical_decoding_ctx->reader;
@@ -2761,7 +2758,7 @@ XLogSendPhysical(void)
enlargeStringInfo(&output_message, nbytes);
retry:
- if (!WALRead(xlogreader,
+ if (!WALRead(xlogreader, WalSndSegmentOpen, wal_segment_close,
&output_message.data[output_message.len],
startptr,
nbytes,
@@ -2859,7 +2856,12 @@ XLogSendLogical(void)
*/
WalSndCaughtUp = false;
- record = XLogReadRecord(logical_decoding_ctx->reader, &errm);
+ while (XLogReadRecord(logical_decoding_ctx->reader, &record, &errm) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!logical_decoding_ctx->page_read(logical_decoding_ctx->reader))
+ break;
+ }
/* xlog record was invalid */
if (errm != NULL)
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index 0b7e73ae79..3dd6df4be6 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -40,15 +40,9 @@ static int xlogreadfd = -1;
static XLogSegNo xlogreadsegno = -1;
static char xlogfpath[MAXPGPATH];
-typedef struct XLogPageReadPrivate
-{
- const char *restoreCommand;
- int tliIndex;
-} XLogPageReadPrivate;
-
-static bool SimpleXLogPageRead(XLogReaderState *xlogreader,
- XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
+static bool SimpleXLogPageRead(XLogReaderState *xlogreader,
+ const char *datadir, int *tliIndex,
+ const char *restoreCommand);
/*
* Read WAL from the datadir/pg_wal, starting from 'startpoint' on timeline
@@ -62,20 +56,22 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
XLogRecord *record;
XLogReaderState *xlogreader;
char *errormsg;
- XLogPageReadPrivate private;
- private.tliIndex = tliIndex;
- private.restoreCommand = restoreCommand;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir,
- XL_ROUTINE(.page_read = &SimpleXLogPageRead),
- &private);
+ xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+
if (xlogreader == NULL)
pg_fatal("out of memory");
XLogBeginRead(xlogreader, startpoint);
do
{
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!SimpleXLogPageRead(xlogreader, datadir,
+ &tliIndex, restoreCommand))
+ break;
+ }
if (record == NULL)
{
@@ -113,19 +109,19 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex,
XLogRecord *record;
XLogReaderState *xlogreader;
char *errormsg;
- XLogPageReadPrivate private;
XLogRecPtr endptr;
- private.tliIndex = tliIndex;
- private.restoreCommand = restoreCommand;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir,
- XL_ROUTINE(.page_read = &SimpleXLogPageRead),
- &private);
+ xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
XLogBeginRead(xlogreader, ptr);
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!SimpleXLogPageRead(xlogreader, datadir, &tliIndex, restoreCommand))
+ break;
+ }
if (record == NULL)
{
if (errormsg)
@@ -160,7 +156,6 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
XLogRecPtr searchptr;
XLogReaderState *xlogreader;
char *errormsg;
- XLogPageReadPrivate private;
/*
* The given fork pointer points to the end of the last common record,
@@ -176,11 +171,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
forkptr += SizeOfXLogShortPHD;
}
- private.tliIndex = tliIndex;
- private.restoreCommand = restoreCommand;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir,
- XL_ROUTINE(.page_read = &SimpleXLogPageRead),
- &private);
+ xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
@@ -190,7 +181,13 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
uint8 info;
XLogBeginRead(xlogreader, searchptr);
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!SimpleXLogPageRead(xlogreader, datadir,
+ &tliIndex, restoreCommand))
+ break;
+ }
if (record == NULL)
{
@@ -237,10 +234,11 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
/* XLogReader callback function, to read a WAL page */
static bool
-SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *readBuf)
+SimpleXLogPageRead(XLogReaderState *xlogreader, const char *datadir,
+ int *tliIndex, const char *restoreCommand)
{
- XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
+ XLogRecPtr targetPagePtr = xlogreader->readPagePtr;
+ char *readBuf = xlogreader->readBuf;
uint32 targetPageOff;
XLogRecPtr targetSegEnd;
XLogSegNo targetSegNo;
@@ -273,14 +271,14 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
* be done both forward and backward, consider also switching timeline
* accordingly.
*/
- while (private->tliIndex < targetNentries - 1 &&
- targetHistory[private->tliIndex].end < targetSegEnd)
- private->tliIndex++;
- while (private->tliIndex > 0 &&
- targetHistory[private->tliIndex].begin >= targetSegEnd)
- private->tliIndex--;
+ while (*tliIndex < targetNentries - 1 &&
+ targetHistory[*tliIndex].end < targetSegEnd)
+ (*tliIndex)++;
+ while (*tliIndex > 0 &&
+ targetHistory[*tliIndex].begin >= targetSegEnd)
+ (*tliIndex)--;
- XLogFileName(xlogfname, targetHistory[private->tliIndex].tli,
+ XLogFileName(xlogfname, targetHistory[*tliIndex].tli,
xlogreadsegno, WalSegSz);
snprintf(xlogfpath, MAXPGPATH, "%s/" XLOGDIR "/%s",
@@ -293,7 +291,7 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
/*
* If we have no restore_command to execute, then exit.
*/
- if (private->restoreCommand == NULL)
+ if (restoreCommand == NULL)
{
pg_log_error("could not open file \"%s\": %m", xlogfpath);
xlogreader->readLen = -1;
@@ -307,7 +305,7 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
xlogreadfd = RestoreArchivedFile(xlogreader->segcxt.ws_dir,
xlogfname,
WalSegSz,
- private->restoreCommand);
+ restoreCommand);
if (xlogreadfd < 0)
{
@@ -349,7 +347,7 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
Assert(targetSegNo == xlogreadsegno);
- xlogreader->seg.ws_tli = targetHistory[private->tliIndex].tli;
+ xlogreader->seg.ws_tli = targetHistory[*tliIndex].tli;
xlogreader->readLen = XLOG_BLCKSZ;
return true;
}
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index ab0c28d9b7..02e71138cc 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -330,12 +330,17 @@ WALDumpCloseSegment(XLogReaderState *state)
state->seg.ws_file = -1;
}
-/* pg_waldump's XLogReaderRoutine->page_read callback */
+/*
+ * pg_waldump's WAL page rader, also used as page_read callback for
+ * XLogFindNextRecord
+ */
static bool
-WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetPtr, char *readBuff)
+WALDumpReadPage(XLogReaderState *state, void *priv)
{
- XLogDumpPrivate *private = state->private_data;
+ XLogRecPtr targetPagePtr = state->readPagePtr;
+ int reqLen = state->readLen;
+ char *readBuff = state->readBuf;
+ XLogDumpPrivate *private = (XLogDumpPrivate *) priv;
int count = XLOG_BLCKSZ;
WALReadError errinfo;
@@ -353,8 +358,8 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
}
}
- if (!WALRead(state, readBuff, targetPagePtr, count, private->timeline,
- &errinfo))
+ if (!WALRead(state, WALDumpOpenSegment, WALDumpCloseSegment,
+ readBuff, targetPagePtr, count, private->timeline, &errinfo))
{
WALOpenSegment *seg = &errinfo.wre_seg;
char fname[MAXPGPATH];
@@ -374,6 +379,7 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
(Size) errinfo.wre_req);
}
+ Assert(count >= state->readLen);
state->readLen = count;
return true;
}
@@ -1042,16 +1048,14 @@ main(int argc, char **argv)
/* we have everything we need, start reading */
xlogreader_state =
- XLogReaderAllocate(WalSegSz, waldir,
- XL_ROUTINE(.page_read = WALDumpReadPage,
- .segment_open = WALDumpOpenSegment,
- .segment_close = WALDumpCloseSegment),
- &private);
+ XLogReaderAllocate(WalSegSz, waldir, WALDumpCloseSegment);
+
if (!xlogreader_state)
fatal_error("out of memory");
/* first find a valid recptr to start from */
- first_record = XLogFindNextRecord(xlogreader_state, private.startptr);
+ first_record = XLogFindNextRecord(xlogreader_state, private.startptr,
+ &WALDumpReadPage, (void*) &private);
if (first_record == InvalidXLogRecPtr)
fatal_error("could not find a valid record after %X/%X",
@@ -1075,7 +1079,13 @@ main(int argc, char **argv)
for (;;)
{
/* try to read the next record */
- record = XLogReadRecord(xlogreader_state, &errormsg);
+ while (XLogReadRecord(xlogreader_state, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!WALDumpReadPage(xlogreader_state, (void *) &private))
+ break;
+ }
+
if (!record)
{
if (!config.follow || private.endptr_reached)
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index 6c8848e14f..6a7fe140cb 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -57,64 +57,15 @@ typedef struct WALSegmentContext
typedef struct XLogReaderState XLogReaderState;
-/* Function type definition for the read_page callback */
-typedef bool (*XLogPageReadCB) (XLogReaderState *xlogreader,
- XLogRecPtr targetPagePtr,
- int reqLen,
- XLogRecPtr targetRecPtr,
- char *readBuf);
+/* Function type definition for the segment cleanup callback */
+typedef void (*WALSegmentCleanupCB) (XLogReaderState *xlogreader);
+
+/* Function type definition for the open/close callbacks for WALRead() */
typedef void (*WALSegmentOpenCB) (XLogReaderState *xlogreader,
XLogSegNo nextSegNo,
TimeLineID *tli_p);
typedef void (*WALSegmentCloseCB) (XLogReaderState *xlogreader);
-typedef struct XLogReaderRoutine
-{
- /*
- * Data input callback
- *
- * This callback shall read at least reqLen valid bytes of the xlog page
- * starting at targetPagePtr, and store them in readBuf. The callback
- * shall return the number of bytes read (never more than XLOG_BLCKSZ), or
- * -1 on failure. The callback shall sleep, if necessary, to wait for the
- * requested bytes to become available. The callback will not be invoked
- * again for the same page unless more than the returned number of bytes
- * are needed.
- *
- * targetRecPtr is the position of the WAL record we're reading. Usually
- * it is equal to targetPagePtr + reqLen, but sometimes xlogreader needs
- * to read and verify the page or segment header, before it reads the
- * actual WAL record it's interested in. In that case, targetRecPtr can
- * be used to determine which timeline to read the page from.
- *
- * The callback shall set ->seg.ws_tli to the TLI of the file the page was
- * read from.
- */
- XLogPageReadCB page_read;
-
- /*
- * Callback to open the specified WAL segment for reading. ->seg.ws_file
- * shall be set to the file descriptor of the opened segment. In case of
- * failure, an error shall be raised by the callback and it shall not
- * return.
- *
- * "nextSegNo" is the number of the segment to be opened.
- *
- * "tli_p" is an input/output argument. WALRead() uses it to pass the
- * timeline in which the new segment should be found, but the callback can
- * use it to return the TLI that it actually opened.
- */
- WALSegmentOpenCB segment_open;
-
- /*
- * WAL segment close callback. ->seg.ws_file shall be set to a negative
- * number.
- */
- WALSegmentCloseCB segment_close;
-} XLogReaderRoutine;
-
-#define XL_ROUTINE(...) &(XLogReaderRoutine){__VA_ARGS__}
-
typedef struct
{
/* Is this block ref in use? */
@@ -144,12 +95,35 @@ typedef struct
uint16 data_bufsz;
} DecodedBkpBlock;
+/* Return code from XLogReadRecord */
+typedef enum XLogReadRecordResult
+{
+ XLREAD_SUCCESS, /* record is successfully read */
+ XLREAD_NEED_DATA, /* need more data. see XLogReadRecord. */
+ XLREAD_FAIL /* failed during reading a record */
+} XLogReadRecordResult;
+
+/*
+ * internal state of XLogReadRecord
+ *
+ * XLogReadState runs a state machine while reading a record. Theses states
+ * are not seen outside the function. Each state may repeat several times
+ * exiting requesting caller for new data. See the comment of XLogReadRecrod
+ * for details.
+ */
+typedef enum XLogReadRecordState {
+ XLREAD_NEXT_RECORD,
+ XLREAD_TOT_LEN,
+ XLREAD_FIRST_FRAGMENT,
+ XLREAD_CONTINUATION
+} XLogReadRecordState;
+
struct XLogReaderState
{
/*
* Operational callbacks
*/
- XLogReaderRoutine routine;
+ WALSegmentCleanupCB cleanup_cb;
/* ----------------------------------------
* Public parameters
@@ -162,18 +136,14 @@ struct XLogReaderState
*/
uint64 system_identifier;
- /*
- * Opaque data for callbacks to use. Not used by XLogReader.
- */
- void *private_data;
-
/*
* Start and end point of last record read. EndRecPtr is also used as the
* position to read next. Calling XLogBeginRead() sets EndRecPtr to the
* starting position and ReadRecPtr to invalid.
*/
- XLogRecPtr ReadRecPtr; /* start of last record read */
+ XLogRecPtr ReadRecPtr; /* start of last record read or being read */
XLogRecPtr EndRecPtr; /* end+1 of last record read */
+ XLogRecPtr PrevRecPtr; /* start of previous record read */
/* ----------------------------------------
* Communication with page reader
@@ -187,7 +157,9 @@ struct XLogReaderState
* the request, or -1 on error */
TimeLineID readPageTLI; /* TLI for data currently in readBuf */
char *readBuf; /* buffer to store data */
- bool page_verified; /* is the page on the buffer verified? */
+ bool page_verified; /* is the page header on the buffer verified? */
+ bool record_verified;/* is the current record header verified? */
+
/* ----------------------------------------
@@ -227,8 +199,6 @@ struct XLogReaderState
XLogRecPtr latestPagePtr;
TimeLineID latestPageTLI;
- /* beginning of the WAL record being read. */
- XLogRecPtr currRecPtr;
/* timeline to read it from, 0 if a lookup is required */
TimeLineID currTLI;
@@ -255,6 +225,15 @@ struct XLogReaderState
char *readRecordBuf;
uint32 readRecordBufSize;
+ /*
+ * XLogReadRecord() state
+ */
+ XLogReadRecordState readRecordState;/* state machine state */
+ int recordGotLen; /* amount of current record that has
+ * already been read */
+ int recordRemainLen; /* length of current record that remains */
+ XLogRecPtr recordContRecPtr; /* where the current record continues */
+
/* Buffer to hold error message */
char *errormsg_buf;
};
@@ -262,9 +241,7 @@ struct XLogReaderState
/* Get a new XLogReader */
extern XLogReaderState *XLogReaderAllocate(int wal_segment_size,
const char *waldir,
- XLogReaderRoutine *routine,
- void *private_data);
-extern XLogReaderRoutine *LocalXLogReaderRoutine(void);
+ WALSegmentCleanupCB cleanup_cb);
/* Free an XLogReader */
extern void XLogReaderFree(XLogReaderState *state);
@@ -272,12 +249,17 @@ extern void XLogReaderFree(XLogReaderState *state);
/* Position the XLogReader to given record */
extern void XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr);
#ifdef FRONTEND
-extern XLogRecPtr XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr);
+/* Function type definition for the read_page callback */
+typedef bool (*XLogFindNextRecordCB) (XLogReaderState *xlogreader,
+ void *private);
+extern XLogRecPtr XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
+ XLogFindNextRecordCB read_page, void *private);
#endif /* FRONTEND */
/* Read the next XLog record. Returns NULL on end-of-WAL or failure */
-extern struct XLogRecord *XLogReadRecord(XLogReaderState *state,
- char **errormsg);
+extern XLogReadRecordResult XLogReadRecord(XLogReaderState *state,
+ XLogRecord **record,
+ char **errormsg);
/* Validate a page */
extern bool XLogReaderValidatePageHeader(XLogReaderState *state,
@@ -297,6 +279,7 @@ typedef struct WALReadError
} WALReadError;
extern bool WALRead(XLogReaderState *state,
+ WALSegmentOpenCB segopenfn, WALSegmentCloseCB sgclosefn,
char *buf, XLogRecPtr startptr, Size count,
TimeLineID tli, WALReadError *errinfo);
diff --git a/src/include/access/xlogutils.h b/src/include/access/xlogutils.h
index c287edf206..482e4ced69 100644
--- a/src/include/access/xlogutils.h
+++ b/src/include/access/xlogutils.h
@@ -47,9 +47,7 @@ extern Buffer XLogReadBufferExtended(RelFileNode rnode, ForkNumber forknum,
extern Relation CreateFakeRelcacheEntry(RelFileNode rnode);
extern void FreeFakeRelcacheEntry(Relation fakerel);
-extern bool read_local_xlog_page(XLogReaderState *state,
- XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetRecPtr, char *cur_page);
+extern bool read_local_xlog_page(XLogReaderState *state);
extern void wal_segment_open(XLogReaderState *state,
XLogSegNo nextSegNo,
TimeLineID *tli_p);
diff --git a/src/include/replication/logical.h b/src/include/replication/logical.h
index c2f2475e5d..a743db49cf 100644
--- a/src/include/replication/logical.h
+++ b/src/include/replication/logical.h
@@ -29,6 +29,10 @@ typedef void (*LogicalOutputPluginWriterUpdateProgress) (struct LogicalDecodingC
TransactionId xid
);
+typedef struct LogicalDecodingContext LogicalDecodingContext;
+
+typedef bool (*LogicalDecodingXLogPageReadCB)(XLogReaderState *ctx);
+
typedef struct LogicalDecodingContext
{
/* memory context this is all allocated in */
@@ -39,6 +43,7 @@ typedef struct LogicalDecodingContext
/* infrastructure pieces for decoding */
XLogReaderState *reader;
+ LogicalDecodingXLogPageReadCB page_read;
struct ReorderBuffer *reorder;
struct SnapBuild *snapshot_builder;
@@ -95,14 +100,16 @@ extern LogicalDecodingContext *CreateInitDecodingContext(char *plugin,
List *output_plugin_options,
bool need_full_snapshot,
XLogRecPtr restart_lsn,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress);
extern LogicalDecodingContext *CreateDecodingContext(XLogRecPtr start_lsn,
List *output_plugin_options,
bool fast_forward,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress);
--
2.18.4
----Next_Part(Thu_Jul__2_13_53_30_2020_072)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
filename="v15-0003-Remove-globals-readOff-readLen-and-readSegNo.patch"
^ permalink raw reply [nested|flat] 70+ messages in thread
* [PATCH v17 2/4] Move page-reader out of XLogReadRecord
@ 2019-09-10 03:58 Kyotaro Horiguchi <[email protected]>
0 siblings, 0 replies; 70+ messages in thread
From: Kyotaro Horiguchi @ 2019-09-10 03:58 UTC (permalink / raw)
This is the second step of removing callbacks from WAL record reader.
Since it is essential to take in additional data while reading a
record, the function have to ask caller for new data while keeping
working state. Thus the function is turned into a state machine.
---
src/backend/access/transam/twophase.c | 15 +-
src/backend/access/transam/xlog.c | 58 +-
src/backend/access/transam/xlogreader.c | 697 +++++++++++-------
src/backend/access/transam/xlogutils.c | 17 +-
src/backend/replication/logical/logical.c | 26 +-
.../replication/logical/logicalfuncs.c | 13 +-
src/backend/replication/slotfuncs.c | 18 +-
src/backend/replication/walsender.c | 32 +-
src/bin/pg_rewind/parsexlog.c | 84 ++-
src/bin/pg_waldump/pg_waldump.c | 36 +-
src/include/access/xlogreader.h | 121 ++-
src/include/access/xlogutils.h | 4 +-
src/include/pg_config_manual.h | 2 +-
src/include/replication/logical.h | 11 +-
14 files changed, 646 insertions(+), 488 deletions(-)
diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index 80d2d20d6c..7d5a96a5f5 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -1325,11 +1325,8 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
XLogReaderState *xlogreader;
char *errormsg;
- xlogreader = XLogReaderAllocate(wal_segment_size, NULL,
- XL_ROUTINE(.page_read = &read_local_xlog_page,
- .segment_open = &wal_segment_open,
- .segment_close = &wal_segment_close),
- NULL);
+ xlogreader = XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+
if (!xlogreader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
@@ -1337,7 +1334,13 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
errdetail("Failed while allocating a WAL reading processor.")));
XLogBeginRead(xlogreader, lsn);
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!read_local_xlog_page(xlogreader))
+ break;
+ }
+
if (record == NULL)
ereport(ERROR,
(errcode_for_file_access(),
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 606328a65a..670996ae67 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -836,13 +836,6 @@ static XLogSource currentSource = XLOG_FROM_ANY;
static bool lastSourceFailed = false;
static bool pendingWalRcvRestart = false;
-typedef struct XLogPageReadPrivate
-{
- int emode;
- bool fetching_ckpt; /* are we fetching a checkpoint record? */
- bool randAccess;
-} XLogPageReadPrivate;
-
/*
* These variables track when we last obtained some WAL data to process,
* and where we got it from. (XLogReceiptSource is initially the same as
@@ -917,8 +910,8 @@ static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath,
static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli,
XLogSource source, bool notfoundOk);
static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, XLogSource source);
-static bool XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
+static bool XLogPageRead(XLogReaderState *xlogreader,
+ bool fetching_ckpt, int emode, bool randAccess);
static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
bool fetching_ckpt, XLogRecPtr tliRecPtr);
static int emode_for_corrupt_record(int emode, XLogRecPtr RecPtr);
@@ -1231,8 +1224,7 @@ XLogInsertRecord(XLogRecData *rdata,
appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len);
if (!debug_reader)
- debug_reader = XLogReaderAllocate(wal_segment_size, NULL,
- XL_ROUTINE(), NULL);
+ debug_reader = XLogReaderAllocate(wal_segment_size, NULL, NULL);
if (!debug_reader)
{
@@ -4344,15 +4336,10 @@ CleanupBackupHistory(void)
* record is available.
*/
static XLogRecord *
-ReadRecord(XLogReaderState *xlogreader, int emode,
- bool fetching_ckpt)
+ReadRecord(XLogReaderState *xlogreader, int emode, bool fetching_ckpt)
{
XLogRecord *record;
- XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
-
- private->fetching_ckpt = fetching_ckpt;
- private->emode = emode;
- private->randAccess = (xlogreader->ReadRecPtr == InvalidXLogRecPtr);
+ bool randAccess = (xlogreader->ReadRecPtr == InvalidXLogRecPtr);
/* This is the first attempt to read this page. */
lastSourceFailed = false;
@@ -4360,8 +4347,16 @@ ReadRecord(XLogReaderState *xlogreader, int emode,
for (;;)
{
char *errormsg;
+ XLogReadRecordResult result;
+
+ while ((result = XLogReadRecord(xlogreader, &record, &errormsg))
+ == XLREAD_NEED_DATA)
+ {
+ if (!XLogPageRead(xlogreader, fetching_ckpt, emode, randAccess))
+ break;
+
+ }
- record = XLogReadRecord(xlogreader, &errormsg);
ReadRecPtr = xlogreader->ReadRecPtr;
EndRecPtr = xlogreader->EndRecPtr;
if (record == NULL)
@@ -6380,7 +6375,6 @@ StartupXLOG(void)
bool backupFromStandby = false;
DBState dbstate_at_startup;
XLogReaderState *xlogreader;
- XLogPageReadPrivate private;
bool promoted = false;
struct stat st;
@@ -6539,13 +6533,9 @@ StartupXLOG(void)
OwnLatch(&XLogCtl->recoveryWakeupLatch);
/* Set up XLOG reader facility */
- MemSet(&private, 0, sizeof(XLogPageReadPrivate));
xlogreader =
- XLogReaderAllocate(wal_segment_size, NULL,
- XL_ROUTINE(.page_read = &XLogPageRead,
- .segment_open = NULL,
- .segment_close = wal_segment_close),
- &private);
+ XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+
if (!xlogreader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
@@ -11987,12 +11977,13 @@ CancelBackup(void)
* sleep and retry.
*/
static bool
-XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetRecPtr, char *readBuf)
+XLogPageRead(XLogReaderState *xlogreader,
+ bool fetching_ckpt, int emode, bool randAccess)
{
- XLogPageReadPrivate *private =
- (XLogPageReadPrivate *) xlogreader->private_data;
- int emode = private->emode;
+ char *readBuf = xlogreader->readBuf;
+ XLogRecPtr targetPagePtr = xlogreader->readPagePtr;
+ int reqLen = xlogreader->readLen;
+ XLogRecPtr targetRecPtr = xlogreader->ReadRecPtr;
uint32 targetPageOff;
XLogSegNo targetSegNo PG_USED_FOR_ASSERTS_ONLY;
int r;
@@ -12035,8 +12026,8 @@ retry:
flushedUpto < targetPagePtr + reqLen))
{
if (!WaitForWALToBecomeAvailable(targetPagePtr + reqLen,
- private->randAccess,
- private->fetching_ckpt,
+ randAccess,
+ fetching_ckpt,
targetRecPtr))
{
if (readFile >= 0)
@@ -12141,6 +12132,7 @@ retry:
goto next_record_is_invalid;
}
+ Assert(xlogreader->readPagePtr == targetPagePtr);
xlogreader->readLen = readLen;
return true;
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index 7d04995b7b..ba3e4868f7 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -40,7 +40,7 @@ static bool XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr,
int reqLen, bool header_inclusive);
static void XLogReaderInvalReadState(XLogReaderState *state);
static bool ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
- XLogRecPtr PrevRecPtr, XLogRecord *record, bool randAccess);
+ XLogRecPtr PrevRecPtr, XLogRecord *record);
static bool ValidXLogRecord(XLogReaderState *state, XLogRecord *record,
XLogRecPtr recptr);
static void ResetDecoder(XLogReaderState *state);
@@ -73,7 +73,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...)
*/
XLogReaderState *
XLogReaderAllocate(int wal_segment_size, const char *waldir,
- XLogReaderRoutine *routine, void *private_data)
+ WALSegmentCleanupCB cleanup_cb)
{
XLogReaderState *state;
@@ -84,7 +84,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
return NULL;
/* initialize caller-provided support functions */
- state->routine = *routine;
+ state->cleanup_cb = cleanup_cb;
state->max_block_id = -1;
@@ -107,8 +107,6 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
WALOpenSegmentInit(&state->seg, &state->segcxt, wal_segment_size,
waldir);
- /* system_identifier initialized to zeroes above */
- state->private_data = private_data;
/* ReadRecPtr, EndRecPtr and readLen initialized to zeroes above */
state->errormsg_buf = palloc_extended(MAX_ERRORMSG_LEN + 1,
MCXT_ALLOC_NO_OOM);
@@ -140,8 +138,8 @@ XLogReaderFree(XLogReaderState *state)
{
int block_id;
- if (state->seg.ws_file != -1)
- state->routine.segment_close(state);
+ if (state->seg.ws_file >= 0)
+ state->cleanup_cb(state);
for (block_id = 0; block_id <= XLR_MAX_BLOCK_ID; block_id++)
{
@@ -246,6 +244,7 @@ XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
/* Begin at the passed-in record pointer. */
state->EndRecPtr = RecPtr;
state->ReadRecPtr = InvalidXLogRecPtr;
+ state->readRecordState = XLREAD_NEXT_RECORD;
}
/*
@@ -254,317 +253,449 @@ XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
* XLogBeginRead() or XLogFindNextRecord() must be called before the first call
* to XLogReadRecord().
*
- * If the page_read callback fails to read the requested data, NULL is
- * returned. The callback is expected to have reported the error; errormsg
- * is set to NULL.
+ * This function may return XLREAD_NEED_DATA several times before returning a
+ * result record. The caller shall read in some new data then call this
+ * function again with the same parameters.
*
- * If the reading fails for some other reason, NULL is also returned, and
- * *errormsg is set to a string with details of the failure.
+ * When a record is successfully read, returns XLREAD_SUCCESS with result
+ * record being stored in *record. Otherwise *record is NULL.
*
- * The returned pointer (or *errormsg) points to an internal buffer that's
- * valid until the next call to XLogReadRecord.
+ * Returns XLREAD_NEED_DATA if more data is needed to finish reading the
+ * current record. In that case, state->readPagePtr and state->readLen inform
+ * the desired position and minimum length of data needed. The caller shall
+ * read in the requested data and set state->readBuf to point to a buffer
+ * containing it. The caller must also set state->readPageTLI and
+ * state->readLen to indicate the timeline that it was read from, and the
+ * length of data that is now available (which must be >= given readLen),
+ * respectively.
+ *
+ * If invalid data is encountered, returns XLREAD_FAIL with *record being set to
+ * NULL. *errormsg is set to a string with details of the failure.
+ * The returned pointer (or *errormsg) points to an internal buffer that's valid
+ * until the next call to XLogReadRecord.
+ *
+ *
+ * This function runs a state machine consists of the following states.
+ *
+ * XLREAD_NEXT_RECORD :
+ * The initial state, if called with valid RecPtr, try to read a record at
+ * that position. If invalid RecPtr is given try to read a record just after
+ * the last one previously read.
+ * This state ens after setting ReadRecPtr. Then goes to XLREAD_TOT_LEN.
+ *
+ * XLREAD_TOT_LEN:
+ * Examining record header. Ends after reading record total
+ * length. recordRemainLen and recordGotLen are initialized.
+ *
+ * XLREAD_FIRST_FRAGMENT:
+ * Reading the first fragment. Ends with finishing reading a single
+ * record. Goes to XLREAD_NEXT_RECORD if that's all or
+ * XLREAD_CONTINUATION if we have continuation.
+
+ * XLREAD_CONTINUATION:
+ * Reading continuation of record. Ends with finishing the whole record then
+ * goes to XLREAD_NEXT_RECORD. During this state, recordRemainLen indicates
+ * how much is left and readRecordBuf holds the partially assert
+ * record.recordContRecPtr points to the beginning of the next page where to
+ * continue.
+ *
+ * If wrong data found in any state, the state machine stays at the current
+ * state. This behavior allows to continue reading a reacord switching among
+ * different souces, while streaming replication.
*/
-XLogRecord *
-XLogReadRecord(XLogReaderState *state, char **errormsg)
+XLogReadRecordResult
+XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
{
- XLogRecPtr RecPtr;
- XLogRecord *record;
- XLogRecPtr targetPagePtr;
- bool randAccess;
- uint32 len,
- total_len;
- uint32 targetRecOff;
- uint32 pageHeaderSize;
- bool gotheader;
+ XLogRecord *prec;
- /*
- * randAccess indicates whether to verify the previous-record pointer of
- * the record we're reading. We only do this if we're reading
- * sequentially, which is what we initially assume.
- */
- randAccess = false;
+ *record = NULL;
/* reset error state */
*errormsg = NULL;
state->errormsg_buf[0] = '\0';
- ResetDecoder(state);
-
- RecPtr = state->EndRecPtr;
-
- if (state->ReadRecPtr != InvalidXLogRecPtr)
- {
- /* read the record after the one we just read */
-
- /*
- * EndRecPtr is pointing to end+1 of the previous WAL record. If
- * we're at a page boundary, no more records can fit on the current
- * page. We must skip over the page header, but we can't do that until
- * we've read in the page, since the header size is variable.
- */
- }
- else
- {
- /*
- * Caller supplied a position to start at.
- *
- * In this case, EndRecPtr should already be pointing to a valid
- * record starting position.
- */
- Assert(XRecOffIsValid(RecPtr));
- randAccess = true;
- }
-
- state->currRecPtr = RecPtr;
-
- targetPagePtr = RecPtr - (RecPtr % XLOG_BLCKSZ);
- targetRecOff = RecPtr % XLOG_BLCKSZ;
-
- /*
- * Read the page containing the record into state->readBuf. Request enough
- * byte to cover the whole record header, or at least the part of it that
- * fits on the same page.
- */
- while (XLogNeedData(state, targetPagePtr,
- Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
- targetRecOff != 0))
- {
- if (!state->routine.page_read(state, state->readPagePtr, state->readLen,
- RecPtr, state->readBuf))
- break;
- }
-
- if (!state->page_verified)
- goto err;
-
- /*
- * We have at least the page header, so we can examine it now.
- */
- pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
- if (targetRecOff == 0)
- {
- /*
- * At page start, so skip over page header.
- */
- RecPtr += pageHeaderSize;
- targetRecOff = pageHeaderSize;
- }
- else if (targetRecOff < pageHeaderSize)
- {
- report_invalid_record(state, "invalid record offset at %X/%X",
- LSN_FORMAT_ARGS(RecPtr));
- goto err;
- }
-
- if ((((XLogPageHeader) state->readBuf)->xlp_info & XLP_FIRST_IS_CONTRECORD) &&
- targetRecOff == pageHeaderSize)
- {
- report_invalid_record(state, "contrecord is requested by %X/%X",
- LSN_FORMAT_ARGS(RecPtr));
- goto err;
- }
-
- /* XLogNeedData has verified the page header */
- Assert(pageHeaderSize <= state->readLen);
-
- /*
- * Read the record length.
- *
- * NB: Even though we use an XLogRecord pointer here, the whole record
- * header might not fit on this page. xl_tot_len is the first field of the
- * struct, so it must be on this page (the records are MAXALIGNed), but we
- * cannot access any other fields until we've verified that we got the
- * whole header.
- */
- record = (XLogRecord *) (state->readBuf + RecPtr % XLOG_BLCKSZ);
- total_len = record->xl_tot_len;
-
- /*
- * If the whole record header is on this page, validate it immediately.
- * Otherwise do just a basic sanity check on xl_tot_len, and validate the
- * rest of the header after reading it from the next page. The xl_tot_len
- * check is necessary here to ensure that we enter the "Need to reassemble
- * record" code path below; otherwise we might fail to apply
- * ValidXLogRecordHeader at all.
- */
- if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
- {
- if (!ValidXLogRecordHeader(state, RecPtr, state->ReadRecPtr, record,
- randAccess))
- goto err;
- gotheader = true;
- }
- else
- {
- /* XXX: more validation should be done here */
- if (total_len < SizeOfXLogRecord)
- {
- report_invalid_record(state,
- "invalid record length at %X/%X: wanted %u, got %u",
- LSN_FORMAT_ARGS(RecPtr),
- (uint32) SizeOfXLogRecord, total_len);
- goto err;
- }
- gotheader = false;
- }
-
- len = XLOG_BLCKSZ - RecPtr % XLOG_BLCKSZ;
- if (total_len > len)
+ switch (state->readRecordState)
{
- /* Need to reassemble record */
- char *contdata;
- XLogPageHeader pageHeader;
- char *buffer;
- uint32 gotlen;
-
- /*
- * Enlarge readRecordBuf as needed.
- */
- if (total_len > state->readRecordBufSize &&
- !allocate_recordbuf(state, total_len))
- {
- /* We treat this as a "bogus data" condition */
- report_invalid_record(state, "record length %u at %X/%X too long",
- total_len, LSN_FORMAT_ARGS(RecPtr));
- goto err;
- }
+ case XLREAD_NEXT_RECORD:
+ ResetDecoder(state);
- /* Copy the first fragment of the record from the first page. */
- memcpy(state->readRecordBuf,
- state->readBuf + RecPtr % XLOG_BLCKSZ, len);
- buffer = state->readRecordBuf + len;
- gotlen = len;
-
- do
- {
- int rest_len = total_len - gotlen;
-
- /* Calculate pointer to beginning of next page */
- targetPagePtr += XLOG_BLCKSZ;
-
- /* Wait for the next page to become available */
- while (XLogNeedData(state, targetPagePtr,
- Min(rest_len, XLOG_BLCKSZ),
- false))
+ if (state->ReadRecPtr != InvalidXLogRecPtr)
{
- if (!state->routine.page_read(state, state->readPagePtr,
- state->readLen,
- state->ReadRecPtr,
- state->readBuf))
- break;
+ /* read the record after the one we just read */
+
+ /*
+ * EndRecPtr is pointing to end+1 of the previous WAL record.
+ * If we're at a page boundary, no more records can fit on the
+ * current page. We must skip over the page header, but we
+ * can't do that until we've read in the page, since the header
+ * size is variable.
+ */
+ state->PrevRecPtr = state->ReadRecPtr;
+ state->ReadRecPtr = state->EndRecPtr;
+ }
+ else
+ {
+ /*
+ * Caller supplied a position to start at.
+ *
+ * In this case, EndRecPtr should already be pointing to a
+ * valid record starting position.
+ */
+ Assert(XRecOffIsValid(state->EndRecPtr));
+ state->ReadRecPtr = state->EndRecPtr;
+
+ /*
+ * We cannot verify the previous-record pointer when we're
+ * seeking to a particular record. Reset PrevRecPtr so that we
+ * won't try doing that.
+ */
+ state->PrevRecPtr = InvalidXLogRecPtr;
+ state->EndRecPtr = InvalidXLogRecPtr; /* to be tidy */
}
+ state->record_verified = false;
+ state->readRecordState = XLREAD_TOT_LEN;
+ /* fall through */
+
+ case XLREAD_TOT_LEN:
+ {
+ uint32 total_len;
+ uint32 pageHeaderSize;
+ XLogRecPtr targetPagePtr;
+ uint32 targetRecOff;
+ XLogPageHeader pageHeader;
+
+ targetPagePtr =
+ state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
+ targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+
+ /*
+ * Check if we have enough data. For the first record in the page,
+ * the requesting length doesn't contain page header.
+ */
+ if (XLogNeedData(state, targetPagePtr,
+ Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+ targetRecOff != 0))
+ return XLREAD_NEED_DATA;
+
+ /* error out if caller supplied bogus page */
if (!state->page_verified)
goto err;
- Assert(SizeOfXLogShortPHD <= state->readLen);
+ /* examine page header now. */
+ pageHeaderSize =
+ XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+ if (targetRecOff == 0)
+ {
+ /* At page start, so skip over page header. */
+ state->ReadRecPtr += pageHeaderSize;
+ targetRecOff = pageHeaderSize;
+ }
+ else if (targetRecOff < pageHeaderSize)
+ {
+ report_invalid_record(state, "invalid record offset at %X/%X",
+ LSN_FORMAT_ARGS(state->ReadRecPtr));
+ goto err;
+ }
- /* Check that the continuation on next page looks valid */
pageHeader = (XLogPageHeader) state->readBuf;
- if (!(pageHeader->xlp_info & XLP_FIRST_IS_CONTRECORD))
+ if ((pageHeader->xlp_info & XLP_FIRST_IS_CONTRECORD) &&
+ targetRecOff == pageHeaderSize)
{
- report_invalid_record(state,
- "there is no contrecord flag at %X/%X",
- LSN_FORMAT_ARGS(RecPtr));
+ report_invalid_record(state, "contrecord is requested by %X/%X",
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr);
goto err;
}
- /*
- * Cross-check that xlp_rem_len agrees with how much of the record
- * we expect there to be left.
- */
- if (pageHeader->xlp_rem_len == 0 ||
- total_len != (pageHeader->xlp_rem_len + gotlen))
- {
- report_invalid_record(state,
- "invalid contrecord length %u (expected %lld) at %X/%X",
- pageHeader->xlp_rem_len,
- ((long long) total_len) - gotlen,
- LSN_FORMAT_ARGS(RecPtr));
- goto err;
- }
-
- /* Append the continuation from this page to the buffer */
- pageHeaderSize = XLogPageHeaderSize(pageHeader);
-
+ /* XLogNeedData has verified the page header */
Assert(pageHeaderSize <= state->readLen);
- contdata = (char *) state->readBuf + pageHeaderSize;
- len = XLOG_BLCKSZ - pageHeaderSize;
- if (pageHeader->xlp_rem_len < len)
- len = pageHeader->xlp_rem_len;
+ /*
+ * Read the record length.
+ *
+ * NB: Even though we use an XLogRecord pointer here, the whole
+ * record header might not fit on this page. xl_tot_len is the first
+ * field of the struct, so it must be on this page (the records are
+ * MAXALIGNed), but we cannot access any other fields until we've
+ * verified that we got the whole header.
+ */
+ prec = (XLogRecord *) (state->readBuf +
+ state->ReadRecPtr % XLOG_BLCKSZ);
+ total_len = prec->xl_tot_len;
- Assert (pageHeaderSize + len <= state->readLen);
- memcpy(buffer, (char *) contdata, len);
- buffer += len;
- gotlen += len;
+ /*
+ * If the whole record header is on this page, validate it
+ * immediately. Otherwise do just a basic sanity check on
+ * xl_tot_len, and validate the rest of the header after reading it
+ * from the next page. The xl_tot_len check is necessary here to
+ * ensure that we enter the XLREAD_CONTINUATION state below;
+ * otherwise we might fail to apply ValidXLogRecordHeader at all.
+ */
+ if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
+ {
+ if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
+ state->PrevRecPtr, prec))
+ goto err;
- /* If we just reassembled the record header, validate it. */
- if (!gotheader)
+ state->record_verified = true;
+ }
+ else
{
- record = (XLogRecord *) state->readRecordBuf;
- if (!ValidXLogRecordHeader(state, RecPtr, state->ReadRecPtr,
- record, randAccess))
+ /* XXX: more validation should be done here */
+ if (total_len < SizeOfXLogRecord)
+ {
+ report_invalid_record(state,
+ "invalid record length at %X/%X: wanted %u, got %u",
+ LSN_FORMAT_ARGS(state->ReadRecPtr),
+ (uint32) SizeOfXLogRecord, total_len);
goto err;
- gotheader = true;
+ }
}
- } while (gotlen < total_len);
-
- Assert(gotheader);
- record = (XLogRecord *) state->readRecordBuf;
- if (!ValidXLogRecord(state, record, RecPtr))
- goto err;
+ /*
+ * Wait for the rest of the record, or the part of the record that
+ * fit on the first page if crossed a page boundary, to become
+ * available.
+ */
+ state->recordGotLen = 0;
+ state->recordRemainLen = total_len;
+ state->readRecordState = XLREAD_FIRST_FRAGMENT;
+ }
+ /* fall through */
- pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
- state->ReadRecPtr = RecPtr;
- state->EndRecPtr = targetPagePtr + pageHeaderSize
- + MAXALIGN(pageHeader->xlp_rem_len);
- }
- else
- {
- /* Wait for the record data to become available */
- while (XLogNeedData(state, targetPagePtr,
- Min(targetRecOff + total_len, XLOG_BLCKSZ), true))
+ case XLREAD_FIRST_FRAGMENT:
{
- if (!state->routine.page_read(state, state->readPagePtr,
- state->readLen,
- state->ReadRecPtr, state->readBuf))
+ uint32 total_len = state->recordRemainLen;
+ uint32 request_len;
+ uint32 record_len;
+ XLogRecPtr targetPagePtr;
+ uint32 targetRecOff;
+
+ /*
+ * Wait for the rest of the record on the first page to become
+ * available
+ */
+ targetPagePtr =
+ state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
+ targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+
+ request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ);
+ record_len = request_len - targetRecOff;
+
+ /* ReadRecPtr contains page header */
+ Assert (targetRecOff != 0);
+ if (XLogNeedData(state, targetPagePtr, request_len, true))
+ return XLREAD_NEED_DATA;
+
+ /* error out if caller supplied bogus page */
+ if (!state->page_verified)
+ goto err;
+
+ prec = (XLogRecord *) (state->readBuf + targetRecOff);
+
+ /* validate record header if not yet */
+ if (!state->record_verified && record_len >= SizeOfXLogRecord)
+ {
+ if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
+ state->PrevRecPtr, prec))
+ goto err;
+
+ state->record_verified = true;
+ }
+
+
+ if (total_len == record_len)
+ {
+ /* Record does not cross a page boundary */
+ Assert(state->record_verified);
+
+ if (!ValidXLogRecord(state, prec, state->ReadRecPtr))
+ goto err;
+
+ state->record_verified = true; /* to be tidy */
+
+ /* We already checked the header earlier */
+ state->EndRecPtr = state->ReadRecPtr + MAXALIGN(record_len);
+
+ *record = prec;
+ state->readRecordState = XLREAD_NEXT_RECORD;
break;
+ }
+
+ /*
+ * The record continues on the next page. Need to reassemble
+ * record
+ */
+ Assert(total_len > record_len);
+
+ /* Enlarge readRecordBuf as needed. */
+ if (total_len > state->readRecordBufSize &&
+ !allocate_recordbuf(state, total_len))
+ {
+ /* We treat this as a "bogus data" condition */
+ report_invalid_record(state,
+ "record length %u at %X/%X too long",
+ total_len,
+ LSN_FORMAT_ARGS(state->ReadRecPtr));
+ goto err;
+ }
+
+ /* Copy the first fragment of the record from the first page. */
+ memcpy(state->readRecordBuf, state->readBuf + targetRecOff,
+ record_len);
+ state->recordGotLen += record_len;
+ state->recordRemainLen -= record_len;
+
+ /* Calculate pointer to beginning of next page */
+ state->recordContRecPtr = state->ReadRecPtr + record_len;
+ Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0);
+
+ state->readRecordState = XLREAD_CONTINUATION;
}
+ /* fall through */
+
+ case XLREAD_CONTINUATION:
+ {
+ XLogPageHeader pageHeader;
+ uint32 pageHeaderSize;
+ XLogRecPtr targetPagePtr;
+
+ /* we enter this state only if we haven't read the whole record. */
+ Assert (state->recordRemainLen > 0);
+
+ while(state->recordRemainLen > 0)
+ {
+ char *contdata;
+ uint32 request_len;
+ uint32 record_len;
+
+ /* Wait for the next page to become available */
+ targetPagePtr = state->recordContRecPtr;
+
+ /* this request contains page header */
+ Assert (targetPagePtr != 0);
+ if (XLogNeedData(state, targetPagePtr,
+ Min(state->recordRemainLen, XLOG_BLCKSZ),
+ false))
+ return XLREAD_NEED_DATA;
+
+ if (!state->page_verified)
+ goto err;
+
+ Assert(SizeOfXLogShortPHD <= state->readLen);
- if (!state->page_verified)
- goto err;
+ /* Check that the continuation on next page looks valid */
+ pageHeader = (XLogPageHeader) state->readBuf;
+ if (!(pageHeader->xlp_info & XLP_FIRST_IS_CONTRECORD))
+ {
+ report_invalid_record(
+ state,
+ "there is no contrecord flag at %X/%X reading %X/%X",
+ (uint32) (state->recordContRecPtr >> 32),
+ (uint32) state->recordContRecPtr,
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr);
+ goto err;
+ }
- /* Record does not cross a page boundary */
- if (!ValidXLogRecord(state, record, RecPtr))
- goto err;
+ /*
+ * Cross-check that xlp_rem_len agrees with how much of the
+ * record we expect there to be left.
+ */
+ if (pageHeader->xlp_rem_len == 0 ||
+ pageHeader->xlp_rem_len != state->recordRemainLen)
+ {
+ report_invalid_record(
+ state,
+ "invalid contrecord length %u at %X/%X reading %X/%X, expected %u",
+ pageHeader->xlp_rem_len,
+ (uint32) (state->recordContRecPtr >> 32),
+ (uint32) state->recordContRecPtr,
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr,
+ state->recordRemainLen);
+ goto err;
+ }
- state->EndRecPtr = RecPtr + MAXALIGN(total_len);
+ /* Append the continuation from this page to the buffer */
+ pageHeaderSize = XLogPageHeaderSize(pageHeader);
- state->ReadRecPtr = RecPtr;
+ /*
+ * XLogNeedData should have ensured that the whole page header
+ * was read
+ */
+ Assert(state->readLen >= pageHeaderSize);
+
+ contdata = (char *) state->readBuf + pageHeaderSize;
+ record_len = XLOG_BLCKSZ - pageHeaderSize;
+ if (pageHeader->xlp_rem_len < record_len)
+ record_len = pageHeader->xlp_rem_len;
+
+ request_len = record_len + pageHeaderSize;
+
+ /* XLogNeedData should have ensured all needed data was read */
+ Assert (state->readLen >= request_len);
+
+ memcpy(state->readRecordBuf + state->recordGotLen,
+ (char *) contdata, record_len);
+ state->recordGotLen += record_len;
+ state->recordRemainLen -= record_len;
+
+ /* If we just reassembled the record header, validate it. */
+ if (!state->record_verified)
+ {
+ Assert(state->recordGotLen >= SizeOfXLogRecord);
+ if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
+ state->PrevRecPtr,
+ (XLogRecord *) state->readRecordBuf))
+ goto err;
+
+ state->record_verified = true;
+ }
+
+ /* Calculate pointer to beginning of next page, and continue */
+ state->recordContRecPtr += XLOG_BLCKSZ;
+ }
+
+ /* targetPagePtr is pointing the last-read page here */
+ prec = (XLogRecord *) state->readRecordBuf;
+ if (!ValidXLogRecord(state, prec, state->ReadRecPtr))
+ goto err;
+
+ pageHeaderSize =
+ XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+ state->EndRecPtr = targetPagePtr + pageHeaderSize
+ + MAXALIGN(pageHeader->xlp_rem_len);
+
+ *record = prec;
+ state->readRecordState = XLREAD_NEXT_RECORD;
+ break;
+ }
}
/*
* Special processing if it's an XLOG SWITCH record
*/
- if (record->xl_rmid == RM_XLOG_ID &&
- (record->xl_info & ~XLR_INFO_MASK) == XLOG_SWITCH)
+ if ((*record)->xl_rmid == RM_XLOG_ID &&
+ ((*record)->xl_info & ~XLR_INFO_MASK) == XLOG_SWITCH)
{
/* Pretend it extends to end of segment */
state->EndRecPtr += state->segcxt.ws_segsize - 1;
state->EndRecPtr -= XLogSegmentOffset(state->EndRecPtr, state->segcxt.ws_segsize);
}
- if (DecodeXLogRecord(state, record, errormsg))
- return record;
- else
- return NULL;
+ Assert (!*record || state->readLen >= 0);
+ if (DecodeXLogRecord(state, *record, errormsg))
+ return XLREAD_SUCCESS;
+
+ *record = NULL;
+ return XLREAD_FAIL;
err:
/*
- * Invalidate the read state. We might read from a different source after
+ * Invalidate the read page. We might read from a different source after
* failure.
*/
XLogReaderInvalReadState(state);
@@ -572,7 +703,8 @@ err:
if (state->errormsg_buf[0] != '\0')
*errormsg = state->errormsg_buf;
- return NULL;
+ *record = NULL;
+ return XLREAD_FAIL;
}
/*
@@ -724,11 +856,12 @@ XLogReaderInvalReadState(XLogReaderState *state)
*
* This is just a convenience subroutine to avoid duplicated code in
* XLogReadRecord. It's not intended for use from anywhere else.
+ *
+ * If PrevRecPtr is valid, the xl_prev is is cross-checked with it.
*/
static bool
ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
- XLogRecPtr PrevRecPtr, XLogRecord *record,
- bool randAccess)
+ XLogRecPtr PrevRecPtr, XLogRecord *record)
{
if (record->xl_tot_len < SizeOfXLogRecord)
{
@@ -745,7 +878,7 @@ ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
record->xl_rmid, LSN_FORMAT_ARGS(RecPtr));
return false;
}
- if (randAccess)
+ if (PrevRecPtr == InvalidXLogRecPtr)
{
/*
* We can't exactly verify the prev-link, but surely it should be less
@@ -975,11 +1108,14 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
* XLogReadRecord() will read the next valid record.
*/
XLogRecPtr
-XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
+XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
+ XLogFindNextRecordCB read_page, void *private)
{
XLogRecPtr tmpRecPtr;
XLogRecPtr found = InvalidXLogRecPtr;
XLogPageHeader header;
+ XLogRecord *record;
+ XLogReadRecordResult result;
char *errormsg;
Assert(!XLogRecPtrIsInvalid(RecPtr));
@@ -1012,9 +1148,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
while(XLogNeedData(state, targetPagePtr, targetRecOff,
targetRecOff != 0))
{
- if (!state->routine.page_read(state, state->readPagePtr,
- state->readLen,
- state->ReadRecPtr, state->readBuf))
+ if (!read_page(state, private))
break;
}
@@ -1066,8 +1200,16 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
* or we just jumped over the remaining data of a continuation.
*/
XLogBeginRead(state, tmpRecPtr);
- while (XLogReadRecord(state, &errormsg) != NULL)
+ while ((result = XLogReadRecord(state, &record, &errormsg)) !=
+ XLREAD_FAIL)
{
+ if (result == XLREAD_NEED_DATA)
+ {
+ if (!read_page(state, private))
+ goto err;
+ continue;
+ }
+
/* past the record we've found, break out */
if (RecPtr <= state->ReadRecPtr)
{
@@ -1087,9 +1229,9 @@ err:
#endif /* FRONTEND */
/*
- * Helper function to ease writing of XLogRoutine->page_read callbacks.
- * If this function is used, caller must supply a segment_open callback in
- * 'state', as that is used here.
+ * Helper function to ease writing of page_read callback.
+ * If this function is used, caller must supply a segment_open callback and
+ * segment_close callback as that is used here.
*
* Read 'count' bytes into 'buf', starting at location 'startptr', from WAL
* fetched from timeline 'tli'.
@@ -1102,6 +1244,7 @@ err:
*/
bool
WALRead(XLogReaderState *state,
+ WALSegmentOpenCB segopenfn, WALSegmentCloseCB segclosefn,
char *buf, XLogRecPtr startptr, Size count, TimeLineID tli,
WALReadError *errinfo)
{
@@ -1133,10 +1276,10 @@ WALRead(XLogReaderState *state,
XLogSegNo nextSegNo;
if (state->seg.ws_file >= 0)
- state->routine.segment_close(state);
+ segclosefn(state);
XLByteToSeg(recptr, nextSegNo, state->segcxt.ws_segsize);
- state->routine.segment_open(state, nextSegNo, &tli);
+ segopenfn(state, nextSegNo, &tli);
/* This shouldn't happen -- indicates a bug in segment_open */
Assert(state->seg.ws_file >= 0);
diff --git a/src/backend/access/transam/xlogutils.c b/src/backend/access/transam/xlogutils.c
index 46eda33f25..b003990745 100644
--- a/src/backend/access/transam/xlogutils.c
+++ b/src/backend/access/transam/xlogutils.c
@@ -686,8 +686,7 @@ XLogTruncateRelation(RelFileNode rnode, ForkNumber forkNum,
void
XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wantLength)
{
- const XLogRecPtr lastReadPage = state->seg.ws_segno *
- state->segcxt.ws_segsize + state->readLen;
+ const XLogRecPtr lastReadPage = state->readPagePtr;
Assert(wantPage != InvalidXLogRecPtr && wantPage % XLOG_BLCKSZ == 0);
Assert(wantLength <= XLOG_BLCKSZ);
@@ -702,7 +701,7 @@ XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wa
* current TLI has since become historical.
*/
if (lastReadPage == wantPage &&
- state->readLen != 0 &&
+ state->page_verified &&
lastReadPage + state->readLen >= wantPage + Min(wantLength, XLOG_BLCKSZ - 1))
return;
@@ -788,6 +787,7 @@ wal_segment_open(XLogReaderState *state, XLogSegNo nextSegNo,
char path[MAXPGPATH];
XLogFilePath(path, tli, nextSegNo, state->segcxt.ws_segsize);
+ elog(LOG, "HOGE: %lu, %d => %s", nextSegNo, tli, path);
state->seg.ws_file = BasicOpenFile(path, O_RDONLY | PG_BINARY);
if (state->seg.ws_file >= 0)
return;
@@ -825,9 +825,11 @@ wal_segment_close(XLogReaderState *state)
* loop for now.
*/
bool
-read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *cur_page)
+read_local_xlog_page(XLogReaderState *state)
{
+ XLogRecPtr targetPagePtr = state->readPagePtr;
+ int reqLen = state->readLen;
+ char *cur_page = state->readBuf;
XLogRecPtr read_upto,
loc;
TimeLineID tli;
@@ -940,11 +942,12 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
* as 'count', read the whole page anyway. It's guaranteed to be
* zero-padded up to the page boundary if it's incomplete.
*/
- if (!WALRead(state, cur_page, targetPagePtr, XLOG_BLCKSZ, tli,
- &errinfo))
+ if (!WALRead(state, wal_segment_open, wal_segment_close,
+ cur_page, targetPagePtr, XLOG_BLCKSZ, tli, &errinfo))
WALReadRaiseError(&errinfo);
/* number of valid bytes in the buffer */
+ state->readPagePtr = targetPagePtr;
state->readLen = count;
return true;
}
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index 37b75deb72..9cbd8afa49 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -148,7 +148,8 @@ StartupDecodingContext(List *output_plugin_options,
TransactionId xmin_horizon,
bool need_full_snapshot,
bool fast_forward,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress)
@@ -198,11 +199,12 @@ StartupDecodingContext(List *output_plugin_options,
ctx->slot = slot;
- ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, xl_routine, ctx);
+ ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, cleanup_cb);
if (!ctx->reader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
errmsg("out of memory")));
+ ctx->page_read = page_read;
ctx->reorder = ReorderBufferAllocate();
ctx->snapshot_builder =
@@ -319,7 +321,8 @@ CreateInitDecodingContext(const char *plugin,
List *output_plugin_options,
bool need_full_snapshot,
XLogRecPtr restart_lsn,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress)
@@ -422,7 +425,7 @@ CreateInitDecodingContext(const char *plugin,
ctx = StartupDecodingContext(NIL, restart_lsn, xmin_horizon,
need_full_snapshot, false,
- xl_routine, prepare_write, do_write,
+ page_read, cleanup_cb, prepare_write, do_write,
update_progress);
/* call output plugin initialization callback */
@@ -476,7 +479,8 @@ LogicalDecodingContext *
CreateDecodingContext(XLogRecPtr start_lsn,
List *output_plugin_options,
bool fast_forward,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress)
@@ -528,8 +532,8 @@ CreateDecodingContext(XLogRecPtr start_lsn,
ctx = StartupDecodingContext(output_plugin_options,
start_lsn, InvalidTransactionId, false,
- fast_forward, xl_routine, prepare_write,
- do_write, update_progress);
+ fast_forward, page_read, cleanup_cb,
+ prepare_write, do_write, update_progress);
/* call output plugin initialization callback */
old_context = MemoryContextSwitchTo(ctx->context);
@@ -585,7 +589,13 @@ DecodingContextFindStartpoint(LogicalDecodingContext *ctx)
char *err = NULL;
/* the read_page callback waits for new WAL */
- record = XLogReadRecord(ctx->reader, &err);
+ while (XLogReadRecord(ctx->reader, &record, &err) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!ctx->page_read(ctx->reader))
+ break;
+ }
+
if (err)
elog(ERROR, "%s", err);
if (!record)
diff --git a/src/backend/replication/logical/logicalfuncs.c b/src/backend/replication/logical/logicalfuncs.c
index f7e055874e..b69f6911c5 100644
--- a/src/backend/replication/logical/logicalfuncs.c
+++ b/src/backend/replication/logical/logicalfuncs.c
@@ -233,9 +233,8 @@ pg_logical_slot_get_changes_guts(FunctionCallInfo fcinfo, bool confirm, bool bin
ctx = CreateDecodingContext(InvalidXLogRecPtr,
options,
false,
- XL_ROUTINE(.page_read = read_local_xlog_page,
- .segment_open = wal_segment_open,
- .segment_close = wal_segment_close),
+ read_local_xlog_page,
+ wal_segment_close,
LogicalOutputPrepareWrite,
LogicalOutputWrite, NULL);
@@ -284,7 +283,13 @@ pg_logical_slot_get_changes_guts(FunctionCallInfo fcinfo, bool confirm, bool bin
XLogRecord *record;
char *errm = NULL;
- record = XLogReadRecord(ctx->reader, &errm);
+ while (XLogReadRecord(ctx->reader, &record, &errm) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!ctx->page_read(ctx->reader))
+ break;
+ }
+
if (errm)
elog(ERROR, "%s", errm);
diff --git a/src/backend/replication/slotfuncs.c b/src/backend/replication/slotfuncs.c
index 9817b44113..deb7cf15eb 100644
--- a/src/backend/replication/slotfuncs.c
+++ b/src/backend/replication/slotfuncs.c
@@ -153,9 +153,8 @@ create_logical_replication_slot(char *name, char *plugin,
ctx = CreateInitDecodingContext(plugin, NIL,
false, /* just catalogs is OK */
restart_lsn,
- XL_ROUTINE(.page_read = read_local_xlog_page,
- .segment_open = wal_segment_open,
- .segment_close = wal_segment_close),
+ read_local_xlog_page,
+ wal_segment_close,
NULL, NULL, NULL);
/*
@@ -512,9 +511,8 @@ pg_logical_replication_slot_advance(XLogRecPtr moveto)
ctx = CreateDecodingContext(InvalidXLogRecPtr,
NIL,
true, /* fast_forward */
- XL_ROUTINE(.page_read = read_local_xlog_page,
- .segment_open = wal_segment_open,
- .segment_close = wal_segment_close),
+ read_local_xlog_page,
+ wal_segment_close,
NULL, NULL, NULL);
/*
@@ -536,7 +534,13 @@ pg_logical_replication_slot_advance(XLogRecPtr moveto)
* Read records. No changes are generated in fast_forward mode,
* but snapbuilder/slot statuses are updated properly.
*/
- record = XLogReadRecord(ctx->reader, &errm);
+ while (XLogReadRecord(ctx->reader, &record, &errm) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!ctx->page_read(ctx->reader))
+ break;
+ }
+
if (errm)
elog(ERROR, "%s", errm);
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index c36b91ae5e..062093727d 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -580,10 +580,7 @@ StartReplication(StartReplicationCmd *cmd)
/* create xlogreader for physical replication */
xlogreader =
- XLogReaderAllocate(wal_segment_size, NULL,
- XL_ROUTINE(.segment_open = WalSndSegmentOpen,
- .segment_close = wal_segment_close),
- NULL);
+ XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
if (!xlogreader)
ereport(ERROR,
@@ -807,9 +804,11 @@ StartReplication(StartReplicationCmd *cmd)
* set every time WAL is flushed.
*/
static bool
-logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetRecPtr, char *cur_page)
+logical_read_xlog_page(XLogReaderState *state)
{
+ XLogRecPtr targetPagePtr = state->readPagePtr;
+ int reqLen = state->readLen;
+ char *cur_page = state->readBuf;
XLogRecPtr flushptr;
int count;
WALReadError errinfo;
@@ -837,7 +836,7 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
count = flushptr - targetPagePtr; /* part of the page available */
/* now actually read the data, we know it's there */
- if (!WALRead(state,
+ if (!WALRead(state, WalSndSegmentOpen, wal_segment_close,
cur_page,
targetPagePtr,
XLOG_BLCKSZ,
@@ -1011,9 +1010,8 @@ CreateReplicationSlot(CreateReplicationSlotCmd *cmd)
ctx = CreateInitDecodingContext(cmd->plugin, NIL, need_full_snapshot,
InvalidXLogRecPtr,
- XL_ROUTINE(.page_read = logical_read_xlog_page,
- .segment_open = WalSndSegmentOpen,
- .segment_close = wal_segment_close),
+ logical_read_xlog_page,
+ wal_segment_close,
WalSndPrepareWrite, WalSndWriteData,
WalSndUpdateProgress);
@@ -1171,9 +1169,8 @@ StartLogicalReplication(StartReplicationCmd *cmd)
*/
logical_decoding_ctx =
CreateDecodingContext(cmd->startpoint, cmd->options, false,
- XL_ROUTINE(.page_read = logical_read_xlog_page,
- .segment_open = WalSndSegmentOpen,
- .segment_close = wal_segment_close),
+ logical_read_xlog_page,
+ wal_segment_close,
WalSndPrepareWrite, WalSndWriteData,
WalSndUpdateProgress);
xlogreader = logical_decoding_ctx->reader;
@@ -2749,7 +2746,7 @@ XLogSendPhysical(void)
enlargeStringInfo(&output_message, nbytes);
retry:
- if (!WALRead(xlogreader,
+ if (!WALRead(xlogreader, WalSndSegmentOpen, wal_segment_close,
&output_message.data[output_message.len],
startptr,
nbytes,
@@ -2847,7 +2844,12 @@ XLogSendLogical(void)
*/
WalSndCaughtUp = false;
- record = XLogReadRecord(logical_decoding_ctx->reader, &errm);
+ while (XLogReadRecord(logical_decoding_ctx->reader, &record, &errm) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!logical_decoding_ctx->page_read(logical_decoding_ctx->reader))
+ break;
+ }
/* xlog record was invalid */
if (errm != NULL)
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index cf119848b0..712c85281c 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -41,15 +41,9 @@ static int xlogreadfd = -1;
static XLogSegNo xlogreadsegno = -1;
static char xlogfpath[MAXPGPATH];
-typedef struct XLogPageReadPrivate
-{
- const char *restoreCommand;
- int tliIndex;
-} XLogPageReadPrivate;
-
-static bool SimpleXLogPageRead(XLogReaderState *xlogreader,
- XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
+static bool SimpleXLogPageRead(XLogReaderState *xlogreader,
+ const char *datadir, int *tliIndex,
+ const char *restoreCommand);
/*
* Read WAL from the datadir/pg_wal, starting from 'startpoint' on timeline
@@ -66,20 +60,22 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
XLogRecord *record;
XLogReaderState *xlogreader;
char *errormsg;
- XLogPageReadPrivate private;
- private.tliIndex = tliIndex;
- private.restoreCommand = restoreCommand;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir,
- XL_ROUTINE(.page_read = &SimpleXLogPageRead),
- &private);
+ xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+
if (xlogreader == NULL)
pg_fatal("out of memory");
XLogBeginRead(xlogreader, startpoint);
do
{
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!SimpleXLogPageRead(xlogreader, datadir,
+ &tliIndex, restoreCommand))
+ break;
+ }
if (record == NULL)
{
@@ -123,19 +119,19 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex,
XLogRecord *record;
XLogReaderState *xlogreader;
char *errormsg;
- XLogPageReadPrivate private;
XLogRecPtr endptr;
- private.tliIndex = tliIndex;
- private.restoreCommand = restoreCommand;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir,
- XL_ROUTINE(.page_read = &SimpleXLogPageRead),
- &private);
+ xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
XLogBeginRead(xlogreader, ptr);
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!SimpleXLogPageRead(xlogreader, datadir, &tliIndex, restoreCommand))
+ break;
+ }
if (record == NULL)
{
if (errormsg)
@@ -170,7 +166,6 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
XLogRecPtr searchptr;
XLogReaderState *xlogreader;
char *errormsg;
- XLogPageReadPrivate private;
/*
* The given fork pointer points to the end of the last common record,
@@ -186,11 +181,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
forkptr += SizeOfXLogShortPHD;
}
- private.tliIndex = tliIndex;
- private.restoreCommand = restoreCommand;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir,
- XL_ROUTINE(.page_read = &SimpleXLogPageRead),
- &private);
+ xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
@@ -200,7 +191,13 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
uint8 info;
XLogBeginRead(xlogreader, searchptr);
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!SimpleXLogPageRead(xlogreader, datadir,
+ &tliIndex, restoreCommand))
+ break;
+ }
if (record == NULL)
{
@@ -247,10 +244,11 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
/* XLogReader callback function, to read a WAL page */
static bool
-SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *readBuf)
+SimpleXLogPageRead(XLogReaderState *xlogreader, const char *datadir,
+ int *tliIndex, const char *restoreCommand)
{
- XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
+ XLogRecPtr targetPagePtr = xlogreader->readPagePtr;
+ char *readBuf = xlogreader->readBuf;
uint32 targetPageOff;
XLogRecPtr targetSegEnd;
XLogSegNo targetSegNo;
@@ -283,14 +281,14 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
* be done both forward and backward, consider also switching timeline
* accordingly.
*/
- while (private->tliIndex < targetNentries - 1 &&
- targetHistory[private->tliIndex].end < targetSegEnd)
- private->tliIndex++;
- while (private->tliIndex > 0 &&
- targetHistory[private->tliIndex].begin >= targetSegEnd)
- private->tliIndex--;
+ while (*tliIndex < targetNentries - 1 &&
+ targetHistory[*tliIndex].end < targetSegEnd)
+ (*tliIndex)++;
+ while (*tliIndex > 0 &&
+ targetHistory[*tliIndex].begin >= targetSegEnd)
+ (*tliIndex)--;
- XLogFileName(xlogfname, targetHistory[private->tliIndex].tli,
+ XLogFileName(xlogfname, targetHistory[*tliIndex].tli,
xlogreadsegno, WalSegSz);
snprintf(xlogfpath, MAXPGPATH, "%s/" XLOGDIR "/%s",
@@ -303,7 +301,7 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
/*
* If we have no restore_command to execute, then exit.
*/
- if (private->restoreCommand == NULL)
+ if (restoreCommand == NULL)
{
pg_log_error("could not open file \"%s\": %m", xlogfpath);
xlogreader->readLen = -1;
@@ -317,7 +315,7 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
xlogreadfd = RestoreArchivedFile(xlogreader->segcxt.ws_dir,
xlogfname,
WalSegSz,
- private->restoreCommand);
+ restoreCommand);
if (xlogreadfd < 0)
{
@@ -359,7 +357,7 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
Assert(targetSegNo == xlogreadsegno);
- xlogreader->seg.ws_tli = targetHistory[private->tliIndex].tli;
+ xlogreader->seg.ws_tli = targetHistory[*tliIndex].tli;
xlogreader->readLen = XLOG_BLCKSZ;
return true;
}
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index 0f0e0723a3..f514de6b13 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -330,12 +330,17 @@ WALDumpCloseSegment(XLogReaderState *state)
state->seg.ws_file = -1;
}
-/* pg_waldump's XLogReaderRoutine->page_read callback */
+/*
+ * pg_waldump's WAL page rader, also used as page_read callback for
+ * XLogFindNextRecord
+ */
static bool
-WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetPtr, char *readBuff)
+WALDumpReadPage(XLogReaderState *state, void *priv)
{
- XLogDumpPrivate *private = state->private_data;
+ XLogRecPtr targetPagePtr = state->readPagePtr;
+ int reqLen = state->readLen;
+ char *readBuff = state->readBuf;
+ XLogDumpPrivate *private = (XLogDumpPrivate *) priv;
int count = XLOG_BLCKSZ;
WALReadError errinfo;
@@ -353,8 +358,8 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
}
}
- if (!WALRead(state, readBuff, targetPagePtr, count, private->timeline,
- &errinfo))
+ if (!WALRead(state, WALDumpOpenSegment, WALDumpCloseSegment,
+ readBuff, targetPagePtr, count, private->timeline, &errinfo))
{
WALOpenSegment *seg = &errinfo.wre_seg;
char fname[MAXPGPATH];
@@ -374,6 +379,7 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
(Size) errinfo.wre_req);
}
+ Assert(count >= state->readLen);
state->readLen = count;
return true;
}
@@ -1035,16 +1041,14 @@ main(int argc, char **argv)
/* we have everything we need, start reading */
xlogreader_state =
- XLogReaderAllocate(WalSegSz, waldir,
- XL_ROUTINE(.page_read = WALDumpReadPage,
- .segment_open = WALDumpOpenSegment,
- .segment_close = WALDumpCloseSegment),
- &private);
+ XLogReaderAllocate(WalSegSz, waldir, WALDumpCloseSegment);
+
if (!xlogreader_state)
fatal_error("out of memory");
/* first find a valid recptr to start from */
- first_record = XLogFindNextRecord(xlogreader_state, private.startptr);
+ first_record = XLogFindNextRecord(xlogreader_state, private.startptr,
+ &WALDumpReadPage, (void*) &private);
if (first_record == InvalidXLogRecPtr)
fatal_error("could not find a valid record after %X/%X",
@@ -1067,7 +1071,13 @@ main(int argc, char **argv)
for (;;)
{
/* try to read the next record */
- record = XLogReadRecord(xlogreader_state, &errormsg);
+ while (XLogReadRecord(xlogreader_state, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!WALDumpReadPage(xlogreader_state, (void *) &private))
+ break;
+ }
+
if (!record)
{
if (!config.follow || private.endptr_reached)
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index 23d9e70a59..4e9268b553 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -57,64 +57,15 @@ typedef struct WALSegmentContext
typedef struct XLogReaderState XLogReaderState;
-/* Function type definition for the read_page callback */
-typedef bool (*XLogPageReadCB) (XLogReaderState *xlogreader,
- XLogRecPtr targetPagePtr,
- int reqLen,
- XLogRecPtr targetRecPtr,
- char *readBuf);
+/* Function type definition for the segment cleanup callback */
+typedef void (*WALSegmentCleanupCB) (XLogReaderState *xlogreader);
+
+/* Function type definition for the open/close callbacks for WALRead() */
typedef void (*WALSegmentOpenCB) (XLogReaderState *xlogreader,
XLogSegNo nextSegNo,
TimeLineID *tli_p);
typedef void (*WALSegmentCloseCB) (XLogReaderState *xlogreader);
-typedef struct XLogReaderRoutine
-{
- /*
- * Data input callback
- *
- * This callback shall read at least reqLen valid bytes of the xlog page
- * starting at targetPagePtr, and store them in readBuf. The callback
- * shall return the number of bytes read (never more than XLOG_BLCKSZ), or
- * -1 on failure. The callback shall sleep, if necessary, to wait for the
- * requested bytes to become available. The callback will not be invoked
- * again for the same page unless more than the returned number of bytes
- * are needed.
- *
- * targetRecPtr is the position of the WAL record we're reading. Usually
- * it is equal to targetPagePtr + reqLen, but sometimes xlogreader needs
- * to read and verify the page or segment header, before it reads the
- * actual WAL record it's interested in. In that case, targetRecPtr can
- * be used to determine which timeline to read the page from.
- *
- * The callback shall set ->seg.ws_tli to the TLI of the file the page was
- * read from.
- */
- XLogPageReadCB page_read;
-
- /*
- * Callback to open the specified WAL segment for reading. ->seg.ws_file
- * shall be set to the file descriptor of the opened segment. In case of
- * failure, an error shall be raised by the callback and it shall not
- * return.
- *
- * "nextSegNo" is the number of the segment to be opened.
- *
- * "tli_p" is an input/output argument. WALRead() uses it to pass the
- * timeline in which the new segment should be found, but the callback can
- * use it to return the TLI that it actually opened.
- */
- WALSegmentOpenCB segment_open;
-
- /*
- * WAL segment close callback. ->seg.ws_file shall be set to a negative
- * number.
- */
- WALSegmentCloseCB segment_close;
-} XLogReaderRoutine;
-
-#define XL_ROUTINE(...) &(XLogReaderRoutine){__VA_ARGS__}
-
typedef struct
{
/* Is this block ref in use? */
@@ -144,12 +95,35 @@ typedef struct
uint16 data_bufsz;
} DecodedBkpBlock;
+/* Return code from XLogReadRecord */
+typedef enum XLogReadRecordResult
+{
+ XLREAD_SUCCESS, /* record is successfully read */
+ XLREAD_NEED_DATA, /* need more data. see XLogReadRecord. */
+ XLREAD_FAIL /* failed during reading a record */
+} XLogReadRecordResult;
+
+/*
+ * internal state of XLogReadRecord
+ *
+ * XLogReadState runs a state machine while reading a record. Theses states
+ * are not seen outside the function. Each state may repeat several times
+ * exiting requesting caller for new data. See the comment of XLogReadRecrod
+ * for details.
+ */
+typedef enum XLogReadRecordState {
+ XLREAD_NEXT_RECORD,
+ XLREAD_TOT_LEN,
+ XLREAD_FIRST_FRAGMENT,
+ XLREAD_CONTINUATION
+} XLogReadRecordState;
+
struct XLogReaderState
{
/*
* Operational callbacks
*/
- XLogReaderRoutine routine;
+ WALSegmentCleanupCB cleanup_cb;
/* ----------------------------------------
* Public parameters
@@ -162,18 +136,14 @@ struct XLogReaderState
*/
uint64 system_identifier;
- /*
- * Opaque data for callbacks to use. Not used by XLogReader.
- */
- void *private_data;
-
/*
* Start and end point of last record read. EndRecPtr is also used as the
* position to read next. Calling XLogBeginRead() sets EndRecPtr to the
* starting position and ReadRecPtr to invalid.
*/
- XLogRecPtr ReadRecPtr; /* start of last record read */
+ XLogRecPtr ReadRecPtr; /* start of last record read or being read */
XLogRecPtr EndRecPtr; /* end+1 of last record read */
+ XLogRecPtr PrevRecPtr; /* start of previous record read */
/* ----------------------------------------
* Communication with page reader
@@ -187,7 +157,9 @@ struct XLogReaderState
* the request, or -1 on error */
TimeLineID readPageTLI; /* TLI for data currently in readBuf */
char *readBuf; /* buffer to store data */
- bool page_verified; /* is the page on the buffer verified? */
+ bool page_verified; /* is the page header on the buffer verified? */
+ bool record_verified;/* is the current record header verified? */
+
/* ----------------------------------------
@@ -229,8 +201,6 @@ struct XLogReaderState
XLogRecPtr latestPagePtr;
TimeLineID latestPageTLI;
- /* beginning of the WAL record being read. */
- XLogRecPtr currRecPtr;
/* timeline to read it from, 0 if a lookup is required */
TimeLineID currTLI;
@@ -257,6 +227,15 @@ struct XLogReaderState
char *readRecordBuf;
uint32 readRecordBufSize;
+ /*
+ * XLogReadRecord() state
+ */
+ XLogReadRecordState readRecordState;/* state machine state */
+ int recordGotLen; /* amount of current record that has
+ * already been read */
+ int recordRemainLen; /* length of current record that remains */
+ XLogRecPtr recordContRecPtr; /* where the current record continues */
+
/* Buffer to hold error message */
char *errormsg_buf;
};
@@ -264,9 +243,7 @@ struct XLogReaderState
/* Get a new XLogReader */
extern XLogReaderState *XLogReaderAllocate(int wal_segment_size,
const char *waldir,
- XLogReaderRoutine *routine,
- void *private_data);
-extern XLogReaderRoutine *LocalXLogReaderRoutine(void);
+ WALSegmentCleanupCB cleanup_cb);
/* Free an XLogReader */
extern void XLogReaderFree(XLogReaderState *state);
@@ -274,12 +251,17 @@ extern void XLogReaderFree(XLogReaderState *state);
/* Position the XLogReader to given record */
extern void XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr);
#ifdef FRONTEND
-extern XLogRecPtr XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr);
+/* Function type definition for the read_page callback */
+typedef bool (*XLogFindNextRecordCB) (XLogReaderState *xlogreader,
+ void *private);
+extern XLogRecPtr XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
+ XLogFindNextRecordCB read_page, void *private);
#endif /* FRONTEND */
/* Read the next XLog record. Returns NULL on end-of-WAL or failure */
-extern struct XLogRecord *XLogReadRecord(XLogReaderState *state,
- char **errormsg);
+extern XLogReadRecordResult XLogReadRecord(XLogReaderState *state,
+ XLogRecord **record,
+ char **errormsg);
/* Validate a page */
extern bool XLogReaderValidatePageHeader(XLogReaderState *state,
@@ -299,6 +281,7 @@ typedef struct WALReadError
} WALReadError;
extern bool WALRead(XLogReaderState *state,
+ WALSegmentOpenCB segopenfn, WALSegmentCloseCB sgclosefn,
char *buf, XLogRecPtr startptr, Size count,
TimeLineID tli, WALReadError *errinfo);
diff --git a/src/include/access/xlogutils.h b/src/include/access/xlogutils.h
index 364a21c4ea..397fb27fc2 100644
--- a/src/include/access/xlogutils.h
+++ b/src/include/access/xlogutils.h
@@ -47,9 +47,7 @@ extern Buffer XLogReadBufferExtended(RelFileNode rnode, ForkNumber forknum,
extern Relation CreateFakeRelcacheEntry(RelFileNode rnode);
extern void FreeFakeRelcacheEntry(Relation fakerel);
-extern bool read_local_xlog_page(XLogReaderState *state,
- XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetRecPtr, char *cur_page);
+extern bool read_local_xlog_page(XLogReaderState *state);
extern void wal_segment_open(XLogReaderState *state,
XLogSegNo nextSegNo,
TimeLineID *tli_p);
diff --git a/src/include/pg_config_manual.h b/src/include/pg_config_manual.h
index f10ad0acd6..94749609b4 100644
--- a/src/include/pg_config_manual.h
+++ b/src/include/pg_config_manual.h
@@ -377,7 +377,7 @@
* Enable debugging print statements for WAL-related operations; see
* also the wal_debug GUC var.
*/
-/* #define WAL_DEBUG */
+#define WAL_DEBUG
/*
* Enable tracing of resource consumption during sort operations;
diff --git a/src/include/replication/logical.h b/src/include/replication/logical.h
index c253403372..b936f31393 100644
--- a/src/include/replication/logical.h
+++ b/src/include/replication/logical.h
@@ -29,6 +29,10 @@ typedef void (*LogicalOutputPluginWriterUpdateProgress) (struct LogicalDecodingC
TransactionId xid
);
+typedef struct LogicalDecodingContext LogicalDecodingContext;
+
+typedef bool (*LogicalDecodingXLogPageReadCB)(XLogReaderState *ctx);
+
typedef struct LogicalDecodingContext
{
/* memory context this is all allocated in */
@@ -39,6 +43,7 @@ typedef struct LogicalDecodingContext
/* infrastructure pieces for decoding */
XLogReaderState *reader;
+ LogicalDecodingXLogPageReadCB page_read;
struct ReorderBuffer *reorder;
struct SnapBuild *snapshot_builder;
@@ -105,14 +110,16 @@ extern LogicalDecodingContext *CreateInitDecodingContext(const char *plugin,
List *output_plugin_options,
bool need_full_snapshot,
XLogRecPtr restart_lsn,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress);
extern LogicalDecodingContext *CreateDecodingContext(XLogRecPtr start_lsn,
List *output_plugin_options,
bool fast_forward,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress);
--
2.27.0
----Next_Part(Thu_Mar__4_11_28_53_2021_014)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
filename="v17-0003-Remove-globals-readOff-readLen-and-readSegNo.patch"
^ permalink raw reply [nested|flat] 70+ messages in thread
* [PATCH v15 2/4] Move page-reader out of XLogReadRecord
@ 2019-09-10 03:58 Kyotaro Horiguchi <[email protected]>
0 siblings, 0 replies; 70+ messages in thread
From: Kyotaro Horiguchi @ 2019-09-10 03:58 UTC (permalink / raw)
This is the second step of removing callbacks from WAL record reader.
Since it is essential to take in additional data while reading a
record, the function have to ask caller for new data while keeping
working state. Thus the function is turned into a state machine.
---
src/backend/access/transam/twophase.c | 15 +-
src/backend/access/transam/xlog.c | 58 +-
src/backend/access/transam/xlogreader.c | 700 +++++++++++-------
src/backend/access/transam/xlogutils.c | 17 +-
src/backend/replication/logical/logical.c | 26 +-
.../replication/logical/logicalfuncs.c | 13 +-
src/backend/replication/slotfuncs.c | 18 +-
src/backend/replication/walsender.c | 32 +-
src/bin/pg_rewind/parsexlog.c | 84 +--
src/bin/pg_waldump/pg_waldump.c | 36 +-
src/include/access/xlogreader.h | 121 ++-
src/include/access/xlogutils.h | 4 +-
src/include/replication/logical.h | 11 +-
13 files changed, 648 insertions(+), 487 deletions(-)
diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index 9b2e59bf0e..b0d60a0d0f 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -1330,11 +1330,8 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
XLogReaderState *xlogreader;
char *errormsg;
- xlogreader = XLogReaderAllocate(wal_segment_size, NULL,
- XL_ROUTINE(.page_read = &read_local_xlog_page,
- .segment_open = &wal_segment_open,
- .segment_close = &wal_segment_close),
- NULL);
+ xlogreader = XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+
if (!xlogreader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
@@ -1342,7 +1339,13 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
errdetail("Failed while allocating a WAL reading processor.")));
XLogBeginRead(xlogreader, lsn);
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!read_local_xlog_page(xlogreader))
+ break;
+ }
+
if (record == NULL)
ereport(ERROR,
(errcode_for_file_access(),
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index e570e56a24..f9b0108602 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -831,13 +831,6 @@ static XLogSource currentSource = XLOG_FROM_ANY;
static bool lastSourceFailed = false;
static bool pendingWalRcvRestart = false;
-typedef struct XLogPageReadPrivate
-{
- int emode;
- bool fetching_ckpt; /* are we fetching a checkpoint record? */
- bool randAccess;
-} XLogPageReadPrivate;
-
/*
* These variables track when we last obtained some WAL data to process,
* and where we got it from. (XLogReceiptSource is initially the same as
@@ -912,8 +905,8 @@ static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath,
static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli,
XLogSource source, bool notfoundOk);
static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, XLogSource source);
-static bool XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
+static bool XLogPageRead(XLogReaderState *xlogreader,
+ bool fetching_ckpt, int emode, bool randAccess);
static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
bool fetching_ckpt, XLogRecPtr tliRecPtr);
static int emode_for_corrupt_record(int emode, XLogRecPtr RecPtr);
@@ -1225,8 +1218,7 @@ XLogInsertRecord(XLogRecData *rdata,
appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len);
if (!debug_reader)
- debug_reader = XLogReaderAllocate(wal_segment_size, NULL,
- XL_ROUTINE(), NULL);
+ debug_reader = XLogReaderAllocate(wal_segment_size, NULL NULL);
if (!debug_reader)
{
@@ -4326,15 +4318,10 @@ CleanupBackupHistory(void)
* record is available.
*/
static XLogRecord *
-ReadRecord(XLogReaderState *xlogreader, int emode,
- bool fetching_ckpt)
+ReadRecord(XLogReaderState *xlogreader, int emode, bool fetching_ckpt)
{
XLogRecord *record;
- XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
-
- private->fetching_ckpt = fetching_ckpt;
- private->emode = emode;
- private->randAccess = (xlogreader->ReadRecPtr == InvalidXLogRecPtr);
+ bool randAccess = (xlogreader->ReadRecPtr == InvalidXLogRecPtr);
/* This is the first attempt to read this page. */
lastSourceFailed = false;
@@ -4342,8 +4329,16 @@ ReadRecord(XLogReaderState *xlogreader, int emode,
for (;;)
{
char *errormsg;
+ XLogReadRecordResult result;
+
+ while ((result = XLogReadRecord(xlogreader, &record, &errormsg))
+ == XLREAD_NEED_DATA)
+ {
+ if (!XLogPageRead(xlogreader, fetching_ckpt, emode, randAccess))
+ break;
+
+ }
- record = XLogReadRecord(xlogreader, &errormsg);
ReadRecPtr = xlogreader->ReadRecPtr;
EndRecPtr = xlogreader->EndRecPtr;
if (record == NULL)
@@ -6320,7 +6315,6 @@ StartupXLOG(void)
bool backupFromStandby = false;
DBState dbstate_at_startup;
XLogReaderState *xlogreader;
- XLogPageReadPrivate private;
bool fast_promoted = false;
struct stat st;
@@ -6480,13 +6474,9 @@ StartupXLOG(void)
OwnLatch(&XLogCtl->recoveryWakeupLatch);
/* Set up XLOG reader facility */
- MemSet(&private, 0, sizeof(XLogPageReadPrivate));
xlogreader =
- XLogReaderAllocate(wal_segment_size, NULL,
- XL_ROUTINE(.page_read = &XLogPageRead,
- .segment_open = NULL,
- .segment_close = wal_segment_close),
- &private);
+ XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+
if (!xlogreader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
@@ -11860,12 +11850,13 @@ CancelBackup(void)
* sleep and retry.
*/
static bool
-XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetRecPtr, char *readBuf)
+XLogPageRead(XLogReaderState *xlogreader,
+ bool fetching_ckpt, int emode, bool randAccess)
{
- XLogPageReadPrivate *private =
- (XLogPageReadPrivate *) xlogreader->private_data;
- int emode = private->emode;
+ char *readBuf = xlogreader->readBuf;
+ XLogRecPtr targetPagePtr = xlogreader->readPagePtr;
+ int reqLen = xlogreader->readLen;
+ XLogRecPtr targetRecPtr = xlogreader->ReadRecPtr;
uint32 targetPageOff;
XLogSegNo targetSegNo PG_USED_FOR_ASSERTS_ONLY;
int r;
@@ -11908,8 +11899,8 @@ retry:
flushedUpto < targetPagePtr + reqLen))
{
if (!WaitForWALToBecomeAvailable(targetPagePtr + reqLen,
- private->randAccess,
- private->fetching_ckpt,
+ randAccess,
+ fetching_ckpt,
targetRecPtr))
{
if (readFile >= 0)
@@ -12014,6 +12005,7 @@ retry:
goto next_record_is_invalid;
}
+ Assert(xlogreader->readPagePtr == targetPagePtr);
xlogreader->readLen = readLen;
return true;
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index 3d599325ee..9281b57379 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -40,7 +40,7 @@ static bool XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr,
int reqLen, bool header_inclusive);
static void XLogReaderInvalReadState(XLogReaderState *state);
static bool ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
- XLogRecPtr PrevRecPtr, XLogRecord *record, bool randAccess);
+ XLogRecPtr PrevRecPtr, XLogRecord *record);
static bool ValidXLogRecord(XLogReaderState *state, XLogRecord *record,
XLogRecPtr recptr);
static void ResetDecoder(XLogReaderState *state);
@@ -73,7 +73,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...)
*/
XLogReaderState *
XLogReaderAllocate(int wal_segment_size, const char *waldir,
- XLogReaderRoutine *routine, void *private_data)
+ WALSegmentCleanupCB cleanup_cb)
{
XLogReaderState *state;
@@ -84,7 +84,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
return NULL;
/* initialize caller-provided support functions */
- state->routine = *routine;
+ state->cleanup_cb = cleanup_cb;
state->max_block_id = -1;
@@ -107,8 +107,6 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
WALOpenSegmentInit(&state->seg, &state->segcxt, wal_segment_size,
waldir);
- /* system_identifier initialized to zeroes above */
- state->private_data = private_data;
/* ReadRecPtr, EndRecPtr and readLen initialized to zeroes above */
state->errormsg_buf = palloc_extended(MAX_ERRORMSG_LEN + 1,
MCXT_ALLOC_NO_OOM);
@@ -140,8 +138,8 @@ XLogReaderFree(XLogReaderState *state)
{
int block_id;
- if (state->seg.ws_file != -1)
- state->routine.segment_close(state);
+ if (state->seg.ws_file >= 0)
+ state->cleanup_cb(state);
for (block_id = 0; block_id <= XLR_MAX_BLOCK_ID; block_id++)
{
@@ -246,6 +244,7 @@ XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
/* Begin at the passed-in record pointer. */
state->EndRecPtr = RecPtr;
state->ReadRecPtr = InvalidXLogRecPtr;
+ state->readRecordState = XLREAD_NEXT_RECORD;
}
/*
@@ -254,317 +253,452 @@ XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
* XLogBeginRead() or XLogFindNextRecord() must be called before the first call
* to XLogReadRecord().
*
- * If the page_read callback fails to read the requested data, NULL is
- * returned. The callback is expected to have reported the error; errormsg
- * is set to NULL.
+ * This function may return XLREAD_NEED_DATA several times before returning a
+ * result record. The caller shall read in some new data then call this
+ * function again with the same parameters.
*
- * If the reading fails for some other reason, NULL is also returned, and
- * *errormsg is set to a string with details of the failure.
+ * When a record is successfully read, returns XLREAD_SUCCESS with result
+ * record being stored in *record. Otherwise *record is NULL.
*
- * The returned pointer (or *errormsg) points to an internal buffer that's
- * valid until the next call to XLogReadRecord.
+ * Returns XLREAD_NEED_DATA if more data is needed to finish reading the
+ * current record. In that case, state->readPagePtr and state->readLen inform
+ * the desired position and minimum length of data needed. The caller shall
+ * read in the requested data and set state->readBuf to point to a buffer
+ * containing it. The caller must also set state->readPageTLI and
+ * state->readLen to indicate the timeline that it was read from, and the
+ * length of data that is now available (which must be >= given readLen),
+ * respectively.
+ *
+ * If invalid data is encountered, returns XLREAD_FAIL with *record being set to
+ * NULL. *errormsg is set to a string with details of the failure.
+ * The returned pointer (or *errormsg) points to an internal buffer that's valid
+ * until the next call to XLogReadRecord.
+ *
+ *
+ * This function runs a state machine consists of the following states.
+ *
+ * XLREAD_NEXT_RECORD :
+ * The initial state, if called with valid RecPtr, try to read a record at
+ * that position. If invalid RecPtr is given try to read a record just after
+ * the last one previously read.
+ * This state ens after setting ReadRecPtr. Then goes to XLREAD_TOT_LEN.
+ *
+ * XLREAD_TOT_LEN:
+ * Examining record header. Ends after reading record total
+ * length. recordRemainLen and recordGotLen are initialized.
+ *
+ * XLREAD_FIRST_FRAGMENT:
+ * Reading the first fragment. Ends with finishing reading a single
+ * record. Goes to XLREAD_NEXT_RECORD if that's all or
+ * XLREAD_CONTINUATION if we have continuation.
+
+ * XLREAD_CONTINUATION:
+ * Reading continuation of record. Ends with finishing the whole record then
+ * goes to XLREAD_NEXT_RECORD. During this state, recordRemainLen indicates
+ * how much is left and readRecordBuf holds the partially assert
+ * record.recordContRecPtr points to the beginning of the next page where to
+ * continue.
+ *
+ * If wrong data found in any state, the state machine stays at the current
+ * state. This behavior allows to continue reading a reacord switching among
+ * different souces, while streaming replication.
*/
-XLogRecord *
-XLogReadRecord(XLogReaderState *state, char **errormsg)
+XLogReadRecordResult
+XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
{
- XLogRecPtr RecPtr;
- XLogRecord *record;
- XLogRecPtr targetPagePtr;
- bool randAccess;
- uint32 len,
- total_len;
- uint32 targetRecOff;
- uint32 pageHeaderSize;
- bool gotheader;
+ XLogRecord *prec;
- /*
- * randAccess indicates whether to verify the previous-record pointer of
- * the record we're reading. We only do this if we're reading
- * sequentially, which is what we initially assume.
- */
- randAccess = false;
+ *record = NULL;
/* reset error state */
*errormsg = NULL;
state->errormsg_buf[0] = '\0';
- ResetDecoder(state);
-
- RecPtr = state->EndRecPtr;
-
- if (state->ReadRecPtr != InvalidXLogRecPtr)
- {
- /* read the record after the one we just read */
-
- /*
- * EndRecPtr is pointing to end+1 of the previous WAL record. If
- * we're at a page boundary, no more records can fit on the current
- * page. We must skip over the page header, but we can't do that until
- * we've read in the page, since the header size is variable.
- */
- }
- else
- {
- /*
- * Caller supplied a position to start at.
- *
- * In this case, EndRecPtr should already be pointing to a valid
- * record starting position.
- */
- Assert(XRecOffIsValid(RecPtr));
- randAccess = true;
- }
-
- state->currRecPtr = RecPtr;
-
- targetPagePtr = RecPtr - (RecPtr % XLOG_BLCKSZ);
- targetRecOff = RecPtr % XLOG_BLCKSZ;
-
- /*
- * Read the page containing the record into state->readBuf. Request enough
- * byte to cover the whole record header, or at least the part of it that
- * fits on the same page.
- */
- while (XLogNeedData(state, targetPagePtr,
- Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
- targetRecOff != 0))
- {
- if (!state->routine.page_read(state, state->readPagePtr, state->readLen,
- RecPtr, state->readBuf))
- break;
- }
-
- if (!state->page_verified)
- goto err;
-
- /*
- * We have at least the page header, so we can examine it now.
- */
- pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
- if (targetRecOff == 0)
- {
- /*
- * At page start, so skip over page header.
- */
- RecPtr += pageHeaderSize;
- targetRecOff = pageHeaderSize;
- }
- else if (targetRecOff < pageHeaderSize)
- {
- report_invalid_record(state, "invalid record offset at %X/%X",
- (uint32) (RecPtr >> 32), (uint32) RecPtr);
- goto err;
- }
-
- if ((((XLogPageHeader) state->readBuf)->xlp_info & XLP_FIRST_IS_CONTRECORD) &&
- targetRecOff == pageHeaderSize)
- {
- report_invalid_record(state, "contrecord is requested by %X/%X",
- (uint32) (RecPtr >> 32), (uint32) RecPtr);
- goto err;
- }
-
- /* XLogNeedData has verified the page header */
- Assert(pageHeaderSize <= state->readLen);
-
- /*
- * Read the record length.
- *
- * NB: Even though we use an XLogRecord pointer here, the whole record
- * header might not fit on this page. xl_tot_len is the first field of the
- * struct, so it must be on this page (the records are MAXALIGNed), but we
- * cannot access any other fields until we've verified that we got the
- * whole header.
- */
- record = (XLogRecord *) (state->readBuf + RecPtr % XLOG_BLCKSZ);
- total_len = record->xl_tot_len;
-
- /*
- * If the whole record header is on this page, validate it immediately.
- * Otherwise do just a basic sanity check on xl_tot_len, and validate the
- * rest of the header after reading it from the next page. The xl_tot_len
- * check is necessary here to ensure that we enter the "Need to reassemble
- * record" code path below; otherwise we might fail to apply
- * ValidXLogRecordHeader at all.
- */
- if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
- {
- if (!ValidXLogRecordHeader(state, RecPtr, state->ReadRecPtr, record,
- randAccess))
- goto err;
- gotheader = true;
- }
- else
- {
- /* XXX: more validation should be done here */
- if (total_len < SizeOfXLogRecord)
- {
- report_invalid_record(state,
- "invalid record length at %X/%X: wanted %u, got %u",
- (uint32) (RecPtr >> 32), (uint32) RecPtr,
- (uint32) SizeOfXLogRecord, total_len);
- goto err;
- }
- gotheader = false;
- }
-
- len = XLOG_BLCKSZ - RecPtr % XLOG_BLCKSZ;
- if (total_len > len)
+ switch (state->readRecordState)
{
- /* Need to reassemble record */
- char *contdata;
- XLogPageHeader pageHeader;
- char *buffer;
- uint32 gotlen;
-
- /*
- * Enlarge readRecordBuf as needed.
- */
- if (total_len > state->readRecordBufSize &&
- !allocate_recordbuf(state, total_len))
- {
- /* We treat this as a "bogus data" condition */
- report_invalid_record(state, "record length %u at %X/%X too long",
- total_len,
- (uint32) (RecPtr >> 32), (uint32) RecPtr);
- goto err;
- }
+ case XLREAD_NEXT_RECORD:
+ ResetDecoder(state);
- /* Copy the first fragment of the record from the first page. */
- memcpy(state->readRecordBuf,
- state->readBuf + RecPtr % XLOG_BLCKSZ, len);
- buffer = state->readRecordBuf + len;
- gotlen = len;
-
- do
- {
- int rest_len = total_len - gotlen;
-
- /* Calculate pointer to beginning of next page */
- targetPagePtr += XLOG_BLCKSZ;
-
- /* Wait for the next page to become available */
- while (XLogNeedData(state, targetPagePtr,
- Min(rest_len, XLOG_BLCKSZ),
- false))
+ if (state->ReadRecPtr != InvalidXLogRecPtr)
{
- if (!state->routine.page_read(state, state->readPagePtr,
- state->readLen,
- state->ReadRecPtr,
- state->readBuf))
- break;
+ /* read the record after the one we just read */
+
+ /*
+ * EndRecPtr is pointing to end+1 of the previous WAL record.
+ * If we're at a page boundary, no more records can fit on the
+ * current page. We must skip over the page header, but we
+ * can't do that until we've read in the page, since the header
+ * size is variable.
+ */
+ state->PrevRecPtr = state->ReadRecPtr;
+ state->ReadRecPtr = state->EndRecPtr;
+ }
+ else
+ {
+ /*
+ * Caller supplied a position to start at.
+ *
+ * In this case, EndRecPtr should already be pointing to a
+ * valid record starting position.
+ */
+ Assert(XRecOffIsValid(state->EndRecPtr));
+ state->ReadRecPtr = state->EndRecPtr;
+
+ /*
+ * We cannot verify the previous-record pointer when we're
+ * seeking to a particular record. Reset PrevRecPtr so that we
+ * won't try doing that.
+ */
+ state->PrevRecPtr = InvalidXLogRecPtr;
+ state->EndRecPtr = InvalidXLogRecPtr; /* to be tidy */
}
+ state->record_verified = false;
+ state->readRecordState = XLREAD_TOT_LEN;
+ /* fall through */
+
+ case XLREAD_TOT_LEN:
+ {
+ uint32 total_len;
+ uint32 pageHeaderSize;
+ XLogRecPtr targetPagePtr;
+ uint32 targetRecOff;
+ XLogPageHeader pageHeader;
+
+ targetPagePtr =
+ state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
+ targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+
+ /*
+ * Check if we have enough data. For the first record in the page,
+ * the requesting length doesn't contain page header.
+ */
+ if (XLogNeedData(state, targetPagePtr,
+ Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+ targetRecOff != 0))
+ return XLREAD_NEED_DATA;
+
+ /* error out if caller supplied bogus page */
if (!state->page_verified)
goto err;
- Assert(SizeOfXLogShortPHD <= state->readLen);
+ /* examine page header now. */
+ pageHeaderSize =
+ XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+ if (targetRecOff == 0)
+ {
+ /* At page start, so skip over page header. */
+ state->ReadRecPtr += pageHeaderSize;
+ targetRecOff = pageHeaderSize;
+ }
+ else if (targetRecOff < pageHeaderSize)
+ {
+ report_invalid_record(state, "invalid record offset at %X/%X",
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr);
+ goto err;
+ }
- /* Check that the continuation on next page looks valid */
pageHeader = (XLogPageHeader) state->readBuf;
- if (!(pageHeader->xlp_info & XLP_FIRST_IS_CONTRECORD))
+ if ((pageHeader->xlp_info & XLP_FIRST_IS_CONTRECORD) &&
+ targetRecOff == pageHeaderSize)
{
- report_invalid_record(state,
- "there is no contrecord flag at %X/%X",
- (uint32) (RecPtr >> 32), (uint32) RecPtr);
+ report_invalid_record(state, "contrecord is requested by %X/%X",
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr);
goto err;
}
- /*
- * Cross-check that xlp_rem_len agrees with how much of the record
- * we expect there to be left.
- */
- if (pageHeader->xlp_rem_len == 0 ||
- total_len != (pageHeader->xlp_rem_len + gotlen))
- {
- report_invalid_record(state,
- "invalid contrecord length %u at %X/%X",
- pageHeader->xlp_rem_len,
- (uint32) (RecPtr >> 32), (uint32) RecPtr);
- goto err;
- }
-
- /* Append the continuation from this page to the buffer */
- pageHeaderSize = XLogPageHeaderSize(pageHeader);
-
+ /* XLogNeedData has verified the page header */
Assert(pageHeaderSize <= state->readLen);
- contdata = (char *) state->readBuf + pageHeaderSize;
- len = XLOG_BLCKSZ - pageHeaderSize;
- if (pageHeader->xlp_rem_len < len)
- len = pageHeader->xlp_rem_len;
+ /*
+ * Read the record length.
+ *
+ * NB: Even though we use an XLogRecord pointer here, the whole
+ * record header might not fit on this page. xl_tot_len is the first
+ * field of the struct, so it must be on this page (the records are
+ * MAXALIGNed), but we cannot access any other fields until we've
+ * verified that we got the whole header.
+ */
+ prec = (XLogRecord *) (state->readBuf +
+ state->ReadRecPtr % XLOG_BLCKSZ);
+ total_len = prec->xl_tot_len;
- Assert (pageHeaderSize + len <= state->readLen);
- memcpy(buffer, (char *) contdata, len);
- buffer += len;
- gotlen += len;
+ /*
+ * If the whole record header is on this page, validate it
+ * immediately. Otherwise do just a basic sanity check on
+ * xl_tot_len, and validate the rest of the header after reading it
+ * from the next page. The xl_tot_len check is necessary here to
+ * ensure that we enter the XLREAD_CONTINUATION state below;
+ * otherwise we might fail to apply ValidXLogRecordHeader at all.
+ */
+ if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
+ {
+ if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
+ state->PrevRecPtr, prec))
+ goto err;
- /* If we just reassembled the record header, validate it. */
- if (!gotheader)
+ state->record_verified = true;
+ }
+ else
{
- record = (XLogRecord *) state->readRecordBuf;
- if (!ValidXLogRecordHeader(state, RecPtr, state->ReadRecPtr,
- record, randAccess))
+ /* XXX: more validation should be done here */
+ if (total_len < SizeOfXLogRecord)
+ {
+ report_invalid_record(state,
+ "invalid record length at %X/%X: wanted %u, got %u",
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr,
+ (uint32) SizeOfXLogRecord, total_len);
goto err;
- gotheader = true;
+ }
}
- } while (gotlen < total_len);
-
- Assert(gotheader);
- record = (XLogRecord *) state->readRecordBuf;
- if (!ValidXLogRecord(state, record, RecPtr))
- goto err;
+ /*
+ * Wait for the rest of the record, or the part of the record that
+ * fit on the first page if crossed a page boundary, to become
+ * available.
+ */
+ state->recordGotLen = 0;
+ state->recordRemainLen = total_len;
+ state->readRecordState = XLREAD_FIRST_FRAGMENT;
+ }
+ /* fall through */
- pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
- state->ReadRecPtr = RecPtr;
- state->EndRecPtr = targetPagePtr + pageHeaderSize
- + MAXALIGN(pageHeader->xlp_rem_len);
- }
- else
- {
- /* Wait for the record data to become available */
- while (XLogNeedData(state, targetPagePtr,
- Min(targetRecOff + total_len, XLOG_BLCKSZ), true))
+ case XLREAD_FIRST_FRAGMENT:
{
- if (!state->routine.page_read(state, state->readPagePtr,
- state->readLen,
- state->ReadRecPtr, state->readBuf))
+ uint32 total_len = state->recordRemainLen;
+ uint32 request_len;
+ uint32 record_len;
+ XLogRecPtr targetPagePtr;
+ uint32 targetRecOff;
+
+ /*
+ * Wait for the rest of the record on the first page to become
+ * available
+ */
+ targetPagePtr =
+ state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
+ targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+
+ request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ);
+ record_len = request_len - targetRecOff;
+
+ /* ReadRecPtr contains page header */
+ Assert (targetRecOff != 0);
+ if (XLogNeedData(state, targetPagePtr, request_len, true))
+ return XLREAD_NEED_DATA;
+
+ /* error out if caller supplied bogus page */
+ if (!state->page_verified)
+ goto err;
+
+ prec = (XLogRecord *) (state->readBuf + targetRecOff);
+
+ /* validate record header if not yet */
+ if (!state->record_verified && record_len >= SizeOfXLogRecord)
+ {
+ if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
+ state->PrevRecPtr, prec))
+ goto err;
+
+ state->record_verified = true;
+ }
+
+
+ if (total_len == record_len)
+ {
+ /* Record does not cross a page boundary */
+ Assert(state->record_verified);
+
+ if (!ValidXLogRecord(state, prec, state->ReadRecPtr))
+ goto err;
+
+ state->record_verified = true; /* to be tidy */
+
+ /* We already checked the header earlier */
+ state->EndRecPtr = state->ReadRecPtr + MAXALIGN(record_len);
+
+ *record = prec;
+ state->readRecordState = XLREAD_NEXT_RECORD;
break;
+ }
+
+ /*
+ * The record continues on the next page. Need to reassemble
+ * record
+ */
+ Assert(total_len > record_len);
+
+ /* Enlarge readRecordBuf as needed. */
+ if (total_len > state->readRecordBufSize &&
+ !allocate_recordbuf(state, total_len))
+ {
+ /* We treat this as a "bogus data" condition */
+ report_invalid_record(state,
+ "record length %u at %X/%X too long",
+ total_len,
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr);
+ goto err;
+ }
+
+ /* Copy the first fragment of the record from the first page. */
+ memcpy(state->readRecordBuf, state->readBuf + targetRecOff,
+ record_len);
+ state->recordGotLen += record_len;
+ state->recordRemainLen -= record_len;
+
+ /* Calculate pointer to beginning of next page */
+ state->recordContRecPtr = state->ReadRecPtr + record_len;
+ Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0);
+
+ state->readRecordState = XLREAD_CONTINUATION;
}
+ /* fall through */
+
+ case XLREAD_CONTINUATION:
+ {
+ XLogPageHeader pageHeader;
+ uint32 pageHeaderSize;
+ XLogRecPtr targetPagePtr;
+
+ /* we enter this state only if we haven't read the whole record. */
+ Assert (state->recordRemainLen > 0);
+
+ while(state->recordRemainLen > 0)
+ {
+ char *contdata;
+ uint32 request_len;
+ uint32 record_len;
+
+ /* Wait for the next page to become available */
+ targetPagePtr = state->recordContRecPtr;
+
+ /* this request contains page header */
+ Assert (targetPagePtr != 0);
+ if (XLogNeedData(state, targetPagePtr,
+ Min(state->recordRemainLen, XLOG_BLCKSZ),
+ false))
+ return XLREAD_NEED_DATA;
+
+ if (!state->page_verified)
+ goto err;
+
+ Assert(SizeOfXLogShortPHD <= state->readLen);
- if (!state->page_verified)
- goto err;
+ /* Check that the continuation on next page looks valid */
+ pageHeader = (XLogPageHeader) state->readBuf;
+ if (!(pageHeader->xlp_info & XLP_FIRST_IS_CONTRECORD))
+ {
+ report_invalid_record(
+ state,
+ "there is no contrecord flag at %X/%X reading %X/%X",
+ (uint32) (state->recordContRecPtr >> 32),
+ (uint32) state->recordContRecPtr,
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr);
+ goto err;
+ }
- /* Record does not cross a page boundary */
- if (!ValidXLogRecord(state, record, RecPtr))
- goto err;
+ /*
+ * Cross-check that xlp_rem_len agrees with how much of the
+ * record we expect there to be left.
+ */
+ if (pageHeader->xlp_rem_len == 0 ||
+ pageHeader->xlp_rem_len != state->recordRemainLen)
+ {
+ report_invalid_record(
+ state,
+ "invalid contrecord length %u at %X/%X reading %X/%X, expected %u",
+ pageHeader->xlp_rem_len,
+ (uint32) (state->recordContRecPtr >> 32),
+ (uint32) state->recordContRecPtr,
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr,
+ state->recordRemainLen);
+ goto err;
+ }
- state->EndRecPtr = RecPtr + MAXALIGN(total_len);
+ /* Append the continuation from this page to the buffer */
+ pageHeaderSize = XLogPageHeaderSize(pageHeader);
- state->ReadRecPtr = RecPtr;
+ /*
+ * XLogNeedData should have ensured that the whole page header
+ * was read
+ */
+ Assert(state->readLen >= pageHeaderSize);
+
+ contdata = (char *) state->readBuf + pageHeaderSize;
+ record_len = XLOG_BLCKSZ - pageHeaderSize;
+ if (pageHeader->xlp_rem_len < record_len)
+ record_len = pageHeader->xlp_rem_len;
+
+ request_len = record_len + pageHeaderSize;
+
+ /* XLogNeedData should have ensured all needed data was read */
+ Assert (state->readLen >= request_len);
+
+ memcpy(state->readRecordBuf + state->recordGotLen,
+ (char *) contdata, record_len);
+ state->recordGotLen += record_len;
+ state->recordRemainLen -= record_len;
+
+ /* If we just reassembled the record header, validate it. */
+ if (!state->record_verified)
+ {
+ Assert(state->recordGotLen >= SizeOfXLogRecord);
+ if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
+ state->PrevRecPtr,
+ (XLogRecord *) state->readRecordBuf))
+ goto err;
+
+ state->record_verified = true;
+ }
+
+ /* Calculate pointer to beginning of next page, and continue */
+ state->recordContRecPtr += XLOG_BLCKSZ;
+ }
+
+ /* targetPagePtr is pointing the last-read page here */
+ prec = (XLogRecord *) state->readRecordBuf;
+ if (!ValidXLogRecord(state, prec, state->ReadRecPtr))
+ goto err;
+
+ pageHeaderSize =
+ XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+ state->EndRecPtr = targetPagePtr + pageHeaderSize
+ + MAXALIGN(pageHeader->xlp_rem_len);
+
+ *record = prec;
+ state->readRecordState = XLREAD_NEXT_RECORD;
+ break;
+ }
}
/*
* Special processing if it's an XLOG SWITCH record
*/
- if (record->xl_rmid == RM_XLOG_ID &&
- (record->xl_info & ~XLR_INFO_MASK) == XLOG_SWITCH)
+ if ((*record)->xl_rmid == RM_XLOG_ID &&
+ ((*record)->xl_info & ~XLR_INFO_MASK) == XLOG_SWITCH)
{
/* Pretend it extends to end of segment */
state->EndRecPtr += state->segcxt.ws_segsize - 1;
state->EndRecPtr -= XLogSegmentOffset(state->EndRecPtr, state->segcxt.ws_segsize);
}
- if (DecodeXLogRecord(state, record, errormsg))
- return record;
- else
- return NULL;
+ Assert (!*record || state->readLen >= 0);
+ if (DecodeXLogRecord(state, *record, errormsg))
+ return XLREAD_SUCCESS;
+
+ *record = NULL;
+ return XLREAD_FAIL;
err:
/*
- * Invalidate the read state. We might read from a different source after
+ * Invalidate the read page. We might read from a different source after
* failure.
*/
XLogReaderInvalReadState(state);
@@ -572,7 +706,8 @@ err:
if (state->errormsg_buf[0] != '\0')
*errormsg = state->errormsg_buf;
- return NULL;
+ *record = NULL;
+ return XLREAD_FAIL;
}
/*
@@ -724,11 +859,12 @@ XLogReaderInvalReadState(XLogReaderState *state)
*
* This is just a convenience subroutine to avoid duplicated code in
* XLogReadRecord. It's not intended for use from anywhere else.
+ *
+ * If PrevRecPtr is valid, the xl_prev is is cross-checked with it.
*/
static bool
ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
- XLogRecPtr PrevRecPtr, XLogRecord *record,
- bool randAccess)
+ XLogRecPtr PrevRecPtr, XLogRecord *record)
{
if (record->xl_tot_len < SizeOfXLogRecord)
{
@@ -746,7 +882,7 @@ ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
(uint32) RecPtr);
return false;
}
- if (randAccess)
+ if (PrevRecPtr == InvalidXLogRecPtr)
{
/*
* We can't exactly verify the prev-link, but surely it should be less
@@ -978,11 +1114,14 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
* XLogReadRecord() will read the next valid record.
*/
XLogRecPtr
-XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
+XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
+ XLogFindNextRecordCB read_page, void *private)
{
XLogRecPtr tmpRecPtr;
XLogRecPtr found = InvalidXLogRecPtr;
XLogPageHeader header;
+ XLogRecord *record;
+ XLogReadRecordResult result;
char *errormsg;
Assert(!XLogRecPtrIsInvalid(RecPtr));
@@ -1015,9 +1154,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
while(XLogNeedData(state, targetPagePtr, targetRecOff,
targetRecOff != 0))
{
- if (!state->routine.page_read(state, state->readPagePtr,
- state->readLen,
- state->ReadRecPtr, state->readBuf))
+ if (!read_page(state, private))
break;
}
@@ -1069,8 +1206,16 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
* or we just jumped over the remaining data of a continuation.
*/
XLogBeginRead(state, tmpRecPtr);
- while (XLogReadRecord(state, &errormsg) != NULL)
+ while ((result = XLogReadRecord(state, &record, &errormsg)) !=
+ XLREAD_FAIL)
{
+ if (result == XLREAD_NEED_DATA)
+ {
+ if (!read_page(state, private))
+ goto err;
+ continue;
+ }
+
/* past the record we've found, break out */
if (RecPtr <= state->ReadRecPtr)
{
@@ -1090,9 +1235,9 @@ err:
#endif /* FRONTEND */
/*
- * Helper function to ease writing of XLogRoutine->page_read callbacks.
- * If this function is used, caller must supply a segment_open callback in
- * 'state', as that is used here.
+ * Helper function to ease writing of page_read callback.
+ * If this function is used, caller must supply a segment_open callback and
+ * segment_close callback as that is used here.
*
* Read 'count' bytes into 'buf', starting at location 'startptr', from WAL
* fetched from timeline 'tli'.
@@ -1105,6 +1250,7 @@ err:
*/
bool
WALRead(XLogReaderState *state,
+ WALSegmentOpenCB segopenfn, WALSegmentCloseCB segclosefn,
char *buf, XLogRecPtr startptr, Size count, TimeLineID tli,
WALReadError *errinfo)
{
@@ -1136,10 +1282,10 @@ WALRead(XLogReaderState *state,
XLogSegNo nextSegNo;
if (state->seg.ws_file >= 0)
- state->routine.segment_close(state);
+ segclosefn(state);
XLByteToSeg(recptr, nextSegNo, state->segcxt.ws_segsize);
- state->routine.segment_open(state, nextSegNo, &tli);
+ segopenfn(state, nextSegNo, &tli);
/* This shouldn't happen -- indicates a bug in segment_open */
Assert(state->seg.ws_file >= 0);
diff --git a/src/backend/access/transam/xlogutils.c b/src/backend/access/transam/xlogutils.c
index 1b10b6df20..303333571f 100644
--- a/src/backend/access/transam/xlogutils.c
+++ b/src/backend/access/transam/xlogutils.c
@@ -689,8 +689,7 @@ XLogTruncateRelation(RelFileNode rnode, ForkNumber forkNum,
void
XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wantLength)
{
- const XLogRecPtr lastReadPage = state->seg.ws_segno *
- state->segcxt.ws_segsize + state->readLen;
+ const XLogRecPtr lastReadPage = state->readPagePtr;
Assert(wantPage != InvalidXLogRecPtr && wantPage % XLOG_BLCKSZ == 0);
Assert(wantLength <= XLOG_BLCKSZ);
@@ -705,7 +704,7 @@ XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wa
* current TLI has since become historical.
*/
if (lastReadPage == wantPage &&
- state->readLen != 0 &&
+ state->page_verified &&
lastReadPage + state->readLen >= wantPage + Min(wantLength, XLOG_BLCKSZ - 1))
return;
@@ -792,6 +791,7 @@ wal_segment_open(XLogReaderState *state, XLogSegNo nextSegNo,
char path[MAXPGPATH];
XLogFilePath(path, tli, nextSegNo, state->segcxt.ws_segsize);
+ elog(LOG, "HOGE: %lu, %d => %s", nextSegNo, tli, path);
state->seg.ws_file = BasicOpenFile(path, O_RDONLY | PG_BINARY);
if (state->seg.ws_file >= 0)
return;
@@ -829,9 +829,11 @@ wal_segment_close(XLogReaderState *state)
* loop for now.
*/
bool
-read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *cur_page)
+read_local_xlog_page(XLogReaderState *state)
{
+ XLogRecPtr targetPagePtr = state->readPagePtr;
+ int reqLen = state->readLen;
+ char *cur_page = state->readBuf;
XLogRecPtr read_upto,
loc;
TimeLineID tli;
@@ -944,11 +946,12 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
* as 'count', read the whole page anyway. It's guaranteed to be
* zero-padded up to the page boundary if it's incomplete.
*/
- if (!WALRead(state, cur_page, targetPagePtr, XLOG_BLCKSZ, tli,
- &errinfo))
+ if (!WALRead(state, wal_segment_open, wal_segment_close,
+ cur_page, targetPagePtr, XLOG_BLCKSZ, tli, &errinfo))
WALReadRaiseError(&errinfo);
/* number of valid bytes in the buffer */
+ state->readPagePtr = targetPagePtr;
state->readLen = count;
return true;
}
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index 61902be3b0..a15b0b3355 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -120,7 +120,8 @@ StartupDecodingContext(List *output_plugin_options,
TransactionId xmin_horizon,
bool need_full_snapshot,
bool fast_forward,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress)
@@ -169,11 +170,12 @@ StartupDecodingContext(List *output_plugin_options,
ctx->slot = slot;
- ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, xl_routine, ctx);
+ ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, cleanup_cb);
if (!ctx->reader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
errmsg("out of memory")));
+ ctx->page_read = page_read;
ctx->reorder = ReorderBufferAllocate();
ctx->snapshot_builder =
@@ -231,7 +233,8 @@ CreateInitDecodingContext(char *plugin,
List *output_plugin_options,
bool need_full_snapshot,
XLogRecPtr restart_lsn,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress)
@@ -328,7 +331,7 @@ CreateInitDecodingContext(char *plugin,
ctx = StartupDecodingContext(NIL, restart_lsn, xmin_horizon,
need_full_snapshot, false,
- xl_routine, prepare_write, do_write,
+ page_read, cleanup_cb, prepare_write, do_write,
update_progress);
/* call output plugin initialization callback */
@@ -376,7 +379,8 @@ LogicalDecodingContext *
CreateDecodingContext(XLogRecPtr start_lsn,
List *output_plugin_options,
bool fast_forward,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress)
@@ -429,8 +433,8 @@ CreateDecodingContext(XLogRecPtr start_lsn,
ctx = StartupDecodingContext(output_plugin_options,
start_lsn, InvalidTransactionId, false,
- fast_forward, xl_routine, prepare_write,
- do_write, update_progress);
+ fast_forward, page_read, cleanup_cb,
+ prepare_write, do_write, update_progress);
/* call output plugin initialization callback */
old_context = MemoryContextSwitchTo(ctx->context);
@@ -483,7 +487,13 @@ DecodingContextFindStartpoint(LogicalDecodingContext *ctx)
char *err = NULL;
/* the read_page callback waits for new WAL */
- record = XLogReadRecord(ctx->reader, &err);
+ while (XLogReadRecord(ctx->reader, &record, &err) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!ctx->page_read(ctx->reader))
+ break;
+ }
+
if (err)
elog(ERROR, "%s", err);
if (!record)
diff --git a/src/backend/replication/logical/logicalfuncs.c b/src/backend/replication/logical/logicalfuncs.c
index b99c94e848..fcc81b358e 100644
--- a/src/backend/replication/logical/logicalfuncs.c
+++ b/src/backend/replication/logical/logicalfuncs.c
@@ -233,9 +233,8 @@ pg_logical_slot_get_changes_guts(FunctionCallInfo fcinfo, bool confirm, bool bin
ctx = CreateDecodingContext(InvalidXLogRecPtr,
options,
false,
- XL_ROUTINE(.page_read = read_local_xlog_page,
- .segment_open = wal_segment_open,
- .segment_close = wal_segment_close),
+ read_local_xlog_page,
+ wal_segment_close,
LogicalOutputPrepareWrite,
LogicalOutputWrite, NULL);
@@ -284,7 +283,13 @@ pg_logical_slot_get_changes_guts(FunctionCallInfo fcinfo, bool confirm, bool bin
XLogRecord *record;
char *errm = NULL;
- record = XLogReadRecord(ctx->reader, &errm);
+ while (XLogReadRecord(ctx->reader, &record, &errm) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!ctx->page_read(ctx->reader))
+ break;
+ }
+
if (errm)
elog(ERROR, "%s", errm);
diff --git a/src/backend/replication/slotfuncs.c b/src/backend/replication/slotfuncs.c
index 88033a79b2..8a678074d5 100644
--- a/src/backend/replication/slotfuncs.c
+++ b/src/backend/replication/slotfuncs.c
@@ -152,9 +152,8 @@ create_logical_replication_slot(char *name, char *plugin,
ctx = CreateInitDecodingContext(plugin, NIL,
false, /* just catalogs is OK */
restart_lsn,
- XL_ROUTINE(.page_read = read_local_xlog_page,
- .segment_open = wal_segment_open,
- .segment_close = wal_segment_close),
+ read_local_xlog_page,
+ wal_segment_close,
NULL, NULL, NULL);
/*
@@ -486,9 +485,8 @@ pg_logical_replication_slot_advance(XLogRecPtr moveto)
ctx = CreateDecodingContext(InvalidXLogRecPtr,
NIL,
true, /* fast_forward */
- XL_ROUTINE(.page_read = read_local_xlog_page,
- .segment_open = wal_segment_open,
- .segment_close = wal_segment_close),
+ read_local_xlog_page,
+ wal_segment_close,
NULL, NULL, NULL);
/*
@@ -513,7 +511,13 @@ pg_logical_replication_slot_advance(XLogRecPtr moveto)
* Read records. No changes are generated in fast_forward mode,
* but snapbuilder/slot statuses are updated properly.
*/
- record = XLogReadRecord(ctx->reader, &errm);
+ while (XLogReadRecord(ctx->reader, &record, &errm) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!ctx->page_read(ctx->reader))
+ break;
+ }
+
if (errm)
elog(ERROR, "%s", errm);
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index bf2711bddd..2fd24a9b14 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -580,10 +580,7 @@ StartReplication(StartReplicationCmd *cmd)
/* create xlogreader for physical replication */
xlogreader =
- XLogReaderAllocate(wal_segment_size, NULL,
- XL_ROUTINE(.segment_open = WalSndSegmentOpen,
- .segment_close = wal_segment_close),
- NULL);
+ XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
if (!xlogreader)
ereport(ERROR,
@@ -812,9 +809,11 @@ StartReplication(StartReplicationCmd *cmd)
* set every time WAL is flushed.
*/
static bool
-logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetRecPtr, char *cur_page)
+logical_read_xlog_page(XLogReaderState *state)
{
+ XLogRecPtr targetPagePtr = state->readPagePtr;
+ int reqLen = state->readLen;
+ char *cur_page = state->readBuf;
XLogRecPtr flushptr;
int count;
WALReadError errinfo;
@@ -842,7 +841,7 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
count = flushptr - targetPagePtr; /* part of the page available */
/* now actually read the data, we know it's there */
- if (!WALRead(state,
+ if (!WALRead(state, WalSndSegmentOpen, wal_segment_close,
cur_page,
targetPagePtr,
XLOG_BLCKSZ,
@@ -1014,9 +1013,8 @@ CreateReplicationSlot(CreateReplicationSlotCmd *cmd)
ctx = CreateInitDecodingContext(cmd->plugin, NIL, need_full_snapshot,
InvalidXLogRecPtr,
- XL_ROUTINE(.page_read = logical_read_xlog_page,
- .segment_open = WalSndSegmentOpen,
- .segment_close = wal_segment_close),
+ logical_read_xlog_page,
+ wal_segment_close,
WalSndPrepareWrite, WalSndWriteData,
WalSndUpdateProgress);
@@ -1179,9 +1177,8 @@ StartLogicalReplication(StartReplicationCmd *cmd)
*/
logical_decoding_ctx =
CreateDecodingContext(cmd->startpoint, cmd->options, false,
- XL_ROUTINE(.page_read = logical_read_xlog_page,
- .segment_open = WalSndSegmentOpen,
- .segment_close = wal_segment_close),
+ logical_read_xlog_page,
+ wal_segment_close,
WalSndPrepareWrite, WalSndWriteData,
WalSndUpdateProgress);
xlogreader = logical_decoding_ctx->reader;
@@ -2761,7 +2758,7 @@ XLogSendPhysical(void)
enlargeStringInfo(&output_message, nbytes);
retry:
- if (!WALRead(xlogreader,
+ if (!WALRead(xlogreader, WalSndSegmentOpen, wal_segment_close,
&output_message.data[output_message.len],
startptr,
nbytes,
@@ -2859,7 +2856,12 @@ XLogSendLogical(void)
*/
WalSndCaughtUp = false;
- record = XLogReadRecord(logical_decoding_ctx->reader, &errm);
+ while (XLogReadRecord(logical_decoding_ctx->reader, &record, &errm) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!logical_decoding_ctx->page_read(logical_decoding_ctx->reader))
+ break;
+ }
/* xlog record was invalid */
if (errm != NULL)
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index 0b7e73ae79..3dd6df4be6 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -40,15 +40,9 @@ static int xlogreadfd = -1;
static XLogSegNo xlogreadsegno = -1;
static char xlogfpath[MAXPGPATH];
-typedef struct XLogPageReadPrivate
-{
- const char *restoreCommand;
- int tliIndex;
-} XLogPageReadPrivate;
-
-static bool SimpleXLogPageRead(XLogReaderState *xlogreader,
- XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
+static bool SimpleXLogPageRead(XLogReaderState *xlogreader,
+ const char *datadir, int *tliIndex,
+ const char *restoreCommand);
/*
* Read WAL from the datadir/pg_wal, starting from 'startpoint' on timeline
@@ -62,20 +56,22 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
XLogRecord *record;
XLogReaderState *xlogreader;
char *errormsg;
- XLogPageReadPrivate private;
- private.tliIndex = tliIndex;
- private.restoreCommand = restoreCommand;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir,
- XL_ROUTINE(.page_read = &SimpleXLogPageRead),
- &private);
+ xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+
if (xlogreader == NULL)
pg_fatal("out of memory");
XLogBeginRead(xlogreader, startpoint);
do
{
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!SimpleXLogPageRead(xlogreader, datadir,
+ &tliIndex, restoreCommand))
+ break;
+ }
if (record == NULL)
{
@@ -113,19 +109,19 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex,
XLogRecord *record;
XLogReaderState *xlogreader;
char *errormsg;
- XLogPageReadPrivate private;
XLogRecPtr endptr;
- private.tliIndex = tliIndex;
- private.restoreCommand = restoreCommand;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir,
- XL_ROUTINE(.page_read = &SimpleXLogPageRead),
- &private);
+ xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
XLogBeginRead(xlogreader, ptr);
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!SimpleXLogPageRead(xlogreader, datadir, &tliIndex, restoreCommand))
+ break;
+ }
if (record == NULL)
{
if (errormsg)
@@ -160,7 +156,6 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
XLogRecPtr searchptr;
XLogReaderState *xlogreader;
char *errormsg;
- XLogPageReadPrivate private;
/*
* The given fork pointer points to the end of the last common record,
@@ -176,11 +171,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
forkptr += SizeOfXLogShortPHD;
}
- private.tliIndex = tliIndex;
- private.restoreCommand = restoreCommand;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir,
- XL_ROUTINE(.page_read = &SimpleXLogPageRead),
- &private);
+ xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
@@ -190,7 +181,13 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
uint8 info;
XLogBeginRead(xlogreader, searchptr);
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!SimpleXLogPageRead(xlogreader, datadir,
+ &tliIndex, restoreCommand))
+ break;
+ }
if (record == NULL)
{
@@ -237,10 +234,11 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
/* XLogReader callback function, to read a WAL page */
static bool
-SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *readBuf)
+SimpleXLogPageRead(XLogReaderState *xlogreader, const char *datadir,
+ int *tliIndex, const char *restoreCommand)
{
- XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
+ XLogRecPtr targetPagePtr = xlogreader->readPagePtr;
+ char *readBuf = xlogreader->readBuf;
uint32 targetPageOff;
XLogRecPtr targetSegEnd;
XLogSegNo targetSegNo;
@@ -273,14 +271,14 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
* be done both forward and backward, consider also switching timeline
* accordingly.
*/
- while (private->tliIndex < targetNentries - 1 &&
- targetHistory[private->tliIndex].end < targetSegEnd)
- private->tliIndex++;
- while (private->tliIndex > 0 &&
- targetHistory[private->tliIndex].begin >= targetSegEnd)
- private->tliIndex--;
+ while (*tliIndex < targetNentries - 1 &&
+ targetHistory[*tliIndex].end < targetSegEnd)
+ (*tliIndex)++;
+ while (*tliIndex > 0 &&
+ targetHistory[*tliIndex].begin >= targetSegEnd)
+ (*tliIndex)--;
- XLogFileName(xlogfname, targetHistory[private->tliIndex].tli,
+ XLogFileName(xlogfname, targetHistory[*tliIndex].tli,
xlogreadsegno, WalSegSz);
snprintf(xlogfpath, MAXPGPATH, "%s/" XLOGDIR "/%s",
@@ -293,7 +291,7 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
/*
* If we have no restore_command to execute, then exit.
*/
- if (private->restoreCommand == NULL)
+ if (restoreCommand == NULL)
{
pg_log_error("could not open file \"%s\": %m", xlogfpath);
xlogreader->readLen = -1;
@@ -307,7 +305,7 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
xlogreadfd = RestoreArchivedFile(xlogreader->segcxt.ws_dir,
xlogfname,
WalSegSz,
- private->restoreCommand);
+ restoreCommand);
if (xlogreadfd < 0)
{
@@ -349,7 +347,7 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
Assert(targetSegNo == xlogreadsegno);
- xlogreader->seg.ws_tli = targetHistory[private->tliIndex].tli;
+ xlogreader->seg.ws_tli = targetHistory[*tliIndex].tli;
xlogreader->readLen = XLOG_BLCKSZ;
return true;
}
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index ab0c28d9b7..02e71138cc 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -330,12 +330,17 @@ WALDumpCloseSegment(XLogReaderState *state)
state->seg.ws_file = -1;
}
-/* pg_waldump's XLogReaderRoutine->page_read callback */
+/*
+ * pg_waldump's WAL page rader, also used as page_read callback for
+ * XLogFindNextRecord
+ */
static bool
-WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetPtr, char *readBuff)
+WALDumpReadPage(XLogReaderState *state, void *priv)
{
- XLogDumpPrivate *private = state->private_data;
+ XLogRecPtr targetPagePtr = state->readPagePtr;
+ int reqLen = state->readLen;
+ char *readBuff = state->readBuf;
+ XLogDumpPrivate *private = (XLogDumpPrivate *) priv;
int count = XLOG_BLCKSZ;
WALReadError errinfo;
@@ -353,8 +358,8 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
}
}
- if (!WALRead(state, readBuff, targetPagePtr, count, private->timeline,
- &errinfo))
+ if (!WALRead(state, WALDumpOpenSegment, WALDumpCloseSegment,
+ readBuff, targetPagePtr, count, private->timeline, &errinfo))
{
WALOpenSegment *seg = &errinfo.wre_seg;
char fname[MAXPGPATH];
@@ -374,6 +379,7 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
(Size) errinfo.wre_req);
}
+ Assert(count >= state->readLen);
state->readLen = count;
return true;
}
@@ -1042,16 +1048,14 @@ main(int argc, char **argv)
/* we have everything we need, start reading */
xlogreader_state =
- XLogReaderAllocate(WalSegSz, waldir,
- XL_ROUTINE(.page_read = WALDumpReadPage,
- .segment_open = WALDumpOpenSegment,
- .segment_close = WALDumpCloseSegment),
- &private);
+ XLogReaderAllocate(WalSegSz, waldir, WALDumpCloseSegment);
+
if (!xlogreader_state)
fatal_error("out of memory");
/* first find a valid recptr to start from */
- first_record = XLogFindNextRecord(xlogreader_state, private.startptr);
+ first_record = XLogFindNextRecord(xlogreader_state, private.startptr,
+ &WALDumpReadPage, (void*) &private);
if (first_record == InvalidXLogRecPtr)
fatal_error("could not find a valid record after %X/%X",
@@ -1075,7 +1079,13 @@ main(int argc, char **argv)
for (;;)
{
/* try to read the next record */
- record = XLogReadRecord(xlogreader_state, &errormsg);
+ while (XLogReadRecord(xlogreader_state, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!WALDumpReadPage(xlogreader_state, (void *) &private))
+ break;
+ }
+
if (!record)
{
if (!config.follow || private.endptr_reached)
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index 6c8848e14f..6a7fe140cb 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -57,64 +57,15 @@ typedef struct WALSegmentContext
typedef struct XLogReaderState XLogReaderState;
-/* Function type definition for the read_page callback */
-typedef bool (*XLogPageReadCB) (XLogReaderState *xlogreader,
- XLogRecPtr targetPagePtr,
- int reqLen,
- XLogRecPtr targetRecPtr,
- char *readBuf);
+/* Function type definition for the segment cleanup callback */
+typedef void (*WALSegmentCleanupCB) (XLogReaderState *xlogreader);
+
+/* Function type definition for the open/close callbacks for WALRead() */
typedef void (*WALSegmentOpenCB) (XLogReaderState *xlogreader,
XLogSegNo nextSegNo,
TimeLineID *tli_p);
typedef void (*WALSegmentCloseCB) (XLogReaderState *xlogreader);
-typedef struct XLogReaderRoutine
-{
- /*
- * Data input callback
- *
- * This callback shall read at least reqLen valid bytes of the xlog page
- * starting at targetPagePtr, and store them in readBuf. The callback
- * shall return the number of bytes read (never more than XLOG_BLCKSZ), or
- * -1 on failure. The callback shall sleep, if necessary, to wait for the
- * requested bytes to become available. The callback will not be invoked
- * again for the same page unless more than the returned number of bytes
- * are needed.
- *
- * targetRecPtr is the position of the WAL record we're reading. Usually
- * it is equal to targetPagePtr + reqLen, but sometimes xlogreader needs
- * to read and verify the page or segment header, before it reads the
- * actual WAL record it's interested in. In that case, targetRecPtr can
- * be used to determine which timeline to read the page from.
- *
- * The callback shall set ->seg.ws_tli to the TLI of the file the page was
- * read from.
- */
- XLogPageReadCB page_read;
-
- /*
- * Callback to open the specified WAL segment for reading. ->seg.ws_file
- * shall be set to the file descriptor of the opened segment. In case of
- * failure, an error shall be raised by the callback and it shall not
- * return.
- *
- * "nextSegNo" is the number of the segment to be opened.
- *
- * "tli_p" is an input/output argument. WALRead() uses it to pass the
- * timeline in which the new segment should be found, but the callback can
- * use it to return the TLI that it actually opened.
- */
- WALSegmentOpenCB segment_open;
-
- /*
- * WAL segment close callback. ->seg.ws_file shall be set to a negative
- * number.
- */
- WALSegmentCloseCB segment_close;
-} XLogReaderRoutine;
-
-#define XL_ROUTINE(...) &(XLogReaderRoutine){__VA_ARGS__}
-
typedef struct
{
/* Is this block ref in use? */
@@ -144,12 +95,35 @@ typedef struct
uint16 data_bufsz;
} DecodedBkpBlock;
+/* Return code from XLogReadRecord */
+typedef enum XLogReadRecordResult
+{
+ XLREAD_SUCCESS, /* record is successfully read */
+ XLREAD_NEED_DATA, /* need more data. see XLogReadRecord. */
+ XLREAD_FAIL /* failed during reading a record */
+} XLogReadRecordResult;
+
+/*
+ * internal state of XLogReadRecord
+ *
+ * XLogReadState runs a state machine while reading a record. Theses states
+ * are not seen outside the function. Each state may repeat several times
+ * exiting requesting caller for new data. See the comment of XLogReadRecrod
+ * for details.
+ */
+typedef enum XLogReadRecordState {
+ XLREAD_NEXT_RECORD,
+ XLREAD_TOT_LEN,
+ XLREAD_FIRST_FRAGMENT,
+ XLREAD_CONTINUATION
+} XLogReadRecordState;
+
struct XLogReaderState
{
/*
* Operational callbacks
*/
- XLogReaderRoutine routine;
+ WALSegmentCleanupCB cleanup_cb;
/* ----------------------------------------
* Public parameters
@@ -162,18 +136,14 @@ struct XLogReaderState
*/
uint64 system_identifier;
- /*
- * Opaque data for callbacks to use. Not used by XLogReader.
- */
- void *private_data;
-
/*
* Start and end point of last record read. EndRecPtr is also used as the
* position to read next. Calling XLogBeginRead() sets EndRecPtr to the
* starting position and ReadRecPtr to invalid.
*/
- XLogRecPtr ReadRecPtr; /* start of last record read */
+ XLogRecPtr ReadRecPtr; /* start of last record read or being read */
XLogRecPtr EndRecPtr; /* end+1 of last record read */
+ XLogRecPtr PrevRecPtr; /* start of previous record read */
/* ----------------------------------------
* Communication with page reader
@@ -187,7 +157,9 @@ struct XLogReaderState
* the request, or -1 on error */
TimeLineID readPageTLI; /* TLI for data currently in readBuf */
char *readBuf; /* buffer to store data */
- bool page_verified; /* is the page on the buffer verified? */
+ bool page_verified; /* is the page header on the buffer verified? */
+ bool record_verified;/* is the current record header verified? */
+
/* ----------------------------------------
@@ -227,8 +199,6 @@ struct XLogReaderState
XLogRecPtr latestPagePtr;
TimeLineID latestPageTLI;
- /* beginning of the WAL record being read. */
- XLogRecPtr currRecPtr;
/* timeline to read it from, 0 if a lookup is required */
TimeLineID currTLI;
@@ -255,6 +225,15 @@ struct XLogReaderState
char *readRecordBuf;
uint32 readRecordBufSize;
+ /*
+ * XLogReadRecord() state
+ */
+ XLogReadRecordState readRecordState;/* state machine state */
+ int recordGotLen; /* amount of current record that has
+ * already been read */
+ int recordRemainLen; /* length of current record that remains */
+ XLogRecPtr recordContRecPtr; /* where the current record continues */
+
/* Buffer to hold error message */
char *errormsg_buf;
};
@@ -262,9 +241,7 @@ struct XLogReaderState
/* Get a new XLogReader */
extern XLogReaderState *XLogReaderAllocate(int wal_segment_size,
const char *waldir,
- XLogReaderRoutine *routine,
- void *private_data);
-extern XLogReaderRoutine *LocalXLogReaderRoutine(void);
+ WALSegmentCleanupCB cleanup_cb);
/* Free an XLogReader */
extern void XLogReaderFree(XLogReaderState *state);
@@ -272,12 +249,17 @@ extern void XLogReaderFree(XLogReaderState *state);
/* Position the XLogReader to given record */
extern void XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr);
#ifdef FRONTEND
-extern XLogRecPtr XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr);
+/* Function type definition for the read_page callback */
+typedef bool (*XLogFindNextRecordCB) (XLogReaderState *xlogreader,
+ void *private);
+extern XLogRecPtr XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
+ XLogFindNextRecordCB read_page, void *private);
#endif /* FRONTEND */
/* Read the next XLog record. Returns NULL on end-of-WAL or failure */
-extern struct XLogRecord *XLogReadRecord(XLogReaderState *state,
- char **errormsg);
+extern XLogReadRecordResult XLogReadRecord(XLogReaderState *state,
+ XLogRecord **record,
+ char **errormsg);
/* Validate a page */
extern bool XLogReaderValidatePageHeader(XLogReaderState *state,
@@ -297,6 +279,7 @@ typedef struct WALReadError
} WALReadError;
extern bool WALRead(XLogReaderState *state,
+ WALSegmentOpenCB segopenfn, WALSegmentCloseCB sgclosefn,
char *buf, XLogRecPtr startptr, Size count,
TimeLineID tli, WALReadError *errinfo);
diff --git a/src/include/access/xlogutils.h b/src/include/access/xlogutils.h
index c287edf206..482e4ced69 100644
--- a/src/include/access/xlogutils.h
+++ b/src/include/access/xlogutils.h
@@ -47,9 +47,7 @@ extern Buffer XLogReadBufferExtended(RelFileNode rnode, ForkNumber forknum,
extern Relation CreateFakeRelcacheEntry(RelFileNode rnode);
extern void FreeFakeRelcacheEntry(Relation fakerel);
-extern bool read_local_xlog_page(XLogReaderState *state,
- XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetRecPtr, char *cur_page);
+extern bool read_local_xlog_page(XLogReaderState *state);
extern void wal_segment_open(XLogReaderState *state,
XLogSegNo nextSegNo,
TimeLineID *tli_p);
diff --git a/src/include/replication/logical.h b/src/include/replication/logical.h
index c2f2475e5d..a743db49cf 100644
--- a/src/include/replication/logical.h
+++ b/src/include/replication/logical.h
@@ -29,6 +29,10 @@ typedef void (*LogicalOutputPluginWriterUpdateProgress) (struct LogicalDecodingC
TransactionId xid
);
+typedef struct LogicalDecodingContext LogicalDecodingContext;
+
+typedef bool (*LogicalDecodingXLogPageReadCB)(XLogReaderState *ctx);
+
typedef struct LogicalDecodingContext
{
/* memory context this is all allocated in */
@@ -39,6 +43,7 @@ typedef struct LogicalDecodingContext
/* infrastructure pieces for decoding */
XLogReaderState *reader;
+ LogicalDecodingXLogPageReadCB page_read;
struct ReorderBuffer *reorder;
struct SnapBuild *snapshot_builder;
@@ -95,14 +100,16 @@ extern LogicalDecodingContext *CreateInitDecodingContext(char *plugin,
List *output_plugin_options,
bool need_full_snapshot,
XLogRecPtr restart_lsn,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress);
extern LogicalDecodingContext *CreateDecodingContext(XLogRecPtr start_lsn,
List *output_plugin_options,
bool fast_forward,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress);
--
2.18.4
----Next_Part(Thu_Jul__2_13_53_30_2020_072)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
filename="v15-0003-Remove-globals-readOff-readLen-and-readSegNo.patch"
^ permalink raw reply [nested|flat] 70+ messages in thread
* [PATCH v15 2/4] Move page-reader out of XLogReadRecord
@ 2019-09-10 03:58 Kyotaro Horiguchi <[email protected]>
0 siblings, 0 replies; 70+ messages in thread
From: Kyotaro Horiguchi @ 2019-09-10 03:58 UTC (permalink / raw)
This is the second step of removing callbacks from WAL record reader.
Since it is essential to take in additional data while reading a
record, the function have to ask caller for new data while keeping
working state. Thus the function is turned into a state machine.
---
src/backend/access/transam/twophase.c | 15 +-
src/backend/access/transam/xlog.c | 58 +-
src/backend/access/transam/xlogreader.c | 700 +++++++++++-------
src/backend/access/transam/xlogutils.c | 17 +-
src/backend/replication/logical/logical.c | 26 +-
.../replication/logical/logicalfuncs.c | 13 +-
src/backend/replication/slotfuncs.c | 18 +-
src/backend/replication/walsender.c | 32 +-
src/bin/pg_rewind/parsexlog.c | 84 +--
src/bin/pg_waldump/pg_waldump.c | 36 +-
src/include/access/xlogreader.h | 121 ++-
src/include/access/xlogutils.h | 4 +-
src/include/replication/logical.h | 11 +-
13 files changed, 648 insertions(+), 487 deletions(-)
diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index 9b2e59bf0e..b0d60a0d0f 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -1330,11 +1330,8 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
XLogReaderState *xlogreader;
char *errormsg;
- xlogreader = XLogReaderAllocate(wal_segment_size, NULL,
- XL_ROUTINE(.page_read = &read_local_xlog_page,
- .segment_open = &wal_segment_open,
- .segment_close = &wal_segment_close),
- NULL);
+ xlogreader = XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+
if (!xlogreader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
@@ -1342,7 +1339,13 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
errdetail("Failed while allocating a WAL reading processor.")));
XLogBeginRead(xlogreader, lsn);
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!read_local_xlog_page(xlogreader))
+ break;
+ }
+
if (record == NULL)
ereport(ERROR,
(errcode_for_file_access(),
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index e570e56a24..f9b0108602 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -831,13 +831,6 @@ static XLogSource currentSource = XLOG_FROM_ANY;
static bool lastSourceFailed = false;
static bool pendingWalRcvRestart = false;
-typedef struct XLogPageReadPrivate
-{
- int emode;
- bool fetching_ckpt; /* are we fetching a checkpoint record? */
- bool randAccess;
-} XLogPageReadPrivate;
-
/*
* These variables track when we last obtained some WAL data to process,
* and where we got it from. (XLogReceiptSource is initially the same as
@@ -912,8 +905,8 @@ static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath,
static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli,
XLogSource source, bool notfoundOk);
static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, XLogSource source);
-static bool XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
+static bool XLogPageRead(XLogReaderState *xlogreader,
+ bool fetching_ckpt, int emode, bool randAccess);
static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
bool fetching_ckpt, XLogRecPtr tliRecPtr);
static int emode_for_corrupt_record(int emode, XLogRecPtr RecPtr);
@@ -1225,8 +1218,7 @@ XLogInsertRecord(XLogRecData *rdata,
appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len);
if (!debug_reader)
- debug_reader = XLogReaderAllocate(wal_segment_size, NULL,
- XL_ROUTINE(), NULL);
+ debug_reader = XLogReaderAllocate(wal_segment_size, NULL NULL);
if (!debug_reader)
{
@@ -4326,15 +4318,10 @@ CleanupBackupHistory(void)
* record is available.
*/
static XLogRecord *
-ReadRecord(XLogReaderState *xlogreader, int emode,
- bool fetching_ckpt)
+ReadRecord(XLogReaderState *xlogreader, int emode, bool fetching_ckpt)
{
XLogRecord *record;
- XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
-
- private->fetching_ckpt = fetching_ckpt;
- private->emode = emode;
- private->randAccess = (xlogreader->ReadRecPtr == InvalidXLogRecPtr);
+ bool randAccess = (xlogreader->ReadRecPtr == InvalidXLogRecPtr);
/* This is the first attempt to read this page. */
lastSourceFailed = false;
@@ -4342,8 +4329,16 @@ ReadRecord(XLogReaderState *xlogreader, int emode,
for (;;)
{
char *errormsg;
+ XLogReadRecordResult result;
+
+ while ((result = XLogReadRecord(xlogreader, &record, &errormsg))
+ == XLREAD_NEED_DATA)
+ {
+ if (!XLogPageRead(xlogreader, fetching_ckpt, emode, randAccess))
+ break;
+
+ }
- record = XLogReadRecord(xlogreader, &errormsg);
ReadRecPtr = xlogreader->ReadRecPtr;
EndRecPtr = xlogreader->EndRecPtr;
if (record == NULL)
@@ -6320,7 +6315,6 @@ StartupXLOG(void)
bool backupFromStandby = false;
DBState dbstate_at_startup;
XLogReaderState *xlogreader;
- XLogPageReadPrivate private;
bool fast_promoted = false;
struct stat st;
@@ -6480,13 +6474,9 @@ StartupXLOG(void)
OwnLatch(&XLogCtl->recoveryWakeupLatch);
/* Set up XLOG reader facility */
- MemSet(&private, 0, sizeof(XLogPageReadPrivate));
xlogreader =
- XLogReaderAllocate(wal_segment_size, NULL,
- XL_ROUTINE(.page_read = &XLogPageRead,
- .segment_open = NULL,
- .segment_close = wal_segment_close),
- &private);
+ XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+
if (!xlogreader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
@@ -11860,12 +11850,13 @@ CancelBackup(void)
* sleep and retry.
*/
static bool
-XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetRecPtr, char *readBuf)
+XLogPageRead(XLogReaderState *xlogreader,
+ bool fetching_ckpt, int emode, bool randAccess)
{
- XLogPageReadPrivate *private =
- (XLogPageReadPrivate *) xlogreader->private_data;
- int emode = private->emode;
+ char *readBuf = xlogreader->readBuf;
+ XLogRecPtr targetPagePtr = xlogreader->readPagePtr;
+ int reqLen = xlogreader->readLen;
+ XLogRecPtr targetRecPtr = xlogreader->ReadRecPtr;
uint32 targetPageOff;
XLogSegNo targetSegNo PG_USED_FOR_ASSERTS_ONLY;
int r;
@@ -11908,8 +11899,8 @@ retry:
flushedUpto < targetPagePtr + reqLen))
{
if (!WaitForWALToBecomeAvailable(targetPagePtr + reqLen,
- private->randAccess,
- private->fetching_ckpt,
+ randAccess,
+ fetching_ckpt,
targetRecPtr))
{
if (readFile >= 0)
@@ -12014,6 +12005,7 @@ retry:
goto next_record_is_invalid;
}
+ Assert(xlogreader->readPagePtr == targetPagePtr);
xlogreader->readLen = readLen;
return true;
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index 3d599325ee..9281b57379 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -40,7 +40,7 @@ static bool XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr,
int reqLen, bool header_inclusive);
static void XLogReaderInvalReadState(XLogReaderState *state);
static bool ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
- XLogRecPtr PrevRecPtr, XLogRecord *record, bool randAccess);
+ XLogRecPtr PrevRecPtr, XLogRecord *record);
static bool ValidXLogRecord(XLogReaderState *state, XLogRecord *record,
XLogRecPtr recptr);
static void ResetDecoder(XLogReaderState *state);
@@ -73,7 +73,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...)
*/
XLogReaderState *
XLogReaderAllocate(int wal_segment_size, const char *waldir,
- XLogReaderRoutine *routine, void *private_data)
+ WALSegmentCleanupCB cleanup_cb)
{
XLogReaderState *state;
@@ -84,7 +84,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
return NULL;
/* initialize caller-provided support functions */
- state->routine = *routine;
+ state->cleanup_cb = cleanup_cb;
state->max_block_id = -1;
@@ -107,8 +107,6 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
WALOpenSegmentInit(&state->seg, &state->segcxt, wal_segment_size,
waldir);
- /* system_identifier initialized to zeroes above */
- state->private_data = private_data;
/* ReadRecPtr, EndRecPtr and readLen initialized to zeroes above */
state->errormsg_buf = palloc_extended(MAX_ERRORMSG_LEN + 1,
MCXT_ALLOC_NO_OOM);
@@ -140,8 +138,8 @@ XLogReaderFree(XLogReaderState *state)
{
int block_id;
- if (state->seg.ws_file != -1)
- state->routine.segment_close(state);
+ if (state->seg.ws_file >= 0)
+ state->cleanup_cb(state);
for (block_id = 0; block_id <= XLR_MAX_BLOCK_ID; block_id++)
{
@@ -246,6 +244,7 @@ XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
/* Begin at the passed-in record pointer. */
state->EndRecPtr = RecPtr;
state->ReadRecPtr = InvalidXLogRecPtr;
+ state->readRecordState = XLREAD_NEXT_RECORD;
}
/*
@@ -254,317 +253,452 @@ XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
* XLogBeginRead() or XLogFindNextRecord() must be called before the first call
* to XLogReadRecord().
*
- * If the page_read callback fails to read the requested data, NULL is
- * returned. The callback is expected to have reported the error; errormsg
- * is set to NULL.
+ * This function may return XLREAD_NEED_DATA several times before returning a
+ * result record. The caller shall read in some new data then call this
+ * function again with the same parameters.
*
- * If the reading fails for some other reason, NULL is also returned, and
- * *errormsg is set to a string with details of the failure.
+ * When a record is successfully read, returns XLREAD_SUCCESS with result
+ * record being stored in *record. Otherwise *record is NULL.
*
- * The returned pointer (or *errormsg) points to an internal buffer that's
- * valid until the next call to XLogReadRecord.
+ * Returns XLREAD_NEED_DATA if more data is needed to finish reading the
+ * current record. In that case, state->readPagePtr and state->readLen inform
+ * the desired position and minimum length of data needed. The caller shall
+ * read in the requested data and set state->readBuf to point to a buffer
+ * containing it. The caller must also set state->readPageTLI and
+ * state->readLen to indicate the timeline that it was read from, and the
+ * length of data that is now available (which must be >= given readLen),
+ * respectively.
+ *
+ * If invalid data is encountered, returns XLREAD_FAIL with *record being set to
+ * NULL. *errormsg is set to a string with details of the failure.
+ * The returned pointer (or *errormsg) points to an internal buffer that's valid
+ * until the next call to XLogReadRecord.
+ *
+ *
+ * This function runs a state machine consists of the following states.
+ *
+ * XLREAD_NEXT_RECORD :
+ * The initial state, if called with valid RecPtr, try to read a record at
+ * that position. If invalid RecPtr is given try to read a record just after
+ * the last one previously read.
+ * This state ens after setting ReadRecPtr. Then goes to XLREAD_TOT_LEN.
+ *
+ * XLREAD_TOT_LEN:
+ * Examining record header. Ends after reading record total
+ * length. recordRemainLen and recordGotLen are initialized.
+ *
+ * XLREAD_FIRST_FRAGMENT:
+ * Reading the first fragment. Ends with finishing reading a single
+ * record. Goes to XLREAD_NEXT_RECORD if that's all or
+ * XLREAD_CONTINUATION if we have continuation.
+
+ * XLREAD_CONTINUATION:
+ * Reading continuation of record. Ends with finishing the whole record then
+ * goes to XLREAD_NEXT_RECORD. During this state, recordRemainLen indicates
+ * how much is left and readRecordBuf holds the partially assert
+ * record.recordContRecPtr points to the beginning of the next page where to
+ * continue.
+ *
+ * If wrong data found in any state, the state machine stays at the current
+ * state. This behavior allows to continue reading a reacord switching among
+ * different souces, while streaming replication.
*/
-XLogRecord *
-XLogReadRecord(XLogReaderState *state, char **errormsg)
+XLogReadRecordResult
+XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
{
- XLogRecPtr RecPtr;
- XLogRecord *record;
- XLogRecPtr targetPagePtr;
- bool randAccess;
- uint32 len,
- total_len;
- uint32 targetRecOff;
- uint32 pageHeaderSize;
- bool gotheader;
+ XLogRecord *prec;
- /*
- * randAccess indicates whether to verify the previous-record pointer of
- * the record we're reading. We only do this if we're reading
- * sequentially, which is what we initially assume.
- */
- randAccess = false;
+ *record = NULL;
/* reset error state */
*errormsg = NULL;
state->errormsg_buf[0] = '\0';
- ResetDecoder(state);
-
- RecPtr = state->EndRecPtr;
-
- if (state->ReadRecPtr != InvalidXLogRecPtr)
- {
- /* read the record after the one we just read */
-
- /*
- * EndRecPtr is pointing to end+1 of the previous WAL record. If
- * we're at a page boundary, no more records can fit on the current
- * page. We must skip over the page header, but we can't do that until
- * we've read in the page, since the header size is variable.
- */
- }
- else
- {
- /*
- * Caller supplied a position to start at.
- *
- * In this case, EndRecPtr should already be pointing to a valid
- * record starting position.
- */
- Assert(XRecOffIsValid(RecPtr));
- randAccess = true;
- }
-
- state->currRecPtr = RecPtr;
-
- targetPagePtr = RecPtr - (RecPtr % XLOG_BLCKSZ);
- targetRecOff = RecPtr % XLOG_BLCKSZ;
-
- /*
- * Read the page containing the record into state->readBuf. Request enough
- * byte to cover the whole record header, or at least the part of it that
- * fits on the same page.
- */
- while (XLogNeedData(state, targetPagePtr,
- Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
- targetRecOff != 0))
- {
- if (!state->routine.page_read(state, state->readPagePtr, state->readLen,
- RecPtr, state->readBuf))
- break;
- }
-
- if (!state->page_verified)
- goto err;
-
- /*
- * We have at least the page header, so we can examine it now.
- */
- pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
- if (targetRecOff == 0)
- {
- /*
- * At page start, so skip over page header.
- */
- RecPtr += pageHeaderSize;
- targetRecOff = pageHeaderSize;
- }
- else if (targetRecOff < pageHeaderSize)
- {
- report_invalid_record(state, "invalid record offset at %X/%X",
- (uint32) (RecPtr >> 32), (uint32) RecPtr);
- goto err;
- }
-
- if ((((XLogPageHeader) state->readBuf)->xlp_info & XLP_FIRST_IS_CONTRECORD) &&
- targetRecOff == pageHeaderSize)
- {
- report_invalid_record(state, "contrecord is requested by %X/%X",
- (uint32) (RecPtr >> 32), (uint32) RecPtr);
- goto err;
- }
-
- /* XLogNeedData has verified the page header */
- Assert(pageHeaderSize <= state->readLen);
-
- /*
- * Read the record length.
- *
- * NB: Even though we use an XLogRecord pointer here, the whole record
- * header might not fit on this page. xl_tot_len is the first field of the
- * struct, so it must be on this page (the records are MAXALIGNed), but we
- * cannot access any other fields until we've verified that we got the
- * whole header.
- */
- record = (XLogRecord *) (state->readBuf + RecPtr % XLOG_BLCKSZ);
- total_len = record->xl_tot_len;
-
- /*
- * If the whole record header is on this page, validate it immediately.
- * Otherwise do just a basic sanity check on xl_tot_len, and validate the
- * rest of the header after reading it from the next page. The xl_tot_len
- * check is necessary here to ensure that we enter the "Need to reassemble
- * record" code path below; otherwise we might fail to apply
- * ValidXLogRecordHeader at all.
- */
- if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
- {
- if (!ValidXLogRecordHeader(state, RecPtr, state->ReadRecPtr, record,
- randAccess))
- goto err;
- gotheader = true;
- }
- else
- {
- /* XXX: more validation should be done here */
- if (total_len < SizeOfXLogRecord)
- {
- report_invalid_record(state,
- "invalid record length at %X/%X: wanted %u, got %u",
- (uint32) (RecPtr >> 32), (uint32) RecPtr,
- (uint32) SizeOfXLogRecord, total_len);
- goto err;
- }
- gotheader = false;
- }
-
- len = XLOG_BLCKSZ - RecPtr % XLOG_BLCKSZ;
- if (total_len > len)
+ switch (state->readRecordState)
{
- /* Need to reassemble record */
- char *contdata;
- XLogPageHeader pageHeader;
- char *buffer;
- uint32 gotlen;
-
- /*
- * Enlarge readRecordBuf as needed.
- */
- if (total_len > state->readRecordBufSize &&
- !allocate_recordbuf(state, total_len))
- {
- /* We treat this as a "bogus data" condition */
- report_invalid_record(state, "record length %u at %X/%X too long",
- total_len,
- (uint32) (RecPtr >> 32), (uint32) RecPtr);
- goto err;
- }
+ case XLREAD_NEXT_RECORD:
+ ResetDecoder(state);
- /* Copy the first fragment of the record from the first page. */
- memcpy(state->readRecordBuf,
- state->readBuf + RecPtr % XLOG_BLCKSZ, len);
- buffer = state->readRecordBuf + len;
- gotlen = len;
-
- do
- {
- int rest_len = total_len - gotlen;
-
- /* Calculate pointer to beginning of next page */
- targetPagePtr += XLOG_BLCKSZ;
-
- /* Wait for the next page to become available */
- while (XLogNeedData(state, targetPagePtr,
- Min(rest_len, XLOG_BLCKSZ),
- false))
+ if (state->ReadRecPtr != InvalidXLogRecPtr)
{
- if (!state->routine.page_read(state, state->readPagePtr,
- state->readLen,
- state->ReadRecPtr,
- state->readBuf))
- break;
+ /* read the record after the one we just read */
+
+ /*
+ * EndRecPtr is pointing to end+1 of the previous WAL record.
+ * If we're at a page boundary, no more records can fit on the
+ * current page. We must skip over the page header, but we
+ * can't do that until we've read in the page, since the header
+ * size is variable.
+ */
+ state->PrevRecPtr = state->ReadRecPtr;
+ state->ReadRecPtr = state->EndRecPtr;
+ }
+ else
+ {
+ /*
+ * Caller supplied a position to start at.
+ *
+ * In this case, EndRecPtr should already be pointing to a
+ * valid record starting position.
+ */
+ Assert(XRecOffIsValid(state->EndRecPtr));
+ state->ReadRecPtr = state->EndRecPtr;
+
+ /*
+ * We cannot verify the previous-record pointer when we're
+ * seeking to a particular record. Reset PrevRecPtr so that we
+ * won't try doing that.
+ */
+ state->PrevRecPtr = InvalidXLogRecPtr;
+ state->EndRecPtr = InvalidXLogRecPtr; /* to be tidy */
}
+ state->record_verified = false;
+ state->readRecordState = XLREAD_TOT_LEN;
+ /* fall through */
+
+ case XLREAD_TOT_LEN:
+ {
+ uint32 total_len;
+ uint32 pageHeaderSize;
+ XLogRecPtr targetPagePtr;
+ uint32 targetRecOff;
+ XLogPageHeader pageHeader;
+
+ targetPagePtr =
+ state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
+ targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+
+ /*
+ * Check if we have enough data. For the first record in the page,
+ * the requesting length doesn't contain page header.
+ */
+ if (XLogNeedData(state, targetPagePtr,
+ Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+ targetRecOff != 0))
+ return XLREAD_NEED_DATA;
+
+ /* error out if caller supplied bogus page */
if (!state->page_verified)
goto err;
- Assert(SizeOfXLogShortPHD <= state->readLen);
+ /* examine page header now. */
+ pageHeaderSize =
+ XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+ if (targetRecOff == 0)
+ {
+ /* At page start, so skip over page header. */
+ state->ReadRecPtr += pageHeaderSize;
+ targetRecOff = pageHeaderSize;
+ }
+ else if (targetRecOff < pageHeaderSize)
+ {
+ report_invalid_record(state, "invalid record offset at %X/%X",
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr);
+ goto err;
+ }
- /* Check that the continuation on next page looks valid */
pageHeader = (XLogPageHeader) state->readBuf;
- if (!(pageHeader->xlp_info & XLP_FIRST_IS_CONTRECORD))
+ if ((pageHeader->xlp_info & XLP_FIRST_IS_CONTRECORD) &&
+ targetRecOff == pageHeaderSize)
{
- report_invalid_record(state,
- "there is no contrecord flag at %X/%X",
- (uint32) (RecPtr >> 32), (uint32) RecPtr);
+ report_invalid_record(state, "contrecord is requested by %X/%X",
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr);
goto err;
}
- /*
- * Cross-check that xlp_rem_len agrees with how much of the record
- * we expect there to be left.
- */
- if (pageHeader->xlp_rem_len == 0 ||
- total_len != (pageHeader->xlp_rem_len + gotlen))
- {
- report_invalid_record(state,
- "invalid contrecord length %u at %X/%X",
- pageHeader->xlp_rem_len,
- (uint32) (RecPtr >> 32), (uint32) RecPtr);
- goto err;
- }
-
- /* Append the continuation from this page to the buffer */
- pageHeaderSize = XLogPageHeaderSize(pageHeader);
-
+ /* XLogNeedData has verified the page header */
Assert(pageHeaderSize <= state->readLen);
- contdata = (char *) state->readBuf + pageHeaderSize;
- len = XLOG_BLCKSZ - pageHeaderSize;
- if (pageHeader->xlp_rem_len < len)
- len = pageHeader->xlp_rem_len;
+ /*
+ * Read the record length.
+ *
+ * NB: Even though we use an XLogRecord pointer here, the whole
+ * record header might not fit on this page. xl_tot_len is the first
+ * field of the struct, so it must be on this page (the records are
+ * MAXALIGNed), but we cannot access any other fields until we've
+ * verified that we got the whole header.
+ */
+ prec = (XLogRecord *) (state->readBuf +
+ state->ReadRecPtr % XLOG_BLCKSZ);
+ total_len = prec->xl_tot_len;
- Assert (pageHeaderSize + len <= state->readLen);
- memcpy(buffer, (char *) contdata, len);
- buffer += len;
- gotlen += len;
+ /*
+ * If the whole record header is on this page, validate it
+ * immediately. Otherwise do just a basic sanity check on
+ * xl_tot_len, and validate the rest of the header after reading it
+ * from the next page. The xl_tot_len check is necessary here to
+ * ensure that we enter the XLREAD_CONTINUATION state below;
+ * otherwise we might fail to apply ValidXLogRecordHeader at all.
+ */
+ if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
+ {
+ if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
+ state->PrevRecPtr, prec))
+ goto err;
- /* If we just reassembled the record header, validate it. */
- if (!gotheader)
+ state->record_verified = true;
+ }
+ else
{
- record = (XLogRecord *) state->readRecordBuf;
- if (!ValidXLogRecordHeader(state, RecPtr, state->ReadRecPtr,
- record, randAccess))
+ /* XXX: more validation should be done here */
+ if (total_len < SizeOfXLogRecord)
+ {
+ report_invalid_record(state,
+ "invalid record length at %X/%X: wanted %u, got %u",
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr,
+ (uint32) SizeOfXLogRecord, total_len);
goto err;
- gotheader = true;
+ }
}
- } while (gotlen < total_len);
-
- Assert(gotheader);
- record = (XLogRecord *) state->readRecordBuf;
- if (!ValidXLogRecord(state, record, RecPtr))
- goto err;
+ /*
+ * Wait for the rest of the record, or the part of the record that
+ * fit on the first page if crossed a page boundary, to become
+ * available.
+ */
+ state->recordGotLen = 0;
+ state->recordRemainLen = total_len;
+ state->readRecordState = XLREAD_FIRST_FRAGMENT;
+ }
+ /* fall through */
- pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
- state->ReadRecPtr = RecPtr;
- state->EndRecPtr = targetPagePtr + pageHeaderSize
- + MAXALIGN(pageHeader->xlp_rem_len);
- }
- else
- {
- /* Wait for the record data to become available */
- while (XLogNeedData(state, targetPagePtr,
- Min(targetRecOff + total_len, XLOG_BLCKSZ), true))
+ case XLREAD_FIRST_FRAGMENT:
{
- if (!state->routine.page_read(state, state->readPagePtr,
- state->readLen,
- state->ReadRecPtr, state->readBuf))
+ uint32 total_len = state->recordRemainLen;
+ uint32 request_len;
+ uint32 record_len;
+ XLogRecPtr targetPagePtr;
+ uint32 targetRecOff;
+
+ /*
+ * Wait for the rest of the record on the first page to become
+ * available
+ */
+ targetPagePtr =
+ state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
+ targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+
+ request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ);
+ record_len = request_len - targetRecOff;
+
+ /* ReadRecPtr contains page header */
+ Assert (targetRecOff != 0);
+ if (XLogNeedData(state, targetPagePtr, request_len, true))
+ return XLREAD_NEED_DATA;
+
+ /* error out if caller supplied bogus page */
+ if (!state->page_verified)
+ goto err;
+
+ prec = (XLogRecord *) (state->readBuf + targetRecOff);
+
+ /* validate record header if not yet */
+ if (!state->record_verified && record_len >= SizeOfXLogRecord)
+ {
+ if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
+ state->PrevRecPtr, prec))
+ goto err;
+
+ state->record_verified = true;
+ }
+
+
+ if (total_len == record_len)
+ {
+ /* Record does not cross a page boundary */
+ Assert(state->record_verified);
+
+ if (!ValidXLogRecord(state, prec, state->ReadRecPtr))
+ goto err;
+
+ state->record_verified = true; /* to be tidy */
+
+ /* We already checked the header earlier */
+ state->EndRecPtr = state->ReadRecPtr + MAXALIGN(record_len);
+
+ *record = prec;
+ state->readRecordState = XLREAD_NEXT_RECORD;
break;
+ }
+
+ /*
+ * The record continues on the next page. Need to reassemble
+ * record
+ */
+ Assert(total_len > record_len);
+
+ /* Enlarge readRecordBuf as needed. */
+ if (total_len > state->readRecordBufSize &&
+ !allocate_recordbuf(state, total_len))
+ {
+ /* We treat this as a "bogus data" condition */
+ report_invalid_record(state,
+ "record length %u at %X/%X too long",
+ total_len,
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr);
+ goto err;
+ }
+
+ /* Copy the first fragment of the record from the first page. */
+ memcpy(state->readRecordBuf, state->readBuf + targetRecOff,
+ record_len);
+ state->recordGotLen += record_len;
+ state->recordRemainLen -= record_len;
+
+ /* Calculate pointer to beginning of next page */
+ state->recordContRecPtr = state->ReadRecPtr + record_len;
+ Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0);
+
+ state->readRecordState = XLREAD_CONTINUATION;
}
+ /* fall through */
+
+ case XLREAD_CONTINUATION:
+ {
+ XLogPageHeader pageHeader;
+ uint32 pageHeaderSize;
+ XLogRecPtr targetPagePtr;
+
+ /* we enter this state only if we haven't read the whole record. */
+ Assert (state->recordRemainLen > 0);
+
+ while(state->recordRemainLen > 0)
+ {
+ char *contdata;
+ uint32 request_len;
+ uint32 record_len;
+
+ /* Wait for the next page to become available */
+ targetPagePtr = state->recordContRecPtr;
+
+ /* this request contains page header */
+ Assert (targetPagePtr != 0);
+ if (XLogNeedData(state, targetPagePtr,
+ Min(state->recordRemainLen, XLOG_BLCKSZ),
+ false))
+ return XLREAD_NEED_DATA;
+
+ if (!state->page_verified)
+ goto err;
+
+ Assert(SizeOfXLogShortPHD <= state->readLen);
- if (!state->page_verified)
- goto err;
+ /* Check that the continuation on next page looks valid */
+ pageHeader = (XLogPageHeader) state->readBuf;
+ if (!(pageHeader->xlp_info & XLP_FIRST_IS_CONTRECORD))
+ {
+ report_invalid_record(
+ state,
+ "there is no contrecord flag at %X/%X reading %X/%X",
+ (uint32) (state->recordContRecPtr >> 32),
+ (uint32) state->recordContRecPtr,
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr);
+ goto err;
+ }
- /* Record does not cross a page boundary */
- if (!ValidXLogRecord(state, record, RecPtr))
- goto err;
+ /*
+ * Cross-check that xlp_rem_len agrees with how much of the
+ * record we expect there to be left.
+ */
+ if (pageHeader->xlp_rem_len == 0 ||
+ pageHeader->xlp_rem_len != state->recordRemainLen)
+ {
+ report_invalid_record(
+ state,
+ "invalid contrecord length %u at %X/%X reading %X/%X, expected %u",
+ pageHeader->xlp_rem_len,
+ (uint32) (state->recordContRecPtr >> 32),
+ (uint32) state->recordContRecPtr,
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr,
+ state->recordRemainLen);
+ goto err;
+ }
- state->EndRecPtr = RecPtr + MAXALIGN(total_len);
+ /* Append the continuation from this page to the buffer */
+ pageHeaderSize = XLogPageHeaderSize(pageHeader);
- state->ReadRecPtr = RecPtr;
+ /*
+ * XLogNeedData should have ensured that the whole page header
+ * was read
+ */
+ Assert(state->readLen >= pageHeaderSize);
+
+ contdata = (char *) state->readBuf + pageHeaderSize;
+ record_len = XLOG_BLCKSZ - pageHeaderSize;
+ if (pageHeader->xlp_rem_len < record_len)
+ record_len = pageHeader->xlp_rem_len;
+
+ request_len = record_len + pageHeaderSize;
+
+ /* XLogNeedData should have ensured all needed data was read */
+ Assert (state->readLen >= request_len);
+
+ memcpy(state->readRecordBuf + state->recordGotLen,
+ (char *) contdata, record_len);
+ state->recordGotLen += record_len;
+ state->recordRemainLen -= record_len;
+
+ /* If we just reassembled the record header, validate it. */
+ if (!state->record_verified)
+ {
+ Assert(state->recordGotLen >= SizeOfXLogRecord);
+ if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
+ state->PrevRecPtr,
+ (XLogRecord *) state->readRecordBuf))
+ goto err;
+
+ state->record_verified = true;
+ }
+
+ /* Calculate pointer to beginning of next page, and continue */
+ state->recordContRecPtr += XLOG_BLCKSZ;
+ }
+
+ /* targetPagePtr is pointing the last-read page here */
+ prec = (XLogRecord *) state->readRecordBuf;
+ if (!ValidXLogRecord(state, prec, state->ReadRecPtr))
+ goto err;
+
+ pageHeaderSize =
+ XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+ state->EndRecPtr = targetPagePtr + pageHeaderSize
+ + MAXALIGN(pageHeader->xlp_rem_len);
+
+ *record = prec;
+ state->readRecordState = XLREAD_NEXT_RECORD;
+ break;
+ }
}
/*
* Special processing if it's an XLOG SWITCH record
*/
- if (record->xl_rmid == RM_XLOG_ID &&
- (record->xl_info & ~XLR_INFO_MASK) == XLOG_SWITCH)
+ if ((*record)->xl_rmid == RM_XLOG_ID &&
+ ((*record)->xl_info & ~XLR_INFO_MASK) == XLOG_SWITCH)
{
/* Pretend it extends to end of segment */
state->EndRecPtr += state->segcxt.ws_segsize - 1;
state->EndRecPtr -= XLogSegmentOffset(state->EndRecPtr, state->segcxt.ws_segsize);
}
- if (DecodeXLogRecord(state, record, errormsg))
- return record;
- else
- return NULL;
+ Assert (!*record || state->readLen >= 0);
+ if (DecodeXLogRecord(state, *record, errormsg))
+ return XLREAD_SUCCESS;
+
+ *record = NULL;
+ return XLREAD_FAIL;
err:
/*
- * Invalidate the read state. We might read from a different source after
+ * Invalidate the read page. We might read from a different source after
* failure.
*/
XLogReaderInvalReadState(state);
@@ -572,7 +706,8 @@ err:
if (state->errormsg_buf[0] != '\0')
*errormsg = state->errormsg_buf;
- return NULL;
+ *record = NULL;
+ return XLREAD_FAIL;
}
/*
@@ -724,11 +859,12 @@ XLogReaderInvalReadState(XLogReaderState *state)
*
* This is just a convenience subroutine to avoid duplicated code in
* XLogReadRecord. It's not intended for use from anywhere else.
+ *
+ * If PrevRecPtr is valid, the xl_prev is is cross-checked with it.
*/
static bool
ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
- XLogRecPtr PrevRecPtr, XLogRecord *record,
- bool randAccess)
+ XLogRecPtr PrevRecPtr, XLogRecord *record)
{
if (record->xl_tot_len < SizeOfXLogRecord)
{
@@ -746,7 +882,7 @@ ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
(uint32) RecPtr);
return false;
}
- if (randAccess)
+ if (PrevRecPtr == InvalidXLogRecPtr)
{
/*
* We can't exactly verify the prev-link, but surely it should be less
@@ -978,11 +1114,14 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
* XLogReadRecord() will read the next valid record.
*/
XLogRecPtr
-XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
+XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
+ XLogFindNextRecordCB read_page, void *private)
{
XLogRecPtr tmpRecPtr;
XLogRecPtr found = InvalidXLogRecPtr;
XLogPageHeader header;
+ XLogRecord *record;
+ XLogReadRecordResult result;
char *errormsg;
Assert(!XLogRecPtrIsInvalid(RecPtr));
@@ -1015,9 +1154,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
while(XLogNeedData(state, targetPagePtr, targetRecOff,
targetRecOff != 0))
{
- if (!state->routine.page_read(state, state->readPagePtr,
- state->readLen,
- state->ReadRecPtr, state->readBuf))
+ if (!read_page(state, private))
break;
}
@@ -1069,8 +1206,16 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
* or we just jumped over the remaining data of a continuation.
*/
XLogBeginRead(state, tmpRecPtr);
- while (XLogReadRecord(state, &errormsg) != NULL)
+ while ((result = XLogReadRecord(state, &record, &errormsg)) !=
+ XLREAD_FAIL)
{
+ if (result == XLREAD_NEED_DATA)
+ {
+ if (!read_page(state, private))
+ goto err;
+ continue;
+ }
+
/* past the record we've found, break out */
if (RecPtr <= state->ReadRecPtr)
{
@@ -1090,9 +1235,9 @@ err:
#endif /* FRONTEND */
/*
- * Helper function to ease writing of XLogRoutine->page_read callbacks.
- * If this function is used, caller must supply a segment_open callback in
- * 'state', as that is used here.
+ * Helper function to ease writing of page_read callback.
+ * If this function is used, caller must supply a segment_open callback and
+ * segment_close callback as that is used here.
*
* Read 'count' bytes into 'buf', starting at location 'startptr', from WAL
* fetched from timeline 'tli'.
@@ -1105,6 +1250,7 @@ err:
*/
bool
WALRead(XLogReaderState *state,
+ WALSegmentOpenCB segopenfn, WALSegmentCloseCB segclosefn,
char *buf, XLogRecPtr startptr, Size count, TimeLineID tli,
WALReadError *errinfo)
{
@@ -1136,10 +1282,10 @@ WALRead(XLogReaderState *state,
XLogSegNo nextSegNo;
if (state->seg.ws_file >= 0)
- state->routine.segment_close(state);
+ segclosefn(state);
XLByteToSeg(recptr, nextSegNo, state->segcxt.ws_segsize);
- state->routine.segment_open(state, nextSegNo, &tli);
+ segopenfn(state, nextSegNo, &tli);
/* This shouldn't happen -- indicates a bug in segment_open */
Assert(state->seg.ws_file >= 0);
diff --git a/src/backend/access/transam/xlogutils.c b/src/backend/access/transam/xlogutils.c
index 1b10b6df20..303333571f 100644
--- a/src/backend/access/transam/xlogutils.c
+++ b/src/backend/access/transam/xlogutils.c
@@ -689,8 +689,7 @@ XLogTruncateRelation(RelFileNode rnode, ForkNumber forkNum,
void
XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wantLength)
{
- const XLogRecPtr lastReadPage = state->seg.ws_segno *
- state->segcxt.ws_segsize + state->readLen;
+ const XLogRecPtr lastReadPage = state->readPagePtr;
Assert(wantPage != InvalidXLogRecPtr && wantPage % XLOG_BLCKSZ == 0);
Assert(wantLength <= XLOG_BLCKSZ);
@@ -705,7 +704,7 @@ XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wa
* current TLI has since become historical.
*/
if (lastReadPage == wantPage &&
- state->readLen != 0 &&
+ state->page_verified &&
lastReadPage + state->readLen >= wantPage + Min(wantLength, XLOG_BLCKSZ - 1))
return;
@@ -792,6 +791,7 @@ wal_segment_open(XLogReaderState *state, XLogSegNo nextSegNo,
char path[MAXPGPATH];
XLogFilePath(path, tli, nextSegNo, state->segcxt.ws_segsize);
+ elog(LOG, "HOGE: %lu, %d => %s", nextSegNo, tli, path);
state->seg.ws_file = BasicOpenFile(path, O_RDONLY | PG_BINARY);
if (state->seg.ws_file >= 0)
return;
@@ -829,9 +829,11 @@ wal_segment_close(XLogReaderState *state)
* loop for now.
*/
bool
-read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *cur_page)
+read_local_xlog_page(XLogReaderState *state)
{
+ XLogRecPtr targetPagePtr = state->readPagePtr;
+ int reqLen = state->readLen;
+ char *cur_page = state->readBuf;
XLogRecPtr read_upto,
loc;
TimeLineID tli;
@@ -944,11 +946,12 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
* as 'count', read the whole page anyway. It's guaranteed to be
* zero-padded up to the page boundary if it's incomplete.
*/
- if (!WALRead(state, cur_page, targetPagePtr, XLOG_BLCKSZ, tli,
- &errinfo))
+ if (!WALRead(state, wal_segment_open, wal_segment_close,
+ cur_page, targetPagePtr, XLOG_BLCKSZ, tli, &errinfo))
WALReadRaiseError(&errinfo);
/* number of valid bytes in the buffer */
+ state->readPagePtr = targetPagePtr;
state->readLen = count;
return true;
}
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index 61902be3b0..a15b0b3355 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -120,7 +120,8 @@ StartupDecodingContext(List *output_plugin_options,
TransactionId xmin_horizon,
bool need_full_snapshot,
bool fast_forward,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress)
@@ -169,11 +170,12 @@ StartupDecodingContext(List *output_plugin_options,
ctx->slot = slot;
- ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, xl_routine, ctx);
+ ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, cleanup_cb);
if (!ctx->reader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
errmsg("out of memory")));
+ ctx->page_read = page_read;
ctx->reorder = ReorderBufferAllocate();
ctx->snapshot_builder =
@@ -231,7 +233,8 @@ CreateInitDecodingContext(char *plugin,
List *output_plugin_options,
bool need_full_snapshot,
XLogRecPtr restart_lsn,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress)
@@ -328,7 +331,7 @@ CreateInitDecodingContext(char *plugin,
ctx = StartupDecodingContext(NIL, restart_lsn, xmin_horizon,
need_full_snapshot, false,
- xl_routine, prepare_write, do_write,
+ page_read, cleanup_cb, prepare_write, do_write,
update_progress);
/* call output plugin initialization callback */
@@ -376,7 +379,8 @@ LogicalDecodingContext *
CreateDecodingContext(XLogRecPtr start_lsn,
List *output_plugin_options,
bool fast_forward,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress)
@@ -429,8 +433,8 @@ CreateDecodingContext(XLogRecPtr start_lsn,
ctx = StartupDecodingContext(output_plugin_options,
start_lsn, InvalidTransactionId, false,
- fast_forward, xl_routine, prepare_write,
- do_write, update_progress);
+ fast_forward, page_read, cleanup_cb,
+ prepare_write, do_write, update_progress);
/* call output plugin initialization callback */
old_context = MemoryContextSwitchTo(ctx->context);
@@ -483,7 +487,13 @@ DecodingContextFindStartpoint(LogicalDecodingContext *ctx)
char *err = NULL;
/* the read_page callback waits for new WAL */
- record = XLogReadRecord(ctx->reader, &err);
+ while (XLogReadRecord(ctx->reader, &record, &err) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!ctx->page_read(ctx->reader))
+ break;
+ }
+
if (err)
elog(ERROR, "%s", err);
if (!record)
diff --git a/src/backend/replication/logical/logicalfuncs.c b/src/backend/replication/logical/logicalfuncs.c
index b99c94e848..fcc81b358e 100644
--- a/src/backend/replication/logical/logicalfuncs.c
+++ b/src/backend/replication/logical/logicalfuncs.c
@@ -233,9 +233,8 @@ pg_logical_slot_get_changes_guts(FunctionCallInfo fcinfo, bool confirm, bool bin
ctx = CreateDecodingContext(InvalidXLogRecPtr,
options,
false,
- XL_ROUTINE(.page_read = read_local_xlog_page,
- .segment_open = wal_segment_open,
- .segment_close = wal_segment_close),
+ read_local_xlog_page,
+ wal_segment_close,
LogicalOutputPrepareWrite,
LogicalOutputWrite, NULL);
@@ -284,7 +283,13 @@ pg_logical_slot_get_changes_guts(FunctionCallInfo fcinfo, bool confirm, bool bin
XLogRecord *record;
char *errm = NULL;
- record = XLogReadRecord(ctx->reader, &errm);
+ while (XLogReadRecord(ctx->reader, &record, &errm) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!ctx->page_read(ctx->reader))
+ break;
+ }
+
if (errm)
elog(ERROR, "%s", errm);
diff --git a/src/backend/replication/slotfuncs.c b/src/backend/replication/slotfuncs.c
index 88033a79b2..8a678074d5 100644
--- a/src/backend/replication/slotfuncs.c
+++ b/src/backend/replication/slotfuncs.c
@@ -152,9 +152,8 @@ create_logical_replication_slot(char *name, char *plugin,
ctx = CreateInitDecodingContext(plugin, NIL,
false, /* just catalogs is OK */
restart_lsn,
- XL_ROUTINE(.page_read = read_local_xlog_page,
- .segment_open = wal_segment_open,
- .segment_close = wal_segment_close),
+ read_local_xlog_page,
+ wal_segment_close,
NULL, NULL, NULL);
/*
@@ -486,9 +485,8 @@ pg_logical_replication_slot_advance(XLogRecPtr moveto)
ctx = CreateDecodingContext(InvalidXLogRecPtr,
NIL,
true, /* fast_forward */
- XL_ROUTINE(.page_read = read_local_xlog_page,
- .segment_open = wal_segment_open,
- .segment_close = wal_segment_close),
+ read_local_xlog_page,
+ wal_segment_close,
NULL, NULL, NULL);
/*
@@ -513,7 +511,13 @@ pg_logical_replication_slot_advance(XLogRecPtr moveto)
* Read records. No changes are generated in fast_forward mode,
* but snapbuilder/slot statuses are updated properly.
*/
- record = XLogReadRecord(ctx->reader, &errm);
+ while (XLogReadRecord(ctx->reader, &record, &errm) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!ctx->page_read(ctx->reader))
+ break;
+ }
+
if (errm)
elog(ERROR, "%s", errm);
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index bf2711bddd..2fd24a9b14 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -580,10 +580,7 @@ StartReplication(StartReplicationCmd *cmd)
/* create xlogreader for physical replication */
xlogreader =
- XLogReaderAllocate(wal_segment_size, NULL,
- XL_ROUTINE(.segment_open = WalSndSegmentOpen,
- .segment_close = wal_segment_close),
- NULL);
+ XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
if (!xlogreader)
ereport(ERROR,
@@ -812,9 +809,11 @@ StartReplication(StartReplicationCmd *cmd)
* set every time WAL is flushed.
*/
static bool
-logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetRecPtr, char *cur_page)
+logical_read_xlog_page(XLogReaderState *state)
{
+ XLogRecPtr targetPagePtr = state->readPagePtr;
+ int reqLen = state->readLen;
+ char *cur_page = state->readBuf;
XLogRecPtr flushptr;
int count;
WALReadError errinfo;
@@ -842,7 +841,7 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
count = flushptr - targetPagePtr; /* part of the page available */
/* now actually read the data, we know it's there */
- if (!WALRead(state,
+ if (!WALRead(state, WalSndSegmentOpen, wal_segment_close,
cur_page,
targetPagePtr,
XLOG_BLCKSZ,
@@ -1014,9 +1013,8 @@ CreateReplicationSlot(CreateReplicationSlotCmd *cmd)
ctx = CreateInitDecodingContext(cmd->plugin, NIL, need_full_snapshot,
InvalidXLogRecPtr,
- XL_ROUTINE(.page_read = logical_read_xlog_page,
- .segment_open = WalSndSegmentOpen,
- .segment_close = wal_segment_close),
+ logical_read_xlog_page,
+ wal_segment_close,
WalSndPrepareWrite, WalSndWriteData,
WalSndUpdateProgress);
@@ -1179,9 +1177,8 @@ StartLogicalReplication(StartReplicationCmd *cmd)
*/
logical_decoding_ctx =
CreateDecodingContext(cmd->startpoint, cmd->options, false,
- XL_ROUTINE(.page_read = logical_read_xlog_page,
- .segment_open = WalSndSegmentOpen,
- .segment_close = wal_segment_close),
+ logical_read_xlog_page,
+ wal_segment_close,
WalSndPrepareWrite, WalSndWriteData,
WalSndUpdateProgress);
xlogreader = logical_decoding_ctx->reader;
@@ -2761,7 +2758,7 @@ XLogSendPhysical(void)
enlargeStringInfo(&output_message, nbytes);
retry:
- if (!WALRead(xlogreader,
+ if (!WALRead(xlogreader, WalSndSegmentOpen, wal_segment_close,
&output_message.data[output_message.len],
startptr,
nbytes,
@@ -2859,7 +2856,12 @@ XLogSendLogical(void)
*/
WalSndCaughtUp = false;
- record = XLogReadRecord(logical_decoding_ctx->reader, &errm);
+ while (XLogReadRecord(logical_decoding_ctx->reader, &record, &errm) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!logical_decoding_ctx->page_read(logical_decoding_ctx->reader))
+ break;
+ }
/* xlog record was invalid */
if (errm != NULL)
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index 0b7e73ae79..3dd6df4be6 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -40,15 +40,9 @@ static int xlogreadfd = -1;
static XLogSegNo xlogreadsegno = -1;
static char xlogfpath[MAXPGPATH];
-typedef struct XLogPageReadPrivate
-{
- const char *restoreCommand;
- int tliIndex;
-} XLogPageReadPrivate;
-
-static bool SimpleXLogPageRead(XLogReaderState *xlogreader,
- XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
+static bool SimpleXLogPageRead(XLogReaderState *xlogreader,
+ const char *datadir, int *tliIndex,
+ const char *restoreCommand);
/*
* Read WAL from the datadir/pg_wal, starting from 'startpoint' on timeline
@@ -62,20 +56,22 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
XLogRecord *record;
XLogReaderState *xlogreader;
char *errormsg;
- XLogPageReadPrivate private;
- private.tliIndex = tliIndex;
- private.restoreCommand = restoreCommand;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir,
- XL_ROUTINE(.page_read = &SimpleXLogPageRead),
- &private);
+ xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+
if (xlogreader == NULL)
pg_fatal("out of memory");
XLogBeginRead(xlogreader, startpoint);
do
{
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!SimpleXLogPageRead(xlogreader, datadir,
+ &tliIndex, restoreCommand))
+ break;
+ }
if (record == NULL)
{
@@ -113,19 +109,19 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex,
XLogRecord *record;
XLogReaderState *xlogreader;
char *errormsg;
- XLogPageReadPrivate private;
XLogRecPtr endptr;
- private.tliIndex = tliIndex;
- private.restoreCommand = restoreCommand;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir,
- XL_ROUTINE(.page_read = &SimpleXLogPageRead),
- &private);
+ xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
XLogBeginRead(xlogreader, ptr);
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!SimpleXLogPageRead(xlogreader, datadir, &tliIndex, restoreCommand))
+ break;
+ }
if (record == NULL)
{
if (errormsg)
@@ -160,7 +156,6 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
XLogRecPtr searchptr;
XLogReaderState *xlogreader;
char *errormsg;
- XLogPageReadPrivate private;
/*
* The given fork pointer points to the end of the last common record,
@@ -176,11 +171,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
forkptr += SizeOfXLogShortPHD;
}
- private.tliIndex = tliIndex;
- private.restoreCommand = restoreCommand;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir,
- XL_ROUTINE(.page_read = &SimpleXLogPageRead),
- &private);
+ xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
@@ -190,7 +181,13 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
uint8 info;
XLogBeginRead(xlogreader, searchptr);
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!SimpleXLogPageRead(xlogreader, datadir,
+ &tliIndex, restoreCommand))
+ break;
+ }
if (record == NULL)
{
@@ -237,10 +234,11 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
/* XLogReader callback function, to read a WAL page */
static bool
-SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *readBuf)
+SimpleXLogPageRead(XLogReaderState *xlogreader, const char *datadir,
+ int *tliIndex, const char *restoreCommand)
{
- XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
+ XLogRecPtr targetPagePtr = xlogreader->readPagePtr;
+ char *readBuf = xlogreader->readBuf;
uint32 targetPageOff;
XLogRecPtr targetSegEnd;
XLogSegNo targetSegNo;
@@ -273,14 +271,14 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
* be done both forward and backward, consider also switching timeline
* accordingly.
*/
- while (private->tliIndex < targetNentries - 1 &&
- targetHistory[private->tliIndex].end < targetSegEnd)
- private->tliIndex++;
- while (private->tliIndex > 0 &&
- targetHistory[private->tliIndex].begin >= targetSegEnd)
- private->tliIndex--;
+ while (*tliIndex < targetNentries - 1 &&
+ targetHistory[*tliIndex].end < targetSegEnd)
+ (*tliIndex)++;
+ while (*tliIndex > 0 &&
+ targetHistory[*tliIndex].begin >= targetSegEnd)
+ (*tliIndex)--;
- XLogFileName(xlogfname, targetHistory[private->tliIndex].tli,
+ XLogFileName(xlogfname, targetHistory[*tliIndex].tli,
xlogreadsegno, WalSegSz);
snprintf(xlogfpath, MAXPGPATH, "%s/" XLOGDIR "/%s",
@@ -293,7 +291,7 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
/*
* If we have no restore_command to execute, then exit.
*/
- if (private->restoreCommand == NULL)
+ if (restoreCommand == NULL)
{
pg_log_error("could not open file \"%s\": %m", xlogfpath);
xlogreader->readLen = -1;
@@ -307,7 +305,7 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
xlogreadfd = RestoreArchivedFile(xlogreader->segcxt.ws_dir,
xlogfname,
WalSegSz,
- private->restoreCommand);
+ restoreCommand);
if (xlogreadfd < 0)
{
@@ -349,7 +347,7 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
Assert(targetSegNo == xlogreadsegno);
- xlogreader->seg.ws_tli = targetHistory[private->tliIndex].tli;
+ xlogreader->seg.ws_tli = targetHistory[*tliIndex].tli;
xlogreader->readLen = XLOG_BLCKSZ;
return true;
}
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index ab0c28d9b7..02e71138cc 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -330,12 +330,17 @@ WALDumpCloseSegment(XLogReaderState *state)
state->seg.ws_file = -1;
}
-/* pg_waldump's XLogReaderRoutine->page_read callback */
+/*
+ * pg_waldump's WAL page rader, also used as page_read callback for
+ * XLogFindNextRecord
+ */
static bool
-WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetPtr, char *readBuff)
+WALDumpReadPage(XLogReaderState *state, void *priv)
{
- XLogDumpPrivate *private = state->private_data;
+ XLogRecPtr targetPagePtr = state->readPagePtr;
+ int reqLen = state->readLen;
+ char *readBuff = state->readBuf;
+ XLogDumpPrivate *private = (XLogDumpPrivate *) priv;
int count = XLOG_BLCKSZ;
WALReadError errinfo;
@@ -353,8 +358,8 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
}
}
- if (!WALRead(state, readBuff, targetPagePtr, count, private->timeline,
- &errinfo))
+ if (!WALRead(state, WALDumpOpenSegment, WALDumpCloseSegment,
+ readBuff, targetPagePtr, count, private->timeline, &errinfo))
{
WALOpenSegment *seg = &errinfo.wre_seg;
char fname[MAXPGPATH];
@@ -374,6 +379,7 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
(Size) errinfo.wre_req);
}
+ Assert(count >= state->readLen);
state->readLen = count;
return true;
}
@@ -1042,16 +1048,14 @@ main(int argc, char **argv)
/* we have everything we need, start reading */
xlogreader_state =
- XLogReaderAllocate(WalSegSz, waldir,
- XL_ROUTINE(.page_read = WALDumpReadPage,
- .segment_open = WALDumpOpenSegment,
- .segment_close = WALDumpCloseSegment),
- &private);
+ XLogReaderAllocate(WalSegSz, waldir, WALDumpCloseSegment);
+
if (!xlogreader_state)
fatal_error("out of memory");
/* first find a valid recptr to start from */
- first_record = XLogFindNextRecord(xlogreader_state, private.startptr);
+ first_record = XLogFindNextRecord(xlogreader_state, private.startptr,
+ &WALDumpReadPage, (void*) &private);
if (first_record == InvalidXLogRecPtr)
fatal_error("could not find a valid record after %X/%X",
@@ -1075,7 +1079,13 @@ main(int argc, char **argv)
for (;;)
{
/* try to read the next record */
- record = XLogReadRecord(xlogreader_state, &errormsg);
+ while (XLogReadRecord(xlogreader_state, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!WALDumpReadPage(xlogreader_state, (void *) &private))
+ break;
+ }
+
if (!record)
{
if (!config.follow || private.endptr_reached)
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index 6c8848e14f..6a7fe140cb 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -57,64 +57,15 @@ typedef struct WALSegmentContext
typedef struct XLogReaderState XLogReaderState;
-/* Function type definition for the read_page callback */
-typedef bool (*XLogPageReadCB) (XLogReaderState *xlogreader,
- XLogRecPtr targetPagePtr,
- int reqLen,
- XLogRecPtr targetRecPtr,
- char *readBuf);
+/* Function type definition for the segment cleanup callback */
+typedef void (*WALSegmentCleanupCB) (XLogReaderState *xlogreader);
+
+/* Function type definition for the open/close callbacks for WALRead() */
typedef void (*WALSegmentOpenCB) (XLogReaderState *xlogreader,
XLogSegNo nextSegNo,
TimeLineID *tli_p);
typedef void (*WALSegmentCloseCB) (XLogReaderState *xlogreader);
-typedef struct XLogReaderRoutine
-{
- /*
- * Data input callback
- *
- * This callback shall read at least reqLen valid bytes of the xlog page
- * starting at targetPagePtr, and store them in readBuf. The callback
- * shall return the number of bytes read (never more than XLOG_BLCKSZ), or
- * -1 on failure. The callback shall sleep, if necessary, to wait for the
- * requested bytes to become available. The callback will not be invoked
- * again for the same page unless more than the returned number of bytes
- * are needed.
- *
- * targetRecPtr is the position of the WAL record we're reading. Usually
- * it is equal to targetPagePtr + reqLen, but sometimes xlogreader needs
- * to read and verify the page or segment header, before it reads the
- * actual WAL record it's interested in. In that case, targetRecPtr can
- * be used to determine which timeline to read the page from.
- *
- * The callback shall set ->seg.ws_tli to the TLI of the file the page was
- * read from.
- */
- XLogPageReadCB page_read;
-
- /*
- * Callback to open the specified WAL segment for reading. ->seg.ws_file
- * shall be set to the file descriptor of the opened segment. In case of
- * failure, an error shall be raised by the callback and it shall not
- * return.
- *
- * "nextSegNo" is the number of the segment to be opened.
- *
- * "tli_p" is an input/output argument. WALRead() uses it to pass the
- * timeline in which the new segment should be found, but the callback can
- * use it to return the TLI that it actually opened.
- */
- WALSegmentOpenCB segment_open;
-
- /*
- * WAL segment close callback. ->seg.ws_file shall be set to a negative
- * number.
- */
- WALSegmentCloseCB segment_close;
-} XLogReaderRoutine;
-
-#define XL_ROUTINE(...) &(XLogReaderRoutine){__VA_ARGS__}
-
typedef struct
{
/* Is this block ref in use? */
@@ -144,12 +95,35 @@ typedef struct
uint16 data_bufsz;
} DecodedBkpBlock;
+/* Return code from XLogReadRecord */
+typedef enum XLogReadRecordResult
+{
+ XLREAD_SUCCESS, /* record is successfully read */
+ XLREAD_NEED_DATA, /* need more data. see XLogReadRecord. */
+ XLREAD_FAIL /* failed during reading a record */
+} XLogReadRecordResult;
+
+/*
+ * internal state of XLogReadRecord
+ *
+ * XLogReadState runs a state machine while reading a record. Theses states
+ * are not seen outside the function. Each state may repeat several times
+ * exiting requesting caller for new data. See the comment of XLogReadRecrod
+ * for details.
+ */
+typedef enum XLogReadRecordState {
+ XLREAD_NEXT_RECORD,
+ XLREAD_TOT_LEN,
+ XLREAD_FIRST_FRAGMENT,
+ XLREAD_CONTINUATION
+} XLogReadRecordState;
+
struct XLogReaderState
{
/*
* Operational callbacks
*/
- XLogReaderRoutine routine;
+ WALSegmentCleanupCB cleanup_cb;
/* ----------------------------------------
* Public parameters
@@ -162,18 +136,14 @@ struct XLogReaderState
*/
uint64 system_identifier;
- /*
- * Opaque data for callbacks to use. Not used by XLogReader.
- */
- void *private_data;
-
/*
* Start and end point of last record read. EndRecPtr is also used as the
* position to read next. Calling XLogBeginRead() sets EndRecPtr to the
* starting position and ReadRecPtr to invalid.
*/
- XLogRecPtr ReadRecPtr; /* start of last record read */
+ XLogRecPtr ReadRecPtr; /* start of last record read or being read */
XLogRecPtr EndRecPtr; /* end+1 of last record read */
+ XLogRecPtr PrevRecPtr; /* start of previous record read */
/* ----------------------------------------
* Communication with page reader
@@ -187,7 +157,9 @@ struct XLogReaderState
* the request, or -1 on error */
TimeLineID readPageTLI; /* TLI for data currently in readBuf */
char *readBuf; /* buffer to store data */
- bool page_verified; /* is the page on the buffer verified? */
+ bool page_verified; /* is the page header on the buffer verified? */
+ bool record_verified;/* is the current record header verified? */
+
/* ----------------------------------------
@@ -227,8 +199,6 @@ struct XLogReaderState
XLogRecPtr latestPagePtr;
TimeLineID latestPageTLI;
- /* beginning of the WAL record being read. */
- XLogRecPtr currRecPtr;
/* timeline to read it from, 0 if a lookup is required */
TimeLineID currTLI;
@@ -255,6 +225,15 @@ struct XLogReaderState
char *readRecordBuf;
uint32 readRecordBufSize;
+ /*
+ * XLogReadRecord() state
+ */
+ XLogReadRecordState readRecordState;/* state machine state */
+ int recordGotLen; /* amount of current record that has
+ * already been read */
+ int recordRemainLen; /* length of current record that remains */
+ XLogRecPtr recordContRecPtr; /* where the current record continues */
+
/* Buffer to hold error message */
char *errormsg_buf;
};
@@ -262,9 +241,7 @@ struct XLogReaderState
/* Get a new XLogReader */
extern XLogReaderState *XLogReaderAllocate(int wal_segment_size,
const char *waldir,
- XLogReaderRoutine *routine,
- void *private_data);
-extern XLogReaderRoutine *LocalXLogReaderRoutine(void);
+ WALSegmentCleanupCB cleanup_cb);
/* Free an XLogReader */
extern void XLogReaderFree(XLogReaderState *state);
@@ -272,12 +249,17 @@ extern void XLogReaderFree(XLogReaderState *state);
/* Position the XLogReader to given record */
extern void XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr);
#ifdef FRONTEND
-extern XLogRecPtr XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr);
+/* Function type definition for the read_page callback */
+typedef bool (*XLogFindNextRecordCB) (XLogReaderState *xlogreader,
+ void *private);
+extern XLogRecPtr XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
+ XLogFindNextRecordCB read_page, void *private);
#endif /* FRONTEND */
/* Read the next XLog record. Returns NULL on end-of-WAL or failure */
-extern struct XLogRecord *XLogReadRecord(XLogReaderState *state,
- char **errormsg);
+extern XLogReadRecordResult XLogReadRecord(XLogReaderState *state,
+ XLogRecord **record,
+ char **errormsg);
/* Validate a page */
extern bool XLogReaderValidatePageHeader(XLogReaderState *state,
@@ -297,6 +279,7 @@ typedef struct WALReadError
} WALReadError;
extern bool WALRead(XLogReaderState *state,
+ WALSegmentOpenCB segopenfn, WALSegmentCloseCB sgclosefn,
char *buf, XLogRecPtr startptr, Size count,
TimeLineID tli, WALReadError *errinfo);
diff --git a/src/include/access/xlogutils.h b/src/include/access/xlogutils.h
index c287edf206..482e4ced69 100644
--- a/src/include/access/xlogutils.h
+++ b/src/include/access/xlogutils.h
@@ -47,9 +47,7 @@ extern Buffer XLogReadBufferExtended(RelFileNode rnode, ForkNumber forknum,
extern Relation CreateFakeRelcacheEntry(RelFileNode rnode);
extern void FreeFakeRelcacheEntry(Relation fakerel);
-extern bool read_local_xlog_page(XLogReaderState *state,
- XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetRecPtr, char *cur_page);
+extern bool read_local_xlog_page(XLogReaderState *state);
extern void wal_segment_open(XLogReaderState *state,
XLogSegNo nextSegNo,
TimeLineID *tli_p);
diff --git a/src/include/replication/logical.h b/src/include/replication/logical.h
index c2f2475e5d..a743db49cf 100644
--- a/src/include/replication/logical.h
+++ b/src/include/replication/logical.h
@@ -29,6 +29,10 @@ typedef void (*LogicalOutputPluginWriterUpdateProgress) (struct LogicalDecodingC
TransactionId xid
);
+typedef struct LogicalDecodingContext LogicalDecodingContext;
+
+typedef bool (*LogicalDecodingXLogPageReadCB)(XLogReaderState *ctx);
+
typedef struct LogicalDecodingContext
{
/* memory context this is all allocated in */
@@ -39,6 +43,7 @@ typedef struct LogicalDecodingContext
/* infrastructure pieces for decoding */
XLogReaderState *reader;
+ LogicalDecodingXLogPageReadCB page_read;
struct ReorderBuffer *reorder;
struct SnapBuild *snapshot_builder;
@@ -95,14 +100,16 @@ extern LogicalDecodingContext *CreateInitDecodingContext(char *plugin,
List *output_plugin_options,
bool need_full_snapshot,
XLogRecPtr restart_lsn,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress);
extern LogicalDecodingContext *CreateDecodingContext(XLogRecPtr start_lsn,
List *output_plugin_options,
bool fast_forward,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress);
--
2.18.4
----Next_Part(Thu_Jul__2_13_53_30_2020_072)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
filename="v15-0003-Remove-globals-readOff-readLen-and-readSegNo.patch"
^ permalink raw reply [nested|flat] 70+ messages in thread
* [PATCH v17 2/4] Move page-reader out of XLogReadRecord
@ 2019-09-10 03:58 Kyotaro Horiguchi <[email protected]>
0 siblings, 0 replies; 70+ messages in thread
From: Kyotaro Horiguchi @ 2019-09-10 03:58 UTC (permalink / raw)
This is the second step of removing callbacks from WAL record reader.
Since it is essential to take in additional data while reading a
record, the function have to ask caller for new data while keeping
working state. Thus the function is turned into a state machine.
---
src/backend/access/transam/twophase.c | 15 +-
src/backend/access/transam/xlog.c | 58 +-
src/backend/access/transam/xlogreader.c | 697 +++++++++++-------
src/backend/access/transam/xlogutils.c | 17 +-
src/backend/replication/logical/logical.c | 26 +-
.../replication/logical/logicalfuncs.c | 13 +-
src/backend/replication/slotfuncs.c | 18 +-
src/backend/replication/walsender.c | 32 +-
src/bin/pg_rewind/parsexlog.c | 84 ++-
src/bin/pg_waldump/pg_waldump.c | 36 +-
src/include/access/xlogreader.h | 121 ++-
src/include/access/xlogutils.h | 4 +-
src/include/pg_config_manual.h | 2 +-
src/include/replication/logical.h | 11 +-
14 files changed, 646 insertions(+), 488 deletions(-)
diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index 80d2d20d6c..7d5a96a5f5 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -1325,11 +1325,8 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
XLogReaderState *xlogreader;
char *errormsg;
- xlogreader = XLogReaderAllocate(wal_segment_size, NULL,
- XL_ROUTINE(.page_read = &read_local_xlog_page,
- .segment_open = &wal_segment_open,
- .segment_close = &wal_segment_close),
- NULL);
+ xlogreader = XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+
if (!xlogreader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
@@ -1337,7 +1334,13 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
errdetail("Failed while allocating a WAL reading processor.")));
XLogBeginRead(xlogreader, lsn);
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!read_local_xlog_page(xlogreader))
+ break;
+ }
+
if (record == NULL)
ereport(ERROR,
(errcode_for_file_access(),
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 606328a65a..670996ae67 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -836,13 +836,6 @@ static XLogSource currentSource = XLOG_FROM_ANY;
static bool lastSourceFailed = false;
static bool pendingWalRcvRestart = false;
-typedef struct XLogPageReadPrivate
-{
- int emode;
- bool fetching_ckpt; /* are we fetching a checkpoint record? */
- bool randAccess;
-} XLogPageReadPrivate;
-
/*
* These variables track when we last obtained some WAL data to process,
* and where we got it from. (XLogReceiptSource is initially the same as
@@ -917,8 +910,8 @@ static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath,
static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli,
XLogSource source, bool notfoundOk);
static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, XLogSource source);
-static bool XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
+static bool XLogPageRead(XLogReaderState *xlogreader,
+ bool fetching_ckpt, int emode, bool randAccess);
static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
bool fetching_ckpt, XLogRecPtr tliRecPtr);
static int emode_for_corrupt_record(int emode, XLogRecPtr RecPtr);
@@ -1231,8 +1224,7 @@ XLogInsertRecord(XLogRecData *rdata,
appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len);
if (!debug_reader)
- debug_reader = XLogReaderAllocate(wal_segment_size, NULL,
- XL_ROUTINE(), NULL);
+ debug_reader = XLogReaderAllocate(wal_segment_size, NULL, NULL);
if (!debug_reader)
{
@@ -4344,15 +4336,10 @@ CleanupBackupHistory(void)
* record is available.
*/
static XLogRecord *
-ReadRecord(XLogReaderState *xlogreader, int emode,
- bool fetching_ckpt)
+ReadRecord(XLogReaderState *xlogreader, int emode, bool fetching_ckpt)
{
XLogRecord *record;
- XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
-
- private->fetching_ckpt = fetching_ckpt;
- private->emode = emode;
- private->randAccess = (xlogreader->ReadRecPtr == InvalidXLogRecPtr);
+ bool randAccess = (xlogreader->ReadRecPtr == InvalidXLogRecPtr);
/* This is the first attempt to read this page. */
lastSourceFailed = false;
@@ -4360,8 +4347,16 @@ ReadRecord(XLogReaderState *xlogreader, int emode,
for (;;)
{
char *errormsg;
+ XLogReadRecordResult result;
+
+ while ((result = XLogReadRecord(xlogreader, &record, &errormsg))
+ == XLREAD_NEED_DATA)
+ {
+ if (!XLogPageRead(xlogreader, fetching_ckpt, emode, randAccess))
+ break;
+
+ }
- record = XLogReadRecord(xlogreader, &errormsg);
ReadRecPtr = xlogreader->ReadRecPtr;
EndRecPtr = xlogreader->EndRecPtr;
if (record == NULL)
@@ -6380,7 +6375,6 @@ StartupXLOG(void)
bool backupFromStandby = false;
DBState dbstate_at_startup;
XLogReaderState *xlogreader;
- XLogPageReadPrivate private;
bool promoted = false;
struct stat st;
@@ -6539,13 +6533,9 @@ StartupXLOG(void)
OwnLatch(&XLogCtl->recoveryWakeupLatch);
/* Set up XLOG reader facility */
- MemSet(&private, 0, sizeof(XLogPageReadPrivate));
xlogreader =
- XLogReaderAllocate(wal_segment_size, NULL,
- XL_ROUTINE(.page_read = &XLogPageRead,
- .segment_open = NULL,
- .segment_close = wal_segment_close),
- &private);
+ XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+
if (!xlogreader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
@@ -11987,12 +11977,13 @@ CancelBackup(void)
* sleep and retry.
*/
static bool
-XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetRecPtr, char *readBuf)
+XLogPageRead(XLogReaderState *xlogreader,
+ bool fetching_ckpt, int emode, bool randAccess)
{
- XLogPageReadPrivate *private =
- (XLogPageReadPrivate *) xlogreader->private_data;
- int emode = private->emode;
+ char *readBuf = xlogreader->readBuf;
+ XLogRecPtr targetPagePtr = xlogreader->readPagePtr;
+ int reqLen = xlogreader->readLen;
+ XLogRecPtr targetRecPtr = xlogreader->ReadRecPtr;
uint32 targetPageOff;
XLogSegNo targetSegNo PG_USED_FOR_ASSERTS_ONLY;
int r;
@@ -12035,8 +12026,8 @@ retry:
flushedUpto < targetPagePtr + reqLen))
{
if (!WaitForWALToBecomeAvailable(targetPagePtr + reqLen,
- private->randAccess,
- private->fetching_ckpt,
+ randAccess,
+ fetching_ckpt,
targetRecPtr))
{
if (readFile >= 0)
@@ -12141,6 +12132,7 @@ retry:
goto next_record_is_invalid;
}
+ Assert(xlogreader->readPagePtr == targetPagePtr);
xlogreader->readLen = readLen;
return true;
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index 7d04995b7b..ba3e4868f7 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -40,7 +40,7 @@ static bool XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr,
int reqLen, bool header_inclusive);
static void XLogReaderInvalReadState(XLogReaderState *state);
static bool ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
- XLogRecPtr PrevRecPtr, XLogRecord *record, bool randAccess);
+ XLogRecPtr PrevRecPtr, XLogRecord *record);
static bool ValidXLogRecord(XLogReaderState *state, XLogRecord *record,
XLogRecPtr recptr);
static void ResetDecoder(XLogReaderState *state);
@@ -73,7 +73,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...)
*/
XLogReaderState *
XLogReaderAllocate(int wal_segment_size, const char *waldir,
- XLogReaderRoutine *routine, void *private_data)
+ WALSegmentCleanupCB cleanup_cb)
{
XLogReaderState *state;
@@ -84,7 +84,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
return NULL;
/* initialize caller-provided support functions */
- state->routine = *routine;
+ state->cleanup_cb = cleanup_cb;
state->max_block_id = -1;
@@ -107,8 +107,6 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
WALOpenSegmentInit(&state->seg, &state->segcxt, wal_segment_size,
waldir);
- /* system_identifier initialized to zeroes above */
- state->private_data = private_data;
/* ReadRecPtr, EndRecPtr and readLen initialized to zeroes above */
state->errormsg_buf = palloc_extended(MAX_ERRORMSG_LEN + 1,
MCXT_ALLOC_NO_OOM);
@@ -140,8 +138,8 @@ XLogReaderFree(XLogReaderState *state)
{
int block_id;
- if (state->seg.ws_file != -1)
- state->routine.segment_close(state);
+ if (state->seg.ws_file >= 0)
+ state->cleanup_cb(state);
for (block_id = 0; block_id <= XLR_MAX_BLOCK_ID; block_id++)
{
@@ -246,6 +244,7 @@ XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
/* Begin at the passed-in record pointer. */
state->EndRecPtr = RecPtr;
state->ReadRecPtr = InvalidXLogRecPtr;
+ state->readRecordState = XLREAD_NEXT_RECORD;
}
/*
@@ -254,317 +253,449 @@ XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
* XLogBeginRead() or XLogFindNextRecord() must be called before the first call
* to XLogReadRecord().
*
- * If the page_read callback fails to read the requested data, NULL is
- * returned. The callback is expected to have reported the error; errormsg
- * is set to NULL.
+ * This function may return XLREAD_NEED_DATA several times before returning a
+ * result record. The caller shall read in some new data then call this
+ * function again with the same parameters.
*
- * If the reading fails for some other reason, NULL is also returned, and
- * *errormsg is set to a string with details of the failure.
+ * When a record is successfully read, returns XLREAD_SUCCESS with result
+ * record being stored in *record. Otherwise *record is NULL.
*
- * The returned pointer (or *errormsg) points to an internal buffer that's
- * valid until the next call to XLogReadRecord.
+ * Returns XLREAD_NEED_DATA if more data is needed to finish reading the
+ * current record. In that case, state->readPagePtr and state->readLen inform
+ * the desired position and minimum length of data needed. The caller shall
+ * read in the requested data and set state->readBuf to point to a buffer
+ * containing it. The caller must also set state->readPageTLI and
+ * state->readLen to indicate the timeline that it was read from, and the
+ * length of data that is now available (which must be >= given readLen),
+ * respectively.
+ *
+ * If invalid data is encountered, returns XLREAD_FAIL with *record being set to
+ * NULL. *errormsg is set to a string with details of the failure.
+ * The returned pointer (or *errormsg) points to an internal buffer that's valid
+ * until the next call to XLogReadRecord.
+ *
+ *
+ * This function runs a state machine consists of the following states.
+ *
+ * XLREAD_NEXT_RECORD :
+ * The initial state, if called with valid RecPtr, try to read a record at
+ * that position. If invalid RecPtr is given try to read a record just after
+ * the last one previously read.
+ * This state ens after setting ReadRecPtr. Then goes to XLREAD_TOT_LEN.
+ *
+ * XLREAD_TOT_LEN:
+ * Examining record header. Ends after reading record total
+ * length. recordRemainLen and recordGotLen are initialized.
+ *
+ * XLREAD_FIRST_FRAGMENT:
+ * Reading the first fragment. Ends with finishing reading a single
+ * record. Goes to XLREAD_NEXT_RECORD if that's all or
+ * XLREAD_CONTINUATION if we have continuation.
+
+ * XLREAD_CONTINUATION:
+ * Reading continuation of record. Ends with finishing the whole record then
+ * goes to XLREAD_NEXT_RECORD. During this state, recordRemainLen indicates
+ * how much is left and readRecordBuf holds the partially assert
+ * record.recordContRecPtr points to the beginning of the next page where to
+ * continue.
+ *
+ * If wrong data found in any state, the state machine stays at the current
+ * state. This behavior allows to continue reading a reacord switching among
+ * different souces, while streaming replication.
*/
-XLogRecord *
-XLogReadRecord(XLogReaderState *state, char **errormsg)
+XLogReadRecordResult
+XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
{
- XLogRecPtr RecPtr;
- XLogRecord *record;
- XLogRecPtr targetPagePtr;
- bool randAccess;
- uint32 len,
- total_len;
- uint32 targetRecOff;
- uint32 pageHeaderSize;
- bool gotheader;
+ XLogRecord *prec;
- /*
- * randAccess indicates whether to verify the previous-record pointer of
- * the record we're reading. We only do this if we're reading
- * sequentially, which is what we initially assume.
- */
- randAccess = false;
+ *record = NULL;
/* reset error state */
*errormsg = NULL;
state->errormsg_buf[0] = '\0';
- ResetDecoder(state);
-
- RecPtr = state->EndRecPtr;
-
- if (state->ReadRecPtr != InvalidXLogRecPtr)
- {
- /* read the record after the one we just read */
-
- /*
- * EndRecPtr is pointing to end+1 of the previous WAL record. If
- * we're at a page boundary, no more records can fit on the current
- * page. We must skip over the page header, but we can't do that until
- * we've read in the page, since the header size is variable.
- */
- }
- else
- {
- /*
- * Caller supplied a position to start at.
- *
- * In this case, EndRecPtr should already be pointing to a valid
- * record starting position.
- */
- Assert(XRecOffIsValid(RecPtr));
- randAccess = true;
- }
-
- state->currRecPtr = RecPtr;
-
- targetPagePtr = RecPtr - (RecPtr % XLOG_BLCKSZ);
- targetRecOff = RecPtr % XLOG_BLCKSZ;
-
- /*
- * Read the page containing the record into state->readBuf. Request enough
- * byte to cover the whole record header, or at least the part of it that
- * fits on the same page.
- */
- while (XLogNeedData(state, targetPagePtr,
- Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
- targetRecOff != 0))
- {
- if (!state->routine.page_read(state, state->readPagePtr, state->readLen,
- RecPtr, state->readBuf))
- break;
- }
-
- if (!state->page_verified)
- goto err;
-
- /*
- * We have at least the page header, so we can examine it now.
- */
- pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
- if (targetRecOff == 0)
- {
- /*
- * At page start, so skip over page header.
- */
- RecPtr += pageHeaderSize;
- targetRecOff = pageHeaderSize;
- }
- else if (targetRecOff < pageHeaderSize)
- {
- report_invalid_record(state, "invalid record offset at %X/%X",
- LSN_FORMAT_ARGS(RecPtr));
- goto err;
- }
-
- if ((((XLogPageHeader) state->readBuf)->xlp_info & XLP_FIRST_IS_CONTRECORD) &&
- targetRecOff == pageHeaderSize)
- {
- report_invalid_record(state, "contrecord is requested by %X/%X",
- LSN_FORMAT_ARGS(RecPtr));
- goto err;
- }
-
- /* XLogNeedData has verified the page header */
- Assert(pageHeaderSize <= state->readLen);
-
- /*
- * Read the record length.
- *
- * NB: Even though we use an XLogRecord pointer here, the whole record
- * header might not fit on this page. xl_tot_len is the first field of the
- * struct, so it must be on this page (the records are MAXALIGNed), but we
- * cannot access any other fields until we've verified that we got the
- * whole header.
- */
- record = (XLogRecord *) (state->readBuf + RecPtr % XLOG_BLCKSZ);
- total_len = record->xl_tot_len;
-
- /*
- * If the whole record header is on this page, validate it immediately.
- * Otherwise do just a basic sanity check on xl_tot_len, and validate the
- * rest of the header after reading it from the next page. The xl_tot_len
- * check is necessary here to ensure that we enter the "Need to reassemble
- * record" code path below; otherwise we might fail to apply
- * ValidXLogRecordHeader at all.
- */
- if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
- {
- if (!ValidXLogRecordHeader(state, RecPtr, state->ReadRecPtr, record,
- randAccess))
- goto err;
- gotheader = true;
- }
- else
- {
- /* XXX: more validation should be done here */
- if (total_len < SizeOfXLogRecord)
- {
- report_invalid_record(state,
- "invalid record length at %X/%X: wanted %u, got %u",
- LSN_FORMAT_ARGS(RecPtr),
- (uint32) SizeOfXLogRecord, total_len);
- goto err;
- }
- gotheader = false;
- }
-
- len = XLOG_BLCKSZ - RecPtr % XLOG_BLCKSZ;
- if (total_len > len)
+ switch (state->readRecordState)
{
- /* Need to reassemble record */
- char *contdata;
- XLogPageHeader pageHeader;
- char *buffer;
- uint32 gotlen;
-
- /*
- * Enlarge readRecordBuf as needed.
- */
- if (total_len > state->readRecordBufSize &&
- !allocate_recordbuf(state, total_len))
- {
- /* We treat this as a "bogus data" condition */
- report_invalid_record(state, "record length %u at %X/%X too long",
- total_len, LSN_FORMAT_ARGS(RecPtr));
- goto err;
- }
+ case XLREAD_NEXT_RECORD:
+ ResetDecoder(state);
- /* Copy the first fragment of the record from the first page. */
- memcpy(state->readRecordBuf,
- state->readBuf + RecPtr % XLOG_BLCKSZ, len);
- buffer = state->readRecordBuf + len;
- gotlen = len;
-
- do
- {
- int rest_len = total_len - gotlen;
-
- /* Calculate pointer to beginning of next page */
- targetPagePtr += XLOG_BLCKSZ;
-
- /* Wait for the next page to become available */
- while (XLogNeedData(state, targetPagePtr,
- Min(rest_len, XLOG_BLCKSZ),
- false))
+ if (state->ReadRecPtr != InvalidXLogRecPtr)
{
- if (!state->routine.page_read(state, state->readPagePtr,
- state->readLen,
- state->ReadRecPtr,
- state->readBuf))
- break;
+ /* read the record after the one we just read */
+
+ /*
+ * EndRecPtr is pointing to end+1 of the previous WAL record.
+ * If we're at a page boundary, no more records can fit on the
+ * current page. We must skip over the page header, but we
+ * can't do that until we've read in the page, since the header
+ * size is variable.
+ */
+ state->PrevRecPtr = state->ReadRecPtr;
+ state->ReadRecPtr = state->EndRecPtr;
+ }
+ else
+ {
+ /*
+ * Caller supplied a position to start at.
+ *
+ * In this case, EndRecPtr should already be pointing to a
+ * valid record starting position.
+ */
+ Assert(XRecOffIsValid(state->EndRecPtr));
+ state->ReadRecPtr = state->EndRecPtr;
+
+ /*
+ * We cannot verify the previous-record pointer when we're
+ * seeking to a particular record. Reset PrevRecPtr so that we
+ * won't try doing that.
+ */
+ state->PrevRecPtr = InvalidXLogRecPtr;
+ state->EndRecPtr = InvalidXLogRecPtr; /* to be tidy */
}
+ state->record_verified = false;
+ state->readRecordState = XLREAD_TOT_LEN;
+ /* fall through */
+
+ case XLREAD_TOT_LEN:
+ {
+ uint32 total_len;
+ uint32 pageHeaderSize;
+ XLogRecPtr targetPagePtr;
+ uint32 targetRecOff;
+ XLogPageHeader pageHeader;
+
+ targetPagePtr =
+ state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
+ targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+
+ /*
+ * Check if we have enough data. For the first record in the page,
+ * the requesting length doesn't contain page header.
+ */
+ if (XLogNeedData(state, targetPagePtr,
+ Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+ targetRecOff != 0))
+ return XLREAD_NEED_DATA;
+
+ /* error out if caller supplied bogus page */
if (!state->page_verified)
goto err;
- Assert(SizeOfXLogShortPHD <= state->readLen);
+ /* examine page header now. */
+ pageHeaderSize =
+ XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+ if (targetRecOff == 0)
+ {
+ /* At page start, so skip over page header. */
+ state->ReadRecPtr += pageHeaderSize;
+ targetRecOff = pageHeaderSize;
+ }
+ else if (targetRecOff < pageHeaderSize)
+ {
+ report_invalid_record(state, "invalid record offset at %X/%X",
+ LSN_FORMAT_ARGS(state->ReadRecPtr));
+ goto err;
+ }
- /* Check that the continuation on next page looks valid */
pageHeader = (XLogPageHeader) state->readBuf;
- if (!(pageHeader->xlp_info & XLP_FIRST_IS_CONTRECORD))
+ if ((pageHeader->xlp_info & XLP_FIRST_IS_CONTRECORD) &&
+ targetRecOff == pageHeaderSize)
{
- report_invalid_record(state,
- "there is no contrecord flag at %X/%X",
- LSN_FORMAT_ARGS(RecPtr));
+ report_invalid_record(state, "contrecord is requested by %X/%X",
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr);
goto err;
}
- /*
- * Cross-check that xlp_rem_len agrees with how much of the record
- * we expect there to be left.
- */
- if (pageHeader->xlp_rem_len == 0 ||
- total_len != (pageHeader->xlp_rem_len + gotlen))
- {
- report_invalid_record(state,
- "invalid contrecord length %u (expected %lld) at %X/%X",
- pageHeader->xlp_rem_len,
- ((long long) total_len) - gotlen,
- LSN_FORMAT_ARGS(RecPtr));
- goto err;
- }
-
- /* Append the continuation from this page to the buffer */
- pageHeaderSize = XLogPageHeaderSize(pageHeader);
-
+ /* XLogNeedData has verified the page header */
Assert(pageHeaderSize <= state->readLen);
- contdata = (char *) state->readBuf + pageHeaderSize;
- len = XLOG_BLCKSZ - pageHeaderSize;
- if (pageHeader->xlp_rem_len < len)
- len = pageHeader->xlp_rem_len;
+ /*
+ * Read the record length.
+ *
+ * NB: Even though we use an XLogRecord pointer here, the whole
+ * record header might not fit on this page. xl_tot_len is the first
+ * field of the struct, so it must be on this page (the records are
+ * MAXALIGNed), but we cannot access any other fields until we've
+ * verified that we got the whole header.
+ */
+ prec = (XLogRecord *) (state->readBuf +
+ state->ReadRecPtr % XLOG_BLCKSZ);
+ total_len = prec->xl_tot_len;
- Assert (pageHeaderSize + len <= state->readLen);
- memcpy(buffer, (char *) contdata, len);
- buffer += len;
- gotlen += len;
+ /*
+ * If the whole record header is on this page, validate it
+ * immediately. Otherwise do just a basic sanity check on
+ * xl_tot_len, and validate the rest of the header after reading it
+ * from the next page. The xl_tot_len check is necessary here to
+ * ensure that we enter the XLREAD_CONTINUATION state below;
+ * otherwise we might fail to apply ValidXLogRecordHeader at all.
+ */
+ if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
+ {
+ if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
+ state->PrevRecPtr, prec))
+ goto err;
- /* If we just reassembled the record header, validate it. */
- if (!gotheader)
+ state->record_verified = true;
+ }
+ else
{
- record = (XLogRecord *) state->readRecordBuf;
- if (!ValidXLogRecordHeader(state, RecPtr, state->ReadRecPtr,
- record, randAccess))
+ /* XXX: more validation should be done here */
+ if (total_len < SizeOfXLogRecord)
+ {
+ report_invalid_record(state,
+ "invalid record length at %X/%X: wanted %u, got %u",
+ LSN_FORMAT_ARGS(state->ReadRecPtr),
+ (uint32) SizeOfXLogRecord, total_len);
goto err;
- gotheader = true;
+ }
}
- } while (gotlen < total_len);
-
- Assert(gotheader);
- record = (XLogRecord *) state->readRecordBuf;
- if (!ValidXLogRecord(state, record, RecPtr))
- goto err;
+ /*
+ * Wait for the rest of the record, or the part of the record that
+ * fit on the first page if crossed a page boundary, to become
+ * available.
+ */
+ state->recordGotLen = 0;
+ state->recordRemainLen = total_len;
+ state->readRecordState = XLREAD_FIRST_FRAGMENT;
+ }
+ /* fall through */
- pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
- state->ReadRecPtr = RecPtr;
- state->EndRecPtr = targetPagePtr + pageHeaderSize
- + MAXALIGN(pageHeader->xlp_rem_len);
- }
- else
- {
- /* Wait for the record data to become available */
- while (XLogNeedData(state, targetPagePtr,
- Min(targetRecOff + total_len, XLOG_BLCKSZ), true))
+ case XLREAD_FIRST_FRAGMENT:
{
- if (!state->routine.page_read(state, state->readPagePtr,
- state->readLen,
- state->ReadRecPtr, state->readBuf))
+ uint32 total_len = state->recordRemainLen;
+ uint32 request_len;
+ uint32 record_len;
+ XLogRecPtr targetPagePtr;
+ uint32 targetRecOff;
+
+ /*
+ * Wait for the rest of the record on the first page to become
+ * available
+ */
+ targetPagePtr =
+ state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
+ targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+
+ request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ);
+ record_len = request_len - targetRecOff;
+
+ /* ReadRecPtr contains page header */
+ Assert (targetRecOff != 0);
+ if (XLogNeedData(state, targetPagePtr, request_len, true))
+ return XLREAD_NEED_DATA;
+
+ /* error out if caller supplied bogus page */
+ if (!state->page_verified)
+ goto err;
+
+ prec = (XLogRecord *) (state->readBuf + targetRecOff);
+
+ /* validate record header if not yet */
+ if (!state->record_verified && record_len >= SizeOfXLogRecord)
+ {
+ if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
+ state->PrevRecPtr, prec))
+ goto err;
+
+ state->record_verified = true;
+ }
+
+
+ if (total_len == record_len)
+ {
+ /* Record does not cross a page boundary */
+ Assert(state->record_verified);
+
+ if (!ValidXLogRecord(state, prec, state->ReadRecPtr))
+ goto err;
+
+ state->record_verified = true; /* to be tidy */
+
+ /* We already checked the header earlier */
+ state->EndRecPtr = state->ReadRecPtr + MAXALIGN(record_len);
+
+ *record = prec;
+ state->readRecordState = XLREAD_NEXT_RECORD;
break;
+ }
+
+ /*
+ * The record continues on the next page. Need to reassemble
+ * record
+ */
+ Assert(total_len > record_len);
+
+ /* Enlarge readRecordBuf as needed. */
+ if (total_len > state->readRecordBufSize &&
+ !allocate_recordbuf(state, total_len))
+ {
+ /* We treat this as a "bogus data" condition */
+ report_invalid_record(state,
+ "record length %u at %X/%X too long",
+ total_len,
+ LSN_FORMAT_ARGS(state->ReadRecPtr));
+ goto err;
+ }
+
+ /* Copy the first fragment of the record from the first page. */
+ memcpy(state->readRecordBuf, state->readBuf + targetRecOff,
+ record_len);
+ state->recordGotLen += record_len;
+ state->recordRemainLen -= record_len;
+
+ /* Calculate pointer to beginning of next page */
+ state->recordContRecPtr = state->ReadRecPtr + record_len;
+ Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0);
+
+ state->readRecordState = XLREAD_CONTINUATION;
}
+ /* fall through */
+
+ case XLREAD_CONTINUATION:
+ {
+ XLogPageHeader pageHeader;
+ uint32 pageHeaderSize;
+ XLogRecPtr targetPagePtr;
+
+ /* we enter this state only if we haven't read the whole record. */
+ Assert (state->recordRemainLen > 0);
+
+ while(state->recordRemainLen > 0)
+ {
+ char *contdata;
+ uint32 request_len;
+ uint32 record_len;
+
+ /* Wait for the next page to become available */
+ targetPagePtr = state->recordContRecPtr;
+
+ /* this request contains page header */
+ Assert (targetPagePtr != 0);
+ if (XLogNeedData(state, targetPagePtr,
+ Min(state->recordRemainLen, XLOG_BLCKSZ),
+ false))
+ return XLREAD_NEED_DATA;
+
+ if (!state->page_verified)
+ goto err;
+
+ Assert(SizeOfXLogShortPHD <= state->readLen);
- if (!state->page_verified)
- goto err;
+ /* Check that the continuation on next page looks valid */
+ pageHeader = (XLogPageHeader) state->readBuf;
+ if (!(pageHeader->xlp_info & XLP_FIRST_IS_CONTRECORD))
+ {
+ report_invalid_record(
+ state,
+ "there is no contrecord flag at %X/%X reading %X/%X",
+ (uint32) (state->recordContRecPtr >> 32),
+ (uint32) state->recordContRecPtr,
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr);
+ goto err;
+ }
- /* Record does not cross a page boundary */
- if (!ValidXLogRecord(state, record, RecPtr))
- goto err;
+ /*
+ * Cross-check that xlp_rem_len agrees with how much of the
+ * record we expect there to be left.
+ */
+ if (pageHeader->xlp_rem_len == 0 ||
+ pageHeader->xlp_rem_len != state->recordRemainLen)
+ {
+ report_invalid_record(
+ state,
+ "invalid contrecord length %u at %X/%X reading %X/%X, expected %u",
+ pageHeader->xlp_rem_len,
+ (uint32) (state->recordContRecPtr >> 32),
+ (uint32) state->recordContRecPtr,
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr,
+ state->recordRemainLen);
+ goto err;
+ }
- state->EndRecPtr = RecPtr + MAXALIGN(total_len);
+ /* Append the continuation from this page to the buffer */
+ pageHeaderSize = XLogPageHeaderSize(pageHeader);
- state->ReadRecPtr = RecPtr;
+ /*
+ * XLogNeedData should have ensured that the whole page header
+ * was read
+ */
+ Assert(state->readLen >= pageHeaderSize);
+
+ contdata = (char *) state->readBuf + pageHeaderSize;
+ record_len = XLOG_BLCKSZ - pageHeaderSize;
+ if (pageHeader->xlp_rem_len < record_len)
+ record_len = pageHeader->xlp_rem_len;
+
+ request_len = record_len + pageHeaderSize;
+
+ /* XLogNeedData should have ensured all needed data was read */
+ Assert (state->readLen >= request_len);
+
+ memcpy(state->readRecordBuf + state->recordGotLen,
+ (char *) contdata, record_len);
+ state->recordGotLen += record_len;
+ state->recordRemainLen -= record_len;
+
+ /* If we just reassembled the record header, validate it. */
+ if (!state->record_verified)
+ {
+ Assert(state->recordGotLen >= SizeOfXLogRecord);
+ if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
+ state->PrevRecPtr,
+ (XLogRecord *) state->readRecordBuf))
+ goto err;
+
+ state->record_verified = true;
+ }
+
+ /* Calculate pointer to beginning of next page, and continue */
+ state->recordContRecPtr += XLOG_BLCKSZ;
+ }
+
+ /* targetPagePtr is pointing the last-read page here */
+ prec = (XLogRecord *) state->readRecordBuf;
+ if (!ValidXLogRecord(state, prec, state->ReadRecPtr))
+ goto err;
+
+ pageHeaderSize =
+ XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+ state->EndRecPtr = targetPagePtr + pageHeaderSize
+ + MAXALIGN(pageHeader->xlp_rem_len);
+
+ *record = prec;
+ state->readRecordState = XLREAD_NEXT_RECORD;
+ break;
+ }
}
/*
* Special processing if it's an XLOG SWITCH record
*/
- if (record->xl_rmid == RM_XLOG_ID &&
- (record->xl_info & ~XLR_INFO_MASK) == XLOG_SWITCH)
+ if ((*record)->xl_rmid == RM_XLOG_ID &&
+ ((*record)->xl_info & ~XLR_INFO_MASK) == XLOG_SWITCH)
{
/* Pretend it extends to end of segment */
state->EndRecPtr += state->segcxt.ws_segsize - 1;
state->EndRecPtr -= XLogSegmentOffset(state->EndRecPtr, state->segcxt.ws_segsize);
}
- if (DecodeXLogRecord(state, record, errormsg))
- return record;
- else
- return NULL;
+ Assert (!*record || state->readLen >= 0);
+ if (DecodeXLogRecord(state, *record, errormsg))
+ return XLREAD_SUCCESS;
+
+ *record = NULL;
+ return XLREAD_FAIL;
err:
/*
- * Invalidate the read state. We might read from a different source after
+ * Invalidate the read page. We might read from a different source after
* failure.
*/
XLogReaderInvalReadState(state);
@@ -572,7 +703,8 @@ err:
if (state->errormsg_buf[0] != '\0')
*errormsg = state->errormsg_buf;
- return NULL;
+ *record = NULL;
+ return XLREAD_FAIL;
}
/*
@@ -724,11 +856,12 @@ XLogReaderInvalReadState(XLogReaderState *state)
*
* This is just a convenience subroutine to avoid duplicated code in
* XLogReadRecord. It's not intended for use from anywhere else.
+ *
+ * If PrevRecPtr is valid, the xl_prev is is cross-checked with it.
*/
static bool
ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
- XLogRecPtr PrevRecPtr, XLogRecord *record,
- bool randAccess)
+ XLogRecPtr PrevRecPtr, XLogRecord *record)
{
if (record->xl_tot_len < SizeOfXLogRecord)
{
@@ -745,7 +878,7 @@ ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
record->xl_rmid, LSN_FORMAT_ARGS(RecPtr));
return false;
}
- if (randAccess)
+ if (PrevRecPtr == InvalidXLogRecPtr)
{
/*
* We can't exactly verify the prev-link, but surely it should be less
@@ -975,11 +1108,14 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
* XLogReadRecord() will read the next valid record.
*/
XLogRecPtr
-XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
+XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
+ XLogFindNextRecordCB read_page, void *private)
{
XLogRecPtr tmpRecPtr;
XLogRecPtr found = InvalidXLogRecPtr;
XLogPageHeader header;
+ XLogRecord *record;
+ XLogReadRecordResult result;
char *errormsg;
Assert(!XLogRecPtrIsInvalid(RecPtr));
@@ -1012,9 +1148,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
while(XLogNeedData(state, targetPagePtr, targetRecOff,
targetRecOff != 0))
{
- if (!state->routine.page_read(state, state->readPagePtr,
- state->readLen,
- state->ReadRecPtr, state->readBuf))
+ if (!read_page(state, private))
break;
}
@@ -1066,8 +1200,16 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
* or we just jumped over the remaining data of a continuation.
*/
XLogBeginRead(state, tmpRecPtr);
- while (XLogReadRecord(state, &errormsg) != NULL)
+ while ((result = XLogReadRecord(state, &record, &errormsg)) !=
+ XLREAD_FAIL)
{
+ if (result == XLREAD_NEED_DATA)
+ {
+ if (!read_page(state, private))
+ goto err;
+ continue;
+ }
+
/* past the record we've found, break out */
if (RecPtr <= state->ReadRecPtr)
{
@@ -1087,9 +1229,9 @@ err:
#endif /* FRONTEND */
/*
- * Helper function to ease writing of XLogRoutine->page_read callbacks.
- * If this function is used, caller must supply a segment_open callback in
- * 'state', as that is used here.
+ * Helper function to ease writing of page_read callback.
+ * If this function is used, caller must supply a segment_open callback and
+ * segment_close callback as that is used here.
*
* Read 'count' bytes into 'buf', starting at location 'startptr', from WAL
* fetched from timeline 'tli'.
@@ -1102,6 +1244,7 @@ err:
*/
bool
WALRead(XLogReaderState *state,
+ WALSegmentOpenCB segopenfn, WALSegmentCloseCB segclosefn,
char *buf, XLogRecPtr startptr, Size count, TimeLineID tli,
WALReadError *errinfo)
{
@@ -1133,10 +1276,10 @@ WALRead(XLogReaderState *state,
XLogSegNo nextSegNo;
if (state->seg.ws_file >= 0)
- state->routine.segment_close(state);
+ segclosefn(state);
XLByteToSeg(recptr, nextSegNo, state->segcxt.ws_segsize);
- state->routine.segment_open(state, nextSegNo, &tli);
+ segopenfn(state, nextSegNo, &tli);
/* This shouldn't happen -- indicates a bug in segment_open */
Assert(state->seg.ws_file >= 0);
diff --git a/src/backend/access/transam/xlogutils.c b/src/backend/access/transam/xlogutils.c
index 46eda33f25..b003990745 100644
--- a/src/backend/access/transam/xlogutils.c
+++ b/src/backend/access/transam/xlogutils.c
@@ -686,8 +686,7 @@ XLogTruncateRelation(RelFileNode rnode, ForkNumber forkNum,
void
XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wantLength)
{
- const XLogRecPtr lastReadPage = state->seg.ws_segno *
- state->segcxt.ws_segsize + state->readLen;
+ const XLogRecPtr lastReadPage = state->readPagePtr;
Assert(wantPage != InvalidXLogRecPtr && wantPage % XLOG_BLCKSZ == 0);
Assert(wantLength <= XLOG_BLCKSZ);
@@ -702,7 +701,7 @@ XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wa
* current TLI has since become historical.
*/
if (lastReadPage == wantPage &&
- state->readLen != 0 &&
+ state->page_verified &&
lastReadPage + state->readLen >= wantPage + Min(wantLength, XLOG_BLCKSZ - 1))
return;
@@ -788,6 +787,7 @@ wal_segment_open(XLogReaderState *state, XLogSegNo nextSegNo,
char path[MAXPGPATH];
XLogFilePath(path, tli, nextSegNo, state->segcxt.ws_segsize);
+ elog(LOG, "HOGE: %lu, %d => %s", nextSegNo, tli, path);
state->seg.ws_file = BasicOpenFile(path, O_RDONLY | PG_BINARY);
if (state->seg.ws_file >= 0)
return;
@@ -825,9 +825,11 @@ wal_segment_close(XLogReaderState *state)
* loop for now.
*/
bool
-read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *cur_page)
+read_local_xlog_page(XLogReaderState *state)
{
+ XLogRecPtr targetPagePtr = state->readPagePtr;
+ int reqLen = state->readLen;
+ char *cur_page = state->readBuf;
XLogRecPtr read_upto,
loc;
TimeLineID tli;
@@ -940,11 +942,12 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
* as 'count', read the whole page anyway. It's guaranteed to be
* zero-padded up to the page boundary if it's incomplete.
*/
- if (!WALRead(state, cur_page, targetPagePtr, XLOG_BLCKSZ, tli,
- &errinfo))
+ if (!WALRead(state, wal_segment_open, wal_segment_close,
+ cur_page, targetPagePtr, XLOG_BLCKSZ, tli, &errinfo))
WALReadRaiseError(&errinfo);
/* number of valid bytes in the buffer */
+ state->readPagePtr = targetPagePtr;
state->readLen = count;
return true;
}
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index 37b75deb72..9cbd8afa49 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -148,7 +148,8 @@ StartupDecodingContext(List *output_plugin_options,
TransactionId xmin_horizon,
bool need_full_snapshot,
bool fast_forward,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress)
@@ -198,11 +199,12 @@ StartupDecodingContext(List *output_plugin_options,
ctx->slot = slot;
- ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, xl_routine, ctx);
+ ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, cleanup_cb);
if (!ctx->reader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
errmsg("out of memory")));
+ ctx->page_read = page_read;
ctx->reorder = ReorderBufferAllocate();
ctx->snapshot_builder =
@@ -319,7 +321,8 @@ CreateInitDecodingContext(const char *plugin,
List *output_plugin_options,
bool need_full_snapshot,
XLogRecPtr restart_lsn,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress)
@@ -422,7 +425,7 @@ CreateInitDecodingContext(const char *plugin,
ctx = StartupDecodingContext(NIL, restart_lsn, xmin_horizon,
need_full_snapshot, false,
- xl_routine, prepare_write, do_write,
+ page_read, cleanup_cb, prepare_write, do_write,
update_progress);
/* call output plugin initialization callback */
@@ -476,7 +479,8 @@ LogicalDecodingContext *
CreateDecodingContext(XLogRecPtr start_lsn,
List *output_plugin_options,
bool fast_forward,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress)
@@ -528,8 +532,8 @@ CreateDecodingContext(XLogRecPtr start_lsn,
ctx = StartupDecodingContext(output_plugin_options,
start_lsn, InvalidTransactionId, false,
- fast_forward, xl_routine, prepare_write,
- do_write, update_progress);
+ fast_forward, page_read, cleanup_cb,
+ prepare_write, do_write, update_progress);
/* call output plugin initialization callback */
old_context = MemoryContextSwitchTo(ctx->context);
@@ -585,7 +589,13 @@ DecodingContextFindStartpoint(LogicalDecodingContext *ctx)
char *err = NULL;
/* the read_page callback waits for new WAL */
- record = XLogReadRecord(ctx->reader, &err);
+ while (XLogReadRecord(ctx->reader, &record, &err) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!ctx->page_read(ctx->reader))
+ break;
+ }
+
if (err)
elog(ERROR, "%s", err);
if (!record)
diff --git a/src/backend/replication/logical/logicalfuncs.c b/src/backend/replication/logical/logicalfuncs.c
index f7e055874e..b69f6911c5 100644
--- a/src/backend/replication/logical/logicalfuncs.c
+++ b/src/backend/replication/logical/logicalfuncs.c
@@ -233,9 +233,8 @@ pg_logical_slot_get_changes_guts(FunctionCallInfo fcinfo, bool confirm, bool bin
ctx = CreateDecodingContext(InvalidXLogRecPtr,
options,
false,
- XL_ROUTINE(.page_read = read_local_xlog_page,
- .segment_open = wal_segment_open,
- .segment_close = wal_segment_close),
+ read_local_xlog_page,
+ wal_segment_close,
LogicalOutputPrepareWrite,
LogicalOutputWrite, NULL);
@@ -284,7 +283,13 @@ pg_logical_slot_get_changes_guts(FunctionCallInfo fcinfo, bool confirm, bool bin
XLogRecord *record;
char *errm = NULL;
- record = XLogReadRecord(ctx->reader, &errm);
+ while (XLogReadRecord(ctx->reader, &record, &errm) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!ctx->page_read(ctx->reader))
+ break;
+ }
+
if (errm)
elog(ERROR, "%s", errm);
diff --git a/src/backend/replication/slotfuncs.c b/src/backend/replication/slotfuncs.c
index 9817b44113..deb7cf15eb 100644
--- a/src/backend/replication/slotfuncs.c
+++ b/src/backend/replication/slotfuncs.c
@@ -153,9 +153,8 @@ create_logical_replication_slot(char *name, char *plugin,
ctx = CreateInitDecodingContext(plugin, NIL,
false, /* just catalogs is OK */
restart_lsn,
- XL_ROUTINE(.page_read = read_local_xlog_page,
- .segment_open = wal_segment_open,
- .segment_close = wal_segment_close),
+ read_local_xlog_page,
+ wal_segment_close,
NULL, NULL, NULL);
/*
@@ -512,9 +511,8 @@ pg_logical_replication_slot_advance(XLogRecPtr moveto)
ctx = CreateDecodingContext(InvalidXLogRecPtr,
NIL,
true, /* fast_forward */
- XL_ROUTINE(.page_read = read_local_xlog_page,
- .segment_open = wal_segment_open,
- .segment_close = wal_segment_close),
+ read_local_xlog_page,
+ wal_segment_close,
NULL, NULL, NULL);
/*
@@ -536,7 +534,13 @@ pg_logical_replication_slot_advance(XLogRecPtr moveto)
* Read records. No changes are generated in fast_forward mode,
* but snapbuilder/slot statuses are updated properly.
*/
- record = XLogReadRecord(ctx->reader, &errm);
+ while (XLogReadRecord(ctx->reader, &record, &errm) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!ctx->page_read(ctx->reader))
+ break;
+ }
+
if (errm)
elog(ERROR, "%s", errm);
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index c36b91ae5e..062093727d 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -580,10 +580,7 @@ StartReplication(StartReplicationCmd *cmd)
/* create xlogreader for physical replication */
xlogreader =
- XLogReaderAllocate(wal_segment_size, NULL,
- XL_ROUTINE(.segment_open = WalSndSegmentOpen,
- .segment_close = wal_segment_close),
- NULL);
+ XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
if (!xlogreader)
ereport(ERROR,
@@ -807,9 +804,11 @@ StartReplication(StartReplicationCmd *cmd)
* set every time WAL is flushed.
*/
static bool
-logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetRecPtr, char *cur_page)
+logical_read_xlog_page(XLogReaderState *state)
{
+ XLogRecPtr targetPagePtr = state->readPagePtr;
+ int reqLen = state->readLen;
+ char *cur_page = state->readBuf;
XLogRecPtr flushptr;
int count;
WALReadError errinfo;
@@ -837,7 +836,7 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
count = flushptr - targetPagePtr; /* part of the page available */
/* now actually read the data, we know it's there */
- if (!WALRead(state,
+ if (!WALRead(state, WalSndSegmentOpen, wal_segment_close,
cur_page,
targetPagePtr,
XLOG_BLCKSZ,
@@ -1011,9 +1010,8 @@ CreateReplicationSlot(CreateReplicationSlotCmd *cmd)
ctx = CreateInitDecodingContext(cmd->plugin, NIL, need_full_snapshot,
InvalidXLogRecPtr,
- XL_ROUTINE(.page_read = logical_read_xlog_page,
- .segment_open = WalSndSegmentOpen,
- .segment_close = wal_segment_close),
+ logical_read_xlog_page,
+ wal_segment_close,
WalSndPrepareWrite, WalSndWriteData,
WalSndUpdateProgress);
@@ -1171,9 +1169,8 @@ StartLogicalReplication(StartReplicationCmd *cmd)
*/
logical_decoding_ctx =
CreateDecodingContext(cmd->startpoint, cmd->options, false,
- XL_ROUTINE(.page_read = logical_read_xlog_page,
- .segment_open = WalSndSegmentOpen,
- .segment_close = wal_segment_close),
+ logical_read_xlog_page,
+ wal_segment_close,
WalSndPrepareWrite, WalSndWriteData,
WalSndUpdateProgress);
xlogreader = logical_decoding_ctx->reader;
@@ -2749,7 +2746,7 @@ XLogSendPhysical(void)
enlargeStringInfo(&output_message, nbytes);
retry:
- if (!WALRead(xlogreader,
+ if (!WALRead(xlogreader, WalSndSegmentOpen, wal_segment_close,
&output_message.data[output_message.len],
startptr,
nbytes,
@@ -2847,7 +2844,12 @@ XLogSendLogical(void)
*/
WalSndCaughtUp = false;
- record = XLogReadRecord(logical_decoding_ctx->reader, &errm);
+ while (XLogReadRecord(logical_decoding_ctx->reader, &record, &errm) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!logical_decoding_ctx->page_read(logical_decoding_ctx->reader))
+ break;
+ }
/* xlog record was invalid */
if (errm != NULL)
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index cf119848b0..712c85281c 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -41,15 +41,9 @@ static int xlogreadfd = -1;
static XLogSegNo xlogreadsegno = -1;
static char xlogfpath[MAXPGPATH];
-typedef struct XLogPageReadPrivate
-{
- const char *restoreCommand;
- int tliIndex;
-} XLogPageReadPrivate;
-
-static bool SimpleXLogPageRead(XLogReaderState *xlogreader,
- XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
+static bool SimpleXLogPageRead(XLogReaderState *xlogreader,
+ const char *datadir, int *tliIndex,
+ const char *restoreCommand);
/*
* Read WAL from the datadir/pg_wal, starting from 'startpoint' on timeline
@@ -66,20 +60,22 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
XLogRecord *record;
XLogReaderState *xlogreader;
char *errormsg;
- XLogPageReadPrivate private;
- private.tliIndex = tliIndex;
- private.restoreCommand = restoreCommand;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir,
- XL_ROUTINE(.page_read = &SimpleXLogPageRead),
- &private);
+ xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+
if (xlogreader == NULL)
pg_fatal("out of memory");
XLogBeginRead(xlogreader, startpoint);
do
{
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!SimpleXLogPageRead(xlogreader, datadir,
+ &tliIndex, restoreCommand))
+ break;
+ }
if (record == NULL)
{
@@ -123,19 +119,19 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex,
XLogRecord *record;
XLogReaderState *xlogreader;
char *errormsg;
- XLogPageReadPrivate private;
XLogRecPtr endptr;
- private.tliIndex = tliIndex;
- private.restoreCommand = restoreCommand;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir,
- XL_ROUTINE(.page_read = &SimpleXLogPageRead),
- &private);
+ xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
XLogBeginRead(xlogreader, ptr);
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!SimpleXLogPageRead(xlogreader, datadir, &tliIndex, restoreCommand))
+ break;
+ }
if (record == NULL)
{
if (errormsg)
@@ -170,7 +166,6 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
XLogRecPtr searchptr;
XLogReaderState *xlogreader;
char *errormsg;
- XLogPageReadPrivate private;
/*
* The given fork pointer points to the end of the last common record,
@@ -186,11 +181,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
forkptr += SizeOfXLogShortPHD;
}
- private.tliIndex = tliIndex;
- private.restoreCommand = restoreCommand;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir,
- XL_ROUTINE(.page_read = &SimpleXLogPageRead),
- &private);
+ xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
@@ -200,7 +191,13 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
uint8 info;
XLogBeginRead(xlogreader, searchptr);
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!SimpleXLogPageRead(xlogreader, datadir,
+ &tliIndex, restoreCommand))
+ break;
+ }
if (record == NULL)
{
@@ -247,10 +244,11 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
/* XLogReader callback function, to read a WAL page */
static bool
-SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *readBuf)
+SimpleXLogPageRead(XLogReaderState *xlogreader, const char *datadir,
+ int *tliIndex, const char *restoreCommand)
{
- XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
+ XLogRecPtr targetPagePtr = xlogreader->readPagePtr;
+ char *readBuf = xlogreader->readBuf;
uint32 targetPageOff;
XLogRecPtr targetSegEnd;
XLogSegNo targetSegNo;
@@ -283,14 +281,14 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
* be done both forward and backward, consider also switching timeline
* accordingly.
*/
- while (private->tliIndex < targetNentries - 1 &&
- targetHistory[private->tliIndex].end < targetSegEnd)
- private->tliIndex++;
- while (private->tliIndex > 0 &&
- targetHistory[private->tliIndex].begin >= targetSegEnd)
- private->tliIndex--;
+ while (*tliIndex < targetNentries - 1 &&
+ targetHistory[*tliIndex].end < targetSegEnd)
+ (*tliIndex)++;
+ while (*tliIndex > 0 &&
+ targetHistory[*tliIndex].begin >= targetSegEnd)
+ (*tliIndex)--;
- XLogFileName(xlogfname, targetHistory[private->tliIndex].tli,
+ XLogFileName(xlogfname, targetHistory[*tliIndex].tli,
xlogreadsegno, WalSegSz);
snprintf(xlogfpath, MAXPGPATH, "%s/" XLOGDIR "/%s",
@@ -303,7 +301,7 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
/*
* If we have no restore_command to execute, then exit.
*/
- if (private->restoreCommand == NULL)
+ if (restoreCommand == NULL)
{
pg_log_error("could not open file \"%s\": %m", xlogfpath);
xlogreader->readLen = -1;
@@ -317,7 +315,7 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
xlogreadfd = RestoreArchivedFile(xlogreader->segcxt.ws_dir,
xlogfname,
WalSegSz,
- private->restoreCommand);
+ restoreCommand);
if (xlogreadfd < 0)
{
@@ -359,7 +357,7 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
Assert(targetSegNo == xlogreadsegno);
- xlogreader->seg.ws_tli = targetHistory[private->tliIndex].tli;
+ xlogreader->seg.ws_tli = targetHistory[*tliIndex].tli;
xlogreader->readLen = XLOG_BLCKSZ;
return true;
}
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index 0f0e0723a3..f514de6b13 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -330,12 +330,17 @@ WALDumpCloseSegment(XLogReaderState *state)
state->seg.ws_file = -1;
}
-/* pg_waldump's XLogReaderRoutine->page_read callback */
+/*
+ * pg_waldump's WAL page rader, also used as page_read callback for
+ * XLogFindNextRecord
+ */
static bool
-WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetPtr, char *readBuff)
+WALDumpReadPage(XLogReaderState *state, void *priv)
{
- XLogDumpPrivate *private = state->private_data;
+ XLogRecPtr targetPagePtr = state->readPagePtr;
+ int reqLen = state->readLen;
+ char *readBuff = state->readBuf;
+ XLogDumpPrivate *private = (XLogDumpPrivate *) priv;
int count = XLOG_BLCKSZ;
WALReadError errinfo;
@@ -353,8 +358,8 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
}
}
- if (!WALRead(state, readBuff, targetPagePtr, count, private->timeline,
- &errinfo))
+ if (!WALRead(state, WALDumpOpenSegment, WALDumpCloseSegment,
+ readBuff, targetPagePtr, count, private->timeline, &errinfo))
{
WALOpenSegment *seg = &errinfo.wre_seg;
char fname[MAXPGPATH];
@@ -374,6 +379,7 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
(Size) errinfo.wre_req);
}
+ Assert(count >= state->readLen);
state->readLen = count;
return true;
}
@@ -1035,16 +1041,14 @@ main(int argc, char **argv)
/* we have everything we need, start reading */
xlogreader_state =
- XLogReaderAllocate(WalSegSz, waldir,
- XL_ROUTINE(.page_read = WALDumpReadPage,
- .segment_open = WALDumpOpenSegment,
- .segment_close = WALDumpCloseSegment),
- &private);
+ XLogReaderAllocate(WalSegSz, waldir, WALDumpCloseSegment);
+
if (!xlogreader_state)
fatal_error("out of memory");
/* first find a valid recptr to start from */
- first_record = XLogFindNextRecord(xlogreader_state, private.startptr);
+ first_record = XLogFindNextRecord(xlogreader_state, private.startptr,
+ &WALDumpReadPage, (void*) &private);
if (first_record == InvalidXLogRecPtr)
fatal_error("could not find a valid record after %X/%X",
@@ -1067,7 +1071,13 @@ main(int argc, char **argv)
for (;;)
{
/* try to read the next record */
- record = XLogReadRecord(xlogreader_state, &errormsg);
+ while (XLogReadRecord(xlogreader_state, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!WALDumpReadPage(xlogreader_state, (void *) &private))
+ break;
+ }
+
if (!record)
{
if (!config.follow || private.endptr_reached)
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index 23d9e70a59..4e9268b553 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -57,64 +57,15 @@ typedef struct WALSegmentContext
typedef struct XLogReaderState XLogReaderState;
-/* Function type definition for the read_page callback */
-typedef bool (*XLogPageReadCB) (XLogReaderState *xlogreader,
- XLogRecPtr targetPagePtr,
- int reqLen,
- XLogRecPtr targetRecPtr,
- char *readBuf);
+/* Function type definition for the segment cleanup callback */
+typedef void (*WALSegmentCleanupCB) (XLogReaderState *xlogreader);
+
+/* Function type definition for the open/close callbacks for WALRead() */
typedef void (*WALSegmentOpenCB) (XLogReaderState *xlogreader,
XLogSegNo nextSegNo,
TimeLineID *tli_p);
typedef void (*WALSegmentCloseCB) (XLogReaderState *xlogreader);
-typedef struct XLogReaderRoutine
-{
- /*
- * Data input callback
- *
- * This callback shall read at least reqLen valid bytes of the xlog page
- * starting at targetPagePtr, and store them in readBuf. The callback
- * shall return the number of bytes read (never more than XLOG_BLCKSZ), or
- * -1 on failure. The callback shall sleep, if necessary, to wait for the
- * requested bytes to become available. The callback will not be invoked
- * again for the same page unless more than the returned number of bytes
- * are needed.
- *
- * targetRecPtr is the position of the WAL record we're reading. Usually
- * it is equal to targetPagePtr + reqLen, but sometimes xlogreader needs
- * to read and verify the page or segment header, before it reads the
- * actual WAL record it's interested in. In that case, targetRecPtr can
- * be used to determine which timeline to read the page from.
- *
- * The callback shall set ->seg.ws_tli to the TLI of the file the page was
- * read from.
- */
- XLogPageReadCB page_read;
-
- /*
- * Callback to open the specified WAL segment for reading. ->seg.ws_file
- * shall be set to the file descriptor of the opened segment. In case of
- * failure, an error shall be raised by the callback and it shall not
- * return.
- *
- * "nextSegNo" is the number of the segment to be opened.
- *
- * "tli_p" is an input/output argument. WALRead() uses it to pass the
- * timeline in which the new segment should be found, but the callback can
- * use it to return the TLI that it actually opened.
- */
- WALSegmentOpenCB segment_open;
-
- /*
- * WAL segment close callback. ->seg.ws_file shall be set to a negative
- * number.
- */
- WALSegmentCloseCB segment_close;
-} XLogReaderRoutine;
-
-#define XL_ROUTINE(...) &(XLogReaderRoutine){__VA_ARGS__}
-
typedef struct
{
/* Is this block ref in use? */
@@ -144,12 +95,35 @@ typedef struct
uint16 data_bufsz;
} DecodedBkpBlock;
+/* Return code from XLogReadRecord */
+typedef enum XLogReadRecordResult
+{
+ XLREAD_SUCCESS, /* record is successfully read */
+ XLREAD_NEED_DATA, /* need more data. see XLogReadRecord. */
+ XLREAD_FAIL /* failed during reading a record */
+} XLogReadRecordResult;
+
+/*
+ * internal state of XLogReadRecord
+ *
+ * XLogReadState runs a state machine while reading a record. Theses states
+ * are not seen outside the function. Each state may repeat several times
+ * exiting requesting caller for new data. See the comment of XLogReadRecrod
+ * for details.
+ */
+typedef enum XLogReadRecordState {
+ XLREAD_NEXT_RECORD,
+ XLREAD_TOT_LEN,
+ XLREAD_FIRST_FRAGMENT,
+ XLREAD_CONTINUATION
+} XLogReadRecordState;
+
struct XLogReaderState
{
/*
* Operational callbacks
*/
- XLogReaderRoutine routine;
+ WALSegmentCleanupCB cleanup_cb;
/* ----------------------------------------
* Public parameters
@@ -162,18 +136,14 @@ struct XLogReaderState
*/
uint64 system_identifier;
- /*
- * Opaque data for callbacks to use. Not used by XLogReader.
- */
- void *private_data;
-
/*
* Start and end point of last record read. EndRecPtr is also used as the
* position to read next. Calling XLogBeginRead() sets EndRecPtr to the
* starting position and ReadRecPtr to invalid.
*/
- XLogRecPtr ReadRecPtr; /* start of last record read */
+ XLogRecPtr ReadRecPtr; /* start of last record read or being read */
XLogRecPtr EndRecPtr; /* end+1 of last record read */
+ XLogRecPtr PrevRecPtr; /* start of previous record read */
/* ----------------------------------------
* Communication with page reader
@@ -187,7 +157,9 @@ struct XLogReaderState
* the request, or -1 on error */
TimeLineID readPageTLI; /* TLI for data currently in readBuf */
char *readBuf; /* buffer to store data */
- bool page_verified; /* is the page on the buffer verified? */
+ bool page_verified; /* is the page header on the buffer verified? */
+ bool record_verified;/* is the current record header verified? */
+
/* ----------------------------------------
@@ -229,8 +201,6 @@ struct XLogReaderState
XLogRecPtr latestPagePtr;
TimeLineID latestPageTLI;
- /* beginning of the WAL record being read. */
- XLogRecPtr currRecPtr;
/* timeline to read it from, 0 if a lookup is required */
TimeLineID currTLI;
@@ -257,6 +227,15 @@ struct XLogReaderState
char *readRecordBuf;
uint32 readRecordBufSize;
+ /*
+ * XLogReadRecord() state
+ */
+ XLogReadRecordState readRecordState;/* state machine state */
+ int recordGotLen; /* amount of current record that has
+ * already been read */
+ int recordRemainLen; /* length of current record that remains */
+ XLogRecPtr recordContRecPtr; /* where the current record continues */
+
/* Buffer to hold error message */
char *errormsg_buf;
};
@@ -264,9 +243,7 @@ struct XLogReaderState
/* Get a new XLogReader */
extern XLogReaderState *XLogReaderAllocate(int wal_segment_size,
const char *waldir,
- XLogReaderRoutine *routine,
- void *private_data);
-extern XLogReaderRoutine *LocalXLogReaderRoutine(void);
+ WALSegmentCleanupCB cleanup_cb);
/* Free an XLogReader */
extern void XLogReaderFree(XLogReaderState *state);
@@ -274,12 +251,17 @@ extern void XLogReaderFree(XLogReaderState *state);
/* Position the XLogReader to given record */
extern void XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr);
#ifdef FRONTEND
-extern XLogRecPtr XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr);
+/* Function type definition for the read_page callback */
+typedef bool (*XLogFindNextRecordCB) (XLogReaderState *xlogreader,
+ void *private);
+extern XLogRecPtr XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
+ XLogFindNextRecordCB read_page, void *private);
#endif /* FRONTEND */
/* Read the next XLog record. Returns NULL on end-of-WAL or failure */
-extern struct XLogRecord *XLogReadRecord(XLogReaderState *state,
- char **errormsg);
+extern XLogReadRecordResult XLogReadRecord(XLogReaderState *state,
+ XLogRecord **record,
+ char **errormsg);
/* Validate a page */
extern bool XLogReaderValidatePageHeader(XLogReaderState *state,
@@ -299,6 +281,7 @@ typedef struct WALReadError
} WALReadError;
extern bool WALRead(XLogReaderState *state,
+ WALSegmentOpenCB segopenfn, WALSegmentCloseCB sgclosefn,
char *buf, XLogRecPtr startptr, Size count,
TimeLineID tli, WALReadError *errinfo);
diff --git a/src/include/access/xlogutils.h b/src/include/access/xlogutils.h
index 364a21c4ea..397fb27fc2 100644
--- a/src/include/access/xlogutils.h
+++ b/src/include/access/xlogutils.h
@@ -47,9 +47,7 @@ extern Buffer XLogReadBufferExtended(RelFileNode rnode, ForkNumber forknum,
extern Relation CreateFakeRelcacheEntry(RelFileNode rnode);
extern void FreeFakeRelcacheEntry(Relation fakerel);
-extern bool read_local_xlog_page(XLogReaderState *state,
- XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetRecPtr, char *cur_page);
+extern bool read_local_xlog_page(XLogReaderState *state);
extern void wal_segment_open(XLogReaderState *state,
XLogSegNo nextSegNo,
TimeLineID *tli_p);
diff --git a/src/include/pg_config_manual.h b/src/include/pg_config_manual.h
index f10ad0acd6..94749609b4 100644
--- a/src/include/pg_config_manual.h
+++ b/src/include/pg_config_manual.h
@@ -377,7 +377,7 @@
* Enable debugging print statements for WAL-related operations; see
* also the wal_debug GUC var.
*/
-/* #define WAL_DEBUG */
+#define WAL_DEBUG
/*
* Enable tracing of resource consumption during sort operations;
diff --git a/src/include/replication/logical.h b/src/include/replication/logical.h
index c253403372..b936f31393 100644
--- a/src/include/replication/logical.h
+++ b/src/include/replication/logical.h
@@ -29,6 +29,10 @@ typedef void (*LogicalOutputPluginWriterUpdateProgress) (struct LogicalDecodingC
TransactionId xid
);
+typedef struct LogicalDecodingContext LogicalDecodingContext;
+
+typedef bool (*LogicalDecodingXLogPageReadCB)(XLogReaderState *ctx);
+
typedef struct LogicalDecodingContext
{
/* memory context this is all allocated in */
@@ -39,6 +43,7 @@ typedef struct LogicalDecodingContext
/* infrastructure pieces for decoding */
XLogReaderState *reader;
+ LogicalDecodingXLogPageReadCB page_read;
struct ReorderBuffer *reorder;
struct SnapBuild *snapshot_builder;
@@ -105,14 +110,16 @@ extern LogicalDecodingContext *CreateInitDecodingContext(const char *plugin,
List *output_plugin_options,
bool need_full_snapshot,
XLogRecPtr restart_lsn,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress);
extern LogicalDecodingContext *CreateDecodingContext(XLogRecPtr start_lsn,
List *output_plugin_options,
bool fast_forward,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress);
--
2.27.0
----Next_Part(Thu_Mar__4_11_28_53_2021_014)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
filename="v17-0003-Remove-globals-readOff-readLen-and-readSegNo.patch"
^ permalink raw reply [nested|flat] 70+ messages in thread
* [PATCH v15 2/4] Move page-reader out of XLogReadRecord
@ 2019-09-10 03:58 Kyotaro Horiguchi <[email protected]>
0 siblings, 0 replies; 70+ messages in thread
From: Kyotaro Horiguchi @ 2019-09-10 03:58 UTC (permalink / raw)
This is the second step of removing callbacks from WAL record reader.
Since it is essential to take in additional data while reading a
record, the function have to ask caller for new data while keeping
working state. Thus the function is turned into a state machine.
---
src/backend/access/transam/twophase.c | 15 +-
src/backend/access/transam/xlog.c | 58 +-
src/backend/access/transam/xlogreader.c | 700 +++++++++++-------
src/backend/access/transam/xlogutils.c | 17 +-
src/backend/replication/logical/logical.c | 26 +-
.../replication/logical/logicalfuncs.c | 13 +-
src/backend/replication/slotfuncs.c | 18 +-
src/backend/replication/walsender.c | 32 +-
src/bin/pg_rewind/parsexlog.c | 84 +--
src/bin/pg_waldump/pg_waldump.c | 36 +-
src/include/access/xlogreader.h | 121 ++-
src/include/access/xlogutils.h | 4 +-
src/include/replication/logical.h | 11 +-
13 files changed, 648 insertions(+), 487 deletions(-)
diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index 9b2e59bf0e..b0d60a0d0f 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -1330,11 +1330,8 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
XLogReaderState *xlogreader;
char *errormsg;
- xlogreader = XLogReaderAllocate(wal_segment_size, NULL,
- XL_ROUTINE(.page_read = &read_local_xlog_page,
- .segment_open = &wal_segment_open,
- .segment_close = &wal_segment_close),
- NULL);
+ xlogreader = XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+
if (!xlogreader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
@@ -1342,7 +1339,13 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
errdetail("Failed while allocating a WAL reading processor.")));
XLogBeginRead(xlogreader, lsn);
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!read_local_xlog_page(xlogreader))
+ break;
+ }
+
if (record == NULL)
ereport(ERROR,
(errcode_for_file_access(),
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index e570e56a24..f9b0108602 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -831,13 +831,6 @@ static XLogSource currentSource = XLOG_FROM_ANY;
static bool lastSourceFailed = false;
static bool pendingWalRcvRestart = false;
-typedef struct XLogPageReadPrivate
-{
- int emode;
- bool fetching_ckpt; /* are we fetching a checkpoint record? */
- bool randAccess;
-} XLogPageReadPrivate;
-
/*
* These variables track when we last obtained some WAL data to process,
* and where we got it from. (XLogReceiptSource is initially the same as
@@ -912,8 +905,8 @@ static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath,
static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli,
XLogSource source, bool notfoundOk);
static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, XLogSource source);
-static bool XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
+static bool XLogPageRead(XLogReaderState *xlogreader,
+ bool fetching_ckpt, int emode, bool randAccess);
static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
bool fetching_ckpt, XLogRecPtr tliRecPtr);
static int emode_for_corrupt_record(int emode, XLogRecPtr RecPtr);
@@ -1225,8 +1218,7 @@ XLogInsertRecord(XLogRecData *rdata,
appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len);
if (!debug_reader)
- debug_reader = XLogReaderAllocate(wal_segment_size, NULL,
- XL_ROUTINE(), NULL);
+ debug_reader = XLogReaderAllocate(wal_segment_size, NULL NULL);
if (!debug_reader)
{
@@ -4326,15 +4318,10 @@ CleanupBackupHistory(void)
* record is available.
*/
static XLogRecord *
-ReadRecord(XLogReaderState *xlogreader, int emode,
- bool fetching_ckpt)
+ReadRecord(XLogReaderState *xlogreader, int emode, bool fetching_ckpt)
{
XLogRecord *record;
- XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
-
- private->fetching_ckpt = fetching_ckpt;
- private->emode = emode;
- private->randAccess = (xlogreader->ReadRecPtr == InvalidXLogRecPtr);
+ bool randAccess = (xlogreader->ReadRecPtr == InvalidXLogRecPtr);
/* This is the first attempt to read this page. */
lastSourceFailed = false;
@@ -4342,8 +4329,16 @@ ReadRecord(XLogReaderState *xlogreader, int emode,
for (;;)
{
char *errormsg;
+ XLogReadRecordResult result;
+
+ while ((result = XLogReadRecord(xlogreader, &record, &errormsg))
+ == XLREAD_NEED_DATA)
+ {
+ if (!XLogPageRead(xlogreader, fetching_ckpt, emode, randAccess))
+ break;
+
+ }
- record = XLogReadRecord(xlogreader, &errormsg);
ReadRecPtr = xlogreader->ReadRecPtr;
EndRecPtr = xlogreader->EndRecPtr;
if (record == NULL)
@@ -6320,7 +6315,6 @@ StartupXLOG(void)
bool backupFromStandby = false;
DBState dbstate_at_startup;
XLogReaderState *xlogreader;
- XLogPageReadPrivate private;
bool fast_promoted = false;
struct stat st;
@@ -6480,13 +6474,9 @@ StartupXLOG(void)
OwnLatch(&XLogCtl->recoveryWakeupLatch);
/* Set up XLOG reader facility */
- MemSet(&private, 0, sizeof(XLogPageReadPrivate));
xlogreader =
- XLogReaderAllocate(wal_segment_size, NULL,
- XL_ROUTINE(.page_read = &XLogPageRead,
- .segment_open = NULL,
- .segment_close = wal_segment_close),
- &private);
+ XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+
if (!xlogreader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
@@ -11860,12 +11850,13 @@ CancelBackup(void)
* sleep and retry.
*/
static bool
-XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetRecPtr, char *readBuf)
+XLogPageRead(XLogReaderState *xlogreader,
+ bool fetching_ckpt, int emode, bool randAccess)
{
- XLogPageReadPrivate *private =
- (XLogPageReadPrivate *) xlogreader->private_data;
- int emode = private->emode;
+ char *readBuf = xlogreader->readBuf;
+ XLogRecPtr targetPagePtr = xlogreader->readPagePtr;
+ int reqLen = xlogreader->readLen;
+ XLogRecPtr targetRecPtr = xlogreader->ReadRecPtr;
uint32 targetPageOff;
XLogSegNo targetSegNo PG_USED_FOR_ASSERTS_ONLY;
int r;
@@ -11908,8 +11899,8 @@ retry:
flushedUpto < targetPagePtr + reqLen))
{
if (!WaitForWALToBecomeAvailable(targetPagePtr + reqLen,
- private->randAccess,
- private->fetching_ckpt,
+ randAccess,
+ fetching_ckpt,
targetRecPtr))
{
if (readFile >= 0)
@@ -12014,6 +12005,7 @@ retry:
goto next_record_is_invalid;
}
+ Assert(xlogreader->readPagePtr == targetPagePtr);
xlogreader->readLen = readLen;
return true;
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index 3d599325ee..9281b57379 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -40,7 +40,7 @@ static bool XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr,
int reqLen, bool header_inclusive);
static void XLogReaderInvalReadState(XLogReaderState *state);
static bool ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
- XLogRecPtr PrevRecPtr, XLogRecord *record, bool randAccess);
+ XLogRecPtr PrevRecPtr, XLogRecord *record);
static bool ValidXLogRecord(XLogReaderState *state, XLogRecord *record,
XLogRecPtr recptr);
static void ResetDecoder(XLogReaderState *state);
@@ -73,7 +73,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...)
*/
XLogReaderState *
XLogReaderAllocate(int wal_segment_size, const char *waldir,
- XLogReaderRoutine *routine, void *private_data)
+ WALSegmentCleanupCB cleanup_cb)
{
XLogReaderState *state;
@@ -84,7 +84,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
return NULL;
/* initialize caller-provided support functions */
- state->routine = *routine;
+ state->cleanup_cb = cleanup_cb;
state->max_block_id = -1;
@@ -107,8 +107,6 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
WALOpenSegmentInit(&state->seg, &state->segcxt, wal_segment_size,
waldir);
- /* system_identifier initialized to zeroes above */
- state->private_data = private_data;
/* ReadRecPtr, EndRecPtr and readLen initialized to zeroes above */
state->errormsg_buf = palloc_extended(MAX_ERRORMSG_LEN + 1,
MCXT_ALLOC_NO_OOM);
@@ -140,8 +138,8 @@ XLogReaderFree(XLogReaderState *state)
{
int block_id;
- if (state->seg.ws_file != -1)
- state->routine.segment_close(state);
+ if (state->seg.ws_file >= 0)
+ state->cleanup_cb(state);
for (block_id = 0; block_id <= XLR_MAX_BLOCK_ID; block_id++)
{
@@ -246,6 +244,7 @@ XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
/* Begin at the passed-in record pointer. */
state->EndRecPtr = RecPtr;
state->ReadRecPtr = InvalidXLogRecPtr;
+ state->readRecordState = XLREAD_NEXT_RECORD;
}
/*
@@ -254,317 +253,452 @@ XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
* XLogBeginRead() or XLogFindNextRecord() must be called before the first call
* to XLogReadRecord().
*
- * If the page_read callback fails to read the requested data, NULL is
- * returned. The callback is expected to have reported the error; errormsg
- * is set to NULL.
+ * This function may return XLREAD_NEED_DATA several times before returning a
+ * result record. The caller shall read in some new data then call this
+ * function again with the same parameters.
*
- * If the reading fails for some other reason, NULL is also returned, and
- * *errormsg is set to a string with details of the failure.
+ * When a record is successfully read, returns XLREAD_SUCCESS with result
+ * record being stored in *record. Otherwise *record is NULL.
*
- * The returned pointer (or *errormsg) points to an internal buffer that's
- * valid until the next call to XLogReadRecord.
+ * Returns XLREAD_NEED_DATA if more data is needed to finish reading the
+ * current record. In that case, state->readPagePtr and state->readLen inform
+ * the desired position and minimum length of data needed. The caller shall
+ * read in the requested data and set state->readBuf to point to a buffer
+ * containing it. The caller must also set state->readPageTLI and
+ * state->readLen to indicate the timeline that it was read from, and the
+ * length of data that is now available (which must be >= given readLen),
+ * respectively.
+ *
+ * If invalid data is encountered, returns XLREAD_FAIL with *record being set to
+ * NULL. *errormsg is set to a string with details of the failure.
+ * The returned pointer (or *errormsg) points to an internal buffer that's valid
+ * until the next call to XLogReadRecord.
+ *
+ *
+ * This function runs a state machine consists of the following states.
+ *
+ * XLREAD_NEXT_RECORD :
+ * The initial state, if called with valid RecPtr, try to read a record at
+ * that position. If invalid RecPtr is given try to read a record just after
+ * the last one previously read.
+ * This state ens after setting ReadRecPtr. Then goes to XLREAD_TOT_LEN.
+ *
+ * XLREAD_TOT_LEN:
+ * Examining record header. Ends after reading record total
+ * length. recordRemainLen and recordGotLen are initialized.
+ *
+ * XLREAD_FIRST_FRAGMENT:
+ * Reading the first fragment. Ends with finishing reading a single
+ * record. Goes to XLREAD_NEXT_RECORD if that's all or
+ * XLREAD_CONTINUATION if we have continuation.
+
+ * XLREAD_CONTINUATION:
+ * Reading continuation of record. Ends with finishing the whole record then
+ * goes to XLREAD_NEXT_RECORD. During this state, recordRemainLen indicates
+ * how much is left and readRecordBuf holds the partially assert
+ * record.recordContRecPtr points to the beginning of the next page where to
+ * continue.
+ *
+ * If wrong data found in any state, the state machine stays at the current
+ * state. This behavior allows to continue reading a reacord switching among
+ * different souces, while streaming replication.
*/
-XLogRecord *
-XLogReadRecord(XLogReaderState *state, char **errormsg)
+XLogReadRecordResult
+XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
{
- XLogRecPtr RecPtr;
- XLogRecord *record;
- XLogRecPtr targetPagePtr;
- bool randAccess;
- uint32 len,
- total_len;
- uint32 targetRecOff;
- uint32 pageHeaderSize;
- bool gotheader;
+ XLogRecord *prec;
- /*
- * randAccess indicates whether to verify the previous-record pointer of
- * the record we're reading. We only do this if we're reading
- * sequentially, which is what we initially assume.
- */
- randAccess = false;
+ *record = NULL;
/* reset error state */
*errormsg = NULL;
state->errormsg_buf[0] = '\0';
- ResetDecoder(state);
-
- RecPtr = state->EndRecPtr;
-
- if (state->ReadRecPtr != InvalidXLogRecPtr)
- {
- /* read the record after the one we just read */
-
- /*
- * EndRecPtr is pointing to end+1 of the previous WAL record. If
- * we're at a page boundary, no more records can fit on the current
- * page. We must skip over the page header, but we can't do that until
- * we've read in the page, since the header size is variable.
- */
- }
- else
- {
- /*
- * Caller supplied a position to start at.
- *
- * In this case, EndRecPtr should already be pointing to a valid
- * record starting position.
- */
- Assert(XRecOffIsValid(RecPtr));
- randAccess = true;
- }
-
- state->currRecPtr = RecPtr;
-
- targetPagePtr = RecPtr - (RecPtr % XLOG_BLCKSZ);
- targetRecOff = RecPtr % XLOG_BLCKSZ;
-
- /*
- * Read the page containing the record into state->readBuf. Request enough
- * byte to cover the whole record header, or at least the part of it that
- * fits on the same page.
- */
- while (XLogNeedData(state, targetPagePtr,
- Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
- targetRecOff != 0))
- {
- if (!state->routine.page_read(state, state->readPagePtr, state->readLen,
- RecPtr, state->readBuf))
- break;
- }
-
- if (!state->page_verified)
- goto err;
-
- /*
- * We have at least the page header, so we can examine it now.
- */
- pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
- if (targetRecOff == 0)
- {
- /*
- * At page start, so skip over page header.
- */
- RecPtr += pageHeaderSize;
- targetRecOff = pageHeaderSize;
- }
- else if (targetRecOff < pageHeaderSize)
- {
- report_invalid_record(state, "invalid record offset at %X/%X",
- (uint32) (RecPtr >> 32), (uint32) RecPtr);
- goto err;
- }
-
- if ((((XLogPageHeader) state->readBuf)->xlp_info & XLP_FIRST_IS_CONTRECORD) &&
- targetRecOff == pageHeaderSize)
- {
- report_invalid_record(state, "contrecord is requested by %X/%X",
- (uint32) (RecPtr >> 32), (uint32) RecPtr);
- goto err;
- }
-
- /* XLogNeedData has verified the page header */
- Assert(pageHeaderSize <= state->readLen);
-
- /*
- * Read the record length.
- *
- * NB: Even though we use an XLogRecord pointer here, the whole record
- * header might not fit on this page. xl_tot_len is the first field of the
- * struct, so it must be on this page (the records are MAXALIGNed), but we
- * cannot access any other fields until we've verified that we got the
- * whole header.
- */
- record = (XLogRecord *) (state->readBuf + RecPtr % XLOG_BLCKSZ);
- total_len = record->xl_tot_len;
-
- /*
- * If the whole record header is on this page, validate it immediately.
- * Otherwise do just a basic sanity check on xl_tot_len, and validate the
- * rest of the header after reading it from the next page. The xl_tot_len
- * check is necessary here to ensure that we enter the "Need to reassemble
- * record" code path below; otherwise we might fail to apply
- * ValidXLogRecordHeader at all.
- */
- if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
- {
- if (!ValidXLogRecordHeader(state, RecPtr, state->ReadRecPtr, record,
- randAccess))
- goto err;
- gotheader = true;
- }
- else
- {
- /* XXX: more validation should be done here */
- if (total_len < SizeOfXLogRecord)
- {
- report_invalid_record(state,
- "invalid record length at %X/%X: wanted %u, got %u",
- (uint32) (RecPtr >> 32), (uint32) RecPtr,
- (uint32) SizeOfXLogRecord, total_len);
- goto err;
- }
- gotheader = false;
- }
-
- len = XLOG_BLCKSZ - RecPtr % XLOG_BLCKSZ;
- if (total_len > len)
+ switch (state->readRecordState)
{
- /* Need to reassemble record */
- char *contdata;
- XLogPageHeader pageHeader;
- char *buffer;
- uint32 gotlen;
-
- /*
- * Enlarge readRecordBuf as needed.
- */
- if (total_len > state->readRecordBufSize &&
- !allocate_recordbuf(state, total_len))
- {
- /* We treat this as a "bogus data" condition */
- report_invalid_record(state, "record length %u at %X/%X too long",
- total_len,
- (uint32) (RecPtr >> 32), (uint32) RecPtr);
- goto err;
- }
+ case XLREAD_NEXT_RECORD:
+ ResetDecoder(state);
- /* Copy the first fragment of the record from the first page. */
- memcpy(state->readRecordBuf,
- state->readBuf + RecPtr % XLOG_BLCKSZ, len);
- buffer = state->readRecordBuf + len;
- gotlen = len;
-
- do
- {
- int rest_len = total_len - gotlen;
-
- /* Calculate pointer to beginning of next page */
- targetPagePtr += XLOG_BLCKSZ;
-
- /* Wait for the next page to become available */
- while (XLogNeedData(state, targetPagePtr,
- Min(rest_len, XLOG_BLCKSZ),
- false))
+ if (state->ReadRecPtr != InvalidXLogRecPtr)
{
- if (!state->routine.page_read(state, state->readPagePtr,
- state->readLen,
- state->ReadRecPtr,
- state->readBuf))
- break;
+ /* read the record after the one we just read */
+
+ /*
+ * EndRecPtr is pointing to end+1 of the previous WAL record.
+ * If we're at a page boundary, no more records can fit on the
+ * current page. We must skip over the page header, but we
+ * can't do that until we've read in the page, since the header
+ * size is variable.
+ */
+ state->PrevRecPtr = state->ReadRecPtr;
+ state->ReadRecPtr = state->EndRecPtr;
+ }
+ else
+ {
+ /*
+ * Caller supplied a position to start at.
+ *
+ * In this case, EndRecPtr should already be pointing to a
+ * valid record starting position.
+ */
+ Assert(XRecOffIsValid(state->EndRecPtr));
+ state->ReadRecPtr = state->EndRecPtr;
+
+ /*
+ * We cannot verify the previous-record pointer when we're
+ * seeking to a particular record. Reset PrevRecPtr so that we
+ * won't try doing that.
+ */
+ state->PrevRecPtr = InvalidXLogRecPtr;
+ state->EndRecPtr = InvalidXLogRecPtr; /* to be tidy */
}
+ state->record_verified = false;
+ state->readRecordState = XLREAD_TOT_LEN;
+ /* fall through */
+
+ case XLREAD_TOT_LEN:
+ {
+ uint32 total_len;
+ uint32 pageHeaderSize;
+ XLogRecPtr targetPagePtr;
+ uint32 targetRecOff;
+ XLogPageHeader pageHeader;
+
+ targetPagePtr =
+ state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
+ targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+
+ /*
+ * Check if we have enough data. For the first record in the page,
+ * the requesting length doesn't contain page header.
+ */
+ if (XLogNeedData(state, targetPagePtr,
+ Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+ targetRecOff != 0))
+ return XLREAD_NEED_DATA;
+
+ /* error out if caller supplied bogus page */
if (!state->page_verified)
goto err;
- Assert(SizeOfXLogShortPHD <= state->readLen);
+ /* examine page header now. */
+ pageHeaderSize =
+ XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+ if (targetRecOff == 0)
+ {
+ /* At page start, so skip over page header. */
+ state->ReadRecPtr += pageHeaderSize;
+ targetRecOff = pageHeaderSize;
+ }
+ else if (targetRecOff < pageHeaderSize)
+ {
+ report_invalid_record(state, "invalid record offset at %X/%X",
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr);
+ goto err;
+ }
- /* Check that the continuation on next page looks valid */
pageHeader = (XLogPageHeader) state->readBuf;
- if (!(pageHeader->xlp_info & XLP_FIRST_IS_CONTRECORD))
+ if ((pageHeader->xlp_info & XLP_FIRST_IS_CONTRECORD) &&
+ targetRecOff == pageHeaderSize)
{
- report_invalid_record(state,
- "there is no contrecord flag at %X/%X",
- (uint32) (RecPtr >> 32), (uint32) RecPtr);
+ report_invalid_record(state, "contrecord is requested by %X/%X",
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr);
goto err;
}
- /*
- * Cross-check that xlp_rem_len agrees with how much of the record
- * we expect there to be left.
- */
- if (pageHeader->xlp_rem_len == 0 ||
- total_len != (pageHeader->xlp_rem_len + gotlen))
- {
- report_invalid_record(state,
- "invalid contrecord length %u at %X/%X",
- pageHeader->xlp_rem_len,
- (uint32) (RecPtr >> 32), (uint32) RecPtr);
- goto err;
- }
-
- /* Append the continuation from this page to the buffer */
- pageHeaderSize = XLogPageHeaderSize(pageHeader);
-
+ /* XLogNeedData has verified the page header */
Assert(pageHeaderSize <= state->readLen);
- contdata = (char *) state->readBuf + pageHeaderSize;
- len = XLOG_BLCKSZ - pageHeaderSize;
- if (pageHeader->xlp_rem_len < len)
- len = pageHeader->xlp_rem_len;
+ /*
+ * Read the record length.
+ *
+ * NB: Even though we use an XLogRecord pointer here, the whole
+ * record header might not fit on this page. xl_tot_len is the first
+ * field of the struct, so it must be on this page (the records are
+ * MAXALIGNed), but we cannot access any other fields until we've
+ * verified that we got the whole header.
+ */
+ prec = (XLogRecord *) (state->readBuf +
+ state->ReadRecPtr % XLOG_BLCKSZ);
+ total_len = prec->xl_tot_len;
- Assert (pageHeaderSize + len <= state->readLen);
- memcpy(buffer, (char *) contdata, len);
- buffer += len;
- gotlen += len;
+ /*
+ * If the whole record header is on this page, validate it
+ * immediately. Otherwise do just a basic sanity check on
+ * xl_tot_len, and validate the rest of the header after reading it
+ * from the next page. The xl_tot_len check is necessary here to
+ * ensure that we enter the XLREAD_CONTINUATION state below;
+ * otherwise we might fail to apply ValidXLogRecordHeader at all.
+ */
+ if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
+ {
+ if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
+ state->PrevRecPtr, prec))
+ goto err;
- /* If we just reassembled the record header, validate it. */
- if (!gotheader)
+ state->record_verified = true;
+ }
+ else
{
- record = (XLogRecord *) state->readRecordBuf;
- if (!ValidXLogRecordHeader(state, RecPtr, state->ReadRecPtr,
- record, randAccess))
+ /* XXX: more validation should be done here */
+ if (total_len < SizeOfXLogRecord)
+ {
+ report_invalid_record(state,
+ "invalid record length at %X/%X: wanted %u, got %u",
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr,
+ (uint32) SizeOfXLogRecord, total_len);
goto err;
- gotheader = true;
+ }
}
- } while (gotlen < total_len);
-
- Assert(gotheader);
- record = (XLogRecord *) state->readRecordBuf;
- if (!ValidXLogRecord(state, record, RecPtr))
- goto err;
+ /*
+ * Wait for the rest of the record, or the part of the record that
+ * fit on the first page if crossed a page boundary, to become
+ * available.
+ */
+ state->recordGotLen = 0;
+ state->recordRemainLen = total_len;
+ state->readRecordState = XLREAD_FIRST_FRAGMENT;
+ }
+ /* fall through */
- pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
- state->ReadRecPtr = RecPtr;
- state->EndRecPtr = targetPagePtr + pageHeaderSize
- + MAXALIGN(pageHeader->xlp_rem_len);
- }
- else
- {
- /* Wait for the record data to become available */
- while (XLogNeedData(state, targetPagePtr,
- Min(targetRecOff + total_len, XLOG_BLCKSZ), true))
+ case XLREAD_FIRST_FRAGMENT:
{
- if (!state->routine.page_read(state, state->readPagePtr,
- state->readLen,
- state->ReadRecPtr, state->readBuf))
+ uint32 total_len = state->recordRemainLen;
+ uint32 request_len;
+ uint32 record_len;
+ XLogRecPtr targetPagePtr;
+ uint32 targetRecOff;
+
+ /*
+ * Wait for the rest of the record on the first page to become
+ * available
+ */
+ targetPagePtr =
+ state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
+ targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+
+ request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ);
+ record_len = request_len - targetRecOff;
+
+ /* ReadRecPtr contains page header */
+ Assert (targetRecOff != 0);
+ if (XLogNeedData(state, targetPagePtr, request_len, true))
+ return XLREAD_NEED_DATA;
+
+ /* error out if caller supplied bogus page */
+ if (!state->page_verified)
+ goto err;
+
+ prec = (XLogRecord *) (state->readBuf + targetRecOff);
+
+ /* validate record header if not yet */
+ if (!state->record_verified && record_len >= SizeOfXLogRecord)
+ {
+ if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
+ state->PrevRecPtr, prec))
+ goto err;
+
+ state->record_verified = true;
+ }
+
+
+ if (total_len == record_len)
+ {
+ /* Record does not cross a page boundary */
+ Assert(state->record_verified);
+
+ if (!ValidXLogRecord(state, prec, state->ReadRecPtr))
+ goto err;
+
+ state->record_verified = true; /* to be tidy */
+
+ /* We already checked the header earlier */
+ state->EndRecPtr = state->ReadRecPtr + MAXALIGN(record_len);
+
+ *record = prec;
+ state->readRecordState = XLREAD_NEXT_RECORD;
break;
+ }
+
+ /*
+ * The record continues on the next page. Need to reassemble
+ * record
+ */
+ Assert(total_len > record_len);
+
+ /* Enlarge readRecordBuf as needed. */
+ if (total_len > state->readRecordBufSize &&
+ !allocate_recordbuf(state, total_len))
+ {
+ /* We treat this as a "bogus data" condition */
+ report_invalid_record(state,
+ "record length %u at %X/%X too long",
+ total_len,
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr);
+ goto err;
+ }
+
+ /* Copy the first fragment of the record from the first page. */
+ memcpy(state->readRecordBuf, state->readBuf + targetRecOff,
+ record_len);
+ state->recordGotLen += record_len;
+ state->recordRemainLen -= record_len;
+
+ /* Calculate pointer to beginning of next page */
+ state->recordContRecPtr = state->ReadRecPtr + record_len;
+ Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0);
+
+ state->readRecordState = XLREAD_CONTINUATION;
}
+ /* fall through */
+
+ case XLREAD_CONTINUATION:
+ {
+ XLogPageHeader pageHeader;
+ uint32 pageHeaderSize;
+ XLogRecPtr targetPagePtr;
+
+ /* we enter this state only if we haven't read the whole record. */
+ Assert (state->recordRemainLen > 0);
+
+ while(state->recordRemainLen > 0)
+ {
+ char *contdata;
+ uint32 request_len;
+ uint32 record_len;
+
+ /* Wait for the next page to become available */
+ targetPagePtr = state->recordContRecPtr;
+
+ /* this request contains page header */
+ Assert (targetPagePtr != 0);
+ if (XLogNeedData(state, targetPagePtr,
+ Min(state->recordRemainLen, XLOG_BLCKSZ),
+ false))
+ return XLREAD_NEED_DATA;
+
+ if (!state->page_verified)
+ goto err;
+
+ Assert(SizeOfXLogShortPHD <= state->readLen);
- if (!state->page_verified)
- goto err;
+ /* Check that the continuation on next page looks valid */
+ pageHeader = (XLogPageHeader) state->readBuf;
+ if (!(pageHeader->xlp_info & XLP_FIRST_IS_CONTRECORD))
+ {
+ report_invalid_record(
+ state,
+ "there is no contrecord flag at %X/%X reading %X/%X",
+ (uint32) (state->recordContRecPtr >> 32),
+ (uint32) state->recordContRecPtr,
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr);
+ goto err;
+ }
- /* Record does not cross a page boundary */
- if (!ValidXLogRecord(state, record, RecPtr))
- goto err;
+ /*
+ * Cross-check that xlp_rem_len agrees with how much of the
+ * record we expect there to be left.
+ */
+ if (pageHeader->xlp_rem_len == 0 ||
+ pageHeader->xlp_rem_len != state->recordRemainLen)
+ {
+ report_invalid_record(
+ state,
+ "invalid contrecord length %u at %X/%X reading %X/%X, expected %u",
+ pageHeader->xlp_rem_len,
+ (uint32) (state->recordContRecPtr >> 32),
+ (uint32) state->recordContRecPtr,
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr,
+ state->recordRemainLen);
+ goto err;
+ }
- state->EndRecPtr = RecPtr + MAXALIGN(total_len);
+ /* Append the continuation from this page to the buffer */
+ pageHeaderSize = XLogPageHeaderSize(pageHeader);
- state->ReadRecPtr = RecPtr;
+ /*
+ * XLogNeedData should have ensured that the whole page header
+ * was read
+ */
+ Assert(state->readLen >= pageHeaderSize);
+
+ contdata = (char *) state->readBuf + pageHeaderSize;
+ record_len = XLOG_BLCKSZ - pageHeaderSize;
+ if (pageHeader->xlp_rem_len < record_len)
+ record_len = pageHeader->xlp_rem_len;
+
+ request_len = record_len + pageHeaderSize;
+
+ /* XLogNeedData should have ensured all needed data was read */
+ Assert (state->readLen >= request_len);
+
+ memcpy(state->readRecordBuf + state->recordGotLen,
+ (char *) contdata, record_len);
+ state->recordGotLen += record_len;
+ state->recordRemainLen -= record_len;
+
+ /* If we just reassembled the record header, validate it. */
+ if (!state->record_verified)
+ {
+ Assert(state->recordGotLen >= SizeOfXLogRecord);
+ if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
+ state->PrevRecPtr,
+ (XLogRecord *) state->readRecordBuf))
+ goto err;
+
+ state->record_verified = true;
+ }
+
+ /* Calculate pointer to beginning of next page, and continue */
+ state->recordContRecPtr += XLOG_BLCKSZ;
+ }
+
+ /* targetPagePtr is pointing the last-read page here */
+ prec = (XLogRecord *) state->readRecordBuf;
+ if (!ValidXLogRecord(state, prec, state->ReadRecPtr))
+ goto err;
+
+ pageHeaderSize =
+ XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+ state->EndRecPtr = targetPagePtr + pageHeaderSize
+ + MAXALIGN(pageHeader->xlp_rem_len);
+
+ *record = prec;
+ state->readRecordState = XLREAD_NEXT_RECORD;
+ break;
+ }
}
/*
* Special processing if it's an XLOG SWITCH record
*/
- if (record->xl_rmid == RM_XLOG_ID &&
- (record->xl_info & ~XLR_INFO_MASK) == XLOG_SWITCH)
+ if ((*record)->xl_rmid == RM_XLOG_ID &&
+ ((*record)->xl_info & ~XLR_INFO_MASK) == XLOG_SWITCH)
{
/* Pretend it extends to end of segment */
state->EndRecPtr += state->segcxt.ws_segsize - 1;
state->EndRecPtr -= XLogSegmentOffset(state->EndRecPtr, state->segcxt.ws_segsize);
}
- if (DecodeXLogRecord(state, record, errormsg))
- return record;
- else
- return NULL;
+ Assert (!*record || state->readLen >= 0);
+ if (DecodeXLogRecord(state, *record, errormsg))
+ return XLREAD_SUCCESS;
+
+ *record = NULL;
+ return XLREAD_FAIL;
err:
/*
- * Invalidate the read state. We might read from a different source after
+ * Invalidate the read page. We might read from a different source after
* failure.
*/
XLogReaderInvalReadState(state);
@@ -572,7 +706,8 @@ err:
if (state->errormsg_buf[0] != '\0')
*errormsg = state->errormsg_buf;
- return NULL;
+ *record = NULL;
+ return XLREAD_FAIL;
}
/*
@@ -724,11 +859,12 @@ XLogReaderInvalReadState(XLogReaderState *state)
*
* This is just a convenience subroutine to avoid duplicated code in
* XLogReadRecord. It's not intended for use from anywhere else.
+ *
+ * If PrevRecPtr is valid, the xl_prev is is cross-checked with it.
*/
static bool
ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
- XLogRecPtr PrevRecPtr, XLogRecord *record,
- bool randAccess)
+ XLogRecPtr PrevRecPtr, XLogRecord *record)
{
if (record->xl_tot_len < SizeOfXLogRecord)
{
@@ -746,7 +882,7 @@ ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
(uint32) RecPtr);
return false;
}
- if (randAccess)
+ if (PrevRecPtr == InvalidXLogRecPtr)
{
/*
* We can't exactly verify the prev-link, but surely it should be less
@@ -978,11 +1114,14 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
* XLogReadRecord() will read the next valid record.
*/
XLogRecPtr
-XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
+XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
+ XLogFindNextRecordCB read_page, void *private)
{
XLogRecPtr tmpRecPtr;
XLogRecPtr found = InvalidXLogRecPtr;
XLogPageHeader header;
+ XLogRecord *record;
+ XLogReadRecordResult result;
char *errormsg;
Assert(!XLogRecPtrIsInvalid(RecPtr));
@@ -1015,9 +1154,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
while(XLogNeedData(state, targetPagePtr, targetRecOff,
targetRecOff != 0))
{
- if (!state->routine.page_read(state, state->readPagePtr,
- state->readLen,
- state->ReadRecPtr, state->readBuf))
+ if (!read_page(state, private))
break;
}
@@ -1069,8 +1206,16 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
* or we just jumped over the remaining data of a continuation.
*/
XLogBeginRead(state, tmpRecPtr);
- while (XLogReadRecord(state, &errormsg) != NULL)
+ while ((result = XLogReadRecord(state, &record, &errormsg)) !=
+ XLREAD_FAIL)
{
+ if (result == XLREAD_NEED_DATA)
+ {
+ if (!read_page(state, private))
+ goto err;
+ continue;
+ }
+
/* past the record we've found, break out */
if (RecPtr <= state->ReadRecPtr)
{
@@ -1090,9 +1235,9 @@ err:
#endif /* FRONTEND */
/*
- * Helper function to ease writing of XLogRoutine->page_read callbacks.
- * If this function is used, caller must supply a segment_open callback in
- * 'state', as that is used here.
+ * Helper function to ease writing of page_read callback.
+ * If this function is used, caller must supply a segment_open callback and
+ * segment_close callback as that is used here.
*
* Read 'count' bytes into 'buf', starting at location 'startptr', from WAL
* fetched from timeline 'tli'.
@@ -1105,6 +1250,7 @@ err:
*/
bool
WALRead(XLogReaderState *state,
+ WALSegmentOpenCB segopenfn, WALSegmentCloseCB segclosefn,
char *buf, XLogRecPtr startptr, Size count, TimeLineID tli,
WALReadError *errinfo)
{
@@ -1136,10 +1282,10 @@ WALRead(XLogReaderState *state,
XLogSegNo nextSegNo;
if (state->seg.ws_file >= 0)
- state->routine.segment_close(state);
+ segclosefn(state);
XLByteToSeg(recptr, nextSegNo, state->segcxt.ws_segsize);
- state->routine.segment_open(state, nextSegNo, &tli);
+ segopenfn(state, nextSegNo, &tli);
/* This shouldn't happen -- indicates a bug in segment_open */
Assert(state->seg.ws_file >= 0);
diff --git a/src/backend/access/transam/xlogutils.c b/src/backend/access/transam/xlogutils.c
index 1b10b6df20..303333571f 100644
--- a/src/backend/access/transam/xlogutils.c
+++ b/src/backend/access/transam/xlogutils.c
@@ -689,8 +689,7 @@ XLogTruncateRelation(RelFileNode rnode, ForkNumber forkNum,
void
XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wantLength)
{
- const XLogRecPtr lastReadPage = state->seg.ws_segno *
- state->segcxt.ws_segsize + state->readLen;
+ const XLogRecPtr lastReadPage = state->readPagePtr;
Assert(wantPage != InvalidXLogRecPtr && wantPage % XLOG_BLCKSZ == 0);
Assert(wantLength <= XLOG_BLCKSZ);
@@ -705,7 +704,7 @@ XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wa
* current TLI has since become historical.
*/
if (lastReadPage == wantPage &&
- state->readLen != 0 &&
+ state->page_verified &&
lastReadPage + state->readLen >= wantPage + Min(wantLength, XLOG_BLCKSZ - 1))
return;
@@ -792,6 +791,7 @@ wal_segment_open(XLogReaderState *state, XLogSegNo nextSegNo,
char path[MAXPGPATH];
XLogFilePath(path, tli, nextSegNo, state->segcxt.ws_segsize);
+ elog(LOG, "HOGE: %lu, %d => %s", nextSegNo, tli, path);
state->seg.ws_file = BasicOpenFile(path, O_RDONLY | PG_BINARY);
if (state->seg.ws_file >= 0)
return;
@@ -829,9 +829,11 @@ wal_segment_close(XLogReaderState *state)
* loop for now.
*/
bool
-read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *cur_page)
+read_local_xlog_page(XLogReaderState *state)
{
+ XLogRecPtr targetPagePtr = state->readPagePtr;
+ int reqLen = state->readLen;
+ char *cur_page = state->readBuf;
XLogRecPtr read_upto,
loc;
TimeLineID tli;
@@ -944,11 +946,12 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
* as 'count', read the whole page anyway. It's guaranteed to be
* zero-padded up to the page boundary if it's incomplete.
*/
- if (!WALRead(state, cur_page, targetPagePtr, XLOG_BLCKSZ, tli,
- &errinfo))
+ if (!WALRead(state, wal_segment_open, wal_segment_close,
+ cur_page, targetPagePtr, XLOG_BLCKSZ, tli, &errinfo))
WALReadRaiseError(&errinfo);
/* number of valid bytes in the buffer */
+ state->readPagePtr = targetPagePtr;
state->readLen = count;
return true;
}
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index 61902be3b0..a15b0b3355 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -120,7 +120,8 @@ StartupDecodingContext(List *output_plugin_options,
TransactionId xmin_horizon,
bool need_full_snapshot,
bool fast_forward,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress)
@@ -169,11 +170,12 @@ StartupDecodingContext(List *output_plugin_options,
ctx->slot = slot;
- ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, xl_routine, ctx);
+ ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, cleanup_cb);
if (!ctx->reader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
errmsg("out of memory")));
+ ctx->page_read = page_read;
ctx->reorder = ReorderBufferAllocate();
ctx->snapshot_builder =
@@ -231,7 +233,8 @@ CreateInitDecodingContext(char *plugin,
List *output_plugin_options,
bool need_full_snapshot,
XLogRecPtr restart_lsn,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress)
@@ -328,7 +331,7 @@ CreateInitDecodingContext(char *plugin,
ctx = StartupDecodingContext(NIL, restart_lsn, xmin_horizon,
need_full_snapshot, false,
- xl_routine, prepare_write, do_write,
+ page_read, cleanup_cb, prepare_write, do_write,
update_progress);
/* call output plugin initialization callback */
@@ -376,7 +379,8 @@ LogicalDecodingContext *
CreateDecodingContext(XLogRecPtr start_lsn,
List *output_plugin_options,
bool fast_forward,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress)
@@ -429,8 +433,8 @@ CreateDecodingContext(XLogRecPtr start_lsn,
ctx = StartupDecodingContext(output_plugin_options,
start_lsn, InvalidTransactionId, false,
- fast_forward, xl_routine, prepare_write,
- do_write, update_progress);
+ fast_forward, page_read, cleanup_cb,
+ prepare_write, do_write, update_progress);
/* call output plugin initialization callback */
old_context = MemoryContextSwitchTo(ctx->context);
@@ -483,7 +487,13 @@ DecodingContextFindStartpoint(LogicalDecodingContext *ctx)
char *err = NULL;
/* the read_page callback waits for new WAL */
- record = XLogReadRecord(ctx->reader, &err);
+ while (XLogReadRecord(ctx->reader, &record, &err) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!ctx->page_read(ctx->reader))
+ break;
+ }
+
if (err)
elog(ERROR, "%s", err);
if (!record)
diff --git a/src/backend/replication/logical/logicalfuncs.c b/src/backend/replication/logical/logicalfuncs.c
index b99c94e848..fcc81b358e 100644
--- a/src/backend/replication/logical/logicalfuncs.c
+++ b/src/backend/replication/logical/logicalfuncs.c
@@ -233,9 +233,8 @@ pg_logical_slot_get_changes_guts(FunctionCallInfo fcinfo, bool confirm, bool bin
ctx = CreateDecodingContext(InvalidXLogRecPtr,
options,
false,
- XL_ROUTINE(.page_read = read_local_xlog_page,
- .segment_open = wal_segment_open,
- .segment_close = wal_segment_close),
+ read_local_xlog_page,
+ wal_segment_close,
LogicalOutputPrepareWrite,
LogicalOutputWrite, NULL);
@@ -284,7 +283,13 @@ pg_logical_slot_get_changes_guts(FunctionCallInfo fcinfo, bool confirm, bool bin
XLogRecord *record;
char *errm = NULL;
- record = XLogReadRecord(ctx->reader, &errm);
+ while (XLogReadRecord(ctx->reader, &record, &errm) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!ctx->page_read(ctx->reader))
+ break;
+ }
+
if (errm)
elog(ERROR, "%s", errm);
diff --git a/src/backend/replication/slotfuncs.c b/src/backend/replication/slotfuncs.c
index 88033a79b2..8a678074d5 100644
--- a/src/backend/replication/slotfuncs.c
+++ b/src/backend/replication/slotfuncs.c
@@ -152,9 +152,8 @@ create_logical_replication_slot(char *name, char *plugin,
ctx = CreateInitDecodingContext(plugin, NIL,
false, /* just catalogs is OK */
restart_lsn,
- XL_ROUTINE(.page_read = read_local_xlog_page,
- .segment_open = wal_segment_open,
- .segment_close = wal_segment_close),
+ read_local_xlog_page,
+ wal_segment_close,
NULL, NULL, NULL);
/*
@@ -486,9 +485,8 @@ pg_logical_replication_slot_advance(XLogRecPtr moveto)
ctx = CreateDecodingContext(InvalidXLogRecPtr,
NIL,
true, /* fast_forward */
- XL_ROUTINE(.page_read = read_local_xlog_page,
- .segment_open = wal_segment_open,
- .segment_close = wal_segment_close),
+ read_local_xlog_page,
+ wal_segment_close,
NULL, NULL, NULL);
/*
@@ -513,7 +511,13 @@ pg_logical_replication_slot_advance(XLogRecPtr moveto)
* Read records. No changes are generated in fast_forward mode,
* but snapbuilder/slot statuses are updated properly.
*/
- record = XLogReadRecord(ctx->reader, &errm);
+ while (XLogReadRecord(ctx->reader, &record, &errm) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!ctx->page_read(ctx->reader))
+ break;
+ }
+
if (errm)
elog(ERROR, "%s", errm);
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index bf2711bddd..2fd24a9b14 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -580,10 +580,7 @@ StartReplication(StartReplicationCmd *cmd)
/* create xlogreader for physical replication */
xlogreader =
- XLogReaderAllocate(wal_segment_size, NULL,
- XL_ROUTINE(.segment_open = WalSndSegmentOpen,
- .segment_close = wal_segment_close),
- NULL);
+ XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
if (!xlogreader)
ereport(ERROR,
@@ -812,9 +809,11 @@ StartReplication(StartReplicationCmd *cmd)
* set every time WAL is flushed.
*/
static bool
-logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetRecPtr, char *cur_page)
+logical_read_xlog_page(XLogReaderState *state)
{
+ XLogRecPtr targetPagePtr = state->readPagePtr;
+ int reqLen = state->readLen;
+ char *cur_page = state->readBuf;
XLogRecPtr flushptr;
int count;
WALReadError errinfo;
@@ -842,7 +841,7 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
count = flushptr - targetPagePtr; /* part of the page available */
/* now actually read the data, we know it's there */
- if (!WALRead(state,
+ if (!WALRead(state, WalSndSegmentOpen, wal_segment_close,
cur_page,
targetPagePtr,
XLOG_BLCKSZ,
@@ -1014,9 +1013,8 @@ CreateReplicationSlot(CreateReplicationSlotCmd *cmd)
ctx = CreateInitDecodingContext(cmd->plugin, NIL, need_full_snapshot,
InvalidXLogRecPtr,
- XL_ROUTINE(.page_read = logical_read_xlog_page,
- .segment_open = WalSndSegmentOpen,
- .segment_close = wal_segment_close),
+ logical_read_xlog_page,
+ wal_segment_close,
WalSndPrepareWrite, WalSndWriteData,
WalSndUpdateProgress);
@@ -1179,9 +1177,8 @@ StartLogicalReplication(StartReplicationCmd *cmd)
*/
logical_decoding_ctx =
CreateDecodingContext(cmd->startpoint, cmd->options, false,
- XL_ROUTINE(.page_read = logical_read_xlog_page,
- .segment_open = WalSndSegmentOpen,
- .segment_close = wal_segment_close),
+ logical_read_xlog_page,
+ wal_segment_close,
WalSndPrepareWrite, WalSndWriteData,
WalSndUpdateProgress);
xlogreader = logical_decoding_ctx->reader;
@@ -2761,7 +2758,7 @@ XLogSendPhysical(void)
enlargeStringInfo(&output_message, nbytes);
retry:
- if (!WALRead(xlogreader,
+ if (!WALRead(xlogreader, WalSndSegmentOpen, wal_segment_close,
&output_message.data[output_message.len],
startptr,
nbytes,
@@ -2859,7 +2856,12 @@ XLogSendLogical(void)
*/
WalSndCaughtUp = false;
- record = XLogReadRecord(logical_decoding_ctx->reader, &errm);
+ while (XLogReadRecord(logical_decoding_ctx->reader, &record, &errm) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!logical_decoding_ctx->page_read(logical_decoding_ctx->reader))
+ break;
+ }
/* xlog record was invalid */
if (errm != NULL)
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index 0b7e73ae79..3dd6df4be6 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -40,15 +40,9 @@ static int xlogreadfd = -1;
static XLogSegNo xlogreadsegno = -1;
static char xlogfpath[MAXPGPATH];
-typedef struct XLogPageReadPrivate
-{
- const char *restoreCommand;
- int tliIndex;
-} XLogPageReadPrivate;
-
-static bool SimpleXLogPageRead(XLogReaderState *xlogreader,
- XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
+static bool SimpleXLogPageRead(XLogReaderState *xlogreader,
+ const char *datadir, int *tliIndex,
+ const char *restoreCommand);
/*
* Read WAL from the datadir/pg_wal, starting from 'startpoint' on timeline
@@ -62,20 +56,22 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
XLogRecord *record;
XLogReaderState *xlogreader;
char *errormsg;
- XLogPageReadPrivate private;
- private.tliIndex = tliIndex;
- private.restoreCommand = restoreCommand;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir,
- XL_ROUTINE(.page_read = &SimpleXLogPageRead),
- &private);
+ xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+
if (xlogreader == NULL)
pg_fatal("out of memory");
XLogBeginRead(xlogreader, startpoint);
do
{
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!SimpleXLogPageRead(xlogreader, datadir,
+ &tliIndex, restoreCommand))
+ break;
+ }
if (record == NULL)
{
@@ -113,19 +109,19 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex,
XLogRecord *record;
XLogReaderState *xlogreader;
char *errormsg;
- XLogPageReadPrivate private;
XLogRecPtr endptr;
- private.tliIndex = tliIndex;
- private.restoreCommand = restoreCommand;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir,
- XL_ROUTINE(.page_read = &SimpleXLogPageRead),
- &private);
+ xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
XLogBeginRead(xlogreader, ptr);
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!SimpleXLogPageRead(xlogreader, datadir, &tliIndex, restoreCommand))
+ break;
+ }
if (record == NULL)
{
if (errormsg)
@@ -160,7 +156,6 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
XLogRecPtr searchptr;
XLogReaderState *xlogreader;
char *errormsg;
- XLogPageReadPrivate private;
/*
* The given fork pointer points to the end of the last common record,
@@ -176,11 +171,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
forkptr += SizeOfXLogShortPHD;
}
- private.tliIndex = tliIndex;
- private.restoreCommand = restoreCommand;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir,
- XL_ROUTINE(.page_read = &SimpleXLogPageRead),
- &private);
+ xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
@@ -190,7 +181,13 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
uint8 info;
XLogBeginRead(xlogreader, searchptr);
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!SimpleXLogPageRead(xlogreader, datadir,
+ &tliIndex, restoreCommand))
+ break;
+ }
if (record == NULL)
{
@@ -237,10 +234,11 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
/* XLogReader callback function, to read a WAL page */
static bool
-SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *readBuf)
+SimpleXLogPageRead(XLogReaderState *xlogreader, const char *datadir,
+ int *tliIndex, const char *restoreCommand)
{
- XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
+ XLogRecPtr targetPagePtr = xlogreader->readPagePtr;
+ char *readBuf = xlogreader->readBuf;
uint32 targetPageOff;
XLogRecPtr targetSegEnd;
XLogSegNo targetSegNo;
@@ -273,14 +271,14 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
* be done both forward and backward, consider also switching timeline
* accordingly.
*/
- while (private->tliIndex < targetNentries - 1 &&
- targetHistory[private->tliIndex].end < targetSegEnd)
- private->tliIndex++;
- while (private->tliIndex > 0 &&
- targetHistory[private->tliIndex].begin >= targetSegEnd)
- private->tliIndex--;
+ while (*tliIndex < targetNentries - 1 &&
+ targetHistory[*tliIndex].end < targetSegEnd)
+ (*tliIndex)++;
+ while (*tliIndex > 0 &&
+ targetHistory[*tliIndex].begin >= targetSegEnd)
+ (*tliIndex)--;
- XLogFileName(xlogfname, targetHistory[private->tliIndex].tli,
+ XLogFileName(xlogfname, targetHistory[*tliIndex].tli,
xlogreadsegno, WalSegSz);
snprintf(xlogfpath, MAXPGPATH, "%s/" XLOGDIR "/%s",
@@ -293,7 +291,7 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
/*
* If we have no restore_command to execute, then exit.
*/
- if (private->restoreCommand == NULL)
+ if (restoreCommand == NULL)
{
pg_log_error("could not open file \"%s\": %m", xlogfpath);
xlogreader->readLen = -1;
@@ -307,7 +305,7 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
xlogreadfd = RestoreArchivedFile(xlogreader->segcxt.ws_dir,
xlogfname,
WalSegSz,
- private->restoreCommand);
+ restoreCommand);
if (xlogreadfd < 0)
{
@@ -349,7 +347,7 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
Assert(targetSegNo == xlogreadsegno);
- xlogreader->seg.ws_tli = targetHistory[private->tliIndex].tli;
+ xlogreader->seg.ws_tli = targetHistory[*tliIndex].tli;
xlogreader->readLen = XLOG_BLCKSZ;
return true;
}
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index ab0c28d9b7..02e71138cc 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -330,12 +330,17 @@ WALDumpCloseSegment(XLogReaderState *state)
state->seg.ws_file = -1;
}
-/* pg_waldump's XLogReaderRoutine->page_read callback */
+/*
+ * pg_waldump's WAL page rader, also used as page_read callback for
+ * XLogFindNextRecord
+ */
static bool
-WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetPtr, char *readBuff)
+WALDumpReadPage(XLogReaderState *state, void *priv)
{
- XLogDumpPrivate *private = state->private_data;
+ XLogRecPtr targetPagePtr = state->readPagePtr;
+ int reqLen = state->readLen;
+ char *readBuff = state->readBuf;
+ XLogDumpPrivate *private = (XLogDumpPrivate *) priv;
int count = XLOG_BLCKSZ;
WALReadError errinfo;
@@ -353,8 +358,8 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
}
}
- if (!WALRead(state, readBuff, targetPagePtr, count, private->timeline,
- &errinfo))
+ if (!WALRead(state, WALDumpOpenSegment, WALDumpCloseSegment,
+ readBuff, targetPagePtr, count, private->timeline, &errinfo))
{
WALOpenSegment *seg = &errinfo.wre_seg;
char fname[MAXPGPATH];
@@ -374,6 +379,7 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
(Size) errinfo.wre_req);
}
+ Assert(count >= state->readLen);
state->readLen = count;
return true;
}
@@ -1042,16 +1048,14 @@ main(int argc, char **argv)
/* we have everything we need, start reading */
xlogreader_state =
- XLogReaderAllocate(WalSegSz, waldir,
- XL_ROUTINE(.page_read = WALDumpReadPage,
- .segment_open = WALDumpOpenSegment,
- .segment_close = WALDumpCloseSegment),
- &private);
+ XLogReaderAllocate(WalSegSz, waldir, WALDumpCloseSegment);
+
if (!xlogreader_state)
fatal_error("out of memory");
/* first find a valid recptr to start from */
- first_record = XLogFindNextRecord(xlogreader_state, private.startptr);
+ first_record = XLogFindNextRecord(xlogreader_state, private.startptr,
+ &WALDumpReadPage, (void*) &private);
if (first_record == InvalidXLogRecPtr)
fatal_error("could not find a valid record after %X/%X",
@@ -1075,7 +1079,13 @@ main(int argc, char **argv)
for (;;)
{
/* try to read the next record */
- record = XLogReadRecord(xlogreader_state, &errormsg);
+ while (XLogReadRecord(xlogreader_state, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!WALDumpReadPage(xlogreader_state, (void *) &private))
+ break;
+ }
+
if (!record)
{
if (!config.follow || private.endptr_reached)
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index 6c8848e14f..6a7fe140cb 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -57,64 +57,15 @@ typedef struct WALSegmentContext
typedef struct XLogReaderState XLogReaderState;
-/* Function type definition for the read_page callback */
-typedef bool (*XLogPageReadCB) (XLogReaderState *xlogreader,
- XLogRecPtr targetPagePtr,
- int reqLen,
- XLogRecPtr targetRecPtr,
- char *readBuf);
+/* Function type definition for the segment cleanup callback */
+typedef void (*WALSegmentCleanupCB) (XLogReaderState *xlogreader);
+
+/* Function type definition for the open/close callbacks for WALRead() */
typedef void (*WALSegmentOpenCB) (XLogReaderState *xlogreader,
XLogSegNo nextSegNo,
TimeLineID *tli_p);
typedef void (*WALSegmentCloseCB) (XLogReaderState *xlogreader);
-typedef struct XLogReaderRoutine
-{
- /*
- * Data input callback
- *
- * This callback shall read at least reqLen valid bytes of the xlog page
- * starting at targetPagePtr, and store them in readBuf. The callback
- * shall return the number of bytes read (never more than XLOG_BLCKSZ), or
- * -1 on failure. The callback shall sleep, if necessary, to wait for the
- * requested bytes to become available. The callback will not be invoked
- * again for the same page unless more than the returned number of bytes
- * are needed.
- *
- * targetRecPtr is the position of the WAL record we're reading. Usually
- * it is equal to targetPagePtr + reqLen, but sometimes xlogreader needs
- * to read and verify the page or segment header, before it reads the
- * actual WAL record it's interested in. In that case, targetRecPtr can
- * be used to determine which timeline to read the page from.
- *
- * The callback shall set ->seg.ws_tli to the TLI of the file the page was
- * read from.
- */
- XLogPageReadCB page_read;
-
- /*
- * Callback to open the specified WAL segment for reading. ->seg.ws_file
- * shall be set to the file descriptor of the opened segment. In case of
- * failure, an error shall be raised by the callback and it shall not
- * return.
- *
- * "nextSegNo" is the number of the segment to be opened.
- *
- * "tli_p" is an input/output argument. WALRead() uses it to pass the
- * timeline in which the new segment should be found, but the callback can
- * use it to return the TLI that it actually opened.
- */
- WALSegmentOpenCB segment_open;
-
- /*
- * WAL segment close callback. ->seg.ws_file shall be set to a negative
- * number.
- */
- WALSegmentCloseCB segment_close;
-} XLogReaderRoutine;
-
-#define XL_ROUTINE(...) &(XLogReaderRoutine){__VA_ARGS__}
-
typedef struct
{
/* Is this block ref in use? */
@@ -144,12 +95,35 @@ typedef struct
uint16 data_bufsz;
} DecodedBkpBlock;
+/* Return code from XLogReadRecord */
+typedef enum XLogReadRecordResult
+{
+ XLREAD_SUCCESS, /* record is successfully read */
+ XLREAD_NEED_DATA, /* need more data. see XLogReadRecord. */
+ XLREAD_FAIL /* failed during reading a record */
+} XLogReadRecordResult;
+
+/*
+ * internal state of XLogReadRecord
+ *
+ * XLogReadState runs a state machine while reading a record. Theses states
+ * are not seen outside the function. Each state may repeat several times
+ * exiting requesting caller for new data. See the comment of XLogReadRecrod
+ * for details.
+ */
+typedef enum XLogReadRecordState {
+ XLREAD_NEXT_RECORD,
+ XLREAD_TOT_LEN,
+ XLREAD_FIRST_FRAGMENT,
+ XLREAD_CONTINUATION
+} XLogReadRecordState;
+
struct XLogReaderState
{
/*
* Operational callbacks
*/
- XLogReaderRoutine routine;
+ WALSegmentCleanupCB cleanup_cb;
/* ----------------------------------------
* Public parameters
@@ -162,18 +136,14 @@ struct XLogReaderState
*/
uint64 system_identifier;
- /*
- * Opaque data for callbacks to use. Not used by XLogReader.
- */
- void *private_data;
-
/*
* Start and end point of last record read. EndRecPtr is also used as the
* position to read next. Calling XLogBeginRead() sets EndRecPtr to the
* starting position and ReadRecPtr to invalid.
*/
- XLogRecPtr ReadRecPtr; /* start of last record read */
+ XLogRecPtr ReadRecPtr; /* start of last record read or being read */
XLogRecPtr EndRecPtr; /* end+1 of last record read */
+ XLogRecPtr PrevRecPtr; /* start of previous record read */
/* ----------------------------------------
* Communication with page reader
@@ -187,7 +157,9 @@ struct XLogReaderState
* the request, or -1 on error */
TimeLineID readPageTLI; /* TLI for data currently in readBuf */
char *readBuf; /* buffer to store data */
- bool page_verified; /* is the page on the buffer verified? */
+ bool page_verified; /* is the page header on the buffer verified? */
+ bool record_verified;/* is the current record header verified? */
+
/* ----------------------------------------
@@ -227,8 +199,6 @@ struct XLogReaderState
XLogRecPtr latestPagePtr;
TimeLineID latestPageTLI;
- /* beginning of the WAL record being read. */
- XLogRecPtr currRecPtr;
/* timeline to read it from, 0 if a lookup is required */
TimeLineID currTLI;
@@ -255,6 +225,15 @@ struct XLogReaderState
char *readRecordBuf;
uint32 readRecordBufSize;
+ /*
+ * XLogReadRecord() state
+ */
+ XLogReadRecordState readRecordState;/* state machine state */
+ int recordGotLen; /* amount of current record that has
+ * already been read */
+ int recordRemainLen; /* length of current record that remains */
+ XLogRecPtr recordContRecPtr; /* where the current record continues */
+
/* Buffer to hold error message */
char *errormsg_buf;
};
@@ -262,9 +241,7 @@ struct XLogReaderState
/* Get a new XLogReader */
extern XLogReaderState *XLogReaderAllocate(int wal_segment_size,
const char *waldir,
- XLogReaderRoutine *routine,
- void *private_data);
-extern XLogReaderRoutine *LocalXLogReaderRoutine(void);
+ WALSegmentCleanupCB cleanup_cb);
/* Free an XLogReader */
extern void XLogReaderFree(XLogReaderState *state);
@@ -272,12 +249,17 @@ extern void XLogReaderFree(XLogReaderState *state);
/* Position the XLogReader to given record */
extern void XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr);
#ifdef FRONTEND
-extern XLogRecPtr XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr);
+/* Function type definition for the read_page callback */
+typedef bool (*XLogFindNextRecordCB) (XLogReaderState *xlogreader,
+ void *private);
+extern XLogRecPtr XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
+ XLogFindNextRecordCB read_page, void *private);
#endif /* FRONTEND */
/* Read the next XLog record. Returns NULL on end-of-WAL or failure */
-extern struct XLogRecord *XLogReadRecord(XLogReaderState *state,
- char **errormsg);
+extern XLogReadRecordResult XLogReadRecord(XLogReaderState *state,
+ XLogRecord **record,
+ char **errormsg);
/* Validate a page */
extern bool XLogReaderValidatePageHeader(XLogReaderState *state,
@@ -297,6 +279,7 @@ typedef struct WALReadError
} WALReadError;
extern bool WALRead(XLogReaderState *state,
+ WALSegmentOpenCB segopenfn, WALSegmentCloseCB sgclosefn,
char *buf, XLogRecPtr startptr, Size count,
TimeLineID tli, WALReadError *errinfo);
diff --git a/src/include/access/xlogutils.h b/src/include/access/xlogutils.h
index c287edf206..482e4ced69 100644
--- a/src/include/access/xlogutils.h
+++ b/src/include/access/xlogutils.h
@@ -47,9 +47,7 @@ extern Buffer XLogReadBufferExtended(RelFileNode rnode, ForkNumber forknum,
extern Relation CreateFakeRelcacheEntry(RelFileNode rnode);
extern void FreeFakeRelcacheEntry(Relation fakerel);
-extern bool read_local_xlog_page(XLogReaderState *state,
- XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetRecPtr, char *cur_page);
+extern bool read_local_xlog_page(XLogReaderState *state);
extern void wal_segment_open(XLogReaderState *state,
XLogSegNo nextSegNo,
TimeLineID *tli_p);
diff --git a/src/include/replication/logical.h b/src/include/replication/logical.h
index c2f2475e5d..a743db49cf 100644
--- a/src/include/replication/logical.h
+++ b/src/include/replication/logical.h
@@ -29,6 +29,10 @@ typedef void (*LogicalOutputPluginWriterUpdateProgress) (struct LogicalDecodingC
TransactionId xid
);
+typedef struct LogicalDecodingContext LogicalDecodingContext;
+
+typedef bool (*LogicalDecodingXLogPageReadCB)(XLogReaderState *ctx);
+
typedef struct LogicalDecodingContext
{
/* memory context this is all allocated in */
@@ -39,6 +43,7 @@ typedef struct LogicalDecodingContext
/* infrastructure pieces for decoding */
XLogReaderState *reader;
+ LogicalDecodingXLogPageReadCB page_read;
struct ReorderBuffer *reorder;
struct SnapBuild *snapshot_builder;
@@ -95,14 +100,16 @@ extern LogicalDecodingContext *CreateInitDecodingContext(char *plugin,
List *output_plugin_options,
bool need_full_snapshot,
XLogRecPtr restart_lsn,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress);
extern LogicalDecodingContext *CreateDecodingContext(XLogRecPtr start_lsn,
List *output_plugin_options,
bool fast_forward,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress);
--
2.18.4
----Next_Part(Thu_Jul__2_13_53_30_2020_072)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
filename="v15-0003-Remove-globals-readOff-readLen-and-readSegNo.patch"
^ permalink raw reply [nested|flat] 70+ messages in thread
* [PATCH v17 2/4] Move page-reader out of XLogReadRecord
@ 2019-09-10 03:58 Kyotaro Horiguchi <[email protected]>
0 siblings, 0 replies; 70+ messages in thread
From: Kyotaro Horiguchi @ 2019-09-10 03:58 UTC (permalink / raw)
This is the second step of removing callbacks from WAL record reader.
Since it is essential to take in additional data while reading a
record, the function have to ask caller for new data while keeping
working state. Thus the function is turned into a state machine.
---
src/backend/access/transam/twophase.c | 15 +-
src/backend/access/transam/xlog.c | 58 +-
src/backend/access/transam/xlogreader.c | 697 +++++++++++-------
src/backend/access/transam/xlogutils.c | 17 +-
src/backend/replication/logical/logical.c | 26 +-
.../replication/logical/logicalfuncs.c | 13 +-
src/backend/replication/slotfuncs.c | 18 +-
src/backend/replication/walsender.c | 32 +-
src/bin/pg_rewind/parsexlog.c | 84 ++-
src/bin/pg_waldump/pg_waldump.c | 36 +-
src/include/access/xlogreader.h | 121 ++-
src/include/access/xlogutils.h | 4 +-
src/include/pg_config_manual.h | 2 +-
src/include/replication/logical.h | 11 +-
14 files changed, 646 insertions(+), 488 deletions(-)
diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index 80d2d20d6c..7d5a96a5f5 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -1325,11 +1325,8 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
XLogReaderState *xlogreader;
char *errormsg;
- xlogreader = XLogReaderAllocate(wal_segment_size, NULL,
- XL_ROUTINE(.page_read = &read_local_xlog_page,
- .segment_open = &wal_segment_open,
- .segment_close = &wal_segment_close),
- NULL);
+ xlogreader = XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+
if (!xlogreader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
@@ -1337,7 +1334,13 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
errdetail("Failed while allocating a WAL reading processor.")));
XLogBeginRead(xlogreader, lsn);
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!read_local_xlog_page(xlogreader))
+ break;
+ }
+
if (record == NULL)
ereport(ERROR,
(errcode_for_file_access(),
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 606328a65a..670996ae67 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -836,13 +836,6 @@ static XLogSource currentSource = XLOG_FROM_ANY;
static bool lastSourceFailed = false;
static bool pendingWalRcvRestart = false;
-typedef struct XLogPageReadPrivate
-{
- int emode;
- bool fetching_ckpt; /* are we fetching a checkpoint record? */
- bool randAccess;
-} XLogPageReadPrivate;
-
/*
* These variables track when we last obtained some WAL data to process,
* and where we got it from. (XLogReceiptSource is initially the same as
@@ -917,8 +910,8 @@ static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath,
static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli,
XLogSource source, bool notfoundOk);
static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, XLogSource source);
-static bool XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
+static bool XLogPageRead(XLogReaderState *xlogreader,
+ bool fetching_ckpt, int emode, bool randAccess);
static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
bool fetching_ckpt, XLogRecPtr tliRecPtr);
static int emode_for_corrupt_record(int emode, XLogRecPtr RecPtr);
@@ -1231,8 +1224,7 @@ XLogInsertRecord(XLogRecData *rdata,
appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len);
if (!debug_reader)
- debug_reader = XLogReaderAllocate(wal_segment_size, NULL,
- XL_ROUTINE(), NULL);
+ debug_reader = XLogReaderAllocate(wal_segment_size, NULL, NULL);
if (!debug_reader)
{
@@ -4344,15 +4336,10 @@ CleanupBackupHistory(void)
* record is available.
*/
static XLogRecord *
-ReadRecord(XLogReaderState *xlogreader, int emode,
- bool fetching_ckpt)
+ReadRecord(XLogReaderState *xlogreader, int emode, bool fetching_ckpt)
{
XLogRecord *record;
- XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
-
- private->fetching_ckpt = fetching_ckpt;
- private->emode = emode;
- private->randAccess = (xlogreader->ReadRecPtr == InvalidXLogRecPtr);
+ bool randAccess = (xlogreader->ReadRecPtr == InvalidXLogRecPtr);
/* This is the first attempt to read this page. */
lastSourceFailed = false;
@@ -4360,8 +4347,16 @@ ReadRecord(XLogReaderState *xlogreader, int emode,
for (;;)
{
char *errormsg;
+ XLogReadRecordResult result;
+
+ while ((result = XLogReadRecord(xlogreader, &record, &errormsg))
+ == XLREAD_NEED_DATA)
+ {
+ if (!XLogPageRead(xlogreader, fetching_ckpt, emode, randAccess))
+ break;
+
+ }
- record = XLogReadRecord(xlogreader, &errormsg);
ReadRecPtr = xlogreader->ReadRecPtr;
EndRecPtr = xlogreader->EndRecPtr;
if (record == NULL)
@@ -6380,7 +6375,6 @@ StartupXLOG(void)
bool backupFromStandby = false;
DBState dbstate_at_startup;
XLogReaderState *xlogreader;
- XLogPageReadPrivate private;
bool promoted = false;
struct stat st;
@@ -6539,13 +6533,9 @@ StartupXLOG(void)
OwnLatch(&XLogCtl->recoveryWakeupLatch);
/* Set up XLOG reader facility */
- MemSet(&private, 0, sizeof(XLogPageReadPrivate));
xlogreader =
- XLogReaderAllocate(wal_segment_size, NULL,
- XL_ROUTINE(.page_read = &XLogPageRead,
- .segment_open = NULL,
- .segment_close = wal_segment_close),
- &private);
+ XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+
if (!xlogreader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
@@ -11987,12 +11977,13 @@ CancelBackup(void)
* sleep and retry.
*/
static bool
-XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetRecPtr, char *readBuf)
+XLogPageRead(XLogReaderState *xlogreader,
+ bool fetching_ckpt, int emode, bool randAccess)
{
- XLogPageReadPrivate *private =
- (XLogPageReadPrivate *) xlogreader->private_data;
- int emode = private->emode;
+ char *readBuf = xlogreader->readBuf;
+ XLogRecPtr targetPagePtr = xlogreader->readPagePtr;
+ int reqLen = xlogreader->readLen;
+ XLogRecPtr targetRecPtr = xlogreader->ReadRecPtr;
uint32 targetPageOff;
XLogSegNo targetSegNo PG_USED_FOR_ASSERTS_ONLY;
int r;
@@ -12035,8 +12026,8 @@ retry:
flushedUpto < targetPagePtr + reqLen))
{
if (!WaitForWALToBecomeAvailable(targetPagePtr + reqLen,
- private->randAccess,
- private->fetching_ckpt,
+ randAccess,
+ fetching_ckpt,
targetRecPtr))
{
if (readFile >= 0)
@@ -12141,6 +12132,7 @@ retry:
goto next_record_is_invalid;
}
+ Assert(xlogreader->readPagePtr == targetPagePtr);
xlogreader->readLen = readLen;
return true;
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index 7d04995b7b..ba3e4868f7 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -40,7 +40,7 @@ static bool XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr,
int reqLen, bool header_inclusive);
static void XLogReaderInvalReadState(XLogReaderState *state);
static bool ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
- XLogRecPtr PrevRecPtr, XLogRecord *record, bool randAccess);
+ XLogRecPtr PrevRecPtr, XLogRecord *record);
static bool ValidXLogRecord(XLogReaderState *state, XLogRecord *record,
XLogRecPtr recptr);
static void ResetDecoder(XLogReaderState *state);
@@ -73,7 +73,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...)
*/
XLogReaderState *
XLogReaderAllocate(int wal_segment_size, const char *waldir,
- XLogReaderRoutine *routine, void *private_data)
+ WALSegmentCleanupCB cleanup_cb)
{
XLogReaderState *state;
@@ -84,7 +84,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
return NULL;
/* initialize caller-provided support functions */
- state->routine = *routine;
+ state->cleanup_cb = cleanup_cb;
state->max_block_id = -1;
@@ -107,8 +107,6 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
WALOpenSegmentInit(&state->seg, &state->segcxt, wal_segment_size,
waldir);
- /* system_identifier initialized to zeroes above */
- state->private_data = private_data;
/* ReadRecPtr, EndRecPtr and readLen initialized to zeroes above */
state->errormsg_buf = palloc_extended(MAX_ERRORMSG_LEN + 1,
MCXT_ALLOC_NO_OOM);
@@ -140,8 +138,8 @@ XLogReaderFree(XLogReaderState *state)
{
int block_id;
- if (state->seg.ws_file != -1)
- state->routine.segment_close(state);
+ if (state->seg.ws_file >= 0)
+ state->cleanup_cb(state);
for (block_id = 0; block_id <= XLR_MAX_BLOCK_ID; block_id++)
{
@@ -246,6 +244,7 @@ XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
/* Begin at the passed-in record pointer. */
state->EndRecPtr = RecPtr;
state->ReadRecPtr = InvalidXLogRecPtr;
+ state->readRecordState = XLREAD_NEXT_RECORD;
}
/*
@@ -254,317 +253,449 @@ XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
* XLogBeginRead() or XLogFindNextRecord() must be called before the first call
* to XLogReadRecord().
*
- * If the page_read callback fails to read the requested data, NULL is
- * returned. The callback is expected to have reported the error; errormsg
- * is set to NULL.
+ * This function may return XLREAD_NEED_DATA several times before returning a
+ * result record. The caller shall read in some new data then call this
+ * function again with the same parameters.
*
- * If the reading fails for some other reason, NULL is also returned, and
- * *errormsg is set to a string with details of the failure.
+ * When a record is successfully read, returns XLREAD_SUCCESS with result
+ * record being stored in *record. Otherwise *record is NULL.
*
- * The returned pointer (or *errormsg) points to an internal buffer that's
- * valid until the next call to XLogReadRecord.
+ * Returns XLREAD_NEED_DATA if more data is needed to finish reading the
+ * current record. In that case, state->readPagePtr and state->readLen inform
+ * the desired position and minimum length of data needed. The caller shall
+ * read in the requested data and set state->readBuf to point to a buffer
+ * containing it. The caller must also set state->readPageTLI and
+ * state->readLen to indicate the timeline that it was read from, and the
+ * length of data that is now available (which must be >= given readLen),
+ * respectively.
+ *
+ * If invalid data is encountered, returns XLREAD_FAIL with *record being set to
+ * NULL. *errormsg is set to a string with details of the failure.
+ * The returned pointer (or *errormsg) points to an internal buffer that's valid
+ * until the next call to XLogReadRecord.
+ *
+ *
+ * This function runs a state machine consists of the following states.
+ *
+ * XLREAD_NEXT_RECORD :
+ * The initial state, if called with valid RecPtr, try to read a record at
+ * that position. If invalid RecPtr is given try to read a record just after
+ * the last one previously read.
+ * This state ens after setting ReadRecPtr. Then goes to XLREAD_TOT_LEN.
+ *
+ * XLREAD_TOT_LEN:
+ * Examining record header. Ends after reading record total
+ * length. recordRemainLen and recordGotLen are initialized.
+ *
+ * XLREAD_FIRST_FRAGMENT:
+ * Reading the first fragment. Ends with finishing reading a single
+ * record. Goes to XLREAD_NEXT_RECORD if that's all or
+ * XLREAD_CONTINUATION if we have continuation.
+
+ * XLREAD_CONTINUATION:
+ * Reading continuation of record. Ends with finishing the whole record then
+ * goes to XLREAD_NEXT_RECORD. During this state, recordRemainLen indicates
+ * how much is left and readRecordBuf holds the partially assert
+ * record.recordContRecPtr points to the beginning of the next page where to
+ * continue.
+ *
+ * If wrong data found in any state, the state machine stays at the current
+ * state. This behavior allows to continue reading a reacord switching among
+ * different souces, while streaming replication.
*/
-XLogRecord *
-XLogReadRecord(XLogReaderState *state, char **errormsg)
+XLogReadRecordResult
+XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
{
- XLogRecPtr RecPtr;
- XLogRecord *record;
- XLogRecPtr targetPagePtr;
- bool randAccess;
- uint32 len,
- total_len;
- uint32 targetRecOff;
- uint32 pageHeaderSize;
- bool gotheader;
+ XLogRecord *prec;
- /*
- * randAccess indicates whether to verify the previous-record pointer of
- * the record we're reading. We only do this if we're reading
- * sequentially, which is what we initially assume.
- */
- randAccess = false;
+ *record = NULL;
/* reset error state */
*errormsg = NULL;
state->errormsg_buf[0] = '\0';
- ResetDecoder(state);
-
- RecPtr = state->EndRecPtr;
-
- if (state->ReadRecPtr != InvalidXLogRecPtr)
- {
- /* read the record after the one we just read */
-
- /*
- * EndRecPtr is pointing to end+1 of the previous WAL record. If
- * we're at a page boundary, no more records can fit on the current
- * page. We must skip over the page header, but we can't do that until
- * we've read in the page, since the header size is variable.
- */
- }
- else
- {
- /*
- * Caller supplied a position to start at.
- *
- * In this case, EndRecPtr should already be pointing to a valid
- * record starting position.
- */
- Assert(XRecOffIsValid(RecPtr));
- randAccess = true;
- }
-
- state->currRecPtr = RecPtr;
-
- targetPagePtr = RecPtr - (RecPtr % XLOG_BLCKSZ);
- targetRecOff = RecPtr % XLOG_BLCKSZ;
-
- /*
- * Read the page containing the record into state->readBuf. Request enough
- * byte to cover the whole record header, or at least the part of it that
- * fits on the same page.
- */
- while (XLogNeedData(state, targetPagePtr,
- Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
- targetRecOff != 0))
- {
- if (!state->routine.page_read(state, state->readPagePtr, state->readLen,
- RecPtr, state->readBuf))
- break;
- }
-
- if (!state->page_verified)
- goto err;
-
- /*
- * We have at least the page header, so we can examine it now.
- */
- pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
- if (targetRecOff == 0)
- {
- /*
- * At page start, so skip over page header.
- */
- RecPtr += pageHeaderSize;
- targetRecOff = pageHeaderSize;
- }
- else if (targetRecOff < pageHeaderSize)
- {
- report_invalid_record(state, "invalid record offset at %X/%X",
- LSN_FORMAT_ARGS(RecPtr));
- goto err;
- }
-
- if ((((XLogPageHeader) state->readBuf)->xlp_info & XLP_FIRST_IS_CONTRECORD) &&
- targetRecOff == pageHeaderSize)
- {
- report_invalid_record(state, "contrecord is requested by %X/%X",
- LSN_FORMAT_ARGS(RecPtr));
- goto err;
- }
-
- /* XLogNeedData has verified the page header */
- Assert(pageHeaderSize <= state->readLen);
-
- /*
- * Read the record length.
- *
- * NB: Even though we use an XLogRecord pointer here, the whole record
- * header might not fit on this page. xl_tot_len is the first field of the
- * struct, so it must be on this page (the records are MAXALIGNed), but we
- * cannot access any other fields until we've verified that we got the
- * whole header.
- */
- record = (XLogRecord *) (state->readBuf + RecPtr % XLOG_BLCKSZ);
- total_len = record->xl_tot_len;
-
- /*
- * If the whole record header is on this page, validate it immediately.
- * Otherwise do just a basic sanity check on xl_tot_len, and validate the
- * rest of the header after reading it from the next page. The xl_tot_len
- * check is necessary here to ensure that we enter the "Need to reassemble
- * record" code path below; otherwise we might fail to apply
- * ValidXLogRecordHeader at all.
- */
- if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
- {
- if (!ValidXLogRecordHeader(state, RecPtr, state->ReadRecPtr, record,
- randAccess))
- goto err;
- gotheader = true;
- }
- else
- {
- /* XXX: more validation should be done here */
- if (total_len < SizeOfXLogRecord)
- {
- report_invalid_record(state,
- "invalid record length at %X/%X: wanted %u, got %u",
- LSN_FORMAT_ARGS(RecPtr),
- (uint32) SizeOfXLogRecord, total_len);
- goto err;
- }
- gotheader = false;
- }
-
- len = XLOG_BLCKSZ - RecPtr % XLOG_BLCKSZ;
- if (total_len > len)
+ switch (state->readRecordState)
{
- /* Need to reassemble record */
- char *contdata;
- XLogPageHeader pageHeader;
- char *buffer;
- uint32 gotlen;
-
- /*
- * Enlarge readRecordBuf as needed.
- */
- if (total_len > state->readRecordBufSize &&
- !allocate_recordbuf(state, total_len))
- {
- /* We treat this as a "bogus data" condition */
- report_invalid_record(state, "record length %u at %X/%X too long",
- total_len, LSN_FORMAT_ARGS(RecPtr));
- goto err;
- }
+ case XLREAD_NEXT_RECORD:
+ ResetDecoder(state);
- /* Copy the first fragment of the record from the first page. */
- memcpy(state->readRecordBuf,
- state->readBuf + RecPtr % XLOG_BLCKSZ, len);
- buffer = state->readRecordBuf + len;
- gotlen = len;
-
- do
- {
- int rest_len = total_len - gotlen;
-
- /* Calculate pointer to beginning of next page */
- targetPagePtr += XLOG_BLCKSZ;
-
- /* Wait for the next page to become available */
- while (XLogNeedData(state, targetPagePtr,
- Min(rest_len, XLOG_BLCKSZ),
- false))
+ if (state->ReadRecPtr != InvalidXLogRecPtr)
{
- if (!state->routine.page_read(state, state->readPagePtr,
- state->readLen,
- state->ReadRecPtr,
- state->readBuf))
- break;
+ /* read the record after the one we just read */
+
+ /*
+ * EndRecPtr is pointing to end+1 of the previous WAL record.
+ * If we're at a page boundary, no more records can fit on the
+ * current page. We must skip over the page header, but we
+ * can't do that until we've read in the page, since the header
+ * size is variable.
+ */
+ state->PrevRecPtr = state->ReadRecPtr;
+ state->ReadRecPtr = state->EndRecPtr;
+ }
+ else
+ {
+ /*
+ * Caller supplied a position to start at.
+ *
+ * In this case, EndRecPtr should already be pointing to a
+ * valid record starting position.
+ */
+ Assert(XRecOffIsValid(state->EndRecPtr));
+ state->ReadRecPtr = state->EndRecPtr;
+
+ /*
+ * We cannot verify the previous-record pointer when we're
+ * seeking to a particular record. Reset PrevRecPtr so that we
+ * won't try doing that.
+ */
+ state->PrevRecPtr = InvalidXLogRecPtr;
+ state->EndRecPtr = InvalidXLogRecPtr; /* to be tidy */
}
+ state->record_verified = false;
+ state->readRecordState = XLREAD_TOT_LEN;
+ /* fall through */
+
+ case XLREAD_TOT_LEN:
+ {
+ uint32 total_len;
+ uint32 pageHeaderSize;
+ XLogRecPtr targetPagePtr;
+ uint32 targetRecOff;
+ XLogPageHeader pageHeader;
+
+ targetPagePtr =
+ state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
+ targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+
+ /*
+ * Check if we have enough data. For the first record in the page,
+ * the requesting length doesn't contain page header.
+ */
+ if (XLogNeedData(state, targetPagePtr,
+ Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+ targetRecOff != 0))
+ return XLREAD_NEED_DATA;
+
+ /* error out if caller supplied bogus page */
if (!state->page_verified)
goto err;
- Assert(SizeOfXLogShortPHD <= state->readLen);
+ /* examine page header now. */
+ pageHeaderSize =
+ XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+ if (targetRecOff == 0)
+ {
+ /* At page start, so skip over page header. */
+ state->ReadRecPtr += pageHeaderSize;
+ targetRecOff = pageHeaderSize;
+ }
+ else if (targetRecOff < pageHeaderSize)
+ {
+ report_invalid_record(state, "invalid record offset at %X/%X",
+ LSN_FORMAT_ARGS(state->ReadRecPtr));
+ goto err;
+ }
- /* Check that the continuation on next page looks valid */
pageHeader = (XLogPageHeader) state->readBuf;
- if (!(pageHeader->xlp_info & XLP_FIRST_IS_CONTRECORD))
+ if ((pageHeader->xlp_info & XLP_FIRST_IS_CONTRECORD) &&
+ targetRecOff == pageHeaderSize)
{
- report_invalid_record(state,
- "there is no contrecord flag at %X/%X",
- LSN_FORMAT_ARGS(RecPtr));
+ report_invalid_record(state, "contrecord is requested by %X/%X",
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr);
goto err;
}
- /*
- * Cross-check that xlp_rem_len agrees with how much of the record
- * we expect there to be left.
- */
- if (pageHeader->xlp_rem_len == 0 ||
- total_len != (pageHeader->xlp_rem_len + gotlen))
- {
- report_invalid_record(state,
- "invalid contrecord length %u (expected %lld) at %X/%X",
- pageHeader->xlp_rem_len,
- ((long long) total_len) - gotlen,
- LSN_FORMAT_ARGS(RecPtr));
- goto err;
- }
-
- /* Append the continuation from this page to the buffer */
- pageHeaderSize = XLogPageHeaderSize(pageHeader);
-
+ /* XLogNeedData has verified the page header */
Assert(pageHeaderSize <= state->readLen);
- contdata = (char *) state->readBuf + pageHeaderSize;
- len = XLOG_BLCKSZ - pageHeaderSize;
- if (pageHeader->xlp_rem_len < len)
- len = pageHeader->xlp_rem_len;
+ /*
+ * Read the record length.
+ *
+ * NB: Even though we use an XLogRecord pointer here, the whole
+ * record header might not fit on this page. xl_tot_len is the first
+ * field of the struct, so it must be on this page (the records are
+ * MAXALIGNed), but we cannot access any other fields until we've
+ * verified that we got the whole header.
+ */
+ prec = (XLogRecord *) (state->readBuf +
+ state->ReadRecPtr % XLOG_BLCKSZ);
+ total_len = prec->xl_tot_len;
- Assert (pageHeaderSize + len <= state->readLen);
- memcpy(buffer, (char *) contdata, len);
- buffer += len;
- gotlen += len;
+ /*
+ * If the whole record header is on this page, validate it
+ * immediately. Otherwise do just a basic sanity check on
+ * xl_tot_len, and validate the rest of the header after reading it
+ * from the next page. The xl_tot_len check is necessary here to
+ * ensure that we enter the XLREAD_CONTINUATION state below;
+ * otherwise we might fail to apply ValidXLogRecordHeader at all.
+ */
+ if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
+ {
+ if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
+ state->PrevRecPtr, prec))
+ goto err;
- /* If we just reassembled the record header, validate it. */
- if (!gotheader)
+ state->record_verified = true;
+ }
+ else
{
- record = (XLogRecord *) state->readRecordBuf;
- if (!ValidXLogRecordHeader(state, RecPtr, state->ReadRecPtr,
- record, randAccess))
+ /* XXX: more validation should be done here */
+ if (total_len < SizeOfXLogRecord)
+ {
+ report_invalid_record(state,
+ "invalid record length at %X/%X: wanted %u, got %u",
+ LSN_FORMAT_ARGS(state->ReadRecPtr),
+ (uint32) SizeOfXLogRecord, total_len);
goto err;
- gotheader = true;
+ }
}
- } while (gotlen < total_len);
-
- Assert(gotheader);
- record = (XLogRecord *) state->readRecordBuf;
- if (!ValidXLogRecord(state, record, RecPtr))
- goto err;
+ /*
+ * Wait for the rest of the record, or the part of the record that
+ * fit on the first page if crossed a page boundary, to become
+ * available.
+ */
+ state->recordGotLen = 0;
+ state->recordRemainLen = total_len;
+ state->readRecordState = XLREAD_FIRST_FRAGMENT;
+ }
+ /* fall through */
- pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
- state->ReadRecPtr = RecPtr;
- state->EndRecPtr = targetPagePtr + pageHeaderSize
- + MAXALIGN(pageHeader->xlp_rem_len);
- }
- else
- {
- /* Wait for the record data to become available */
- while (XLogNeedData(state, targetPagePtr,
- Min(targetRecOff + total_len, XLOG_BLCKSZ), true))
+ case XLREAD_FIRST_FRAGMENT:
{
- if (!state->routine.page_read(state, state->readPagePtr,
- state->readLen,
- state->ReadRecPtr, state->readBuf))
+ uint32 total_len = state->recordRemainLen;
+ uint32 request_len;
+ uint32 record_len;
+ XLogRecPtr targetPagePtr;
+ uint32 targetRecOff;
+
+ /*
+ * Wait for the rest of the record on the first page to become
+ * available
+ */
+ targetPagePtr =
+ state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
+ targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+
+ request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ);
+ record_len = request_len - targetRecOff;
+
+ /* ReadRecPtr contains page header */
+ Assert (targetRecOff != 0);
+ if (XLogNeedData(state, targetPagePtr, request_len, true))
+ return XLREAD_NEED_DATA;
+
+ /* error out if caller supplied bogus page */
+ if (!state->page_verified)
+ goto err;
+
+ prec = (XLogRecord *) (state->readBuf + targetRecOff);
+
+ /* validate record header if not yet */
+ if (!state->record_verified && record_len >= SizeOfXLogRecord)
+ {
+ if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
+ state->PrevRecPtr, prec))
+ goto err;
+
+ state->record_verified = true;
+ }
+
+
+ if (total_len == record_len)
+ {
+ /* Record does not cross a page boundary */
+ Assert(state->record_verified);
+
+ if (!ValidXLogRecord(state, prec, state->ReadRecPtr))
+ goto err;
+
+ state->record_verified = true; /* to be tidy */
+
+ /* We already checked the header earlier */
+ state->EndRecPtr = state->ReadRecPtr + MAXALIGN(record_len);
+
+ *record = prec;
+ state->readRecordState = XLREAD_NEXT_RECORD;
break;
+ }
+
+ /*
+ * The record continues on the next page. Need to reassemble
+ * record
+ */
+ Assert(total_len > record_len);
+
+ /* Enlarge readRecordBuf as needed. */
+ if (total_len > state->readRecordBufSize &&
+ !allocate_recordbuf(state, total_len))
+ {
+ /* We treat this as a "bogus data" condition */
+ report_invalid_record(state,
+ "record length %u at %X/%X too long",
+ total_len,
+ LSN_FORMAT_ARGS(state->ReadRecPtr));
+ goto err;
+ }
+
+ /* Copy the first fragment of the record from the first page. */
+ memcpy(state->readRecordBuf, state->readBuf + targetRecOff,
+ record_len);
+ state->recordGotLen += record_len;
+ state->recordRemainLen -= record_len;
+
+ /* Calculate pointer to beginning of next page */
+ state->recordContRecPtr = state->ReadRecPtr + record_len;
+ Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0);
+
+ state->readRecordState = XLREAD_CONTINUATION;
}
+ /* fall through */
+
+ case XLREAD_CONTINUATION:
+ {
+ XLogPageHeader pageHeader;
+ uint32 pageHeaderSize;
+ XLogRecPtr targetPagePtr;
+
+ /* we enter this state only if we haven't read the whole record. */
+ Assert (state->recordRemainLen > 0);
+
+ while(state->recordRemainLen > 0)
+ {
+ char *contdata;
+ uint32 request_len;
+ uint32 record_len;
+
+ /* Wait for the next page to become available */
+ targetPagePtr = state->recordContRecPtr;
+
+ /* this request contains page header */
+ Assert (targetPagePtr != 0);
+ if (XLogNeedData(state, targetPagePtr,
+ Min(state->recordRemainLen, XLOG_BLCKSZ),
+ false))
+ return XLREAD_NEED_DATA;
+
+ if (!state->page_verified)
+ goto err;
+
+ Assert(SizeOfXLogShortPHD <= state->readLen);
- if (!state->page_verified)
- goto err;
+ /* Check that the continuation on next page looks valid */
+ pageHeader = (XLogPageHeader) state->readBuf;
+ if (!(pageHeader->xlp_info & XLP_FIRST_IS_CONTRECORD))
+ {
+ report_invalid_record(
+ state,
+ "there is no contrecord flag at %X/%X reading %X/%X",
+ (uint32) (state->recordContRecPtr >> 32),
+ (uint32) state->recordContRecPtr,
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr);
+ goto err;
+ }
- /* Record does not cross a page boundary */
- if (!ValidXLogRecord(state, record, RecPtr))
- goto err;
+ /*
+ * Cross-check that xlp_rem_len agrees with how much of the
+ * record we expect there to be left.
+ */
+ if (pageHeader->xlp_rem_len == 0 ||
+ pageHeader->xlp_rem_len != state->recordRemainLen)
+ {
+ report_invalid_record(
+ state,
+ "invalid contrecord length %u at %X/%X reading %X/%X, expected %u",
+ pageHeader->xlp_rem_len,
+ (uint32) (state->recordContRecPtr >> 32),
+ (uint32) state->recordContRecPtr,
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr,
+ state->recordRemainLen);
+ goto err;
+ }
- state->EndRecPtr = RecPtr + MAXALIGN(total_len);
+ /* Append the continuation from this page to the buffer */
+ pageHeaderSize = XLogPageHeaderSize(pageHeader);
- state->ReadRecPtr = RecPtr;
+ /*
+ * XLogNeedData should have ensured that the whole page header
+ * was read
+ */
+ Assert(state->readLen >= pageHeaderSize);
+
+ contdata = (char *) state->readBuf + pageHeaderSize;
+ record_len = XLOG_BLCKSZ - pageHeaderSize;
+ if (pageHeader->xlp_rem_len < record_len)
+ record_len = pageHeader->xlp_rem_len;
+
+ request_len = record_len + pageHeaderSize;
+
+ /* XLogNeedData should have ensured all needed data was read */
+ Assert (state->readLen >= request_len);
+
+ memcpy(state->readRecordBuf + state->recordGotLen,
+ (char *) contdata, record_len);
+ state->recordGotLen += record_len;
+ state->recordRemainLen -= record_len;
+
+ /* If we just reassembled the record header, validate it. */
+ if (!state->record_verified)
+ {
+ Assert(state->recordGotLen >= SizeOfXLogRecord);
+ if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
+ state->PrevRecPtr,
+ (XLogRecord *) state->readRecordBuf))
+ goto err;
+
+ state->record_verified = true;
+ }
+
+ /* Calculate pointer to beginning of next page, and continue */
+ state->recordContRecPtr += XLOG_BLCKSZ;
+ }
+
+ /* targetPagePtr is pointing the last-read page here */
+ prec = (XLogRecord *) state->readRecordBuf;
+ if (!ValidXLogRecord(state, prec, state->ReadRecPtr))
+ goto err;
+
+ pageHeaderSize =
+ XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+ state->EndRecPtr = targetPagePtr + pageHeaderSize
+ + MAXALIGN(pageHeader->xlp_rem_len);
+
+ *record = prec;
+ state->readRecordState = XLREAD_NEXT_RECORD;
+ break;
+ }
}
/*
* Special processing if it's an XLOG SWITCH record
*/
- if (record->xl_rmid == RM_XLOG_ID &&
- (record->xl_info & ~XLR_INFO_MASK) == XLOG_SWITCH)
+ if ((*record)->xl_rmid == RM_XLOG_ID &&
+ ((*record)->xl_info & ~XLR_INFO_MASK) == XLOG_SWITCH)
{
/* Pretend it extends to end of segment */
state->EndRecPtr += state->segcxt.ws_segsize - 1;
state->EndRecPtr -= XLogSegmentOffset(state->EndRecPtr, state->segcxt.ws_segsize);
}
- if (DecodeXLogRecord(state, record, errormsg))
- return record;
- else
- return NULL;
+ Assert (!*record || state->readLen >= 0);
+ if (DecodeXLogRecord(state, *record, errormsg))
+ return XLREAD_SUCCESS;
+
+ *record = NULL;
+ return XLREAD_FAIL;
err:
/*
- * Invalidate the read state. We might read from a different source after
+ * Invalidate the read page. We might read from a different source after
* failure.
*/
XLogReaderInvalReadState(state);
@@ -572,7 +703,8 @@ err:
if (state->errormsg_buf[0] != '\0')
*errormsg = state->errormsg_buf;
- return NULL;
+ *record = NULL;
+ return XLREAD_FAIL;
}
/*
@@ -724,11 +856,12 @@ XLogReaderInvalReadState(XLogReaderState *state)
*
* This is just a convenience subroutine to avoid duplicated code in
* XLogReadRecord. It's not intended for use from anywhere else.
+ *
+ * If PrevRecPtr is valid, the xl_prev is is cross-checked with it.
*/
static bool
ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
- XLogRecPtr PrevRecPtr, XLogRecord *record,
- bool randAccess)
+ XLogRecPtr PrevRecPtr, XLogRecord *record)
{
if (record->xl_tot_len < SizeOfXLogRecord)
{
@@ -745,7 +878,7 @@ ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
record->xl_rmid, LSN_FORMAT_ARGS(RecPtr));
return false;
}
- if (randAccess)
+ if (PrevRecPtr == InvalidXLogRecPtr)
{
/*
* We can't exactly verify the prev-link, but surely it should be less
@@ -975,11 +1108,14 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
* XLogReadRecord() will read the next valid record.
*/
XLogRecPtr
-XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
+XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
+ XLogFindNextRecordCB read_page, void *private)
{
XLogRecPtr tmpRecPtr;
XLogRecPtr found = InvalidXLogRecPtr;
XLogPageHeader header;
+ XLogRecord *record;
+ XLogReadRecordResult result;
char *errormsg;
Assert(!XLogRecPtrIsInvalid(RecPtr));
@@ -1012,9 +1148,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
while(XLogNeedData(state, targetPagePtr, targetRecOff,
targetRecOff != 0))
{
- if (!state->routine.page_read(state, state->readPagePtr,
- state->readLen,
- state->ReadRecPtr, state->readBuf))
+ if (!read_page(state, private))
break;
}
@@ -1066,8 +1200,16 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
* or we just jumped over the remaining data of a continuation.
*/
XLogBeginRead(state, tmpRecPtr);
- while (XLogReadRecord(state, &errormsg) != NULL)
+ while ((result = XLogReadRecord(state, &record, &errormsg)) !=
+ XLREAD_FAIL)
{
+ if (result == XLREAD_NEED_DATA)
+ {
+ if (!read_page(state, private))
+ goto err;
+ continue;
+ }
+
/* past the record we've found, break out */
if (RecPtr <= state->ReadRecPtr)
{
@@ -1087,9 +1229,9 @@ err:
#endif /* FRONTEND */
/*
- * Helper function to ease writing of XLogRoutine->page_read callbacks.
- * If this function is used, caller must supply a segment_open callback in
- * 'state', as that is used here.
+ * Helper function to ease writing of page_read callback.
+ * If this function is used, caller must supply a segment_open callback and
+ * segment_close callback as that is used here.
*
* Read 'count' bytes into 'buf', starting at location 'startptr', from WAL
* fetched from timeline 'tli'.
@@ -1102,6 +1244,7 @@ err:
*/
bool
WALRead(XLogReaderState *state,
+ WALSegmentOpenCB segopenfn, WALSegmentCloseCB segclosefn,
char *buf, XLogRecPtr startptr, Size count, TimeLineID tli,
WALReadError *errinfo)
{
@@ -1133,10 +1276,10 @@ WALRead(XLogReaderState *state,
XLogSegNo nextSegNo;
if (state->seg.ws_file >= 0)
- state->routine.segment_close(state);
+ segclosefn(state);
XLByteToSeg(recptr, nextSegNo, state->segcxt.ws_segsize);
- state->routine.segment_open(state, nextSegNo, &tli);
+ segopenfn(state, nextSegNo, &tli);
/* This shouldn't happen -- indicates a bug in segment_open */
Assert(state->seg.ws_file >= 0);
diff --git a/src/backend/access/transam/xlogutils.c b/src/backend/access/transam/xlogutils.c
index 46eda33f25..b003990745 100644
--- a/src/backend/access/transam/xlogutils.c
+++ b/src/backend/access/transam/xlogutils.c
@@ -686,8 +686,7 @@ XLogTruncateRelation(RelFileNode rnode, ForkNumber forkNum,
void
XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wantLength)
{
- const XLogRecPtr lastReadPage = state->seg.ws_segno *
- state->segcxt.ws_segsize + state->readLen;
+ const XLogRecPtr lastReadPage = state->readPagePtr;
Assert(wantPage != InvalidXLogRecPtr && wantPage % XLOG_BLCKSZ == 0);
Assert(wantLength <= XLOG_BLCKSZ);
@@ -702,7 +701,7 @@ XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wa
* current TLI has since become historical.
*/
if (lastReadPage == wantPage &&
- state->readLen != 0 &&
+ state->page_verified &&
lastReadPage + state->readLen >= wantPage + Min(wantLength, XLOG_BLCKSZ - 1))
return;
@@ -788,6 +787,7 @@ wal_segment_open(XLogReaderState *state, XLogSegNo nextSegNo,
char path[MAXPGPATH];
XLogFilePath(path, tli, nextSegNo, state->segcxt.ws_segsize);
+ elog(LOG, "HOGE: %lu, %d => %s", nextSegNo, tli, path);
state->seg.ws_file = BasicOpenFile(path, O_RDONLY | PG_BINARY);
if (state->seg.ws_file >= 0)
return;
@@ -825,9 +825,11 @@ wal_segment_close(XLogReaderState *state)
* loop for now.
*/
bool
-read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *cur_page)
+read_local_xlog_page(XLogReaderState *state)
{
+ XLogRecPtr targetPagePtr = state->readPagePtr;
+ int reqLen = state->readLen;
+ char *cur_page = state->readBuf;
XLogRecPtr read_upto,
loc;
TimeLineID tli;
@@ -940,11 +942,12 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
* as 'count', read the whole page anyway. It's guaranteed to be
* zero-padded up to the page boundary if it's incomplete.
*/
- if (!WALRead(state, cur_page, targetPagePtr, XLOG_BLCKSZ, tli,
- &errinfo))
+ if (!WALRead(state, wal_segment_open, wal_segment_close,
+ cur_page, targetPagePtr, XLOG_BLCKSZ, tli, &errinfo))
WALReadRaiseError(&errinfo);
/* number of valid bytes in the buffer */
+ state->readPagePtr = targetPagePtr;
state->readLen = count;
return true;
}
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index 37b75deb72..9cbd8afa49 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -148,7 +148,8 @@ StartupDecodingContext(List *output_plugin_options,
TransactionId xmin_horizon,
bool need_full_snapshot,
bool fast_forward,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress)
@@ -198,11 +199,12 @@ StartupDecodingContext(List *output_plugin_options,
ctx->slot = slot;
- ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, xl_routine, ctx);
+ ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, cleanup_cb);
if (!ctx->reader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
errmsg("out of memory")));
+ ctx->page_read = page_read;
ctx->reorder = ReorderBufferAllocate();
ctx->snapshot_builder =
@@ -319,7 +321,8 @@ CreateInitDecodingContext(const char *plugin,
List *output_plugin_options,
bool need_full_snapshot,
XLogRecPtr restart_lsn,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress)
@@ -422,7 +425,7 @@ CreateInitDecodingContext(const char *plugin,
ctx = StartupDecodingContext(NIL, restart_lsn, xmin_horizon,
need_full_snapshot, false,
- xl_routine, prepare_write, do_write,
+ page_read, cleanup_cb, prepare_write, do_write,
update_progress);
/* call output plugin initialization callback */
@@ -476,7 +479,8 @@ LogicalDecodingContext *
CreateDecodingContext(XLogRecPtr start_lsn,
List *output_plugin_options,
bool fast_forward,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress)
@@ -528,8 +532,8 @@ CreateDecodingContext(XLogRecPtr start_lsn,
ctx = StartupDecodingContext(output_plugin_options,
start_lsn, InvalidTransactionId, false,
- fast_forward, xl_routine, prepare_write,
- do_write, update_progress);
+ fast_forward, page_read, cleanup_cb,
+ prepare_write, do_write, update_progress);
/* call output plugin initialization callback */
old_context = MemoryContextSwitchTo(ctx->context);
@@ -585,7 +589,13 @@ DecodingContextFindStartpoint(LogicalDecodingContext *ctx)
char *err = NULL;
/* the read_page callback waits for new WAL */
- record = XLogReadRecord(ctx->reader, &err);
+ while (XLogReadRecord(ctx->reader, &record, &err) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!ctx->page_read(ctx->reader))
+ break;
+ }
+
if (err)
elog(ERROR, "%s", err);
if (!record)
diff --git a/src/backend/replication/logical/logicalfuncs.c b/src/backend/replication/logical/logicalfuncs.c
index f7e055874e..b69f6911c5 100644
--- a/src/backend/replication/logical/logicalfuncs.c
+++ b/src/backend/replication/logical/logicalfuncs.c
@@ -233,9 +233,8 @@ pg_logical_slot_get_changes_guts(FunctionCallInfo fcinfo, bool confirm, bool bin
ctx = CreateDecodingContext(InvalidXLogRecPtr,
options,
false,
- XL_ROUTINE(.page_read = read_local_xlog_page,
- .segment_open = wal_segment_open,
- .segment_close = wal_segment_close),
+ read_local_xlog_page,
+ wal_segment_close,
LogicalOutputPrepareWrite,
LogicalOutputWrite, NULL);
@@ -284,7 +283,13 @@ pg_logical_slot_get_changes_guts(FunctionCallInfo fcinfo, bool confirm, bool bin
XLogRecord *record;
char *errm = NULL;
- record = XLogReadRecord(ctx->reader, &errm);
+ while (XLogReadRecord(ctx->reader, &record, &errm) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!ctx->page_read(ctx->reader))
+ break;
+ }
+
if (errm)
elog(ERROR, "%s", errm);
diff --git a/src/backend/replication/slotfuncs.c b/src/backend/replication/slotfuncs.c
index 9817b44113..deb7cf15eb 100644
--- a/src/backend/replication/slotfuncs.c
+++ b/src/backend/replication/slotfuncs.c
@@ -153,9 +153,8 @@ create_logical_replication_slot(char *name, char *plugin,
ctx = CreateInitDecodingContext(plugin, NIL,
false, /* just catalogs is OK */
restart_lsn,
- XL_ROUTINE(.page_read = read_local_xlog_page,
- .segment_open = wal_segment_open,
- .segment_close = wal_segment_close),
+ read_local_xlog_page,
+ wal_segment_close,
NULL, NULL, NULL);
/*
@@ -512,9 +511,8 @@ pg_logical_replication_slot_advance(XLogRecPtr moveto)
ctx = CreateDecodingContext(InvalidXLogRecPtr,
NIL,
true, /* fast_forward */
- XL_ROUTINE(.page_read = read_local_xlog_page,
- .segment_open = wal_segment_open,
- .segment_close = wal_segment_close),
+ read_local_xlog_page,
+ wal_segment_close,
NULL, NULL, NULL);
/*
@@ -536,7 +534,13 @@ pg_logical_replication_slot_advance(XLogRecPtr moveto)
* Read records. No changes are generated in fast_forward mode,
* but snapbuilder/slot statuses are updated properly.
*/
- record = XLogReadRecord(ctx->reader, &errm);
+ while (XLogReadRecord(ctx->reader, &record, &errm) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!ctx->page_read(ctx->reader))
+ break;
+ }
+
if (errm)
elog(ERROR, "%s", errm);
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index c36b91ae5e..062093727d 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -580,10 +580,7 @@ StartReplication(StartReplicationCmd *cmd)
/* create xlogreader for physical replication */
xlogreader =
- XLogReaderAllocate(wal_segment_size, NULL,
- XL_ROUTINE(.segment_open = WalSndSegmentOpen,
- .segment_close = wal_segment_close),
- NULL);
+ XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
if (!xlogreader)
ereport(ERROR,
@@ -807,9 +804,11 @@ StartReplication(StartReplicationCmd *cmd)
* set every time WAL is flushed.
*/
static bool
-logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetRecPtr, char *cur_page)
+logical_read_xlog_page(XLogReaderState *state)
{
+ XLogRecPtr targetPagePtr = state->readPagePtr;
+ int reqLen = state->readLen;
+ char *cur_page = state->readBuf;
XLogRecPtr flushptr;
int count;
WALReadError errinfo;
@@ -837,7 +836,7 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
count = flushptr - targetPagePtr; /* part of the page available */
/* now actually read the data, we know it's there */
- if (!WALRead(state,
+ if (!WALRead(state, WalSndSegmentOpen, wal_segment_close,
cur_page,
targetPagePtr,
XLOG_BLCKSZ,
@@ -1011,9 +1010,8 @@ CreateReplicationSlot(CreateReplicationSlotCmd *cmd)
ctx = CreateInitDecodingContext(cmd->plugin, NIL, need_full_snapshot,
InvalidXLogRecPtr,
- XL_ROUTINE(.page_read = logical_read_xlog_page,
- .segment_open = WalSndSegmentOpen,
- .segment_close = wal_segment_close),
+ logical_read_xlog_page,
+ wal_segment_close,
WalSndPrepareWrite, WalSndWriteData,
WalSndUpdateProgress);
@@ -1171,9 +1169,8 @@ StartLogicalReplication(StartReplicationCmd *cmd)
*/
logical_decoding_ctx =
CreateDecodingContext(cmd->startpoint, cmd->options, false,
- XL_ROUTINE(.page_read = logical_read_xlog_page,
- .segment_open = WalSndSegmentOpen,
- .segment_close = wal_segment_close),
+ logical_read_xlog_page,
+ wal_segment_close,
WalSndPrepareWrite, WalSndWriteData,
WalSndUpdateProgress);
xlogreader = logical_decoding_ctx->reader;
@@ -2749,7 +2746,7 @@ XLogSendPhysical(void)
enlargeStringInfo(&output_message, nbytes);
retry:
- if (!WALRead(xlogreader,
+ if (!WALRead(xlogreader, WalSndSegmentOpen, wal_segment_close,
&output_message.data[output_message.len],
startptr,
nbytes,
@@ -2847,7 +2844,12 @@ XLogSendLogical(void)
*/
WalSndCaughtUp = false;
- record = XLogReadRecord(logical_decoding_ctx->reader, &errm);
+ while (XLogReadRecord(logical_decoding_ctx->reader, &record, &errm) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!logical_decoding_ctx->page_read(logical_decoding_ctx->reader))
+ break;
+ }
/* xlog record was invalid */
if (errm != NULL)
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index cf119848b0..712c85281c 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -41,15 +41,9 @@ static int xlogreadfd = -1;
static XLogSegNo xlogreadsegno = -1;
static char xlogfpath[MAXPGPATH];
-typedef struct XLogPageReadPrivate
-{
- const char *restoreCommand;
- int tliIndex;
-} XLogPageReadPrivate;
-
-static bool SimpleXLogPageRead(XLogReaderState *xlogreader,
- XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
+static bool SimpleXLogPageRead(XLogReaderState *xlogreader,
+ const char *datadir, int *tliIndex,
+ const char *restoreCommand);
/*
* Read WAL from the datadir/pg_wal, starting from 'startpoint' on timeline
@@ -66,20 +60,22 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
XLogRecord *record;
XLogReaderState *xlogreader;
char *errormsg;
- XLogPageReadPrivate private;
- private.tliIndex = tliIndex;
- private.restoreCommand = restoreCommand;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir,
- XL_ROUTINE(.page_read = &SimpleXLogPageRead),
- &private);
+ xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+
if (xlogreader == NULL)
pg_fatal("out of memory");
XLogBeginRead(xlogreader, startpoint);
do
{
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!SimpleXLogPageRead(xlogreader, datadir,
+ &tliIndex, restoreCommand))
+ break;
+ }
if (record == NULL)
{
@@ -123,19 +119,19 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex,
XLogRecord *record;
XLogReaderState *xlogreader;
char *errormsg;
- XLogPageReadPrivate private;
XLogRecPtr endptr;
- private.tliIndex = tliIndex;
- private.restoreCommand = restoreCommand;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir,
- XL_ROUTINE(.page_read = &SimpleXLogPageRead),
- &private);
+ xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
XLogBeginRead(xlogreader, ptr);
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!SimpleXLogPageRead(xlogreader, datadir, &tliIndex, restoreCommand))
+ break;
+ }
if (record == NULL)
{
if (errormsg)
@@ -170,7 +166,6 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
XLogRecPtr searchptr;
XLogReaderState *xlogreader;
char *errormsg;
- XLogPageReadPrivate private;
/*
* The given fork pointer points to the end of the last common record,
@@ -186,11 +181,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
forkptr += SizeOfXLogShortPHD;
}
- private.tliIndex = tliIndex;
- private.restoreCommand = restoreCommand;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir,
- XL_ROUTINE(.page_read = &SimpleXLogPageRead),
- &private);
+ xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
@@ -200,7 +191,13 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
uint8 info;
XLogBeginRead(xlogreader, searchptr);
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!SimpleXLogPageRead(xlogreader, datadir,
+ &tliIndex, restoreCommand))
+ break;
+ }
if (record == NULL)
{
@@ -247,10 +244,11 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
/* XLogReader callback function, to read a WAL page */
static bool
-SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *readBuf)
+SimpleXLogPageRead(XLogReaderState *xlogreader, const char *datadir,
+ int *tliIndex, const char *restoreCommand)
{
- XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
+ XLogRecPtr targetPagePtr = xlogreader->readPagePtr;
+ char *readBuf = xlogreader->readBuf;
uint32 targetPageOff;
XLogRecPtr targetSegEnd;
XLogSegNo targetSegNo;
@@ -283,14 +281,14 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
* be done both forward and backward, consider also switching timeline
* accordingly.
*/
- while (private->tliIndex < targetNentries - 1 &&
- targetHistory[private->tliIndex].end < targetSegEnd)
- private->tliIndex++;
- while (private->tliIndex > 0 &&
- targetHistory[private->tliIndex].begin >= targetSegEnd)
- private->tliIndex--;
+ while (*tliIndex < targetNentries - 1 &&
+ targetHistory[*tliIndex].end < targetSegEnd)
+ (*tliIndex)++;
+ while (*tliIndex > 0 &&
+ targetHistory[*tliIndex].begin >= targetSegEnd)
+ (*tliIndex)--;
- XLogFileName(xlogfname, targetHistory[private->tliIndex].tli,
+ XLogFileName(xlogfname, targetHistory[*tliIndex].tli,
xlogreadsegno, WalSegSz);
snprintf(xlogfpath, MAXPGPATH, "%s/" XLOGDIR "/%s",
@@ -303,7 +301,7 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
/*
* If we have no restore_command to execute, then exit.
*/
- if (private->restoreCommand == NULL)
+ if (restoreCommand == NULL)
{
pg_log_error("could not open file \"%s\": %m", xlogfpath);
xlogreader->readLen = -1;
@@ -317,7 +315,7 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
xlogreadfd = RestoreArchivedFile(xlogreader->segcxt.ws_dir,
xlogfname,
WalSegSz,
- private->restoreCommand);
+ restoreCommand);
if (xlogreadfd < 0)
{
@@ -359,7 +357,7 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
Assert(targetSegNo == xlogreadsegno);
- xlogreader->seg.ws_tli = targetHistory[private->tliIndex].tli;
+ xlogreader->seg.ws_tli = targetHistory[*tliIndex].tli;
xlogreader->readLen = XLOG_BLCKSZ;
return true;
}
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index 0f0e0723a3..f514de6b13 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -330,12 +330,17 @@ WALDumpCloseSegment(XLogReaderState *state)
state->seg.ws_file = -1;
}
-/* pg_waldump's XLogReaderRoutine->page_read callback */
+/*
+ * pg_waldump's WAL page rader, also used as page_read callback for
+ * XLogFindNextRecord
+ */
static bool
-WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetPtr, char *readBuff)
+WALDumpReadPage(XLogReaderState *state, void *priv)
{
- XLogDumpPrivate *private = state->private_data;
+ XLogRecPtr targetPagePtr = state->readPagePtr;
+ int reqLen = state->readLen;
+ char *readBuff = state->readBuf;
+ XLogDumpPrivate *private = (XLogDumpPrivate *) priv;
int count = XLOG_BLCKSZ;
WALReadError errinfo;
@@ -353,8 +358,8 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
}
}
- if (!WALRead(state, readBuff, targetPagePtr, count, private->timeline,
- &errinfo))
+ if (!WALRead(state, WALDumpOpenSegment, WALDumpCloseSegment,
+ readBuff, targetPagePtr, count, private->timeline, &errinfo))
{
WALOpenSegment *seg = &errinfo.wre_seg;
char fname[MAXPGPATH];
@@ -374,6 +379,7 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
(Size) errinfo.wre_req);
}
+ Assert(count >= state->readLen);
state->readLen = count;
return true;
}
@@ -1035,16 +1041,14 @@ main(int argc, char **argv)
/* we have everything we need, start reading */
xlogreader_state =
- XLogReaderAllocate(WalSegSz, waldir,
- XL_ROUTINE(.page_read = WALDumpReadPage,
- .segment_open = WALDumpOpenSegment,
- .segment_close = WALDumpCloseSegment),
- &private);
+ XLogReaderAllocate(WalSegSz, waldir, WALDumpCloseSegment);
+
if (!xlogreader_state)
fatal_error("out of memory");
/* first find a valid recptr to start from */
- first_record = XLogFindNextRecord(xlogreader_state, private.startptr);
+ first_record = XLogFindNextRecord(xlogreader_state, private.startptr,
+ &WALDumpReadPage, (void*) &private);
if (first_record == InvalidXLogRecPtr)
fatal_error("could not find a valid record after %X/%X",
@@ -1067,7 +1071,13 @@ main(int argc, char **argv)
for (;;)
{
/* try to read the next record */
- record = XLogReadRecord(xlogreader_state, &errormsg);
+ while (XLogReadRecord(xlogreader_state, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!WALDumpReadPage(xlogreader_state, (void *) &private))
+ break;
+ }
+
if (!record)
{
if (!config.follow || private.endptr_reached)
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index 23d9e70a59..4e9268b553 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -57,64 +57,15 @@ typedef struct WALSegmentContext
typedef struct XLogReaderState XLogReaderState;
-/* Function type definition for the read_page callback */
-typedef bool (*XLogPageReadCB) (XLogReaderState *xlogreader,
- XLogRecPtr targetPagePtr,
- int reqLen,
- XLogRecPtr targetRecPtr,
- char *readBuf);
+/* Function type definition for the segment cleanup callback */
+typedef void (*WALSegmentCleanupCB) (XLogReaderState *xlogreader);
+
+/* Function type definition for the open/close callbacks for WALRead() */
typedef void (*WALSegmentOpenCB) (XLogReaderState *xlogreader,
XLogSegNo nextSegNo,
TimeLineID *tli_p);
typedef void (*WALSegmentCloseCB) (XLogReaderState *xlogreader);
-typedef struct XLogReaderRoutine
-{
- /*
- * Data input callback
- *
- * This callback shall read at least reqLen valid bytes of the xlog page
- * starting at targetPagePtr, and store them in readBuf. The callback
- * shall return the number of bytes read (never more than XLOG_BLCKSZ), or
- * -1 on failure. The callback shall sleep, if necessary, to wait for the
- * requested bytes to become available. The callback will not be invoked
- * again for the same page unless more than the returned number of bytes
- * are needed.
- *
- * targetRecPtr is the position of the WAL record we're reading. Usually
- * it is equal to targetPagePtr + reqLen, but sometimes xlogreader needs
- * to read and verify the page or segment header, before it reads the
- * actual WAL record it's interested in. In that case, targetRecPtr can
- * be used to determine which timeline to read the page from.
- *
- * The callback shall set ->seg.ws_tli to the TLI of the file the page was
- * read from.
- */
- XLogPageReadCB page_read;
-
- /*
- * Callback to open the specified WAL segment for reading. ->seg.ws_file
- * shall be set to the file descriptor of the opened segment. In case of
- * failure, an error shall be raised by the callback and it shall not
- * return.
- *
- * "nextSegNo" is the number of the segment to be opened.
- *
- * "tli_p" is an input/output argument. WALRead() uses it to pass the
- * timeline in which the new segment should be found, but the callback can
- * use it to return the TLI that it actually opened.
- */
- WALSegmentOpenCB segment_open;
-
- /*
- * WAL segment close callback. ->seg.ws_file shall be set to a negative
- * number.
- */
- WALSegmentCloseCB segment_close;
-} XLogReaderRoutine;
-
-#define XL_ROUTINE(...) &(XLogReaderRoutine){__VA_ARGS__}
-
typedef struct
{
/* Is this block ref in use? */
@@ -144,12 +95,35 @@ typedef struct
uint16 data_bufsz;
} DecodedBkpBlock;
+/* Return code from XLogReadRecord */
+typedef enum XLogReadRecordResult
+{
+ XLREAD_SUCCESS, /* record is successfully read */
+ XLREAD_NEED_DATA, /* need more data. see XLogReadRecord. */
+ XLREAD_FAIL /* failed during reading a record */
+} XLogReadRecordResult;
+
+/*
+ * internal state of XLogReadRecord
+ *
+ * XLogReadState runs a state machine while reading a record. Theses states
+ * are not seen outside the function. Each state may repeat several times
+ * exiting requesting caller for new data. See the comment of XLogReadRecrod
+ * for details.
+ */
+typedef enum XLogReadRecordState {
+ XLREAD_NEXT_RECORD,
+ XLREAD_TOT_LEN,
+ XLREAD_FIRST_FRAGMENT,
+ XLREAD_CONTINUATION
+} XLogReadRecordState;
+
struct XLogReaderState
{
/*
* Operational callbacks
*/
- XLogReaderRoutine routine;
+ WALSegmentCleanupCB cleanup_cb;
/* ----------------------------------------
* Public parameters
@@ -162,18 +136,14 @@ struct XLogReaderState
*/
uint64 system_identifier;
- /*
- * Opaque data for callbacks to use. Not used by XLogReader.
- */
- void *private_data;
-
/*
* Start and end point of last record read. EndRecPtr is also used as the
* position to read next. Calling XLogBeginRead() sets EndRecPtr to the
* starting position and ReadRecPtr to invalid.
*/
- XLogRecPtr ReadRecPtr; /* start of last record read */
+ XLogRecPtr ReadRecPtr; /* start of last record read or being read */
XLogRecPtr EndRecPtr; /* end+1 of last record read */
+ XLogRecPtr PrevRecPtr; /* start of previous record read */
/* ----------------------------------------
* Communication with page reader
@@ -187,7 +157,9 @@ struct XLogReaderState
* the request, or -1 on error */
TimeLineID readPageTLI; /* TLI for data currently in readBuf */
char *readBuf; /* buffer to store data */
- bool page_verified; /* is the page on the buffer verified? */
+ bool page_verified; /* is the page header on the buffer verified? */
+ bool record_verified;/* is the current record header verified? */
+
/* ----------------------------------------
@@ -229,8 +201,6 @@ struct XLogReaderState
XLogRecPtr latestPagePtr;
TimeLineID latestPageTLI;
- /* beginning of the WAL record being read. */
- XLogRecPtr currRecPtr;
/* timeline to read it from, 0 if a lookup is required */
TimeLineID currTLI;
@@ -257,6 +227,15 @@ struct XLogReaderState
char *readRecordBuf;
uint32 readRecordBufSize;
+ /*
+ * XLogReadRecord() state
+ */
+ XLogReadRecordState readRecordState;/* state machine state */
+ int recordGotLen; /* amount of current record that has
+ * already been read */
+ int recordRemainLen; /* length of current record that remains */
+ XLogRecPtr recordContRecPtr; /* where the current record continues */
+
/* Buffer to hold error message */
char *errormsg_buf;
};
@@ -264,9 +243,7 @@ struct XLogReaderState
/* Get a new XLogReader */
extern XLogReaderState *XLogReaderAllocate(int wal_segment_size,
const char *waldir,
- XLogReaderRoutine *routine,
- void *private_data);
-extern XLogReaderRoutine *LocalXLogReaderRoutine(void);
+ WALSegmentCleanupCB cleanup_cb);
/* Free an XLogReader */
extern void XLogReaderFree(XLogReaderState *state);
@@ -274,12 +251,17 @@ extern void XLogReaderFree(XLogReaderState *state);
/* Position the XLogReader to given record */
extern void XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr);
#ifdef FRONTEND
-extern XLogRecPtr XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr);
+/* Function type definition for the read_page callback */
+typedef bool (*XLogFindNextRecordCB) (XLogReaderState *xlogreader,
+ void *private);
+extern XLogRecPtr XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
+ XLogFindNextRecordCB read_page, void *private);
#endif /* FRONTEND */
/* Read the next XLog record. Returns NULL on end-of-WAL or failure */
-extern struct XLogRecord *XLogReadRecord(XLogReaderState *state,
- char **errormsg);
+extern XLogReadRecordResult XLogReadRecord(XLogReaderState *state,
+ XLogRecord **record,
+ char **errormsg);
/* Validate a page */
extern bool XLogReaderValidatePageHeader(XLogReaderState *state,
@@ -299,6 +281,7 @@ typedef struct WALReadError
} WALReadError;
extern bool WALRead(XLogReaderState *state,
+ WALSegmentOpenCB segopenfn, WALSegmentCloseCB sgclosefn,
char *buf, XLogRecPtr startptr, Size count,
TimeLineID tli, WALReadError *errinfo);
diff --git a/src/include/access/xlogutils.h b/src/include/access/xlogutils.h
index 364a21c4ea..397fb27fc2 100644
--- a/src/include/access/xlogutils.h
+++ b/src/include/access/xlogutils.h
@@ -47,9 +47,7 @@ extern Buffer XLogReadBufferExtended(RelFileNode rnode, ForkNumber forknum,
extern Relation CreateFakeRelcacheEntry(RelFileNode rnode);
extern void FreeFakeRelcacheEntry(Relation fakerel);
-extern bool read_local_xlog_page(XLogReaderState *state,
- XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetRecPtr, char *cur_page);
+extern bool read_local_xlog_page(XLogReaderState *state);
extern void wal_segment_open(XLogReaderState *state,
XLogSegNo nextSegNo,
TimeLineID *tli_p);
diff --git a/src/include/pg_config_manual.h b/src/include/pg_config_manual.h
index f10ad0acd6..94749609b4 100644
--- a/src/include/pg_config_manual.h
+++ b/src/include/pg_config_manual.h
@@ -377,7 +377,7 @@
* Enable debugging print statements for WAL-related operations; see
* also the wal_debug GUC var.
*/
-/* #define WAL_DEBUG */
+#define WAL_DEBUG
/*
* Enable tracing of resource consumption during sort operations;
diff --git a/src/include/replication/logical.h b/src/include/replication/logical.h
index c253403372..b936f31393 100644
--- a/src/include/replication/logical.h
+++ b/src/include/replication/logical.h
@@ -29,6 +29,10 @@ typedef void (*LogicalOutputPluginWriterUpdateProgress) (struct LogicalDecodingC
TransactionId xid
);
+typedef struct LogicalDecodingContext LogicalDecodingContext;
+
+typedef bool (*LogicalDecodingXLogPageReadCB)(XLogReaderState *ctx);
+
typedef struct LogicalDecodingContext
{
/* memory context this is all allocated in */
@@ -39,6 +43,7 @@ typedef struct LogicalDecodingContext
/* infrastructure pieces for decoding */
XLogReaderState *reader;
+ LogicalDecodingXLogPageReadCB page_read;
struct ReorderBuffer *reorder;
struct SnapBuild *snapshot_builder;
@@ -105,14 +110,16 @@ extern LogicalDecodingContext *CreateInitDecodingContext(const char *plugin,
List *output_plugin_options,
bool need_full_snapshot,
XLogRecPtr restart_lsn,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress);
extern LogicalDecodingContext *CreateDecodingContext(XLogRecPtr start_lsn,
List *output_plugin_options,
bool fast_forward,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress);
--
2.27.0
----Next_Part(Thu_Mar__4_11_28_53_2021_014)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
filename="v17-0003-Remove-globals-readOff-readLen-and-readSegNo.patch"
^ permalink raw reply [nested|flat] 70+ messages in thread
* [PATCH v15 2/4] Move page-reader out of XLogReadRecord
@ 2019-09-10 03:58 Kyotaro Horiguchi <[email protected]>
0 siblings, 0 replies; 70+ messages in thread
From: Kyotaro Horiguchi @ 2019-09-10 03:58 UTC (permalink / raw)
This is the second step of removing callbacks from WAL record reader.
Since it is essential to take in additional data while reading a
record, the function have to ask caller for new data while keeping
working state. Thus the function is turned into a state machine.
---
src/backend/access/transam/twophase.c | 15 +-
src/backend/access/transam/xlog.c | 58 +-
src/backend/access/transam/xlogreader.c | 700 +++++++++++-------
src/backend/access/transam/xlogutils.c | 17 +-
src/backend/replication/logical/logical.c | 26 +-
.../replication/logical/logicalfuncs.c | 13 +-
src/backend/replication/slotfuncs.c | 18 +-
src/backend/replication/walsender.c | 32 +-
src/bin/pg_rewind/parsexlog.c | 84 +--
src/bin/pg_waldump/pg_waldump.c | 36 +-
src/include/access/xlogreader.h | 121 ++-
src/include/access/xlogutils.h | 4 +-
src/include/replication/logical.h | 11 +-
13 files changed, 648 insertions(+), 487 deletions(-)
diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index 9b2e59bf0e..b0d60a0d0f 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -1330,11 +1330,8 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
XLogReaderState *xlogreader;
char *errormsg;
- xlogreader = XLogReaderAllocate(wal_segment_size, NULL,
- XL_ROUTINE(.page_read = &read_local_xlog_page,
- .segment_open = &wal_segment_open,
- .segment_close = &wal_segment_close),
- NULL);
+ xlogreader = XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+
if (!xlogreader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
@@ -1342,7 +1339,13 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
errdetail("Failed while allocating a WAL reading processor.")));
XLogBeginRead(xlogreader, lsn);
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!read_local_xlog_page(xlogreader))
+ break;
+ }
+
if (record == NULL)
ereport(ERROR,
(errcode_for_file_access(),
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index e570e56a24..f9b0108602 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -831,13 +831,6 @@ static XLogSource currentSource = XLOG_FROM_ANY;
static bool lastSourceFailed = false;
static bool pendingWalRcvRestart = false;
-typedef struct XLogPageReadPrivate
-{
- int emode;
- bool fetching_ckpt; /* are we fetching a checkpoint record? */
- bool randAccess;
-} XLogPageReadPrivate;
-
/*
* These variables track when we last obtained some WAL data to process,
* and where we got it from. (XLogReceiptSource is initially the same as
@@ -912,8 +905,8 @@ static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath,
static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli,
XLogSource source, bool notfoundOk);
static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, XLogSource source);
-static bool XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
+static bool XLogPageRead(XLogReaderState *xlogreader,
+ bool fetching_ckpt, int emode, bool randAccess);
static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
bool fetching_ckpt, XLogRecPtr tliRecPtr);
static int emode_for_corrupt_record(int emode, XLogRecPtr RecPtr);
@@ -1225,8 +1218,7 @@ XLogInsertRecord(XLogRecData *rdata,
appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len);
if (!debug_reader)
- debug_reader = XLogReaderAllocate(wal_segment_size, NULL,
- XL_ROUTINE(), NULL);
+ debug_reader = XLogReaderAllocate(wal_segment_size, NULL NULL);
if (!debug_reader)
{
@@ -4326,15 +4318,10 @@ CleanupBackupHistory(void)
* record is available.
*/
static XLogRecord *
-ReadRecord(XLogReaderState *xlogreader, int emode,
- bool fetching_ckpt)
+ReadRecord(XLogReaderState *xlogreader, int emode, bool fetching_ckpt)
{
XLogRecord *record;
- XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
-
- private->fetching_ckpt = fetching_ckpt;
- private->emode = emode;
- private->randAccess = (xlogreader->ReadRecPtr == InvalidXLogRecPtr);
+ bool randAccess = (xlogreader->ReadRecPtr == InvalidXLogRecPtr);
/* This is the first attempt to read this page. */
lastSourceFailed = false;
@@ -4342,8 +4329,16 @@ ReadRecord(XLogReaderState *xlogreader, int emode,
for (;;)
{
char *errormsg;
+ XLogReadRecordResult result;
+
+ while ((result = XLogReadRecord(xlogreader, &record, &errormsg))
+ == XLREAD_NEED_DATA)
+ {
+ if (!XLogPageRead(xlogreader, fetching_ckpt, emode, randAccess))
+ break;
+
+ }
- record = XLogReadRecord(xlogreader, &errormsg);
ReadRecPtr = xlogreader->ReadRecPtr;
EndRecPtr = xlogreader->EndRecPtr;
if (record == NULL)
@@ -6320,7 +6315,6 @@ StartupXLOG(void)
bool backupFromStandby = false;
DBState dbstate_at_startup;
XLogReaderState *xlogreader;
- XLogPageReadPrivate private;
bool fast_promoted = false;
struct stat st;
@@ -6480,13 +6474,9 @@ StartupXLOG(void)
OwnLatch(&XLogCtl->recoveryWakeupLatch);
/* Set up XLOG reader facility */
- MemSet(&private, 0, sizeof(XLogPageReadPrivate));
xlogreader =
- XLogReaderAllocate(wal_segment_size, NULL,
- XL_ROUTINE(.page_read = &XLogPageRead,
- .segment_open = NULL,
- .segment_close = wal_segment_close),
- &private);
+ XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+
if (!xlogreader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
@@ -11860,12 +11850,13 @@ CancelBackup(void)
* sleep and retry.
*/
static bool
-XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetRecPtr, char *readBuf)
+XLogPageRead(XLogReaderState *xlogreader,
+ bool fetching_ckpt, int emode, bool randAccess)
{
- XLogPageReadPrivate *private =
- (XLogPageReadPrivate *) xlogreader->private_data;
- int emode = private->emode;
+ char *readBuf = xlogreader->readBuf;
+ XLogRecPtr targetPagePtr = xlogreader->readPagePtr;
+ int reqLen = xlogreader->readLen;
+ XLogRecPtr targetRecPtr = xlogreader->ReadRecPtr;
uint32 targetPageOff;
XLogSegNo targetSegNo PG_USED_FOR_ASSERTS_ONLY;
int r;
@@ -11908,8 +11899,8 @@ retry:
flushedUpto < targetPagePtr + reqLen))
{
if (!WaitForWALToBecomeAvailable(targetPagePtr + reqLen,
- private->randAccess,
- private->fetching_ckpt,
+ randAccess,
+ fetching_ckpt,
targetRecPtr))
{
if (readFile >= 0)
@@ -12014,6 +12005,7 @@ retry:
goto next_record_is_invalid;
}
+ Assert(xlogreader->readPagePtr == targetPagePtr);
xlogreader->readLen = readLen;
return true;
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index 3d599325ee..9281b57379 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -40,7 +40,7 @@ static bool XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr,
int reqLen, bool header_inclusive);
static void XLogReaderInvalReadState(XLogReaderState *state);
static bool ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
- XLogRecPtr PrevRecPtr, XLogRecord *record, bool randAccess);
+ XLogRecPtr PrevRecPtr, XLogRecord *record);
static bool ValidXLogRecord(XLogReaderState *state, XLogRecord *record,
XLogRecPtr recptr);
static void ResetDecoder(XLogReaderState *state);
@@ -73,7 +73,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...)
*/
XLogReaderState *
XLogReaderAllocate(int wal_segment_size, const char *waldir,
- XLogReaderRoutine *routine, void *private_data)
+ WALSegmentCleanupCB cleanup_cb)
{
XLogReaderState *state;
@@ -84,7 +84,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
return NULL;
/* initialize caller-provided support functions */
- state->routine = *routine;
+ state->cleanup_cb = cleanup_cb;
state->max_block_id = -1;
@@ -107,8 +107,6 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
WALOpenSegmentInit(&state->seg, &state->segcxt, wal_segment_size,
waldir);
- /* system_identifier initialized to zeroes above */
- state->private_data = private_data;
/* ReadRecPtr, EndRecPtr and readLen initialized to zeroes above */
state->errormsg_buf = palloc_extended(MAX_ERRORMSG_LEN + 1,
MCXT_ALLOC_NO_OOM);
@@ -140,8 +138,8 @@ XLogReaderFree(XLogReaderState *state)
{
int block_id;
- if (state->seg.ws_file != -1)
- state->routine.segment_close(state);
+ if (state->seg.ws_file >= 0)
+ state->cleanup_cb(state);
for (block_id = 0; block_id <= XLR_MAX_BLOCK_ID; block_id++)
{
@@ -246,6 +244,7 @@ XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
/* Begin at the passed-in record pointer. */
state->EndRecPtr = RecPtr;
state->ReadRecPtr = InvalidXLogRecPtr;
+ state->readRecordState = XLREAD_NEXT_RECORD;
}
/*
@@ -254,317 +253,452 @@ XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
* XLogBeginRead() or XLogFindNextRecord() must be called before the first call
* to XLogReadRecord().
*
- * If the page_read callback fails to read the requested data, NULL is
- * returned. The callback is expected to have reported the error; errormsg
- * is set to NULL.
+ * This function may return XLREAD_NEED_DATA several times before returning a
+ * result record. The caller shall read in some new data then call this
+ * function again with the same parameters.
*
- * If the reading fails for some other reason, NULL is also returned, and
- * *errormsg is set to a string with details of the failure.
+ * When a record is successfully read, returns XLREAD_SUCCESS with result
+ * record being stored in *record. Otherwise *record is NULL.
*
- * The returned pointer (or *errormsg) points to an internal buffer that's
- * valid until the next call to XLogReadRecord.
+ * Returns XLREAD_NEED_DATA if more data is needed to finish reading the
+ * current record. In that case, state->readPagePtr and state->readLen inform
+ * the desired position and minimum length of data needed. The caller shall
+ * read in the requested data and set state->readBuf to point to a buffer
+ * containing it. The caller must also set state->readPageTLI and
+ * state->readLen to indicate the timeline that it was read from, and the
+ * length of data that is now available (which must be >= given readLen),
+ * respectively.
+ *
+ * If invalid data is encountered, returns XLREAD_FAIL with *record being set to
+ * NULL. *errormsg is set to a string with details of the failure.
+ * The returned pointer (or *errormsg) points to an internal buffer that's valid
+ * until the next call to XLogReadRecord.
+ *
+ *
+ * This function runs a state machine consists of the following states.
+ *
+ * XLREAD_NEXT_RECORD :
+ * The initial state, if called with valid RecPtr, try to read a record at
+ * that position. If invalid RecPtr is given try to read a record just after
+ * the last one previously read.
+ * This state ens after setting ReadRecPtr. Then goes to XLREAD_TOT_LEN.
+ *
+ * XLREAD_TOT_LEN:
+ * Examining record header. Ends after reading record total
+ * length. recordRemainLen and recordGotLen are initialized.
+ *
+ * XLREAD_FIRST_FRAGMENT:
+ * Reading the first fragment. Ends with finishing reading a single
+ * record. Goes to XLREAD_NEXT_RECORD if that's all or
+ * XLREAD_CONTINUATION if we have continuation.
+
+ * XLREAD_CONTINUATION:
+ * Reading continuation of record. Ends with finishing the whole record then
+ * goes to XLREAD_NEXT_RECORD. During this state, recordRemainLen indicates
+ * how much is left and readRecordBuf holds the partially assert
+ * record.recordContRecPtr points to the beginning of the next page where to
+ * continue.
+ *
+ * If wrong data found in any state, the state machine stays at the current
+ * state. This behavior allows to continue reading a reacord switching among
+ * different souces, while streaming replication.
*/
-XLogRecord *
-XLogReadRecord(XLogReaderState *state, char **errormsg)
+XLogReadRecordResult
+XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
{
- XLogRecPtr RecPtr;
- XLogRecord *record;
- XLogRecPtr targetPagePtr;
- bool randAccess;
- uint32 len,
- total_len;
- uint32 targetRecOff;
- uint32 pageHeaderSize;
- bool gotheader;
+ XLogRecord *prec;
- /*
- * randAccess indicates whether to verify the previous-record pointer of
- * the record we're reading. We only do this if we're reading
- * sequentially, which is what we initially assume.
- */
- randAccess = false;
+ *record = NULL;
/* reset error state */
*errormsg = NULL;
state->errormsg_buf[0] = '\0';
- ResetDecoder(state);
-
- RecPtr = state->EndRecPtr;
-
- if (state->ReadRecPtr != InvalidXLogRecPtr)
- {
- /* read the record after the one we just read */
-
- /*
- * EndRecPtr is pointing to end+1 of the previous WAL record. If
- * we're at a page boundary, no more records can fit on the current
- * page. We must skip over the page header, but we can't do that until
- * we've read in the page, since the header size is variable.
- */
- }
- else
- {
- /*
- * Caller supplied a position to start at.
- *
- * In this case, EndRecPtr should already be pointing to a valid
- * record starting position.
- */
- Assert(XRecOffIsValid(RecPtr));
- randAccess = true;
- }
-
- state->currRecPtr = RecPtr;
-
- targetPagePtr = RecPtr - (RecPtr % XLOG_BLCKSZ);
- targetRecOff = RecPtr % XLOG_BLCKSZ;
-
- /*
- * Read the page containing the record into state->readBuf. Request enough
- * byte to cover the whole record header, or at least the part of it that
- * fits on the same page.
- */
- while (XLogNeedData(state, targetPagePtr,
- Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
- targetRecOff != 0))
- {
- if (!state->routine.page_read(state, state->readPagePtr, state->readLen,
- RecPtr, state->readBuf))
- break;
- }
-
- if (!state->page_verified)
- goto err;
-
- /*
- * We have at least the page header, so we can examine it now.
- */
- pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
- if (targetRecOff == 0)
- {
- /*
- * At page start, so skip over page header.
- */
- RecPtr += pageHeaderSize;
- targetRecOff = pageHeaderSize;
- }
- else if (targetRecOff < pageHeaderSize)
- {
- report_invalid_record(state, "invalid record offset at %X/%X",
- (uint32) (RecPtr >> 32), (uint32) RecPtr);
- goto err;
- }
-
- if ((((XLogPageHeader) state->readBuf)->xlp_info & XLP_FIRST_IS_CONTRECORD) &&
- targetRecOff == pageHeaderSize)
- {
- report_invalid_record(state, "contrecord is requested by %X/%X",
- (uint32) (RecPtr >> 32), (uint32) RecPtr);
- goto err;
- }
-
- /* XLogNeedData has verified the page header */
- Assert(pageHeaderSize <= state->readLen);
-
- /*
- * Read the record length.
- *
- * NB: Even though we use an XLogRecord pointer here, the whole record
- * header might not fit on this page. xl_tot_len is the first field of the
- * struct, so it must be on this page (the records are MAXALIGNed), but we
- * cannot access any other fields until we've verified that we got the
- * whole header.
- */
- record = (XLogRecord *) (state->readBuf + RecPtr % XLOG_BLCKSZ);
- total_len = record->xl_tot_len;
-
- /*
- * If the whole record header is on this page, validate it immediately.
- * Otherwise do just a basic sanity check on xl_tot_len, and validate the
- * rest of the header after reading it from the next page. The xl_tot_len
- * check is necessary here to ensure that we enter the "Need to reassemble
- * record" code path below; otherwise we might fail to apply
- * ValidXLogRecordHeader at all.
- */
- if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
- {
- if (!ValidXLogRecordHeader(state, RecPtr, state->ReadRecPtr, record,
- randAccess))
- goto err;
- gotheader = true;
- }
- else
- {
- /* XXX: more validation should be done here */
- if (total_len < SizeOfXLogRecord)
- {
- report_invalid_record(state,
- "invalid record length at %X/%X: wanted %u, got %u",
- (uint32) (RecPtr >> 32), (uint32) RecPtr,
- (uint32) SizeOfXLogRecord, total_len);
- goto err;
- }
- gotheader = false;
- }
-
- len = XLOG_BLCKSZ - RecPtr % XLOG_BLCKSZ;
- if (total_len > len)
+ switch (state->readRecordState)
{
- /* Need to reassemble record */
- char *contdata;
- XLogPageHeader pageHeader;
- char *buffer;
- uint32 gotlen;
-
- /*
- * Enlarge readRecordBuf as needed.
- */
- if (total_len > state->readRecordBufSize &&
- !allocate_recordbuf(state, total_len))
- {
- /* We treat this as a "bogus data" condition */
- report_invalid_record(state, "record length %u at %X/%X too long",
- total_len,
- (uint32) (RecPtr >> 32), (uint32) RecPtr);
- goto err;
- }
+ case XLREAD_NEXT_RECORD:
+ ResetDecoder(state);
- /* Copy the first fragment of the record from the first page. */
- memcpy(state->readRecordBuf,
- state->readBuf + RecPtr % XLOG_BLCKSZ, len);
- buffer = state->readRecordBuf + len;
- gotlen = len;
-
- do
- {
- int rest_len = total_len - gotlen;
-
- /* Calculate pointer to beginning of next page */
- targetPagePtr += XLOG_BLCKSZ;
-
- /* Wait for the next page to become available */
- while (XLogNeedData(state, targetPagePtr,
- Min(rest_len, XLOG_BLCKSZ),
- false))
+ if (state->ReadRecPtr != InvalidXLogRecPtr)
{
- if (!state->routine.page_read(state, state->readPagePtr,
- state->readLen,
- state->ReadRecPtr,
- state->readBuf))
- break;
+ /* read the record after the one we just read */
+
+ /*
+ * EndRecPtr is pointing to end+1 of the previous WAL record.
+ * If we're at a page boundary, no more records can fit on the
+ * current page. We must skip over the page header, but we
+ * can't do that until we've read in the page, since the header
+ * size is variable.
+ */
+ state->PrevRecPtr = state->ReadRecPtr;
+ state->ReadRecPtr = state->EndRecPtr;
+ }
+ else
+ {
+ /*
+ * Caller supplied a position to start at.
+ *
+ * In this case, EndRecPtr should already be pointing to a
+ * valid record starting position.
+ */
+ Assert(XRecOffIsValid(state->EndRecPtr));
+ state->ReadRecPtr = state->EndRecPtr;
+
+ /*
+ * We cannot verify the previous-record pointer when we're
+ * seeking to a particular record. Reset PrevRecPtr so that we
+ * won't try doing that.
+ */
+ state->PrevRecPtr = InvalidXLogRecPtr;
+ state->EndRecPtr = InvalidXLogRecPtr; /* to be tidy */
}
+ state->record_verified = false;
+ state->readRecordState = XLREAD_TOT_LEN;
+ /* fall through */
+
+ case XLREAD_TOT_LEN:
+ {
+ uint32 total_len;
+ uint32 pageHeaderSize;
+ XLogRecPtr targetPagePtr;
+ uint32 targetRecOff;
+ XLogPageHeader pageHeader;
+
+ targetPagePtr =
+ state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
+ targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+
+ /*
+ * Check if we have enough data. For the first record in the page,
+ * the requesting length doesn't contain page header.
+ */
+ if (XLogNeedData(state, targetPagePtr,
+ Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+ targetRecOff != 0))
+ return XLREAD_NEED_DATA;
+
+ /* error out if caller supplied bogus page */
if (!state->page_verified)
goto err;
- Assert(SizeOfXLogShortPHD <= state->readLen);
+ /* examine page header now. */
+ pageHeaderSize =
+ XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+ if (targetRecOff == 0)
+ {
+ /* At page start, so skip over page header. */
+ state->ReadRecPtr += pageHeaderSize;
+ targetRecOff = pageHeaderSize;
+ }
+ else if (targetRecOff < pageHeaderSize)
+ {
+ report_invalid_record(state, "invalid record offset at %X/%X",
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr);
+ goto err;
+ }
- /* Check that the continuation on next page looks valid */
pageHeader = (XLogPageHeader) state->readBuf;
- if (!(pageHeader->xlp_info & XLP_FIRST_IS_CONTRECORD))
+ if ((pageHeader->xlp_info & XLP_FIRST_IS_CONTRECORD) &&
+ targetRecOff == pageHeaderSize)
{
- report_invalid_record(state,
- "there is no contrecord flag at %X/%X",
- (uint32) (RecPtr >> 32), (uint32) RecPtr);
+ report_invalid_record(state, "contrecord is requested by %X/%X",
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr);
goto err;
}
- /*
- * Cross-check that xlp_rem_len agrees with how much of the record
- * we expect there to be left.
- */
- if (pageHeader->xlp_rem_len == 0 ||
- total_len != (pageHeader->xlp_rem_len + gotlen))
- {
- report_invalid_record(state,
- "invalid contrecord length %u at %X/%X",
- pageHeader->xlp_rem_len,
- (uint32) (RecPtr >> 32), (uint32) RecPtr);
- goto err;
- }
-
- /* Append the continuation from this page to the buffer */
- pageHeaderSize = XLogPageHeaderSize(pageHeader);
-
+ /* XLogNeedData has verified the page header */
Assert(pageHeaderSize <= state->readLen);
- contdata = (char *) state->readBuf + pageHeaderSize;
- len = XLOG_BLCKSZ - pageHeaderSize;
- if (pageHeader->xlp_rem_len < len)
- len = pageHeader->xlp_rem_len;
+ /*
+ * Read the record length.
+ *
+ * NB: Even though we use an XLogRecord pointer here, the whole
+ * record header might not fit on this page. xl_tot_len is the first
+ * field of the struct, so it must be on this page (the records are
+ * MAXALIGNed), but we cannot access any other fields until we've
+ * verified that we got the whole header.
+ */
+ prec = (XLogRecord *) (state->readBuf +
+ state->ReadRecPtr % XLOG_BLCKSZ);
+ total_len = prec->xl_tot_len;
- Assert (pageHeaderSize + len <= state->readLen);
- memcpy(buffer, (char *) contdata, len);
- buffer += len;
- gotlen += len;
+ /*
+ * If the whole record header is on this page, validate it
+ * immediately. Otherwise do just a basic sanity check on
+ * xl_tot_len, and validate the rest of the header after reading it
+ * from the next page. The xl_tot_len check is necessary here to
+ * ensure that we enter the XLREAD_CONTINUATION state below;
+ * otherwise we might fail to apply ValidXLogRecordHeader at all.
+ */
+ if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
+ {
+ if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
+ state->PrevRecPtr, prec))
+ goto err;
- /* If we just reassembled the record header, validate it. */
- if (!gotheader)
+ state->record_verified = true;
+ }
+ else
{
- record = (XLogRecord *) state->readRecordBuf;
- if (!ValidXLogRecordHeader(state, RecPtr, state->ReadRecPtr,
- record, randAccess))
+ /* XXX: more validation should be done here */
+ if (total_len < SizeOfXLogRecord)
+ {
+ report_invalid_record(state,
+ "invalid record length at %X/%X: wanted %u, got %u",
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr,
+ (uint32) SizeOfXLogRecord, total_len);
goto err;
- gotheader = true;
+ }
}
- } while (gotlen < total_len);
-
- Assert(gotheader);
- record = (XLogRecord *) state->readRecordBuf;
- if (!ValidXLogRecord(state, record, RecPtr))
- goto err;
+ /*
+ * Wait for the rest of the record, or the part of the record that
+ * fit on the first page if crossed a page boundary, to become
+ * available.
+ */
+ state->recordGotLen = 0;
+ state->recordRemainLen = total_len;
+ state->readRecordState = XLREAD_FIRST_FRAGMENT;
+ }
+ /* fall through */
- pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
- state->ReadRecPtr = RecPtr;
- state->EndRecPtr = targetPagePtr + pageHeaderSize
- + MAXALIGN(pageHeader->xlp_rem_len);
- }
- else
- {
- /* Wait for the record data to become available */
- while (XLogNeedData(state, targetPagePtr,
- Min(targetRecOff + total_len, XLOG_BLCKSZ), true))
+ case XLREAD_FIRST_FRAGMENT:
{
- if (!state->routine.page_read(state, state->readPagePtr,
- state->readLen,
- state->ReadRecPtr, state->readBuf))
+ uint32 total_len = state->recordRemainLen;
+ uint32 request_len;
+ uint32 record_len;
+ XLogRecPtr targetPagePtr;
+ uint32 targetRecOff;
+
+ /*
+ * Wait for the rest of the record on the first page to become
+ * available
+ */
+ targetPagePtr =
+ state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
+ targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+
+ request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ);
+ record_len = request_len - targetRecOff;
+
+ /* ReadRecPtr contains page header */
+ Assert (targetRecOff != 0);
+ if (XLogNeedData(state, targetPagePtr, request_len, true))
+ return XLREAD_NEED_DATA;
+
+ /* error out if caller supplied bogus page */
+ if (!state->page_verified)
+ goto err;
+
+ prec = (XLogRecord *) (state->readBuf + targetRecOff);
+
+ /* validate record header if not yet */
+ if (!state->record_verified && record_len >= SizeOfXLogRecord)
+ {
+ if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
+ state->PrevRecPtr, prec))
+ goto err;
+
+ state->record_verified = true;
+ }
+
+
+ if (total_len == record_len)
+ {
+ /* Record does not cross a page boundary */
+ Assert(state->record_verified);
+
+ if (!ValidXLogRecord(state, prec, state->ReadRecPtr))
+ goto err;
+
+ state->record_verified = true; /* to be tidy */
+
+ /* We already checked the header earlier */
+ state->EndRecPtr = state->ReadRecPtr + MAXALIGN(record_len);
+
+ *record = prec;
+ state->readRecordState = XLREAD_NEXT_RECORD;
break;
+ }
+
+ /*
+ * The record continues on the next page. Need to reassemble
+ * record
+ */
+ Assert(total_len > record_len);
+
+ /* Enlarge readRecordBuf as needed. */
+ if (total_len > state->readRecordBufSize &&
+ !allocate_recordbuf(state, total_len))
+ {
+ /* We treat this as a "bogus data" condition */
+ report_invalid_record(state,
+ "record length %u at %X/%X too long",
+ total_len,
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr);
+ goto err;
+ }
+
+ /* Copy the first fragment of the record from the first page. */
+ memcpy(state->readRecordBuf, state->readBuf + targetRecOff,
+ record_len);
+ state->recordGotLen += record_len;
+ state->recordRemainLen -= record_len;
+
+ /* Calculate pointer to beginning of next page */
+ state->recordContRecPtr = state->ReadRecPtr + record_len;
+ Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0);
+
+ state->readRecordState = XLREAD_CONTINUATION;
}
+ /* fall through */
+
+ case XLREAD_CONTINUATION:
+ {
+ XLogPageHeader pageHeader;
+ uint32 pageHeaderSize;
+ XLogRecPtr targetPagePtr;
+
+ /* we enter this state only if we haven't read the whole record. */
+ Assert (state->recordRemainLen > 0);
+
+ while(state->recordRemainLen > 0)
+ {
+ char *contdata;
+ uint32 request_len;
+ uint32 record_len;
+
+ /* Wait for the next page to become available */
+ targetPagePtr = state->recordContRecPtr;
+
+ /* this request contains page header */
+ Assert (targetPagePtr != 0);
+ if (XLogNeedData(state, targetPagePtr,
+ Min(state->recordRemainLen, XLOG_BLCKSZ),
+ false))
+ return XLREAD_NEED_DATA;
+
+ if (!state->page_verified)
+ goto err;
+
+ Assert(SizeOfXLogShortPHD <= state->readLen);
- if (!state->page_verified)
- goto err;
+ /* Check that the continuation on next page looks valid */
+ pageHeader = (XLogPageHeader) state->readBuf;
+ if (!(pageHeader->xlp_info & XLP_FIRST_IS_CONTRECORD))
+ {
+ report_invalid_record(
+ state,
+ "there is no contrecord flag at %X/%X reading %X/%X",
+ (uint32) (state->recordContRecPtr >> 32),
+ (uint32) state->recordContRecPtr,
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr);
+ goto err;
+ }
- /* Record does not cross a page boundary */
- if (!ValidXLogRecord(state, record, RecPtr))
- goto err;
+ /*
+ * Cross-check that xlp_rem_len agrees with how much of the
+ * record we expect there to be left.
+ */
+ if (pageHeader->xlp_rem_len == 0 ||
+ pageHeader->xlp_rem_len != state->recordRemainLen)
+ {
+ report_invalid_record(
+ state,
+ "invalid contrecord length %u at %X/%X reading %X/%X, expected %u",
+ pageHeader->xlp_rem_len,
+ (uint32) (state->recordContRecPtr >> 32),
+ (uint32) state->recordContRecPtr,
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr,
+ state->recordRemainLen);
+ goto err;
+ }
- state->EndRecPtr = RecPtr + MAXALIGN(total_len);
+ /* Append the continuation from this page to the buffer */
+ pageHeaderSize = XLogPageHeaderSize(pageHeader);
- state->ReadRecPtr = RecPtr;
+ /*
+ * XLogNeedData should have ensured that the whole page header
+ * was read
+ */
+ Assert(state->readLen >= pageHeaderSize);
+
+ contdata = (char *) state->readBuf + pageHeaderSize;
+ record_len = XLOG_BLCKSZ - pageHeaderSize;
+ if (pageHeader->xlp_rem_len < record_len)
+ record_len = pageHeader->xlp_rem_len;
+
+ request_len = record_len + pageHeaderSize;
+
+ /* XLogNeedData should have ensured all needed data was read */
+ Assert (state->readLen >= request_len);
+
+ memcpy(state->readRecordBuf + state->recordGotLen,
+ (char *) contdata, record_len);
+ state->recordGotLen += record_len;
+ state->recordRemainLen -= record_len;
+
+ /* If we just reassembled the record header, validate it. */
+ if (!state->record_verified)
+ {
+ Assert(state->recordGotLen >= SizeOfXLogRecord);
+ if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
+ state->PrevRecPtr,
+ (XLogRecord *) state->readRecordBuf))
+ goto err;
+
+ state->record_verified = true;
+ }
+
+ /* Calculate pointer to beginning of next page, and continue */
+ state->recordContRecPtr += XLOG_BLCKSZ;
+ }
+
+ /* targetPagePtr is pointing the last-read page here */
+ prec = (XLogRecord *) state->readRecordBuf;
+ if (!ValidXLogRecord(state, prec, state->ReadRecPtr))
+ goto err;
+
+ pageHeaderSize =
+ XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+ state->EndRecPtr = targetPagePtr + pageHeaderSize
+ + MAXALIGN(pageHeader->xlp_rem_len);
+
+ *record = prec;
+ state->readRecordState = XLREAD_NEXT_RECORD;
+ break;
+ }
}
/*
* Special processing if it's an XLOG SWITCH record
*/
- if (record->xl_rmid == RM_XLOG_ID &&
- (record->xl_info & ~XLR_INFO_MASK) == XLOG_SWITCH)
+ if ((*record)->xl_rmid == RM_XLOG_ID &&
+ ((*record)->xl_info & ~XLR_INFO_MASK) == XLOG_SWITCH)
{
/* Pretend it extends to end of segment */
state->EndRecPtr += state->segcxt.ws_segsize - 1;
state->EndRecPtr -= XLogSegmentOffset(state->EndRecPtr, state->segcxt.ws_segsize);
}
- if (DecodeXLogRecord(state, record, errormsg))
- return record;
- else
- return NULL;
+ Assert (!*record || state->readLen >= 0);
+ if (DecodeXLogRecord(state, *record, errormsg))
+ return XLREAD_SUCCESS;
+
+ *record = NULL;
+ return XLREAD_FAIL;
err:
/*
- * Invalidate the read state. We might read from a different source after
+ * Invalidate the read page. We might read from a different source after
* failure.
*/
XLogReaderInvalReadState(state);
@@ -572,7 +706,8 @@ err:
if (state->errormsg_buf[0] != '\0')
*errormsg = state->errormsg_buf;
- return NULL;
+ *record = NULL;
+ return XLREAD_FAIL;
}
/*
@@ -724,11 +859,12 @@ XLogReaderInvalReadState(XLogReaderState *state)
*
* This is just a convenience subroutine to avoid duplicated code in
* XLogReadRecord. It's not intended for use from anywhere else.
+ *
+ * If PrevRecPtr is valid, the xl_prev is is cross-checked with it.
*/
static bool
ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
- XLogRecPtr PrevRecPtr, XLogRecord *record,
- bool randAccess)
+ XLogRecPtr PrevRecPtr, XLogRecord *record)
{
if (record->xl_tot_len < SizeOfXLogRecord)
{
@@ -746,7 +882,7 @@ ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
(uint32) RecPtr);
return false;
}
- if (randAccess)
+ if (PrevRecPtr == InvalidXLogRecPtr)
{
/*
* We can't exactly verify the prev-link, but surely it should be less
@@ -978,11 +1114,14 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
* XLogReadRecord() will read the next valid record.
*/
XLogRecPtr
-XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
+XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
+ XLogFindNextRecordCB read_page, void *private)
{
XLogRecPtr tmpRecPtr;
XLogRecPtr found = InvalidXLogRecPtr;
XLogPageHeader header;
+ XLogRecord *record;
+ XLogReadRecordResult result;
char *errormsg;
Assert(!XLogRecPtrIsInvalid(RecPtr));
@@ -1015,9 +1154,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
while(XLogNeedData(state, targetPagePtr, targetRecOff,
targetRecOff != 0))
{
- if (!state->routine.page_read(state, state->readPagePtr,
- state->readLen,
- state->ReadRecPtr, state->readBuf))
+ if (!read_page(state, private))
break;
}
@@ -1069,8 +1206,16 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
* or we just jumped over the remaining data of a continuation.
*/
XLogBeginRead(state, tmpRecPtr);
- while (XLogReadRecord(state, &errormsg) != NULL)
+ while ((result = XLogReadRecord(state, &record, &errormsg)) !=
+ XLREAD_FAIL)
{
+ if (result == XLREAD_NEED_DATA)
+ {
+ if (!read_page(state, private))
+ goto err;
+ continue;
+ }
+
/* past the record we've found, break out */
if (RecPtr <= state->ReadRecPtr)
{
@@ -1090,9 +1235,9 @@ err:
#endif /* FRONTEND */
/*
- * Helper function to ease writing of XLogRoutine->page_read callbacks.
- * If this function is used, caller must supply a segment_open callback in
- * 'state', as that is used here.
+ * Helper function to ease writing of page_read callback.
+ * If this function is used, caller must supply a segment_open callback and
+ * segment_close callback as that is used here.
*
* Read 'count' bytes into 'buf', starting at location 'startptr', from WAL
* fetched from timeline 'tli'.
@@ -1105,6 +1250,7 @@ err:
*/
bool
WALRead(XLogReaderState *state,
+ WALSegmentOpenCB segopenfn, WALSegmentCloseCB segclosefn,
char *buf, XLogRecPtr startptr, Size count, TimeLineID tli,
WALReadError *errinfo)
{
@@ -1136,10 +1282,10 @@ WALRead(XLogReaderState *state,
XLogSegNo nextSegNo;
if (state->seg.ws_file >= 0)
- state->routine.segment_close(state);
+ segclosefn(state);
XLByteToSeg(recptr, nextSegNo, state->segcxt.ws_segsize);
- state->routine.segment_open(state, nextSegNo, &tli);
+ segopenfn(state, nextSegNo, &tli);
/* This shouldn't happen -- indicates a bug in segment_open */
Assert(state->seg.ws_file >= 0);
diff --git a/src/backend/access/transam/xlogutils.c b/src/backend/access/transam/xlogutils.c
index 1b10b6df20..303333571f 100644
--- a/src/backend/access/transam/xlogutils.c
+++ b/src/backend/access/transam/xlogutils.c
@@ -689,8 +689,7 @@ XLogTruncateRelation(RelFileNode rnode, ForkNumber forkNum,
void
XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wantLength)
{
- const XLogRecPtr lastReadPage = state->seg.ws_segno *
- state->segcxt.ws_segsize + state->readLen;
+ const XLogRecPtr lastReadPage = state->readPagePtr;
Assert(wantPage != InvalidXLogRecPtr && wantPage % XLOG_BLCKSZ == 0);
Assert(wantLength <= XLOG_BLCKSZ);
@@ -705,7 +704,7 @@ XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wa
* current TLI has since become historical.
*/
if (lastReadPage == wantPage &&
- state->readLen != 0 &&
+ state->page_verified &&
lastReadPage + state->readLen >= wantPage + Min(wantLength, XLOG_BLCKSZ - 1))
return;
@@ -792,6 +791,7 @@ wal_segment_open(XLogReaderState *state, XLogSegNo nextSegNo,
char path[MAXPGPATH];
XLogFilePath(path, tli, nextSegNo, state->segcxt.ws_segsize);
+ elog(LOG, "HOGE: %lu, %d => %s", nextSegNo, tli, path);
state->seg.ws_file = BasicOpenFile(path, O_RDONLY | PG_BINARY);
if (state->seg.ws_file >= 0)
return;
@@ -829,9 +829,11 @@ wal_segment_close(XLogReaderState *state)
* loop for now.
*/
bool
-read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *cur_page)
+read_local_xlog_page(XLogReaderState *state)
{
+ XLogRecPtr targetPagePtr = state->readPagePtr;
+ int reqLen = state->readLen;
+ char *cur_page = state->readBuf;
XLogRecPtr read_upto,
loc;
TimeLineID tli;
@@ -944,11 +946,12 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
* as 'count', read the whole page anyway. It's guaranteed to be
* zero-padded up to the page boundary if it's incomplete.
*/
- if (!WALRead(state, cur_page, targetPagePtr, XLOG_BLCKSZ, tli,
- &errinfo))
+ if (!WALRead(state, wal_segment_open, wal_segment_close,
+ cur_page, targetPagePtr, XLOG_BLCKSZ, tli, &errinfo))
WALReadRaiseError(&errinfo);
/* number of valid bytes in the buffer */
+ state->readPagePtr = targetPagePtr;
state->readLen = count;
return true;
}
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index 61902be3b0..a15b0b3355 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -120,7 +120,8 @@ StartupDecodingContext(List *output_plugin_options,
TransactionId xmin_horizon,
bool need_full_snapshot,
bool fast_forward,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress)
@@ -169,11 +170,12 @@ StartupDecodingContext(List *output_plugin_options,
ctx->slot = slot;
- ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, xl_routine, ctx);
+ ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, cleanup_cb);
if (!ctx->reader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
errmsg("out of memory")));
+ ctx->page_read = page_read;
ctx->reorder = ReorderBufferAllocate();
ctx->snapshot_builder =
@@ -231,7 +233,8 @@ CreateInitDecodingContext(char *plugin,
List *output_plugin_options,
bool need_full_snapshot,
XLogRecPtr restart_lsn,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress)
@@ -328,7 +331,7 @@ CreateInitDecodingContext(char *plugin,
ctx = StartupDecodingContext(NIL, restart_lsn, xmin_horizon,
need_full_snapshot, false,
- xl_routine, prepare_write, do_write,
+ page_read, cleanup_cb, prepare_write, do_write,
update_progress);
/* call output plugin initialization callback */
@@ -376,7 +379,8 @@ LogicalDecodingContext *
CreateDecodingContext(XLogRecPtr start_lsn,
List *output_plugin_options,
bool fast_forward,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress)
@@ -429,8 +433,8 @@ CreateDecodingContext(XLogRecPtr start_lsn,
ctx = StartupDecodingContext(output_plugin_options,
start_lsn, InvalidTransactionId, false,
- fast_forward, xl_routine, prepare_write,
- do_write, update_progress);
+ fast_forward, page_read, cleanup_cb,
+ prepare_write, do_write, update_progress);
/* call output plugin initialization callback */
old_context = MemoryContextSwitchTo(ctx->context);
@@ -483,7 +487,13 @@ DecodingContextFindStartpoint(LogicalDecodingContext *ctx)
char *err = NULL;
/* the read_page callback waits for new WAL */
- record = XLogReadRecord(ctx->reader, &err);
+ while (XLogReadRecord(ctx->reader, &record, &err) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!ctx->page_read(ctx->reader))
+ break;
+ }
+
if (err)
elog(ERROR, "%s", err);
if (!record)
diff --git a/src/backend/replication/logical/logicalfuncs.c b/src/backend/replication/logical/logicalfuncs.c
index b99c94e848..fcc81b358e 100644
--- a/src/backend/replication/logical/logicalfuncs.c
+++ b/src/backend/replication/logical/logicalfuncs.c
@@ -233,9 +233,8 @@ pg_logical_slot_get_changes_guts(FunctionCallInfo fcinfo, bool confirm, bool bin
ctx = CreateDecodingContext(InvalidXLogRecPtr,
options,
false,
- XL_ROUTINE(.page_read = read_local_xlog_page,
- .segment_open = wal_segment_open,
- .segment_close = wal_segment_close),
+ read_local_xlog_page,
+ wal_segment_close,
LogicalOutputPrepareWrite,
LogicalOutputWrite, NULL);
@@ -284,7 +283,13 @@ pg_logical_slot_get_changes_guts(FunctionCallInfo fcinfo, bool confirm, bool bin
XLogRecord *record;
char *errm = NULL;
- record = XLogReadRecord(ctx->reader, &errm);
+ while (XLogReadRecord(ctx->reader, &record, &errm) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!ctx->page_read(ctx->reader))
+ break;
+ }
+
if (errm)
elog(ERROR, "%s", errm);
diff --git a/src/backend/replication/slotfuncs.c b/src/backend/replication/slotfuncs.c
index 88033a79b2..8a678074d5 100644
--- a/src/backend/replication/slotfuncs.c
+++ b/src/backend/replication/slotfuncs.c
@@ -152,9 +152,8 @@ create_logical_replication_slot(char *name, char *plugin,
ctx = CreateInitDecodingContext(plugin, NIL,
false, /* just catalogs is OK */
restart_lsn,
- XL_ROUTINE(.page_read = read_local_xlog_page,
- .segment_open = wal_segment_open,
- .segment_close = wal_segment_close),
+ read_local_xlog_page,
+ wal_segment_close,
NULL, NULL, NULL);
/*
@@ -486,9 +485,8 @@ pg_logical_replication_slot_advance(XLogRecPtr moveto)
ctx = CreateDecodingContext(InvalidXLogRecPtr,
NIL,
true, /* fast_forward */
- XL_ROUTINE(.page_read = read_local_xlog_page,
- .segment_open = wal_segment_open,
- .segment_close = wal_segment_close),
+ read_local_xlog_page,
+ wal_segment_close,
NULL, NULL, NULL);
/*
@@ -513,7 +511,13 @@ pg_logical_replication_slot_advance(XLogRecPtr moveto)
* Read records. No changes are generated in fast_forward mode,
* but snapbuilder/slot statuses are updated properly.
*/
- record = XLogReadRecord(ctx->reader, &errm);
+ while (XLogReadRecord(ctx->reader, &record, &errm) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!ctx->page_read(ctx->reader))
+ break;
+ }
+
if (errm)
elog(ERROR, "%s", errm);
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index bf2711bddd..2fd24a9b14 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -580,10 +580,7 @@ StartReplication(StartReplicationCmd *cmd)
/* create xlogreader for physical replication */
xlogreader =
- XLogReaderAllocate(wal_segment_size, NULL,
- XL_ROUTINE(.segment_open = WalSndSegmentOpen,
- .segment_close = wal_segment_close),
- NULL);
+ XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
if (!xlogreader)
ereport(ERROR,
@@ -812,9 +809,11 @@ StartReplication(StartReplicationCmd *cmd)
* set every time WAL is flushed.
*/
static bool
-logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetRecPtr, char *cur_page)
+logical_read_xlog_page(XLogReaderState *state)
{
+ XLogRecPtr targetPagePtr = state->readPagePtr;
+ int reqLen = state->readLen;
+ char *cur_page = state->readBuf;
XLogRecPtr flushptr;
int count;
WALReadError errinfo;
@@ -842,7 +841,7 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
count = flushptr - targetPagePtr; /* part of the page available */
/* now actually read the data, we know it's there */
- if (!WALRead(state,
+ if (!WALRead(state, WalSndSegmentOpen, wal_segment_close,
cur_page,
targetPagePtr,
XLOG_BLCKSZ,
@@ -1014,9 +1013,8 @@ CreateReplicationSlot(CreateReplicationSlotCmd *cmd)
ctx = CreateInitDecodingContext(cmd->plugin, NIL, need_full_snapshot,
InvalidXLogRecPtr,
- XL_ROUTINE(.page_read = logical_read_xlog_page,
- .segment_open = WalSndSegmentOpen,
- .segment_close = wal_segment_close),
+ logical_read_xlog_page,
+ wal_segment_close,
WalSndPrepareWrite, WalSndWriteData,
WalSndUpdateProgress);
@@ -1179,9 +1177,8 @@ StartLogicalReplication(StartReplicationCmd *cmd)
*/
logical_decoding_ctx =
CreateDecodingContext(cmd->startpoint, cmd->options, false,
- XL_ROUTINE(.page_read = logical_read_xlog_page,
- .segment_open = WalSndSegmentOpen,
- .segment_close = wal_segment_close),
+ logical_read_xlog_page,
+ wal_segment_close,
WalSndPrepareWrite, WalSndWriteData,
WalSndUpdateProgress);
xlogreader = logical_decoding_ctx->reader;
@@ -2761,7 +2758,7 @@ XLogSendPhysical(void)
enlargeStringInfo(&output_message, nbytes);
retry:
- if (!WALRead(xlogreader,
+ if (!WALRead(xlogreader, WalSndSegmentOpen, wal_segment_close,
&output_message.data[output_message.len],
startptr,
nbytes,
@@ -2859,7 +2856,12 @@ XLogSendLogical(void)
*/
WalSndCaughtUp = false;
- record = XLogReadRecord(logical_decoding_ctx->reader, &errm);
+ while (XLogReadRecord(logical_decoding_ctx->reader, &record, &errm) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!logical_decoding_ctx->page_read(logical_decoding_ctx->reader))
+ break;
+ }
/* xlog record was invalid */
if (errm != NULL)
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index 0b7e73ae79..3dd6df4be6 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -40,15 +40,9 @@ static int xlogreadfd = -1;
static XLogSegNo xlogreadsegno = -1;
static char xlogfpath[MAXPGPATH];
-typedef struct XLogPageReadPrivate
-{
- const char *restoreCommand;
- int tliIndex;
-} XLogPageReadPrivate;
-
-static bool SimpleXLogPageRead(XLogReaderState *xlogreader,
- XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
+static bool SimpleXLogPageRead(XLogReaderState *xlogreader,
+ const char *datadir, int *tliIndex,
+ const char *restoreCommand);
/*
* Read WAL from the datadir/pg_wal, starting from 'startpoint' on timeline
@@ -62,20 +56,22 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
XLogRecord *record;
XLogReaderState *xlogreader;
char *errormsg;
- XLogPageReadPrivate private;
- private.tliIndex = tliIndex;
- private.restoreCommand = restoreCommand;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir,
- XL_ROUTINE(.page_read = &SimpleXLogPageRead),
- &private);
+ xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+
if (xlogreader == NULL)
pg_fatal("out of memory");
XLogBeginRead(xlogreader, startpoint);
do
{
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!SimpleXLogPageRead(xlogreader, datadir,
+ &tliIndex, restoreCommand))
+ break;
+ }
if (record == NULL)
{
@@ -113,19 +109,19 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex,
XLogRecord *record;
XLogReaderState *xlogreader;
char *errormsg;
- XLogPageReadPrivate private;
XLogRecPtr endptr;
- private.tliIndex = tliIndex;
- private.restoreCommand = restoreCommand;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir,
- XL_ROUTINE(.page_read = &SimpleXLogPageRead),
- &private);
+ xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
XLogBeginRead(xlogreader, ptr);
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!SimpleXLogPageRead(xlogreader, datadir, &tliIndex, restoreCommand))
+ break;
+ }
if (record == NULL)
{
if (errormsg)
@@ -160,7 +156,6 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
XLogRecPtr searchptr;
XLogReaderState *xlogreader;
char *errormsg;
- XLogPageReadPrivate private;
/*
* The given fork pointer points to the end of the last common record,
@@ -176,11 +171,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
forkptr += SizeOfXLogShortPHD;
}
- private.tliIndex = tliIndex;
- private.restoreCommand = restoreCommand;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir,
- XL_ROUTINE(.page_read = &SimpleXLogPageRead),
- &private);
+ xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
@@ -190,7 +181,13 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
uint8 info;
XLogBeginRead(xlogreader, searchptr);
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!SimpleXLogPageRead(xlogreader, datadir,
+ &tliIndex, restoreCommand))
+ break;
+ }
if (record == NULL)
{
@@ -237,10 +234,11 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
/* XLogReader callback function, to read a WAL page */
static bool
-SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *readBuf)
+SimpleXLogPageRead(XLogReaderState *xlogreader, const char *datadir,
+ int *tliIndex, const char *restoreCommand)
{
- XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
+ XLogRecPtr targetPagePtr = xlogreader->readPagePtr;
+ char *readBuf = xlogreader->readBuf;
uint32 targetPageOff;
XLogRecPtr targetSegEnd;
XLogSegNo targetSegNo;
@@ -273,14 +271,14 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
* be done both forward and backward, consider also switching timeline
* accordingly.
*/
- while (private->tliIndex < targetNentries - 1 &&
- targetHistory[private->tliIndex].end < targetSegEnd)
- private->tliIndex++;
- while (private->tliIndex > 0 &&
- targetHistory[private->tliIndex].begin >= targetSegEnd)
- private->tliIndex--;
+ while (*tliIndex < targetNentries - 1 &&
+ targetHistory[*tliIndex].end < targetSegEnd)
+ (*tliIndex)++;
+ while (*tliIndex > 0 &&
+ targetHistory[*tliIndex].begin >= targetSegEnd)
+ (*tliIndex)--;
- XLogFileName(xlogfname, targetHistory[private->tliIndex].tli,
+ XLogFileName(xlogfname, targetHistory[*tliIndex].tli,
xlogreadsegno, WalSegSz);
snprintf(xlogfpath, MAXPGPATH, "%s/" XLOGDIR "/%s",
@@ -293,7 +291,7 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
/*
* If we have no restore_command to execute, then exit.
*/
- if (private->restoreCommand == NULL)
+ if (restoreCommand == NULL)
{
pg_log_error("could not open file \"%s\": %m", xlogfpath);
xlogreader->readLen = -1;
@@ -307,7 +305,7 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
xlogreadfd = RestoreArchivedFile(xlogreader->segcxt.ws_dir,
xlogfname,
WalSegSz,
- private->restoreCommand);
+ restoreCommand);
if (xlogreadfd < 0)
{
@@ -349,7 +347,7 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
Assert(targetSegNo == xlogreadsegno);
- xlogreader->seg.ws_tli = targetHistory[private->tliIndex].tli;
+ xlogreader->seg.ws_tli = targetHistory[*tliIndex].tli;
xlogreader->readLen = XLOG_BLCKSZ;
return true;
}
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index ab0c28d9b7..02e71138cc 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -330,12 +330,17 @@ WALDumpCloseSegment(XLogReaderState *state)
state->seg.ws_file = -1;
}
-/* pg_waldump's XLogReaderRoutine->page_read callback */
+/*
+ * pg_waldump's WAL page rader, also used as page_read callback for
+ * XLogFindNextRecord
+ */
static bool
-WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetPtr, char *readBuff)
+WALDumpReadPage(XLogReaderState *state, void *priv)
{
- XLogDumpPrivate *private = state->private_data;
+ XLogRecPtr targetPagePtr = state->readPagePtr;
+ int reqLen = state->readLen;
+ char *readBuff = state->readBuf;
+ XLogDumpPrivate *private = (XLogDumpPrivate *) priv;
int count = XLOG_BLCKSZ;
WALReadError errinfo;
@@ -353,8 +358,8 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
}
}
- if (!WALRead(state, readBuff, targetPagePtr, count, private->timeline,
- &errinfo))
+ if (!WALRead(state, WALDumpOpenSegment, WALDumpCloseSegment,
+ readBuff, targetPagePtr, count, private->timeline, &errinfo))
{
WALOpenSegment *seg = &errinfo.wre_seg;
char fname[MAXPGPATH];
@@ -374,6 +379,7 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
(Size) errinfo.wre_req);
}
+ Assert(count >= state->readLen);
state->readLen = count;
return true;
}
@@ -1042,16 +1048,14 @@ main(int argc, char **argv)
/* we have everything we need, start reading */
xlogreader_state =
- XLogReaderAllocate(WalSegSz, waldir,
- XL_ROUTINE(.page_read = WALDumpReadPage,
- .segment_open = WALDumpOpenSegment,
- .segment_close = WALDumpCloseSegment),
- &private);
+ XLogReaderAllocate(WalSegSz, waldir, WALDumpCloseSegment);
+
if (!xlogreader_state)
fatal_error("out of memory");
/* first find a valid recptr to start from */
- first_record = XLogFindNextRecord(xlogreader_state, private.startptr);
+ first_record = XLogFindNextRecord(xlogreader_state, private.startptr,
+ &WALDumpReadPage, (void*) &private);
if (first_record == InvalidXLogRecPtr)
fatal_error("could not find a valid record after %X/%X",
@@ -1075,7 +1079,13 @@ main(int argc, char **argv)
for (;;)
{
/* try to read the next record */
- record = XLogReadRecord(xlogreader_state, &errormsg);
+ while (XLogReadRecord(xlogreader_state, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!WALDumpReadPage(xlogreader_state, (void *) &private))
+ break;
+ }
+
if (!record)
{
if (!config.follow || private.endptr_reached)
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index 6c8848e14f..6a7fe140cb 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -57,64 +57,15 @@ typedef struct WALSegmentContext
typedef struct XLogReaderState XLogReaderState;
-/* Function type definition for the read_page callback */
-typedef bool (*XLogPageReadCB) (XLogReaderState *xlogreader,
- XLogRecPtr targetPagePtr,
- int reqLen,
- XLogRecPtr targetRecPtr,
- char *readBuf);
+/* Function type definition for the segment cleanup callback */
+typedef void (*WALSegmentCleanupCB) (XLogReaderState *xlogreader);
+
+/* Function type definition for the open/close callbacks for WALRead() */
typedef void (*WALSegmentOpenCB) (XLogReaderState *xlogreader,
XLogSegNo nextSegNo,
TimeLineID *tli_p);
typedef void (*WALSegmentCloseCB) (XLogReaderState *xlogreader);
-typedef struct XLogReaderRoutine
-{
- /*
- * Data input callback
- *
- * This callback shall read at least reqLen valid bytes of the xlog page
- * starting at targetPagePtr, and store them in readBuf. The callback
- * shall return the number of bytes read (never more than XLOG_BLCKSZ), or
- * -1 on failure. The callback shall sleep, if necessary, to wait for the
- * requested bytes to become available. The callback will not be invoked
- * again for the same page unless more than the returned number of bytes
- * are needed.
- *
- * targetRecPtr is the position of the WAL record we're reading. Usually
- * it is equal to targetPagePtr + reqLen, but sometimes xlogreader needs
- * to read and verify the page or segment header, before it reads the
- * actual WAL record it's interested in. In that case, targetRecPtr can
- * be used to determine which timeline to read the page from.
- *
- * The callback shall set ->seg.ws_tli to the TLI of the file the page was
- * read from.
- */
- XLogPageReadCB page_read;
-
- /*
- * Callback to open the specified WAL segment for reading. ->seg.ws_file
- * shall be set to the file descriptor of the opened segment. In case of
- * failure, an error shall be raised by the callback and it shall not
- * return.
- *
- * "nextSegNo" is the number of the segment to be opened.
- *
- * "tli_p" is an input/output argument. WALRead() uses it to pass the
- * timeline in which the new segment should be found, but the callback can
- * use it to return the TLI that it actually opened.
- */
- WALSegmentOpenCB segment_open;
-
- /*
- * WAL segment close callback. ->seg.ws_file shall be set to a negative
- * number.
- */
- WALSegmentCloseCB segment_close;
-} XLogReaderRoutine;
-
-#define XL_ROUTINE(...) &(XLogReaderRoutine){__VA_ARGS__}
-
typedef struct
{
/* Is this block ref in use? */
@@ -144,12 +95,35 @@ typedef struct
uint16 data_bufsz;
} DecodedBkpBlock;
+/* Return code from XLogReadRecord */
+typedef enum XLogReadRecordResult
+{
+ XLREAD_SUCCESS, /* record is successfully read */
+ XLREAD_NEED_DATA, /* need more data. see XLogReadRecord. */
+ XLREAD_FAIL /* failed during reading a record */
+} XLogReadRecordResult;
+
+/*
+ * internal state of XLogReadRecord
+ *
+ * XLogReadState runs a state machine while reading a record. Theses states
+ * are not seen outside the function. Each state may repeat several times
+ * exiting requesting caller for new data. See the comment of XLogReadRecrod
+ * for details.
+ */
+typedef enum XLogReadRecordState {
+ XLREAD_NEXT_RECORD,
+ XLREAD_TOT_LEN,
+ XLREAD_FIRST_FRAGMENT,
+ XLREAD_CONTINUATION
+} XLogReadRecordState;
+
struct XLogReaderState
{
/*
* Operational callbacks
*/
- XLogReaderRoutine routine;
+ WALSegmentCleanupCB cleanup_cb;
/* ----------------------------------------
* Public parameters
@@ -162,18 +136,14 @@ struct XLogReaderState
*/
uint64 system_identifier;
- /*
- * Opaque data for callbacks to use. Not used by XLogReader.
- */
- void *private_data;
-
/*
* Start and end point of last record read. EndRecPtr is also used as the
* position to read next. Calling XLogBeginRead() sets EndRecPtr to the
* starting position and ReadRecPtr to invalid.
*/
- XLogRecPtr ReadRecPtr; /* start of last record read */
+ XLogRecPtr ReadRecPtr; /* start of last record read or being read */
XLogRecPtr EndRecPtr; /* end+1 of last record read */
+ XLogRecPtr PrevRecPtr; /* start of previous record read */
/* ----------------------------------------
* Communication with page reader
@@ -187,7 +157,9 @@ struct XLogReaderState
* the request, or -1 on error */
TimeLineID readPageTLI; /* TLI for data currently in readBuf */
char *readBuf; /* buffer to store data */
- bool page_verified; /* is the page on the buffer verified? */
+ bool page_verified; /* is the page header on the buffer verified? */
+ bool record_verified;/* is the current record header verified? */
+
/* ----------------------------------------
@@ -227,8 +199,6 @@ struct XLogReaderState
XLogRecPtr latestPagePtr;
TimeLineID latestPageTLI;
- /* beginning of the WAL record being read. */
- XLogRecPtr currRecPtr;
/* timeline to read it from, 0 if a lookup is required */
TimeLineID currTLI;
@@ -255,6 +225,15 @@ struct XLogReaderState
char *readRecordBuf;
uint32 readRecordBufSize;
+ /*
+ * XLogReadRecord() state
+ */
+ XLogReadRecordState readRecordState;/* state machine state */
+ int recordGotLen; /* amount of current record that has
+ * already been read */
+ int recordRemainLen; /* length of current record that remains */
+ XLogRecPtr recordContRecPtr; /* where the current record continues */
+
/* Buffer to hold error message */
char *errormsg_buf;
};
@@ -262,9 +241,7 @@ struct XLogReaderState
/* Get a new XLogReader */
extern XLogReaderState *XLogReaderAllocate(int wal_segment_size,
const char *waldir,
- XLogReaderRoutine *routine,
- void *private_data);
-extern XLogReaderRoutine *LocalXLogReaderRoutine(void);
+ WALSegmentCleanupCB cleanup_cb);
/* Free an XLogReader */
extern void XLogReaderFree(XLogReaderState *state);
@@ -272,12 +249,17 @@ extern void XLogReaderFree(XLogReaderState *state);
/* Position the XLogReader to given record */
extern void XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr);
#ifdef FRONTEND
-extern XLogRecPtr XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr);
+/* Function type definition for the read_page callback */
+typedef bool (*XLogFindNextRecordCB) (XLogReaderState *xlogreader,
+ void *private);
+extern XLogRecPtr XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
+ XLogFindNextRecordCB read_page, void *private);
#endif /* FRONTEND */
/* Read the next XLog record. Returns NULL on end-of-WAL or failure */
-extern struct XLogRecord *XLogReadRecord(XLogReaderState *state,
- char **errormsg);
+extern XLogReadRecordResult XLogReadRecord(XLogReaderState *state,
+ XLogRecord **record,
+ char **errormsg);
/* Validate a page */
extern bool XLogReaderValidatePageHeader(XLogReaderState *state,
@@ -297,6 +279,7 @@ typedef struct WALReadError
} WALReadError;
extern bool WALRead(XLogReaderState *state,
+ WALSegmentOpenCB segopenfn, WALSegmentCloseCB sgclosefn,
char *buf, XLogRecPtr startptr, Size count,
TimeLineID tli, WALReadError *errinfo);
diff --git a/src/include/access/xlogutils.h b/src/include/access/xlogutils.h
index c287edf206..482e4ced69 100644
--- a/src/include/access/xlogutils.h
+++ b/src/include/access/xlogutils.h
@@ -47,9 +47,7 @@ extern Buffer XLogReadBufferExtended(RelFileNode rnode, ForkNumber forknum,
extern Relation CreateFakeRelcacheEntry(RelFileNode rnode);
extern void FreeFakeRelcacheEntry(Relation fakerel);
-extern bool read_local_xlog_page(XLogReaderState *state,
- XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetRecPtr, char *cur_page);
+extern bool read_local_xlog_page(XLogReaderState *state);
extern void wal_segment_open(XLogReaderState *state,
XLogSegNo nextSegNo,
TimeLineID *tli_p);
diff --git a/src/include/replication/logical.h b/src/include/replication/logical.h
index c2f2475e5d..a743db49cf 100644
--- a/src/include/replication/logical.h
+++ b/src/include/replication/logical.h
@@ -29,6 +29,10 @@ typedef void (*LogicalOutputPluginWriterUpdateProgress) (struct LogicalDecodingC
TransactionId xid
);
+typedef struct LogicalDecodingContext LogicalDecodingContext;
+
+typedef bool (*LogicalDecodingXLogPageReadCB)(XLogReaderState *ctx);
+
typedef struct LogicalDecodingContext
{
/* memory context this is all allocated in */
@@ -39,6 +43,7 @@ typedef struct LogicalDecodingContext
/* infrastructure pieces for decoding */
XLogReaderState *reader;
+ LogicalDecodingXLogPageReadCB page_read;
struct ReorderBuffer *reorder;
struct SnapBuild *snapshot_builder;
@@ -95,14 +100,16 @@ extern LogicalDecodingContext *CreateInitDecodingContext(char *plugin,
List *output_plugin_options,
bool need_full_snapshot,
XLogRecPtr restart_lsn,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress);
extern LogicalDecodingContext *CreateDecodingContext(XLogRecPtr start_lsn,
List *output_plugin_options,
bool fast_forward,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress);
--
2.18.4
----Next_Part(Thu_Jul__2_13_53_30_2020_072)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
filename="v15-0003-Remove-globals-readOff-readLen-and-readSegNo.patch"
^ permalink raw reply [nested|flat] 70+ messages in thread
* [PATCH v15 2/4] Move page-reader out of XLogReadRecord
@ 2019-09-10 03:58 Kyotaro Horiguchi <[email protected]>
0 siblings, 0 replies; 70+ messages in thread
From: Kyotaro Horiguchi @ 2019-09-10 03:58 UTC (permalink / raw)
This is the second step of removing callbacks from WAL record reader.
Since it is essential to take in additional data while reading a
record, the function have to ask caller for new data while keeping
working state. Thus the function is turned into a state machine.
---
src/backend/access/transam/twophase.c | 15 +-
src/backend/access/transam/xlog.c | 58 +-
src/backend/access/transam/xlogreader.c | 700 +++++++++++-------
src/backend/access/transam/xlogutils.c | 17 +-
src/backend/replication/logical/logical.c | 26 +-
.../replication/logical/logicalfuncs.c | 13 +-
src/backend/replication/slotfuncs.c | 18 +-
src/backend/replication/walsender.c | 32 +-
src/bin/pg_rewind/parsexlog.c | 84 +--
src/bin/pg_waldump/pg_waldump.c | 36 +-
src/include/access/xlogreader.h | 121 ++-
src/include/access/xlogutils.h | 4 +-
src/include/replication/logical.h | 11 +-
13 files changed, 648 insertions(+), 487 deletions(-)
diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index 9b2e59bf0e..b0d60a0d0f 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -1330,11 +1330,8 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
XLogReaderState *xlogreader;
char *errormsg;
- xlogreader = XLogReaderAllocate(wal_segment_size, NULL,
- XL_ROUTINE(.page_read = &read_local_xlog_page,
- .segment_open = &wal_segment_open,
- .segment_close = &wal_segment_close),
- NULL);
+ xlogreader = XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+
if (!xlogreader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
@@ -1342,7 +1339,13 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
errdetail("Failed while allocating a WAL reading processor.")));
XLogBeginRead(xlogreader, lsn);
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!read_local_xlog_page(xlogreader))
+ break;
+ }
+
if (record == NULL)
ereport(ERROR,
(errcode_for_file_access(),
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index e570e56a24..f9b0108602 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -831,13 +831,6 @@ static XLogSource currentSource = XLOG_FROM_ANY;
static bool lastSourceFailed = false;
static bool pendingWalRcvRestart = false;
-typedef struct XLogPageReadPrivate
-{
- int emode;
- bool fetching_ckpt; /* are we fetching a checkpoint record? */
- bool randAccess;
-} XLogPageReadPrivate;
-
/*
* These variables track when we last obtained some WAL data to process,
* and where we got it from. (XLogReceiptSource is initially the same as
@@ -912,8 +905,8 @@ static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath,
static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli,
XLogSource source, bool notfoundOk);
static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, XLogSource source);
-static bool XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
+static bool XLogPageRead(XLogReaderState *xlogreader,
+ bool fetching_ckpt, int emode, bool randAccess);
static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
bool fetching_ckpt, XLogRecPtr tliRecPtr);
static int emode_for_corrupt_record(int emode, XLogRecPtr RecPtr);
@@ -1225,8 +1218,7 @@ XLogInsertRecord(XLogRecData *rdata,
appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len);
if (!debug_reader)
- debug_reader = XLogReaderAllocate(wal_segment_size, NULL,
- XL_ROUTINE(), NULL);
+ debug_reader = XLogReaderAllocate(wal_segment_size, NULL NULL);
if (!debug_reader)
{
@@ -4326,15 +4318,10 @@ CleanupBackupHistory(void)
* record is available.
*/
static XLogRecord *
-ReadRecord(XLogReaderState *xlogreader, int emode,
- bool fetching_ckpt)
+ReadRecord(XLogReaderState *xlogreader, int emode, bool fetching_ckpt)
{
XLogRecord *record;
- XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
-
- private->fetching_ckpt = fetching_ckpt;
- private->emode = emode;
- private->randAccess = (xlogreader->ReadRecPtr == InvalidXLogRecPtr);
+ bool randAccess = (xlogreader->ReadRecPtr == InvalidXLogRecPtr);
/* This is the first attempt to read this page. */
lastSourceFailed = false;
@@ -4342,8 +4329,16 @@ ReadRecord(XLogReaderState *xlogreader, int emode,
for (;;)
{
char *errormsg;
+ XLogReadRecordResult result;
+
+ while ((result = XLogReadRecord(xlogreader, &record, &errormsg))
+ == XLREAD_NEED_DATA)
+ {
+ if (!XLogPageRead(xlogreader, fetching_ckpt, emode, randAccess))
+ break;
+
+ }
- record = XLogReadRecord(xlogreader, &errormsg);
ReadRecPtr = xlogreader->ReadRecPtr;
EndRecPtr = xlogreader->EndRecPtr;
if (record == NULL)
@@ -6320,7 +6315,6 @@ StartupXLOG(void)
bool backupFromStandby = false;
DBState dbstate_at_startup;
XLogReaderState *xlogreader;
- XLogPageReadPrivate private;
bool fast_promoted = false;
struct stat st;
@@ -6480,13 +6474,9 @@ StartupXLOG(void)
OwnLatch(&XLogCtl->recoveryWakeupLatch);
/* Set up XLOG reader facility */
- MemSet(&private, 0, sizeof(XLogPageReadPrivate));
xlogreader =
- XLogReaderAllocate(wal_segment_size, NULL,
- XL_ROUTINE(.page_read = &XLogPageRead,
- .segment_open = NULL,
- .segment_close = wal_segment_close),
- &private);
+ XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+
if (!xlogreader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
@@ -11860,12 +11850,13 @@ CancelBackup(void)
* sleep and retry.
*/
static bool
-XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetRecPtr, char *readBuf)
+XLogPageRead(XLogReaderState *xlogreader,
+ bool fetching_ckpt, int emode, bool randAccess)
{
- XLogPageReadPrivate *private =
- (XLogPageReadPrivate *) xlogreader->private_data;
- int emode = private->emode;
+ char *readBuf = xlogreader->readBuf;
+ XLogRecPtr targetPagePtr = xlogreader->readPagePtr;
+ int reqLen = xlogreader->readLen;
+ XLogRecPtr targetRecPtr = xlogreader->ReadRecPtr;
uint32 targetPageOff;
XLogSegNo targetSegNo PG_USED_FOR_ASSERTS_ONLY;
int r;
@@ -11908,8 +11899,8 @@ retry:
flushedUpto < targetPagePtr + reqLen))
{
if (!WaitForWALToBecomeAvailable(targetPagePtr + reqLen,
- private->randAccess,
- private->fetching_ckpt,
+ randAccess,
+ fetching_ckpt,
targetRecPtr))
{
if (readFile >= 0)
@@ -12014,6 +12005,7 @@ retry:
goto next_record_is_invalid;
}
+ Assert(xlogreader->readPagePtr == targetPagePtr);
xlogreader->readLen = readLen;
return true;
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index 3d599325ee..9281b57379 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -40,7 +40,7 @@ static bool XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr,
int reqLen, bool header_inclusive);
static void XLogReaderInvalReadState(XLogReaderState *state);
static bool ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
- XLogRecPtr PrevRecPtr, XLogRecord *record, bool randAccess);
+ XLogRecPtr PrevRecPtr, XLogRecord *record);
static bool ValidXLogRecord(XLogReaderState *state, XLogRecord *record,
XLogRecPtr recptr);
static void ResetDecoder(XLogReaderState *state);
@@ -73,7 +73,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...)
*/
XLogReaderState *
XLogReaderAllocate(int wal_segment_size, const char *waldir,
- XLogReaderRoutine *routine, void *private_data)
+ WALSegmentCleanupCB cleanup_cb)
{
XLogReaderState *state;
@@ -84,7 +84,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
return NULL;
/* initialize caller-provided support functions */
- state->routine = *routine;
+ state->cleanup_cb = cleanup_cb;
state->max_block_id = -1;
@@ -107,8 +107,6 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
WALOpenSegmentInit(&state->seg, &state->segcxt, wal_segment_size,
waldir);
- /* system_identifier initialized to zeroes above */
- state->private_data = private_data;
/* ReadRecPtr, EndRecPtr and readLen initialized to zeroes above */
state->errormsg_buf = palloc_extended(MAX_ERRORMSG_LEN + 1,
MCXT_ALLOC_NO_OOM);
@@ -140,8 +138,8 @@ XLogReaderFree(XLogReaderState *state)
{
int block_id;
- if (state->seg.ws_file != -1)
- state->routine.segment_close(state);
+ if (state->seg.ws_file >= 0)
+ state->cleanup_cb(state);
for (block_id = 0; block_id <= XLR_MAX_BLOCK_ID; block_id++)
{
@@ -246,6 +244,7 @@ XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
/* Begin at the passed-in record pointer. */
state->EndRecPtr = RecPtr;
state->ReadRecPtr = InvalidXLogRecPtr;
+ state->readRecordState = XLREAD_NEXT_RECORD;
}
/*
@@ -254,317 +253,452 @@ XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
* XLogBeginRead() or XLogFindNextRecord() must be called before the first call
* to XLogReadRecord().
*
- * If the page_read callback fails to read the requested data, NULL is
- * returned. The callback is expected to have reported the error; errormsg
- * is set to NULL.
+ * This function may return XLREAD_NEED_DATA several times before returning a
+ * result record. The caller shall read in some new data then call this
+ * function again with the same parameters.
*
- * If the reading fails for some other reason, NULL is also returned, and
- * *errormsg is set to a string with details of the failure.
+ * When a record is successfully read, returns XLREAD_SUCCESS with result
+ * record being stored in *record. Otherwise *record is NULL.
*
- * The returned pointer (or *errormsg) points to an internal buffer that's
- * valid until the next call to XLogReadRecord.
+ * Returns XLREAD_NEED_DATA if more data is needed to finish reading the
+ * current record. In that case, state->readPagePtr and state->readLen inform
+ * the desired position and minimum length of data needed. The caller shall
+ * read in the requested data and set state->readBuf to point to a buffer
+ * containing it. The caller must also set state->readPageTLI and
+ * state->readLen to indicate the timeline that it was read from, and the
+ * length of data that is now available (which must be >= given readLen),
+ * respectively.
+ *
+ * If invalid data is encountered, returns XLREAD_FAIL with *record being set to
+ * NULL. *errormsg is set to a string with details of the failure.
+ * The returned pointer (or *errormsg) points to an internal buffer that's valid
+ * until the next call to XLogReadRecord.
+ *
+ *
+ * This function runs a state machine consists of the following states.
+ *
+ * XLREAD_NEXT_RECORD :
+ * The initial state, if called with valid RecPtr, try to read a record at
+ * that position. If invalid RecPtr is given try to read a record just after
+ * the last one previously read.
+ * This state ens after setting ReadRecPtr. Then goes to XLREAD_TOT_LEN.
+ *
+ * XLREAD_TOT_LEN:
+ * Examining record header. Ends after reading record total
+ * length. recordRemainLen and recordGotLen are initialized.
+ *
+ * XLREAD_FIRST_FRAGMENT:
+ * Reading the first fragment. Ends with finishing reading a single
+ * record. Goes to XLREAD_NEXT_RECORD if that's all or
+ * XLREAD_CONTINUATION if we have continuation.
+
+ * XLREAD_CONTINUATION:
+ * Reading continuation of record. Ends with finishing the whole record then
+ * goes to XLREAD_NEXT_RECORD. During this state, recordRemainLen indicates
+ * how much is left and readRecordBuf holds the partially assert
+ * record.recordContRecPtr points to the beginning of the next page where to
+ * continue.
+ *
+ * If wrong data found in any state, the state machine stays at the current
+ * state. This behavior allows to continue reading a reacord switching among
+ * different souces, while streaming replication.
*/
-XLogRecord *
-XLogReadRecord(XLogReaderState *state, char **errormsg)
+XLogReadRecordResult
+XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
{
- XLogRecPtr RecPtr;
- XLogRecord *record;
- XLogRecPtr targetPagePtr;
- bool randAccess;
- uint32 len,
- total_len;
- uint32 targetRecOff;
- uint32 pageHeaderSize;
- bool gotheader;
+ XLogRecord *prec;
- /*
- * randAccess indicates whether to verify the previous-record pointer of
- * the record we're reading. We only do this if we're reading
- * sequentially, which is what we initially assume.
- */
- randAccess = false;
+ *record = NULL;
/* reset error state */
*errormsg = NULL;
state->errormsg_buf[0] = '\0';
- ResetDecoder(state);
-
- RecPtr = state->EndRecPtr;
-
- if (state->ReadRecPtr != InvalidXLogRecPtr)
- {
- /* read the record after the one we just read */
-
- /*
- * EndRecPtr is pointing to end+1 of the previous WAL record. If
- * we're at a page boundary, no more records can fit on the current
- * page. We must skip over the page header, but we can't do that until
- * we've read in the page, since the header size is variable.
- */
- }
- else
- {
- /*
- * Caller supplied a position to start at.
- *
- * In this case, EndRecPtr should already be pointing to a valid
- * record starting position.
- */
- Assert(XRecOffIsValid(RecPtr));
- randAccess = true;
- }
-
- state->currRecPtr = RecPtr;
-
- targetPagePtr = RecPtr - (RecPtr % XLOG_BLCKSZ);
- targetRecOff = RecPtr % XLOG_BLCKSZ;
-
- /*
- * Read the page containing the record into state->readBuf. Request enough
- * byte to cover the whole record header, or at least the part of it that
- * fits on the same page.
- */
- while (XLogNeedData(state, targetPagePtr,
- Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
- targetRecOff != 0))
- {
- if (!state->routine.page_read(state, state->readPagePtr, state->readLen,
- RecPtr, state->readBuf))
- break;
- }
-
- if (!state->page_verified)
- goto err;
-
- /*
- * We have at least the page header, so we can examine it now.
- */
- pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
- if (targetRecOff == 0)
- {
- /*
- * At page start, so skip over page header.
- */
- RecPtr += pageHeaderSize;
- targetRecOff = pageHeaderSize;
- }
- else if (targetRecOff < pageHeaderSize)
- {
- report_invalid_record(state, "invalid record offset at %X/%X",
- (uint32) (RecPtr >> 32), (uint32) RecPtr);
- goto err;
- }
-
- if ((((XLogPageHeader) state->readBuf)->xlp_info & XLP_FIRST_IS_CONTRECORD) &&
- targetRecOff == pageHeaderSize)
- {
- report_invalid_record(state, "contrecord is requested by %X/%X",
- (uint32) (RecPtr >> 32), (uint32) RecPtr);
- goto err;
- }
-
- /* XLogNeedData has verified the page header */
- Assert(pageHeaderSize <= state->readLen);
-
- /*
- * Read the record length.
- *
- * NB: Even though we use an XLogRecord pointer here, the whole record
- * header might not fit on this page. xl_tot_len is the first field of the
- * struct, so it must be on this page (the records are MAXALIGNed), but we
- * cannot access any other fields until we've verified that we got the
- * whole header.
- */
- record = (XLogRecord *) (state->readBuf + RecPtr % XLOG_BLCKSZ);
- total_len = record->xl_tot_len;
-
- /*
- * If the whole record header is on this page, validate it immediately.
- * Otherwise do just a basic sanity check on xl_tot_len, and validate the
- * rest of the header after reading it from the next page. The xl_tot_len
- * check is necessary here to ensure that we enter the "Need to reassemble
- * record" code path below; otherwise we might fail to apply
- * ValidXLogRecordHeader at all.
- */
- if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
- {
- if (!ValidXLogRecordHeader(state, RecPtr, state->ReadRecPtr, record,
- randAccess))
- goto err;
- gotheader = true;
- }
- else
- {
- /* XXX: more validation should be done here */
- if (total_len < SizeOfXLogRecord)
- {
- report_invalid_record(state,
- "invalid record length at %X/%X: wanted %u, got %u",
- (uint32) (RecPtr >> 32), (uint32) RecPtr,
- (uint32) SizeOfXLogRecord, total_len);
- goto err;
- }
- gotheader = false;
- }
-
- len = XLOG_BLCKSZ - RecPtr % XLOG_BLCKSZ;
- if (total_len > len)
+ switch (state->readRecordState)
{
- /* Need to reassemble record */
- char *contdata;
- XLogPageHeader pageHeader;
- char *buffer;
- uint32 gotlen;
-
- /*
- * Enlarge readRecordBuf as needed.
- */
- if (total_len > state->readRecordBufSize &&
- !allocate_recordbuf(state, total_len))
- {
- /* We treat this as a "bogus data" condition */
- report_invalid_record(state, "record length %u at %X/%X too long",
- total_len,
- (uint32) (RecPtr >> 32), (uint32) RecPtr);
- goto err;
- }
+ case XLREAD_NEXT_RECORD:
+ ResetDecoder(state);
- /* Copy the first fragment of the record from the first page. */
- memcpy(state->readRecordBuf,
- state->readBuf + RecPtr % XLOG_BLCKSZ, len);
- buffer = state->readRecordBuf + len;
- gotlen = len;
-
- do
- {
- int rest_len = total_len - gotlen;
-
- /* Calculate pointer to beginning of next page */
- targetPagePtr += XLOG_BLCKSZ;
-
- /* Wait for the next page to become available */
- while (XLogNeedData(state, targetPagePtr,
- Min(rest_len, XLOG_BLCKSZ),
- false))
+ if (state->ReadRecPtr != InvalidXLogRecPtr)
{
- if (!state->routine.page_read(state, state->readPagePtr,
- state->readLen,
- state->ReadRecPtr,
- state->readBuf))
- break;
+ /* read the record after the one we just read */
+
+ /*
+ * EndRecPtr is pointing to end+1 of the previous WAL record.
+ * If we're at a page boundary, no more records can fit on the
+ * current page. We must skip over the page header, but we
+ * can't do that until we've read in the page, since the header
+ * size is variable.
+ */
+ state->PrevRecPtr = state->ReadRecPtr;
+ state->ReadRecPtr = state->EndRecPtr;
+ }
+ else
+ {
+ /*
+ * Caller supplied a position to start at.
+ *
+ * In this case, EndRecPtr should already be pointing to a
+ * valid record starting position.
+ */
+ Assert(XRecOffIsValid(state->EndRecPtr));
+ state->ReadRecPtr = state->EndRecPtr;
+
+ /*
+ * We cannot verify the previous-record pointer when we're
+ * seeking to a particular record. Reset PrevRecPtr so that we
+ * won't try doing that.
+ */
+ state->PrevRecPtr = InvalidXLogRecPtr;
+ state->EndRecPtr = InvalidXLogRecPtr; /* to be tidy */
}
+ state->record_verified = false;
+ state->readRecordState = XLREAD_TOT_LEN;
+ /* fall through */
+
+ case XLREAD_TOT_LEN:
+ {
+ uint32 total_len;
+ uint32 pageHeaderSize;
+ XLogRecPtr targetPagePtr;
+ uint32 targetRecOff;
+ XLogPageHeader pageHeader;
+
+ targetPagePtr =
+ state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
+ targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+
+ /*
+ * Check if we have enough data. For the first record in the page,
+ * the requesting length doesn't contain page header.
+ */
+ if (XLogNeedData(state, targetPagePtr,
+ Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+ targetRecOff != 0))
+ return XLREAD_NEED_DATA;
+
+ /* error out if caller supplied bogus page */
if (!state->page_verified)
goto err;
- Assert(SizeOfXLogShortPHD <= state->readLen);
+ /* examine page header now. */
+ pageHeaderSize =
+ XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+ if (targetRecOff == 0)
+ {
+ /* At page start, so skip over page header. */
+ state->ReadRecPtr += pageHeaderSize;
+ targetRecOff = pageHeaderSize;
+ }
+ else if (targetRecOff < pageHeaderSize)
+ {
+ report_invalid_record(state, "invalid record offset at %X/%X",
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr);
+ goto err;
+ }
- /* Check that the continuation on next page looks valid */
pageHeader = (XLogPageHeader) state->readBuf;
- if (!(pageHeader->xlp_info & XLP_FIRST_IS_CONTRECORD))
+ if ((pageHeader->xlp_info & XLP_FIRST_IS_CONTRECORD) &&
+ targetRecOff == pageHeaderSize)
{
- report_invalid_record(state,
- "there is no contrecord flag at %X/%X",
- (uint32) (RecPtr >> 32), (uint32) RecPtr);
+ report_invalid_record(state, "contrecord is requested by %X/%X",
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr);
goto err;
}
- /*
- * Cross-check that xlp_rem_len agrees with how much of the record
- * we expect there to be left.
- */
- if (pageHeader->xlp_rem_len == 0 ||
- total_len != (pageHeader->xlp_rem_len + gotlen))
- {
- report_invalid_record(state,
- "invalid contrecord length %u at %X/%X",
- pageHeader->xlp_rem_len,
- (uint32) (RecPtr >> 32), (uint32) RecPtr);
- goto err;
- }
-
- /* Append the continuation from this page to the buffer */
- pageHeaderSize = XLogPageHeaderSize(pageHeader);
-
+ /* XLogNeedData has verified the page header */
Assert(pageHeaderSize <= state->readLen);
- contdata = (char *) state->readBuf + pageHeaderSize;
- len = XLOG_BLCKSZ - pageHeaderSize;
- if (pageHeader->xlp_rem_len < len)
- len = pageHeader->xlp_rem_len;
+ /*
+ * Read the record length.
+ *
+ * NB: Even though we use an XLogRecord pointer here, the whole
+ * record header might not fit on this page. xl_tot_len is the first
+ * field of the struct, so it must be on this page (the records are
+ * MAXALIGNed), but we cannot access any other fields until we've
+ * verified that we got the whole header.
+ */
+ prec = (XLogRecord *) (state->readBuf +
+ state->ReadRecPtr % XLOG_BLCKSZ);
+ total_len = prec->xl_tot_len;
- Assert (pageHeaderSize + len <= state->readLen);
- memcpy(buffer, (char *) contdata, len);
- buffer += len;
- gotlen += len;
+ /*
+ * If the whole record header is on this page, validate it
+ * immediately. Otherwise do just a basic sanity check on
+ * xl_tot_len, and validate the rest of the header after reading it
+ * from the next page. The xl_tot_len check is necessary here to
+ * ensure that we enter the XLREAD_CONTINUATION state below;
+ * otherwise we might fail to apply ValidXLogRecordHeader at all.
+ */
+ if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
+ {
+ if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
+ state->PrevRecPtr, prec))
+ goto err;
- /* If we just reassembled the record header, validate it. */
- if (!gotheader)
+ state->record_verified = true;
+ }
+ else
{
- record = (XLogRecord *) state->readRecordBuf;
- if (!ValidXLogRecordHeader(state, RecPtr, state->ReadRecPtr,
- record, randAccess))
+ /* XXX: more validation should be done here */
+ if (total_len < SizeOfXLogRecord)
+ {
+ report_invalid_record(state,
+ "invalid record length at %X/%X: wanted %u, got %u",
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr,
+ (uint32) SizeOfXLogRecord, total_len);
goto err;
- gotheader = true;
+ }
}
- } while (gotlen < total_len);
-
- Assert(gotheader);
- record = (XLogRecord *) state->readRecordBuf;
- if (!ValidXLogRecord(state, record, RecPtr))
- goto err;
+ /*
+ * Wait for the rest of the record, or the part of the record that
+ * fit on the first page if crossed a page boundary, to become
+ * available.
+ */
+ state->recordGotLen = 0;
+ state->recordRemainLen = total_len;
+ state->readRecordState = XLREAD_FIRST_FRAGMENT;
+ }
+ /* fall through */
- pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
- state->ReadRecPtr = RecPtr;
- state->EndRecPtr = targetPagePtr + pageHeaderSize
- + MAXALIGN(pageHeader->xlp_rem_len);
- }
- else
- {
- /* Wait for the record data to become available */
- while (XLogNeedData(state, targetPagePtr,
- Min(targetRecOff + total_len, XLOG_BLCKSZ), true))
+ case XLREAD_FIRST_FRAGMENT:
{
- if (!state->routine.page_read(state, state->readPagePtr,
- state->readLen,
- state->ReadRecPtr, state->readBuf))
+ uint32 total_len = state->recordRemainLen;
+ uint32 request_len;
+ uint32 record_len;
+ XLogRecPtr targetPagePtr;
+ uint32 targetRecOff;
+
+ /*
+ * Wait for the rest of the record on the first page to become
+ * available
+ */
+ targetPagePtr =
+ state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
+ targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+
+ request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ);
+ record_len = request_len - targetRecOff;
+
+ /* ReadRecPtr contains page header */
+ Assert (targetRecOff != 0);
+ if (XLogNeedData(state, targetPagePtr, request_len, true))
+ return XLREAD_NEED_DATA;
+
+ /* error out if caller supplied bogus page */
+ if (!state->page_verified)
+ goto err;
+
+ prec = (XLogRecord *) (state->readBuf + targetRecOff);
+
+ /* validate record header if not yet */
+ if (!state->record_verified && record_len >= SizeOfXLogRecord)
+ {
+ if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
+ state->PrevRecPtr, prec))
+ goto err;
+
+ state->record_verified = true;
+ }
+
+
+ if (total_len == record_len)
+ {
+ /* Record does not cross a page boundary */
+ Assert(state->record_verified);
+
+ if (!ValidXLogRecord(state, prec, state->ReadRecPtr))
+ goto err;
+
+ state->record_verified = true; /* to be tidy */
+
+ /* We already checked the header earlier */
+ state->EndRecPtr = state->ReadRecPtr + MAXALIGN(record_len);
+
+ *record = prec;
+ state->readRecordState = XLREAD_NEXT_RECORD;
break;
+ }
+
+ /*
+ * The record continues on the next page. Need to reassemble
+ * record
+ */
+ Assert(total_len > record_len);
+
+ /* Enlarge readRecordBuf as needed. */
+ if (total_len > state->readRecordBufSize &&
+ !allocate_recordbuf(state, total_len))
+ {
+ /* We treat this as a "bogus data" condition */
+ report_invalid_record(state,
+ "record length %u at %X/%X too long",
+ total_len,
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr);
+ goto err;
+ }
+
+ /* Copy the first fragment of the record from the first page. */
+ memcpy(state->readRecordBuf, state->readBuf + targetRecOff,
+ record_len);
+ state->recordGotLen += record_len;
+ state->recordRemainLen -= record_len;
+
+ /* Calculate pointer to beginning of next page */
+ state->recordContRecPtr = state->ReadRecPtr + record_len;
+ Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0);
+
+ state->readRecordState = XLREAD_CONTINUATION;
}
+ /* fall through */
+
+ case XLREAD_CONTINUATION:
+ {
+ XLogPageHeader pageHeader;
+ uint32 pageHeaderSize;
+ XLogRecPtr targetPagePtr;
+
+ /* we enter this state only if we haven't read the whole record. */
+ Assert (state->recordRemainLen > 0);
+
+ while(state->recordRemainLen > 0)
+ {
+ char *contdata;
+ uint32 request_len;
+ uint32 record_len;
+
+ /* Wait for the next page to become available */
+ targetPagePtr = state->recordContRecPtr;
+
+ /* this request contains page header */
+ Assert (targetPagePtr != 0);
+ if (XLogNeedData(state, targetPagePtr,
+ Min(state->recordRemainLen, XLOG_BLCKSZ),
+ false))
+ return XLREAD_NEED_DATA;
+
+ if (!state->page_verified)
+ goto err;
+
+ Assert(SizeOfXLogShortPHD <= state->readLen);
- if (!state->page_verified)
- goto err;
+ /* Check that the continuation on next page looks valid */
+ pageHeader = (XLogPageHeader) state->readBuf;
+ if (!(pageHeader->xlp_info & XLP_FIRST_IS_CONTRECORD))
+ {
+ report_invalid_record(
+ state,
+ "there is no contrecord flag at %X/%X reading %X/%X",
+ (uint32) (state->recordContRecPtr >> 32),
+ (uint32) state->recordContRecPtr,
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr);
+ goto err;
+ }
- /* Record does not cross a page boundary */
- if (!ValidXLogRecord(state, record, RecPtr))
- goto err;
+ /*
+ * Cross-check that xlp_rem_len agrees with how much of the
+ * record we expect there to be left.
+ */
+ if (pageHeader->xlp_rem_len == 0 ||
+ pageHeader->xlp_rem_len != state->recordRemainLen)
+ {
+ report_invalid_record(
+ state,
+ "invalid contrecord length %u at %X/%X reading %X/%X, expected %u",
+ pageHeader->xlp_rem_len,
+ (uint32) (state->recordContRecPtr >> 32),
+ (uint32) state->recordContRecPtr,
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr,
+ state->recordRemainLen);
+ goto err;
+ }
- state->EndRecPtr = RecPtr + MAXALIGN(total_len);
+ /* Append the continuation from this page to the buffer */
+ pageHeaderSize = XLogPageHeaderSize(pageHeader);
- state->ReadRecPtr = RecPtr;
+ /*
+ * XLogNeedData should have ensured that the whole page header
+ * was read
+ */
+ Assert(state->readLen >= pageHeaderSize);
+
+ contdata = (char *) state->readBuf + pageHeaderSize;
+ record_len = XLOG_BLCKSZ - pageHeaderSize;
+ if (pageHeader->xlp_rem_len < record_len)
+ record_len = pageHeader->xlp_rem_len;
+
+ request_len = record_len + pageHeaderSize;
+
+ /* XLogNeedData should have ensured all needed data was read */
+ Assert (state->readLen >= request_len);
+
+ memcpy(state->readRecordBuf + state->recordGotLen,
+ (char *) contdata, record_len);
+ state->recordGotLen += record_len;
+ state->recordRemainLen -= record_len;
+
+ /* If we just reassembled the record header, validate it. */
+ if (!state->record_verified)
+ {
+ Assert(state->recordGotLen >= SizeOfXLogRecord);
+ if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
+ state->PrevRecPtr,
+ (XLogRecord *) state->readRecordBuf))
+ goto err;
+
+ state->record_verified = true;
+ }
+
+ /* Calculate pointer to beginning of next page, and continue */
+ state->recordContRecPtr += XLOG_BLCKSZ;
+ }
+
+ /* targetPagePtr is pointing the last-read page here */
+ prec = (XLogRecord *) state->readRecordBuf;
+ if (!ValidXLogRecord(state, prec, state->ReadRecPtr))
+ goto err;
+
+ pageHeaderSize =
+ XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+ state->EndRecPtr = targetPagePtr + pageHeaderSize
+ + MAXALIGN(pageHeader->xlp_rem_len);
+
+ *record = prec;
+ state->readRecordState = XLREAD_NEXT_RECORD;
+ break;
+ }
}
/*
* Special processing if it's an XLOG SWITCH record
*/
- if (record->xl_rmid == RM_XLOG_ID &&
- (record->xl_info & ~XLR_INFO_MASK) == XLOG_SWITCH)
+ if ((*record)->xl_rmid == RM_XLOG_ID &&
+ ((*record)->xl_info & ~XLR_INFO_MASK) == XLOG_SWITCH)
{
/* Pretend it extends to end of segment */
state->EndRecPtr += state->segcxt.ws_segsize - 1;
state->EndRecPtr -= XLogSegmentOffset(state->EndRecPtr, state->segcxt.ws_segsize);
}
- if (DecodeXLogRecord(state, record, errormsg))
- return record;
- else
- return NULL;
+ Assert (!*record || state->readLen >= 0);
+ if (DecodeXLogRecord(state, *record, errormsg))
+ return XLREAD_SUCCESS;
+
+ *record = NULL;
+ return XLREAD_FAIL;
err:
/*
- * Invalidate the read state. We might read from a different source after
+ * Invalidate the read page. We might read from a different source after
* failure.
*/
XLogReaderInvalReadState(state);
@@ -572,7 +706,8 @@ err:
if (state->errormsg_buf[0] != '\0')
*errormsg = state->errormsg_buf;
- return NULL;
+ *record = NULL;
+ return XLREAD_FAIL;
}
/*
@@ -724,11 +859,12 @@ XLogReaderInvalReadState(XLogReaderState *state)
*
* This is just a convenience subroutine to avoid duplicated code in
* XLogReadRecord. It's not intended for use from anywhere else.
+ *
+ * If PrevRecPtr is valid, the xl_prev is is cross-checked with it.
*/
static bool
ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
- XLogRecPtr PrevRecPtr, XLogRecord *record,
- bool randAccess)
+ XLogRecPtr PrevRecPtr, XLogRecord *record)
{
if (record->xl_tot_len < SizeOfXLogRecord)
{
@@ -746,7 +882,7 @@ ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
(uint32) RecPtr);
return false;
}
- if (randAccess)
+ if (PrevRecPtr == InvalidXLogRecPtr)
{
/*
* We can't exactly verify the prev-link, but surely it should be less
@@ -978,11 +1114,14 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
* XLogReadRecord() will read the next valid record.
*/
XLogRecPtr
-XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
+XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
+ XLogFindNextRecordCB read_page, void *private)
{
XLogRecPtr tmpRecPtr;
XLogRecPtr found = InvalidXLogRecPtr;
XLogPageHeader header;
+ XLogRecord *record;
+ XLogReadRecordResult result;
char *errormsg;
Assert(!XLogRecPtrIsInvalid(RecPtr));
@@ -1015,9 +1154,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
while(XLogNeedData(state, targetPagePtr, targetRecOff,
targetRecOff != 0))
{
- if (!state->routine.page_read(state, state->readPagePtr,
- state->readLen,
- state->ReadRecPtr, state->readBuf))
+ if (!read_page(state, private))
break;
}
@@ -1069,8 +1206,16 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
* or we just jumped over the remaining data of a continuation.
*/
XLogBeginRead(state, tmpRecPtr);
- while (XLogReadRecord(state, &errormsg) != NULL)
+ while ((result = XLogReadRecord(state, &record, &errormsg)) !=
+ XLREAD_FAIL)
{
+ if (result == XLREAD_NEED_DATA)
+ {
+ if (!read_page(state, private))
+ goto err;
+ continue;
+ }
+
/* past the record we've found, break out */
if (RecPtr <= state->ReadRecPtr)
{
@@ -1090,9 +1235,9 @@ err:
#endif /* FRONTEND */
/*
- * Helper function to ease writing of XLogRoutine->page_read callbacks.
- * If this function is used, caller must supply a segment_open callback in
- * 'state', as that is used here.
+ * Helper function to ease writing of page_read callback.
+ * If this function is used, caller must supply a segment_open callback and
+ * segment_close callback as that is used here.
*
* Read 'count' bytes into 'buf', starting at location 'startptr', from WAL
* fetched from timeline 'tli'.
@@ -1105,6 +1250,7 @@ err:
*/
bool
WALRead(XLogReaderState *state,
+ WALSegmentOpenCB segopenfn, WALSegmentCloseCB segclosefn,
char *buf, XLogRecPtr startptr, Size count, TimeLineID tli,
WALReadError *errinfo)
{
@@ -1136,10 +1282,10 @@ WALRead(XLogReaderState *state,
XLogSegNo nextSegNo;
if (state->seg.ws_file >= 0)
- state->routine.segment_close(state);
+ segclosefn(state);
XLByteToSeg(recptr, nextSegNo, state->segcxt.ws_segsize);
- state->routine.segment_open(state, nextSegNo, &tli);
+ segopenfn(state, nextSegNo, &tli);
/* This shouldn't happen -- indicates a bug in segment_open */
Assert(state->seg.ws_file >= 0);
diff --git a/src/backend/access/transam/xlogutils.c b/src/backend/access/transam/xlogutils.c
index 1b10b6df20..303333571f 100644
--- a/src/backend/access/transam/xlogutils.c
+++ b/src/backend/access/transam/xlogutils.c
@@ -689,8 +689,7 @@ XLogTruncateRelation(RelFileNode rnode, ForkNumber forkNum,
void
XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wantLength)
{
- const XLogRecPtr lastReadPage = state->seg.ws_segno *
- state->segcxt.ws_segsize + state->readLen;
+ const XLogRecPtr lastReadPage = state->readPagePtr;
Assert(wantPage != InvalidXLogRecPtr && wantPage % XLOG_BLCKSZ == 0);
Assert(wantLength <= XLOG_BLCKSZ);
@@ -705,7 +704,7 @@ XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wa
* current TLI has since become historical.
*/
if (lastReadPage == wantPage &&
- state->readLen != 0 &&
+ state->page_verified &&
lastReadPage + state->readLen >= wantPage + Min(wantLength, XLOG_BLCKSZ - 1))
return;
@@ -792,6 +791,7 @@ wal_segment_open(XLogReaderState *state, XLogSegNo nextSegNo,
char path[MAXPGPATH];
XLogFilePath(path, tli, nextSegNo, state->segcxt.ws_segsize);
+ elog(LOG, "HOGE: %lu, %d => %s", nextSegNo, tli, path);
state->seg.ws_file = BasicOpenFile(path, O_RDONLY | PG_BINARY);
if (state->seg.ws_file >= 0)
return;
@@ -829,9 +829,11 @@ wal_segment_close(XLogReaderState *state)
* loop for now.
*/
bool
-read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *cur_page)
+read_local_xlog_page(XLogReaderState *state)
{
+ XLogRecPtr targetPagePtr = state->readPagePtr;
+ int reqLen = state->readLen;
+ char *cur_page = state->readBuf;
XLogRecPtr read_upto,
loc;
TimeLineID tli;
@@ -944,11 +946,12 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
* as 'count', read the whole page anyway. It's guaranteed to be
* zero-padded up to the page boundary if it's incomplete.
*/
- if (!WALRead(state, cur_page, targetPagePtr, XLOG_BLCKSZ, tli,
- &errinfo))
+ if (!WALRead(state, wal_segment_open, wal_segment_close,
+ cur_page, targetPagePtr, XLOG_BLCKSZ, tli, &errinfo))
WALReadRaiseError(&errinfo);
/* number of valid bytes in the buffer */
+ state->readPagePtr = targetPagePtr;
state->readLen = count;
return true;
}
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index 61902be3b0..a15b0b3355 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -120,7 +120,8 @@ StartupDecodingContext(List *output_plugin_options,
TransactionId xmin_horizon,
bool need_full_snapshot,
bool fast_forward,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress)
@@ -169,11 +170,12 @@ StartupDecodingContext(List *output_plugin_options,
ctx->slot = slot;
- ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, xl_routine, ctx);
+ ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, cleanup_cb);
if (!ctx->reader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
errmsg("out of memory")));
+ ctx->page_read = page_read;
ctx->reorder = ReorderBufferAllocate();
ctx->snapshot_builder =
@@ -231,7 +233,8 @@ CreateInitDecodingContext(char *plugin,
List *output_plugin_options,
bool need_full_snapshot,
XLogRecPtr restart_lsn,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress)
@@ -328,7 +331,7 @@ CreateInitDecodingContext(char *plugin,
ctx = StartupDecodingContext(NIL, restart_lsn, xmin_horizon,
need_full_snapshot, false,
- xl_routine, prepare_write, do_write,
+ page_read, cleanup_cb, prepare_write, do_write,
update_progress);
/* call output plugin initialization callback */
@@ -376,7 +379,8 @@ LogicalDecodingContext *
CreateDecodingContext(XLogRecPtr start_lsn,
List *output_plugin_options,
bool fast_forward,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress)
@@ -429,8 +433,8 @@ CreateDecodingContext(XLogRecPtr start_lsn,
ctx = StartupDecodingContext(output_plugin_options,
start_lsn, InvalidTransactionId, false,
- fast_forward, xl_routine, prepare_write,
- do_write, update_progress);
+ fast_forward, page_read, cleanup_cb,
+ prepare_write, do_write, update_progress);
/* call output plugin initialization callback */
old_context = MemoryContextSwitchTo(ctx->context);
@@ -483,7 +487,13 @@ DecodingContextFindStartpoint(LogicalDecodingContext *ctx)
char *err = NULL;
/* the read_page callback waits for new WAL */
- record = XLogReadRecord(ctx->reader, &err);
+ while (XLogReadRecord(ctx->reader, &record, &err) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!ctx->page_read(ctx->reader))
+ break;
+ }
+
if (err)
elog(ERROR, "%s", err);
if (!record)
diff --git a/src/backend/replication/logical/logicalfuncs.c b/src/backend/replication/logical/logicalfuncs.c
index b99c94e848..fcc81b358e 100644
--- a/src/backend/replication/logical/logicalfuncs.c
+++ b/src/backend/replication/logical/logicalfuncs.c
@@ -233,9 +233,8 @@ pg_logical_slot_get_changes_guts(FunctionCallInfo fcinfo, bool confirm, bool bin
ctx = CreateDecodingContext(InvalidXLogRecPtr,
options,
false,
- XL_ROUTINE(.page_read = read_local_xlog_page,
- .segment_open = wal_segment_open,
- .segment_close = wal_segment_close),
+ read_local_xlog_page,
+ wal_segment_close,
LogicalOutputPrepareWrite,
LogicalOutputWrite, NULL);
@@ -284,7 +283,13 @@ pg_logical_slot_get_changes_guts(FunctionCallInfo fcinfo, bool confirm, bool bin
XLogRecord *record;
char *errm = NULL;
- record = XLogReadRecord(ctx->reader, &errm);
+ while (XLogReadRecord(ctx->reader, &record, &errm) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!ctx->page_read(ctx->reader))
+ break;
+ }
+
if (errm)
elog(ERROR, "%s", errm);
diff --git a/src/backend/replication/slotfuncs.c b/src/backend/replication/slotfuncs.c
index 88033a79b2..8a678074d5 100644
--- a/src/backend/replication/slotfuncs.c
+++ b/src/backend/replication/slotfuncs.c
@@ -152,9 +152,8 @@ create_logical_replication_slot(char *name, char *plugin,
ctx = CreateInitDecodingContext(plugin, NIL,
false, /* just catalogs is OK */
restart_lsn,
- XL_ROUTINE(.page_read = read_local_xlog_page,
- .segment_open = wal_segment_open,
- .segment_close = wal_segment_close),
+ read_local_xlog_page,
+ wal_segment_close,
NULL, NULL, NULL);
/*
@@ -486,9 +485,8 @@ pg_logical_replication_slot_advance(XLogRecPtr moveto)
ctx = CreateDecodingContext(InvalidXLogRecPtr,
NIL,
true, /* fast_forward */
- XL_ROUTINE(.page_read = read_local_xlog_page,
- .segment_open = wal_segment_open,
- .segment_close = wal_segment_close),
+ read_local_xlog_page,
+ wal_segment_close,
NULL, NULL, NULL);
/*
@@ -513,7 +511,13 @@ pg_logical_replication_slot_advance(XLogRecPtr moveto)
* Read records. No changes are generated in fast_forward mode,
* but snapbuilder/slot statuses are updated properly.
*/
- record = XLogReadRecord(ctx->reader, &errm);
+ while (XLogReadRecord(ctx->reader, &record, &errm) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!ctx->page_read(ctx->reader))
+ break;
+ }
+
if (errm)
elog(ERROR, "%s", errm);
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index bf2711bddd..2fd24a9b14 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -580,10 +580,7 @@ StartReplication(StartReplicationCmd *cmd)
/* create xlogreader for physical replication */
xlogreader =
- XLogReaderAllocate(wal_segment_size, NULL,
- XL_ROUTINE(.segment_open = WalSndSegmentOpen,
- .segment_close = wal_segment_close),
- NULL);
+ XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
if (!xlogreader)
ereport(ERROR,
@@ -812,9 +809,11 @@ StartReplication(StartReplicationCmd *cmd)
* set every time WAL is flushed.
*/
static bool
-logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetRecPtr, char *cur_page)
+logical_read_xlog_page(XLogReaderState *state)
{
+ XLogRecPtr targetPagePtr = state->readPagePtr;
+ int reqLen = state->readLen;
+ char *cur_page = state->readBuf;
XLogRecPtr flushptr;
int count;
WALReadError errinfo;
@@ -842,7 +841,7 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
count = flushptr - targetPagePtr; /* part of the page available */
/* now actually read the data, we know it's there */
- if (!WALRead(state,
+ if (!WALRead(state, WalSndSegmentOpen, wal_segment_close,
cur_page,
targetPagePtr,
XLOG_BLCKSZ,
@@ -1014,9 +1013,8 @@ CreateReplicationSlot(CreateReplicationSlotCmd *cmd)
ctx = CreateInitDecodingContext(cmd->plugin, NIL, need_full_snapshot,
InvalidXLogRecPtr,
- XL_ROUTINE(.page_read = logical_read_xlog_page,
- .segment_open = WalSndSegmentOpen,
- .segment_close = wal_segment_close),
+ logical_read_xlog_page,
+ wal_segment_close,
WalSndPrepareWrite, WalSndWriteData,
WalSndUpdateProgress);
@@ -1179,9 +1177,8 @@ StartLogicalReplication(StartReplicationCmd *cmd)
*/
logical_decoding_ctx =
CreateDecodingContext(cmd->startpoint, cmd->options, false,
- XL_ROUTINE(.page_read = logical_read_xlog_page,
- .segment_open = WalSndSegmentOpen,
- .segment_close = wal_segment_close),
+ logical_read_xlog_page,
+ wal_segment_close,
WalSndPrepareWrite, WalSndWriteData,
WalSndUpdateProgress);
xlogreader = logical_decoding_ctx->reader;
@@ -2761,7 +2758,7 @@ XLogSendPhysical(void)
enlargeStringInfo(&output_message, nbytes);
retry:
- if (!WALRead(xlogreader,
+ if (!WALRead(xlogreader, WalSndSegmentOpen, wal_segment_close,
&output_message.data[output_message.len],
startptr,
nbytes,
@@ -2859,7 +2856,12 @@ XLogSendLogical(void)
*/
WalSndCaughtUp = false;
- record = XLogReadRecord(logical_decoding_ctx->reader, &errm);
+ while (XLogReadRecord(logical_decoding_ctx->reader, &record, &errm) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!logical_decoding_ctx->page_read(logical_decoding_ctx->reader))
+ break;
+ }
/* xlog record was invalid */
if (errm != NULL)
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index 0b7e73ae79..3dd6df4be6 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -40,15 +40,9 @@ static int xlogreadfd = -1;
static XLogSegNo xlogreadsegno = -1;
static char xlogfpath[MAXPGPATH];
-typedef struct XLogPageReadPrivate
-{
- const char *restoreCommand;
- int tliIndex;
-} XLogPageReadPrivate;
-
-static bool SimpleXLogPageRead(XLogReaderState *xlogreader,
- XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
+static bool SimpleXLogPageRead(XLogReaderState *xlogreader,
+ const char *datadir, int *tliIndex,
+ const char *restoreCommand);
/*
* Read WAL from the datadir/pg_wal, starting from 'startpoint' on timeline
@@ -62,20 +56,22 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
XLogRecord *record;
XLogReaderState *xlogreader;
char *errormsg;
- XLogPageReadPrivate private;
- private.tliIndex = tliIndex;
- private.restoreCommand = restoreCommand;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir,
- XL_ROUTINE(.page_read = &SimpleXLogPageRead),
- &private);
+ xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+
if (xlogreader == NULL)
pg_fatal("out of memory");
XLogBeginRead(xlogreader, startpoint);
do
{
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!SimpleXLogPageRead(xlogreader, datadir,
+ &tliIndex, restoreCommand))
+ break;
+ }
if (record == NULL)
{
@@ -113,19 +109,19 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex,
XLogRecord *record;
XLogReaderState *xlogreader;
char *errormsg;
- XLogPageReadPrivate private;
XLogRecPtr endptr;
- private.tliIndex = tliIndex;
- private.restoreCommand = restoreCommand;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir,
- XL_ROUTINE(.page_read = &SimpleXLogPageRead),
- &private);
+ xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
XLogBeginRead(xlogreader, ptr);
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!SimpleXLogPageRead(xlogreader, datadir, &tliIndex, restoreCommand))
+ break;
+ }
if (record == NULL)
{
if (errormsg)
@@ -160,7 +156,6 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
XLogRecPtr searchptr;
XLogReaderState *xlogreader;
char *errormsg;
- XLogPageReadPrivate private;
/*
* The given fork pointer points to the end of the last common record,
@@ -176,11 +171,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
forkptr += SizeOfXLogShortPHD;
}
- private.tliIndex = tliIndex;
- private.restoreCommand = restoreCommand;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir,
- XL_ROUTINE(.page_read = &SimpleXLogPageRead),
- &private);
+ xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
@@ -190,7 +181,13 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
uint8 info;
XLogBeginRead(xlogreader, searchptr);
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!SimpleXLogPageRead(xlogreader, datadir,
+ &tliIndex, restoreCommand))
+ break;
+ }
if (record == NULL)
{
@@ -237,10 +234,11 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
/* XLogReader callback function, to read a WAL page */
static bool
-SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *readBuf)
+SimpleXLogPageRead(XLogReaderState *xlogreader, const char *datadir,
+ int *tliIndex, const char *restoreCommand)
{
- XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
+ XLogRecPtr targetPagePtr = xlogreader->readPagePtr;
+ char *readBuf = xlogreader->readBuf;
uint32 targetPageOff;
XLogRecPtr targetSegEnd;
XLogSegNo targetSegNo;
@@ -273,14 +271,14 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
* be done both forward and backward, consider also switching timeline
* accordingly.
*/
- while (private->tliIndex < targetNentries - 1 &&
- targetHistory[private->tliIndex].end < targetSegEnd)
- private->tliIndex++;
- while (private->tliIndex > 0 &&
- targetHistory[private->tliIndex].begin >= targetSegEnd)
- private->tliIndex--;
+ while (*tliIndex < targetNentries - 1 &&
+ targetHistory[*tliIndex].end < targetSegEnd)
+ (*tliIndex)++;
+ while (*tliIndex > 0 &&
+ targetHistory[*tliIndex].begin >= targetSegEnd)
+ (*tliIndex)--;
- XLogFileName(xlogfname, targetHistory[private->tliIndex].tli,
+ XLogFileName(xlogfname, targetHistory[*tliIndex].tli,
xlogreadsegno, WalSegSz);
snprintf(xlogfpath, MAXPGPATH, "%s/" XLOGDIR "/%s",
@@ -293,7 +291,7 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
/*
* If we have no restore_command to execute, then exit.
*/
- if (private->restoreCommand == NULL)
+ if (restoreCommand == NULL)
{
pg_log_error("could not open file \"%s\": %m", xlogfpath);
xlogreader->readLen = -1;
@@ -307,7 +305,7 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
xlogreadfd = RestoreArchivedFile(xlogreader->segcxt.ws_dir,
xlogfname,
WalSegSz,
- private->restoreCommand);
+ restoreCommand);
if (xlogreadfd < 0)
{
@@ -349,7 +347,7 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
Assert(targetSegNo == xlogreadsegno);
- xlogreader->seg.ws_tli = targetHistory[private->tliIndex].tli;
+ xlogreader->seg.ws_tli = targetHistory[*tliIndex].tli;
xlogreader->readLen = XLOG_BLCKSZ;
return true;
}
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index ab0c28d9b7..02e71138cc 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -330,12 +330,17 @@ WALDumpCloseSegment(XLogReaderState *state)
state->seg.ws_file = -1;
}
-/* pg_waldump's XLogReaderRoutine->page_read callback */
+/*
+ * pg_waldump's WAL page rader, also used as page_read callback for
+ * XLogFindNextRecord
+ */
static bool
-WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetPtr, char *readBuff)
+WALDumpReadPage(XLogReaderState *state, void *priv)
{
- XLogDumpPrivate *private = state->private_data;
+ XLogRecPtr targetPagePtr = state->readPagePtr;
+ int reqLen = state->readLen;
+ char *readBuff = state->readBuf;
+ XLogDumpPrivate *private = (XLogDumpPrivate *) priv;
int count = XLOG_BLCKSZ;
WALReadError errinfo;
@@ -353,8 +358,8 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
}
}
- if (!WALRead(state, readBuff, targetPagePtr, count, private->timeline,
- &errinfo))
+ if (!WALRead(state, WALDumpOpenSegment, WALDumpCloseSegment,
+ readBuff, targetPagePtr, count, private->timeline, &errinfo))
{
WALOpenSegment *seg = &errinfo.wre_seg;
char fname[MAXPGPATH];
@@ -374,6 +379,7 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
(Size) errinfo.wre_req);
}
+ Assert(count >= state->readLen);
state->readLen = count;
return true;
}
@@ -1042,16 +1048,14 @@ main(int argc, char **argv)
/* we have everything we need, start reading */
xlogreader_state =
- XLogReaderAllocate(WalSegSz, waldir,
- XL_ROUTINE(.page_read = WALDumpReadPage,
- .segment_open = WALDumpOpenSegment,
- .segment_close = WALDumpCloseSegment),
- &private);
+ XLogReaderAllocate(WalSegSz, waldir, WALDumpCloseSegment);
+
if (!xlogreader_state)
fatal_error("out of memory");
/* first find a valid recptr to start from */
- first_record = XLogFindNextRecord(xlogreader_state, private.startptr);
+ first_record = XLogFindNextRecord(xlogreader_state, private.startptr,
+ &WALDumpReadPage, (void*) &private);
if (first_record == InvalidXLogRecPtr)
fatal_error("could not find a valid record after %X/%X",
@@ -1075,7 +1079,13 @@ main(int argc, char **argv)
for (;;)
{
/* try to read the next record */
- record = XLogReadRecord(xlogreader_state, &errormsg);
+ while (XLogReadRecord(xlogreader_state, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!WALDumpReadPage(xlogreader_state, (void *) &private))
+ break;
+ }
+
if (!record)
{
if (!config.follow || private.endptr_reached)
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index 6c8848e14f..6a7fe140cb 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -57,64 +57,15 @@ typedef struct WALSegmentContext
typedef struct XLogReaderState XLogReaderState;
-/* Function type definition for the read_page callback */
-typedef bool (*XLogPageReadCB) (XLogReaderState *xlogreader,
- XLogRecPtr targetPagePtr,
- int reqLen,
- XLogRecPtr targetRecPtr,
- char *readBuf);
+/* Function type definition for the segment cleanup callback */
+typedef void (*WALSegmentCleanupCB) (XLogReaderState *xlogreader);
+
+/* Function type definition for the open/close callbacks for WALRead() */
typedef void (*WALSegmentOpenCB) (XLogReaderState *xlogreader,
XLogSegNo nextSegNo,
TimeLineID *tli_p);
typedef void (*WALSegmentCloseCB) (XLogReaderState *xlogreader);
-typedef struct XLogReaderRoutine
-{
- /*
- * Data input callback
- *
- * This callback shall read at least reqLen valid bytes of the xlog page
- * starting at targetPagePtr, and store them in readBuf. The callback
- * shall return the number of bytes read (never more than XLOG_BLCKSZ), or
- * -1 on failure. The callback shall sleep, if necessary, to wait for the
- * requested bytes to become available. The callback will not be invoked
- * again for the same page unless more than the returned number of bytes
- * are needed.
- *
- * targetRecPtr is the position of the WAL record we're reading. Usually
- * it is equal to targetPagePtr + reqLen, but sometimes xlogreader needs
- * to read and verify the page or segment header, before it reads the
- * actual WAL record it's interested in. In that case, targetRecPtr can
- * be used to determine which timeline to read the page from.
- *
- * The callback shall set ->seg.ws_tli to the TLI of the file the page was
- * read from.
- */
- XLogPageReadCB page_read;
-
- /*
- * Callback to open the specified WAL segment for reading. ->seg.ws_file
- * shall be set to the file descriptor of the opened segment. In case of
- * failure, an error shall be raised by the callback and it shall not
- * return.
- *
- * "nextSegNo" is the number of the segment to be opened.
- *
- * "tli_p" is an input/output argument. WALRead() uses it to pass the
- * timeline in which the new segment should be found, but the callback can
- * use it to return the TLI that it actually opened.
- */
- WALSegmentOpenCB segment_open;
-
- /*
- * WAL segment close callback. ->seg.ws_file shall be set to a negative
- * number.
- */
- WALSegmentCloseCB segment_close;
-} XLogReaderRoutine;
-
-#define XL_ROUTINE(...) &(XLogReaderRoutine){__VA_ARGS__}
-
typedef struct
{
/* Is this block ref in use? */
@@ -144,12 +95,35 @@ typedef struct
uint16 data_bufsz;
} DecodedBkpBlock;
+/* Return code from XLogReadRecord */
+typedef enum XLogReadRecordResult
+{
+ XLREAD_SUCCESS, /* record is successfully read */
+ XLREAD_NEED_DATA, /* need more data. see XLogReadRecord. */
+ XLREAD_FAIL /* failed during reading a record */
+} XLogReadRecordResult;
+
+/*
+ * internal state of XLogReadRecord
+ *
+ * XLogReadState runs a state machine while reading a record. Theses states
+ * are not seen outside the function. Each state may repeat several times
+ * exiting requesting caller for new data. See the comment of XLogReadRecrod
+ * for details.
+ */
+typedef enum XLogReadRecordState {
+ XLREAD_NEXT_RECORD,
+ XLREAD_TOT_LEN,
+ XLREAD_FIRST_FRAGMENT,
+ XLREAD_CONTINUATION
+} XLogReadRecordState;
+
struct XLogReaderState
{
/*
* Operational callbacks
*/
- XLogReaderRoutine routine;
+ WALSegmentCleanupCB cleanup_cb;
/* ----------------------------------------
* Public parameters
@@ -162,18 +136,14 @@ struct XLogReaderState
*/
uint64 system_identifier;
- /*
- * Opaque data for callbacks to use. Not used by XLogReader.
- */
- void *private_data;
-
/*
* Start and end point of last record read. EndRecPtr is also used as the
* position to read next. Calling XLogBeginRead() sets EndRecPtr to the
* starting position and ReadRecPtr to invalid.
*/
- XLogRecPtr ReadRecPtr; /* start of last record read */
+ XLogRecPtr ReadRecPtr; /* start of last record read or being read */
XLogRecPtr EndRecPtr; /* end+1 of last record read */
+ XLogRecPtr PrevRecPtr; /* start of previous record read */
/* ----------------------------------------
* Communication with page reader
@@ -187,7 +157,9 @@ struct XLogReaderState
* the request, or -1 on error */
TimeLineID readPageTLI; /* TLI for data currently in readBuf */
char *readBuf; /* buffer to store data */
- bool page_verified; /* is the page on the buffer verified? */
+ bool page_verified; /* is the page header on the buffer verified? */
+ bool record_verified;/* is the current record header verified? */
+
/* ----------------------------------------
@@ -227,8 +199,6 @@ struct XLogReaderState
XLogRecPtr latestPagePtr;
TimeLineID latestPageTLI;
- /* beginning of the WAL record being read. */
- XLogRecPtr currRecPtr;
/* timeline to read it from, 0 if a lookup is required */
TimeLineID currTLI;
@@ -255,6 +225,15 @@ struct XLogReaderState
char *readRecordBuf;
uint32 readRecordBufSize;
+ /*
+ * XLogReadRecord() state
+ */
+ XLogReadRecordState readRecordState;/* state machine state */
+ int recordGotLen; /* amount of current record that has
+ * already been read */
+ int recordRemainLen; /* length of current record that remains */
+ XLogRecPtr recordContRecPtr; /* where the current record continues */
+
/* Buffer to hold error message */
char *errormsg_buf;
};
@@ -262,9 +241,7 @@ struct XLogReaderState
/* Get a new XLogReader */
extern XLogReaderState *XLogReaderAllocate(int wal_segment_size,
const char *waldir,
- XLogReaderRoutine *routine,
- void *private_data);
-extern XLogReaderRoutine *LocalXLogReaderRoutine(void);
+ WALSegmentCleanupCB cleanup_cb);
/* Free an XLogReader */
extern void XLogReaderFree(XLogReaderState *state);
@@ -272,12 +249,17 @@ extern void XLogReaderFree(XLogReaderState *state);
/* Position the XLogReader to given record */
extern void XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr);
#ifdef FRONTEND
-extern XLogRecPtr XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr);
+/* Function type definition for the read_page callback */
+typedef bool (*XLogFindNextRecordCB) (XLogReaderState *xlogreader,
+ void *private);
+extern XLogRecPtr XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
+ XLogFindNextRecordCB read_page, void *private);
#endif /* FRONTEND */
/* Read the next XLog record. Returns NULL on end-of-WAL or failure */
-extern struct XLogRecord *XLogReadRecord(XLogReaderState *state,
- char **errormsg);
+extern XLogReadRecordResult XLogReadRecord(XLogReaderState *state,
+ XLogRecord **record,
+ char **errormsg);
/* Validate a page */
extern bool XLogReaderValidatePageHeader(XLogReaderState *state,
@@ -297,6 +279,7 @@ typedef struct WALReadError
} WALReadError;
extern bool WALRead(XLogReaderState *state,
+ WALSegmentOpenCB segopenfn, WALSegmentCloseCB sgclosefn,
char *buf, XLogRecPtr startptr, Size count,
TimeLineID tli, WALReadError *errinfo);
diff --git a/src/include/access/xlogutils.h b/src/include/access/xlogutils.h
index c287edf206..482e4ced69 100644
--- a/src/include/access/xlogutils.h
+++ b/src/include/access/xlogutils.h
@@ -47,9 +47,7 @@ extern Buffer XLogReadBufferExtended(RelFileNode rnode, ForkNumber forknum,
extern Relation CreateFakeRelcacheEntry(RelFileNode rnode);
extern void FreeFakeRelcacheEntry(Relation fakerel);
-extern bool read_local_xlog_page(XLogReaderState *state,
- XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetRecPtr, char *cur_page);
+extern bool read_local_xlog_page(XLogReaderState *state);
extern void wal_segment_open(XLogReaderState *state,
XLogSegNo nextSegNo,
TimeLineID *tli_p);
diff --git a/src/include/replication/logical.h b/src/include/replication/logical.h
index c2f2475e5d..a743db49cf 100644
--- a/src/include/replication/logical.h
+++ b/src/include/replication/logical.h
@@ -29,6 +29,10 @@ typedef void (*LogicalOutputPluginWriterUpdateProgress) (struct LogicalDecodingC
TransactionId xid
);
+typedef struct LogicalDecodingContext LogicalDecodingContext;
+
+typedef bool (*LogicalDecodingXLogPageReadCB)(XLogReaderState *ctx);
+
typedef struct LogicalDecodingContext
{
/* memory context this is all allocated in */
@@ -39,6 +43,7 @@ typedef struct LogicalDecodingContext
/* infrastructure pieces for decoding */
XLogReaderState *reader;
+ LogicalDecodingXLogPageReadCB page_read;
struct ReorderBuffer *reorder;
struct SnapBuild *snapshot_builder;
@@ -95,14 +100,16 @@ extern LogicalDecodingContext *CreateInitDecodingContext(char *plugin,
List *output_plugin_options,
bool need_full_snapshot,
XLogRecPtr restart_lsn,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress);
extern LogicalDecodingContext *CreateDecodingContext(XLogRecPtr start_lsn,
List *output_plugin_options,
bool fast_forward,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress);
--
2.18.4
----Next_Part(Thu_Jul__2_13_53_30_2020_072)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
filename="v15-0003-Remove-globals-readOff-readLen-and-readSegNo.patch"
^ permalink raw reply [nested|flat] 70+ messages in thread
* [PATCH v17 2/4] Move page-reader out of XLogReadRecord
@ 2019-09-10 03:58 Kyotaro Horiguchi <[email protected]>
0 siblings, 0 replies; 70+ messages in thread
From: Kyotaro Horiguchi @ 2019-09-10 03:58 UTC (permalink / raw)
This is the second step of removing callbacks from WAL record reader.
Since it is essential to take in additional data while reading a
record, the function have to ask caller for new data while keeping
working state. Thus the function is turned into a state machine.
---
src/backend/access/transam/twophase.c | 15 +-
src/backend/access/transam/xlog.c | 58 +-
src/backend/access/transam/xlogreader.c | 697 +++++++++++-------
src/backend/access/transam/xlogutils.c | 17 +-
src/backend/replication/logical/logical.c | 26 +-
.../replication/logical/logicalfuncs.c | 13 +-
src/backend/replication/slotfuncs.c | 18 +-
src/backend/replication/walsender.c | 32 +-
src/bin/pg_rewind/parsexlog.c | 84 ++-
src/bin/pg_waldump/pg_waldump.c | 36 +-
src/include/access/xlogreader.h | 121 ++-
src/include/access/xlogutils.h | 4 +-
src/include/pg_config_manual.h | 2 +-
src/include/replication/logical.h | 11 +-
14 files changed, 646 insertions(+), 488 deletions(-)
diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index 80d2d20d6c..7d5a96a5f5 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -1325,11 +1325,8 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
XLogReaderState *xlogreader;
char *errormsg;
- xlogreader = XLogReaderAllocate(wal_segment_size, NULL,
- XL_ROUTINE(.page_read = &read_local_xlog_page,
- .segment_open = &wal_segment_open,
- .segment_close = &wal_segment_close),
- NULL);
+ xlogreader = XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+
if (!xlogreader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
@@ -1337,7 +1334,13 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
errdetail("Failed while allocating a WAL reading processor.")));
XLogBeginRead(xlogreader, lsn);
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!read_local_xlog_page(xlogreader))
+ break;
+ }
+
if (record == NULL)
ereport(ERROR,
(errcode_for_file_access(),
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 606328a65a..670996ae67 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -836,13 +836,6 @@ static XLogSource currentSource = XLOG_FROM_ANY;
static bool lastSourceFailed = false;
static bool pendingWalRcvRestart = false;
-typedef struct XLogPageReadPrivate
-{
- int emode;
- bool fetching_ckpt; /* are we fetching a checkpoint record? */
- bool randAccess;
-} XLogPageReadPrivate;
-
/*
* These variables track when we last obtained some WAL data to process,
* and where we got it from. (XLogReceiptSource is initially the same as
@@ -917,8 +910,8 @@ static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath,
static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli,
XLogSource source, bool notfoundOk);
static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, XLogSource source);
-static bool XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
+static bool XLogPageRead(XLogReaderState *xlogreader,
+ bool fetching_ckpt, int emode, bool randAccess);
static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
bool fetching_ckpt, XLogRecPtr tliRecPtr);
static int emode_for_corrupt_record(int emode, XLogRecPtr RecPtr);
@@ -1231,8 +1224,7 @@ XLogInsertRecord(XLogRecData *rdata,
appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len);
if (!debug_reader)
- debug_reader = XLogReaderAllocate(wal_segment_size, NULL,
- XL_ROUTINE(), NULL);
+ debug_reader = XLogReaderAllocate(wal_segment_size, NULL, NULL);
if (!debug_reader)
{
@@ -4344,15 +4336,10 @@ CleanupBackupHistory(void)
* record is available.
*/
static XLogRecord *
-ReadRecord(XLogReaderState *xlogreader, int emode,
- bool fetching_ckpt)
+ReadRecord(XLogReaderState *xlogreader, int emode, bool fetching_ckpt)
{
XLogRecord *record;
- XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
-
- private->fetching_ckpt = fetching_ckpt;
- private->emode = emode;
- private->randAccess = (xlogreader->ReadRecPtr == InvalidXLogRecPtr);
+ bool randAccess = (xlogreader->ReadRecPtr == InvalidXLogRecPtr);
/* This is the first attempt to read this page. */
lastSourceFailed = false;
@@ -4360,8 +4347,16 @@ ReadRecord(XLogReaderState *xlogreader, int emode,
for (;;)
{
char *errormsg;
+ XLogReadRecordResult result;
+
+ while ((result = XLogReadRecord(xlogreader, &record, &errormsg))
+ == XLREAD_NEED_DATA)
+ {
+ if (!XLogPageRead(xlogreader, fetching_ckpt, emode, randAccess))
+ break;
+
+ }
- record = XLogReadRecord(xlogreader, &errormsg);
ReadRecPtr = xlogreader->ReadRecPtr;
EndRecPtr = xlogreader->EndRecPtr;
if (record == NULL)
@@ -6380,7 +6375,6 @@ StartupXLOG(void)
bool backupFromStandby = false;
DBState dbstate_at_startup;
XLogReaderState *xlogreader;
- XLogPageReadPrivate private;
bool promoted = false;
struct stat st;
@@ -6539,13 +6533,9 @@ StartupXLOG(void)
OwnLatch(&XLogCtl->recoveryWakeupLatch);
/* Set up XLOG reader facility */
- MemSet(&private, 0, sizeof(XLogPageReadPrivate));
xlogreader =
- XLogReaderAllocate(wal_segment_size, NULL,
- XL_ROUTINE(.page_read = &XLogPageRead,
- .segment_open = NULL,
- .segment_close = wal_segment_close),
- &private);
+ XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+
if (!xlogreader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
@@ -11987,12 +11977,13 @@ CancelBackup(void)
* sleep and retry.
*/
static bool
-XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetRecPtr, char *readBuf)
+XLogPageRead(XLogReaderState *xlogreader,
+ bool fetching_ckpt, int emode, bool randAccess)
{
- XLogPageReadPrivate *private =
- (XLogPageReadPrivate *) xlogreader->private_data;
- int emode = private->emode;
+ char *readBuf = xlogreader->readBuf;
+ XLogRecPtr targetPagePtr = xlogreader->readPagePtr;
+ int reqLen = xlogreader->readLen;
+ XLogRecPtr targetRecPtr = xlogreader->ReadRecPtr;
uint32 targetPageOff;
XLogSegNo targetSegNo PG_USED_FOR_ASSERTS_ONLY;
int r;
@@ -12035,8 +12026,8 @@ retry:
flushedUpto < targetPagePtr + reqLen))
{
if (!WaitForWALToBecomeAvailable(targetPagePtr + reqLen,
- private->randAccess,
- private->fetching_ckpt,
+ randAccess,
+ fetching_ckpt,
targetRecPtr))
{
if (readFile >= 0)
@@ -12141,6 +12132,7 @@ retry:
goto next_record_is_invalid;
}
+ Assert(xlogreader->readPagePtr == targetPagePtr);
xlogreader->readLen = readLen;
return true;
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index 7d04995b7b..ba3e4868f7 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -40,7 +40,7 @@ static bool XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr,
int reqLen, bool header_inclusive);
static void XLogReaderInvalReadState(XLogReaderState *state);
static bool ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
- XLogRecPtr PrevRecPtr, XLogRecord *record, bool randAccess);
+ XLogRecPtr PrevRecPtr, XLogRecord *record);
static bool ValidXLogRecord(XLogReaderState *state, XLogRecord *record,
XLogRecPtr recptr);
static void ResetDecoder(XLogReaderState *state);
@@ -73,7 +73,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...)
*/
XLogReaderState *
XLogReaderAllocate(int wal_segment_size, const char *waldir,
- XLogReaderRoutine *routine, void *private_data)
+ WALSegmentCleanupCB cleanup_cb)
{
XLogReaderState *state;
@@ -84,7 +84,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
return NULL;
/* initialize caller-provided support functions */
- state->routine = *routine;
+ state->cleanup_cb = cleanup_cb;
state->max_block_id = -1;
@@ -107,8 +107,6 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
WALOpenSegmentInit(&state->seg, &state->segcxt, wal_segment_size,
waldir);
- /* system_identifier initialized to zeroes above */
- state->private_data = private_data;
/* ReadRecPtr, EndRecPtr and readLen initialized to zeroes above */
state->errormsg_buf = palloc_extended(MAX_ERRORMSG_LEN + 1,
MCXT_ALLOC_NO_OOM);
@@ -140,8 +138,8 @@ XLogReaderFree(XLogReaderState *state)
{
int block_id;
- if (state->seg.ws_file != -1)
- state->routine.segment_close(state);
+ if (state->seg.ws_file >= 0)
+ state->cleanup_cb(state);
for (block_id = 0; block_id <= XLR_MAX_BLOCK_ID; block_id++)
{
@@ -246,6 +244,7 @@ XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
/* Begin at the passed-in record pointer. */
state->EndRecPtr = RecPtr;
state->ReadRecPtr = InvalidXLogRecPtr;
+ state->readRecordState = XLREAD_NEXT_RECORD;
}
/*
@@ -254,317 +253,449 @@ XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
* XLogBeginRead() or XLogFindNextRecord() must be called before the first call
* to XLogReadRecord().
*
- * If the page_read callback fails to read the requested data, NULL is
- * returned. The callback is expected to have reported the error; errormsg
- * is set to NULL.
+ * This function may return XLREAD_NEED_DATA several times before returning a
+ * result record. The caller shall read in some new data then call this
+ * function again with the same parameters.
*
- * If the reading fails for some other reason, NULL is also returned, and
- * *errormsg is set to a string with details of the failure.
+ * When a record is successfully read, returns XLREAD_SUCCESS with result
+ * record being stored in *record. Otherwise *record is NULL.
*
- * The returned pointer (or *errormsg) points to an internal buffer that's
- * valid until the next call to XLogReadRecord.
+ * Returns XLREAD_NEED_DATA if more data is needed to finish reading the
+ * current record. In that case, state->readPagePtr and state->readLen inform
+ * the desired position and minimum length of data needed. The caller shall
+ * read in the requested data and set state->readBuf to point to a buffer
+ * containing it. The caller must also set state->readPageTLI and
+ * state->readLen to indicate the timeline that it was read from, and the
+ * length of data that is now available (which must be >= given readLen),
+ * respectively.
+ *
+ * If invalid data is encountered, returns XLREAD_FAIL with *record being set to
+ * NULL. *errormsg is set to a string with details of the failure.
+ * The returned pointer (or *errormsg) points to an internal buffer that's valid
+ * until the next call to XLogReadRecord.
+ *
+ *
+ * This function runs a state machine consists of the following states.
+ *
+ * XLREAD_NEXT_RECORD :
+ * The initial state, if called with valid RecPtr, try to read a record at
+ * that position. If invalid RecPtr is given try to read a record just after
+ * the last one previously read.
+ * This state ens after setting ReadRecPtr. Then goes to XLREAD_TOT_LEN.
+ *
+ * XLREAD_TOT_LEN:
+ * Examining record header. Ends after reading record total
+ * length. recordRemainLen and recordGotLen are initialized.
+ *
+ * XLREAD_FIRST_FRAGMENT:
+ * Reading the first fragment. Ends with finishing reading a single
+ * record. Goes to XLREAD_NEXT_RECORD if that's all or
+ * XLREAD_CONTINUATION if we have continuation.
+
+ * XLREAD_CONTINUATION:
+ * Reading continuation of record. Ends with finishing the whole record then
+ * goes to XLREAD_NEXT_RECORD. During this state, recordRemainLen indicates
+ * how much is left and readRecordBuf holds the partially assert
+ * record.recordContRecPtr points to the beginning of the next page where to
+ * continue.
+ *
+ * If wrong data found in any state, the state machine stays at the current
+ * state. This behavior allows to continue reading a reacord switching among
+ * different souces, while streaming replication.
*/
-XLogRecord *
-XLogReadRecord(XLogReaderState *state, char **errormsg)
+XLogReadRecordResult
+XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
{
- XLogRecPtr RecPtr;
- XLogRecord *record;
- XLogRecPtr targetPagePtr;
- bool randAccess;
- uint32 len,
- total_len;
- uint32 targetRecOff;
- uint32 pageHeaderSize;
- bool gotheader;
+ XLogRecord *prec;
- /*
- * randAccess indicates whether to verify the previous-record pointer of
- * the record we're reading. We only do this if we're reading
- * sequentially, which is what we initially assume.
- */
- randAccess = false;
+ *record = NULL;
/* reset error state */
*errormsg = NULL;
state->errormsg_buf[0] = '\0';
- ResetDecoder(state);
-
- RecPtr = state->EndRecPtr;
-
- if (state->ReadRecPtr != InvalidXLogRecPtr)
- {
- /* read the record after the one we just read */
-
- /*
- * EndRecPtr is pointing to end+1 of the previous WAL record. If
- * we're at a page boundary, no more records can fit on the current
- * page. We must skip over the page header, but we can't do that until
- * we've read in the page, since the header size is variable.
- */
- }
- else
- {
- /*
- * Caller supplied a position to start at.
- *
- * In this case, EndRecPtr should already be pointing to a valid
- * record starting position.
- */
- Assert(XRecOffIsValid(RecPtr));
- randAccess = true;
- }
-
- state->currRecPtr = RecPtr;
-
- targetPagePtr = RecPtr - (RecPtr % XLOG_BLCKSZ);
- targetRecOff = RecPtr % XLOG_BLCKSZ;
-
- /*
- * Read the page containing the record into state->readBuf. Request enough
- * byte to cover the whole record header, or at least the part of it that
- * fits on the same page.
- */
- while (XLogNeedData(state, targetPagePtr,
- Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
- targetRecOff != 0))
- {
- if (!state->routine.page_read(state, state->readPagePtr, state->readLen,
- RecPtr, state->readBuf))
- break;
- }
-
- if (!state->page_verified)
- goto err;
-
- /*
- * We have at least the page header, so we can examine it now.
- */
- pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
- if (targetRecOff == 0)
- {
- /*
- * At page start, so skip over page header.
- */
- RecPtr += pageHeaderSize;
- targetRecOff = pageHeaderSize;
- }
- else if (targetRecOff < pageHeaderSize)
- {
- report_invalid_record(state, "invalid record offset at %X/%X",
- LSN_FORMAT_ARGS(RecPtr));
- goto err;
- }
-
- if ((((XLogPageHeader) state->readBuf)->xlp_info & XLP_FIRST_IS_CONTRECORD) &&
- targetRecOff == pageHeaderSize)
- {
- report_invalid_record(state, "contrecord is requested by %X/%X",
- LSN_FORMAT_ARGS(RecPtr));
- goto err;
- }
-
- /* XLogNeedData has verified the page header */
- Assert(pageHeaderSize <= state->readLen);
-
- /*
- * Read the record length.
- *
- * NB: Even though we use an XLogRecord pointer here, the whole record
- * header might not fit on this page. xl_tot_len is the first field of the
- * struct, so it must be on this page (the records are MAXALIGNed), but we
- * cannot access any other fields until we've verified that we got the
- * whole header.
- */
- record = (XLogRecord *) (state->readBuf + RecPtr % XLOG_BLCKSZ);
- total_len = record->xl_tot_len;
-
- /*
- * If the whole record header is on this page, validate it immediately.
- * Otherwise do just a basic sanity check on xl_tot_len, and validate the
- * rest of the header after reading it from the next page. The xl_tot_len
- * check is necessary here to ensure that we enter the "Need to reassemble
- * record" code path below; otherwise we might fail to apply
- * ValidXLogRecordHeader at all.
- */
- if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
- {
- if (!ValidXLogRecordHeader(state, RecPtr, state->ReadRecPtr, record,
- randAccess))
- goto err;
- gotheader = true;
- }
- else
- {
- /* XXX: more validation should be done here */
- if (total_len < SizeOfXLogRecord)
- {
- report_invalid_record(state,
- "invalid record length at %X/%X: wanted %u, got %u",
- LSN_FORMAT_ARGS(RecPtr),
- (uint32) SizeOfXLogRecord, total_len);
- goto err;
- }
- gotheader = false;
- }
-
- len = XLOG_BLCKSZ - RecPtr % XLOG_BLCKSZ;
- if (total_len > len)
+ switch (state->readRecordState)
{
- /* Need to reassemble record */
- char *contdata;
- XLogPageHeader pageHeader;
- char *buffer;
- uint32 gotlen;
-
- /*
- * Enlarge readRecordBuf as needed.
- */
- if (total_len > state->readRecordBufSize &&
- !allocate_recordbuf(state, total_len))
- {
- /* We treat this as a "bogus data" condition */
- report_invalid_record(state, "record length %u at %X/%X too long",
- total_len, LSN_FORMAT_ARGS(RecPtr));
- goto err;
- }
+ case XLREAD_NEXT_RECORD:
+ ResetDecoder(state);
- /* Copy the first fragment of the record from the first page. */
- memcpy(state->readRecordBuf,
- state->readBuf + RecPtr % XLOG_BLCKSZ, len);
- buffer = state->readRecordBuf + len;
- gotlen = len;
-
- do
- {
- int rest_len = total_len - gotlen;
-
- /* Calculate pointer to beginning of next page */
- targetPagePtr += XLOG_BLCKSZ;
-
- /* Wait for the next page to become available */
- while (XLogNeedData(state, targetPagePtr,
- Min(rest_len, XLOG_BLCKSZ),
- false))
+ if (state->ReadRecPtr != InvalidXLogRecPtr)
{
- if (!state->routine.page_read(state, state->readPagePtr,
- state->readLen,
- state->ReadRecPtr,
- state->readBuf))
- break;
+ /* read the record after the one we just read */
+
+ /*
+ * EndRecPtr is pointing to end+1 of the previous WAL record.
+ * If we're at a page boundary, no more records can fit on the
+ * current page. We must skip over the page header, but we
+ * can't do that until we've read in the page, since the header
+ * size is variable.
+ */
+ state->PrevRecPtr = state->ReadRecPtr;
+ state->ReadRecPtr = state->EndRecPtr;
+ }
+ else
+ {
+ /*
+ * Caller supplied a position to start at.
+ *
+ * In this case, EndRecPtr should already be pointing to a
+ * valid record starting position.
+ */
+ Assert(XRecOffIsValid(state->EndRecPtr));
+ state->ReadRecPtr = state->EndRecPtr;
+
+ /*
+ * We cannot verify the previous-record pointer when we're
+ * seeking to a particular record. Reset PrevRecPtr so that we
+ * won't try doing that.
+ */
+ state->PrevRecPtr = InvalidXLogRecPtr;
+ state->EndRecPtr = InvalidXLogRecPtr; /* to be tidy */
}
+ state->record_verified = false;
+ state->readRecordState = XLREAD_TOT_LEN;
+ /* fall through */
+
+ case XLREAD_TOT_LEN:
+ {
+ uint32 total_len;
+ uint32 pageHeaderSize;
+ XLogRecPtr targetPagePtr;
+ uint32 targetRecOff;
+ XLogPageHeader pageHeader;
+
+ targetPagePtr =
+ state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
+ targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+
+ /*
+ * Check if we have enough data. For the first record in the page,
+ * the requesting length doesn't contain page header.
+ */
+ if (XLogNeedData(state, targetPagePtr,
+ Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+ targetRecOff != 0))
+ return XLREAD_NEED_DATA;
+
+ /* error out if caller supplied bogus page */
if (!state->page_verified)
goto err;
- Assert(SizeOfXLogShortPHD <= state->readLen);
+ /* examine page header now. */
+ pageHeaderSize =
+ XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+ if (targetRecOff == 0)
+ {
+ /* At page start, so skip over page header. */
+ state->ReadRecPtr += pageHeaderSize;
+ targetRecOff = pageHeaderSize;
+ }
+ else if (targetRecOff < pageHeaderSize)
+ {
+ report_invalid_record(state, "invalid record offset at %X/%X",
+ LSN_FORMAT_ARGS(state->ReadRecPtr));
+ goto err;
+ }
- /* Check that the continuation on next page looks valid */
pageHeader = (XLogPageHeader) state->readBuf;
- if (!(pageHeader->xlp_info & XLP_FIRST_IS_CONTRECORD))
+ if ((pageHeader->xlp_info & XLP_FIRST_IS_CONTRECORD) &&
+ targetRecOff == pageHeaderSize)
{
- report_invalid_record(state,
- "there is no contrecord flag at %X/%X",
- LSN_FORMAT_ARGS(RecPtr));
+ report_invalid_record(state, "contrecord is requested by %X/%X",
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr);
goto err;
}
- /*
- * Cross-check that xlp_rem_len agrees with how much of the record
- * we expect there to be left.
- */
- if (pageHeader->xlp_rem_len == 0 ||
- total_len != (pageHeader->xlp_rem_len + gotlen))
- {
- report_invalid_record(state,
- "invalid contrecord length %u (expected %lld) at %X/%X",
- pageHeader->xlp_rem_len,
- ((long long) total_len) - gotlen,
- LSN_FORMAT_ARGS(RecPtr));
- goto err;
- }
-
- /* Append the continuation from this page to the buffer */
- pageHeaderSize = XLogPageHeaderSize(pageHeader);
-
+ /* XLogNeedData has verified the page header */
Assert(pageHeaderSize <= state->readLen);
- contdata = (char *) state->readBuf + pageHeaderSize;
- len = XLOG_BLCKSZ - pageHeaderSize;
- if (pageHeader->xlp_rem_len < len)
- len = pageHeader->xlp_rem_len;
+ /*
+ * Read the record length.
+ *
+ * NB: Even though we use an XLogRecord pointer here, the whole
+ * record header might not fit on this page. xl_tot_len is the first
+ * field of the struct, so it must be on this page (the records are
+ * MAXALIGNed), but we cannot access any other fields until we've
+ * verified that we got the whole header.
+ */
+ prec = (XLogRecord *) (state->readBuf +
+ state->ReadRecPtr % XLOG_BLCKSZ);
+ total_len = prec->xl_tot_len;
- Assert (pageHeaderSize + len <= state->readLen);
- memcpy(buffer, (char *) contdata, len);
- buffer += len;
- gotlen += len;
+ /*
+ * If the whole record header is on this page, validate it
+ * immediately. Otherwise do just a basic sanity check on
+ * xl_tot_len, and validate the rest of the header after reading it
+ * from the next page. The xl_tot_len check is necessary here to
+ * ensure that we enter the XLREAD_CONTINUATION state below;
+ * otherwise we might fail to apply ValidXLogRecordHeader at all.
+ */
+ if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
+ {
+ if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
+ state->PrevRecPtr, prec))
+ goto err;
- /* If we just reassembled the record header, validate it. */
- if (!gotheader)
+ state->record_verified = true;
+ }
+ else
{
- record = (XLogRecord *) state->readRecordBuf;
- if (!ValidXLogRecordHeader(state, RecPtr, state->ReadRecPtr,
- record, randAccess))
+ /* XXX: more validation should be done here */
+ if (total_len < SizeOfXLogRecord)
+ {
+ report_invalid_record(state,
+ "invalid record length at %X/%X: wanted %u, got %u",
+ LSN_FORMAT_ARGS(state->ReadRecPtr),
+ (uint32) SizeOfXLogRecord, total_len);
goto err;
- gotheader = true;
+ }
}
- } while (gotlen < total_len);
-
- Assert(gotheader);
- record = (XLogRecord *) state->readRecordBuf;
- if (!ValidXLogRecord(state, record, RecPtr))
- goto err;
+ /*
+ * Wait for the rest of the record, or the part of the record that
+ * fit on the first page if crossed a page boundary, to become
+ * available.
+ */
+ state->recordGotLen = 0;
+ state->recordRemainLen = total_len;
+ state->readRecordState = XLREAD_FIRST_FRAGMENT;
+ }
+ /* fall through */
- pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
- state->ReadRecPtr = RecPtr;
- state->EndRecPtr = targetPagePtr + pageHeaderSize
- + MAXALIGN(pageHeader->xlp_rem_len);
- }
- else
- {
- /* Wait for the record data to become available */
- while (XLogNeedData(state, targetPagePtr,
- Min(targetRecOff + total_len, XLOG_BLCKSZ), true))
+ case XLREAD_FIRST_FRAGMENT:
{
- if (!state->routine.page_read(state, state->readPagePtr,
- state->readLen,
- state->ReadRecPtr, state->readBuf))
+ uint32 total_len = state->recordRemainLen;
+ uint32 request_len;
+ uint32 record_len;
+ XLogRecPtr targetPagePtr;
+ uint32 targetRecOff;
+
+ /*
+ * Wait for the rest of the record on the first page to become
+ * available
+ */
+ targetPagePtr =
+ state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
+ targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+
+ request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ);
+ record_len = request_len - targetRecOff;
+
+ /* ReadRecPtr contains page header */
+ Assert (targetRecOff != 0);
+ if (XLogNeedData(state, targetPagePtr, request_len, true))
+ return XLREAD_NEED_DATA;
+
+ /* error out if caller supplied bogus page */
+ if (!state->page_verified)
+ goto err;
+
+ prec = (XLogRecord *) (state->readBuf + targetRecOff);
+
+ /* validate record header if not yet */
+ if (!state->record_verified && record_len >= SizeOfXLogRecord)
+ {
+ if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
+ state->PrevRecPtr, prec))
+ goto err;
+
+ state->record_verified = true;
+ }
+
+
+ if (total_len == record_len)
+ {
+ /* Record does not cross a page boundary */
+ Assert(state->record_verified);
+
+ if (!ValidXLogRecord(state, prec, state->ReadRecPtr))
+ goto err;
+
+ state->record_verified = true; /* to be tidy */
+
+ /* We already checked the header earlier */
+ state->EndRecPtr = state->ReadRecPtr + MAXALIGN(record_len);
+
+ *record = prec;
+ state->readRecordState = XLREAD_NEXT_RECORD;
break;
+ }
+
+ /*
+ * The record continues on the next page. Need to reassemble
+ * record
+ */
+ Assert(total_len > record_len);
+
+ /* Enlarge readRecordBuf as needed. */
+ if (total_len > state->readRecordBufSize &&
+ !allocate_recordbuf(state, total_len))
+ {
+ /* We treat this as a "bogus data" condition */
+ report_invalid_record(state,
+ "record length %u at %X/%X too long",
+ total_len,
+ LSN_FORMAT_ARGS(state->ReadRecPtr));
+ goto err;
+ }
+
+ /* Copy the first fragment of the record from the first page. */
+ memcpy(state->readRecordBuf, state->readBuf + targetRecOff,
+ record_len);
+ state->recordGotLen += record_len;
+ state->recordRemainLen -= record_len;
+
+ /* Calculate pointer to beginning of next page */
+ state->recordContRecPtr = state->ReadRecPtr + record_len;
+ Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0);
+
+ state->readRecordState = XLREAD_CONTINUATION;
}
+ /* fall through */
+
+ case XLREAD_CONTINUATION:
+ {
+ XLogPageHeader pageHeader;
+ uint32 pageHeaderSize;
+ XLogRecPtr targetPagePtr;
+
+ /* we enter this state only if we haven't read the whole record. */
+ Assert (state->recordRemainLen > 0);
+
+ while(state->recordRemainLen > 0)
+ {
+ char *contdata;
+ uint32 request_len;
+ uint32 record_len;
+
+ /* Wait for the next page to become available */
+ targetPagePtr = state->recordContRecPtr;
+
+ /* this request contains page header */
+ Assert (targetPagePtr != 0);
+ if (XLogNeedData(state, targetPagePtr,
+ Min(state->recordRemainLen, XLOG_BLCKSZ),
+ false))
+ return XLREAD_NEED_DATA;
+
+ if (!state->page_verified)
+ goto err;
+
+ Assert(SizeOfXLogShortPHD <= state->readLen);
- if (!state->page_verified)
- goto err;
+ /* Check that the continuation on next page looks valid */
+ pageHeader = (XLogPageHeader) state->readBuf;
+ if (!(pageHeader->xlp_info & XLP_FIRST_IS_CONTRECORD))
+ {
+ report_invalid_record(
+ state,
+ "there is no contrecord flag at %X/%X reading %X/%X",
+ (uint32) (state->recordContRecPtr >> 32),
+ (uint32) state->recordContRecPtr,
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr);
+ goto err;
+ }
- /* Record does not cross a page boundary */
- if (!ValidXLogRecord(state, record, RecPtr))
- goto err;
+ /*
+ * Cross-check that xlp_rem_len agrees with how much of the
+ * record we expect there to be left.
+ */
+ if (pageHeader->xlp_rem_len == 0 ||
+ pageHeader->xlp_rem_len != state->recordRemainLen)
+ {
+ report_invalid_record(
+ state,
+ "invalid contrecord length %u at %X/%X reading %X/%X, expected %u",
+ pageHeader->xlp_rem_len,
+ (uint32) (state->recordContRecPtr >> 32),
+ (uint32) state->recordContRecPtr,
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr,
+ state->recordRemainLen);
+ goto err;
+ }
- state->EndRecPtr = RecPtr + MAXALIGN(total_len);
+ /* Append the continuation from this page to the buffer */
+ pageHeaderSize = XLogPageHeaderSize(pageHeader);
- state->ReadRecPtr = RecPtr;
+ /*
+ * XLogNeedData should have ensured that the whole page header
+ * was read
+ */
+ Assert(state->readLen >= pageHeaderSize);
+
+ contdata = (char *) state->readBuf + pageHeaderSize;
+ record_len = XLOG_BLCKSZ - pageHeaderSize;
+ if (pageHeader->xlp_rem_len < record_len)
+ record_len = pageHeader->xlp_rem_len;
+
+ request_len = record_len + pageHeaderSize;
+
+ /* XLogNeedData should have ensured all needed data was read */
+ Assert (state->readLen >= request_len);
+
+ memcpy(state->readRecordBuf + state->recordGotLen,
+ (char *) contdata, record_len);
+ state->recordGotLen += record_len;
+ state->recordRemainLen -= record_len;
+
+ /* If we just reassembled the record header, validate it. */
+ if (!state->record_verified)
+ {
+ Assert(state->recordGotLen >= SizeOfXLogRecord);
+ if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
+ state->PrevRecPtr,
+ (XLogRecord *) state->readRecordBuf))
+ goto err;
+
+ state->record_verified = true;
+ }
+
+ /* Calculate pointer to beginning of next page, and continue */
+ state->recordContRecPtr += XLOG_BLCKSZ;
+ }
+
+ /* targetPagePtr is pointing the last-read page here */
+ prec = (XLogRecord *) state->readRecordBuf;
+ if (!ValidXLogRecord(state, prec, state->ReadRecPtr))
+ goto err;
+
+ pageHeaderSize =
+ XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+ state->EndRecPtr = targetPagePtr + pageHeaderSize
+ + MAXALIGN(pageHeader->xlp_rem_len);
+
+ *record = prec;
+ state->readRecordState = XLREAD_NEXT_RECORD;
+ break;
+ }
}
/*
* Special processing if it's an XLOG SWITCH record
*/
- if (record->xl_rmid == RM_XLOG_ID &&
- (record->xl_info & ~XLR_INFO_MASK) == XLOG_SWITCH)
+ if ((*record)->xl_rmid == RM_XLOG_ID &&
+ ((*record)->xl_info & ~XLR_INFO_MASK) == XLOG_SWITCH)
{
/* Pretend it extends to end of segment */
state->EndRecPtr += state->segcxt.ws_segsize - 1;
state->EndRecPtr -= XLogSegmentOffset(state->EndRecPtr, state->segcxt.ws_segsize);
}
- if (DecodeXLogRecord(state, record, errormsg))
- return record;
- else
- return NULL;
+ Assert (!*record || state->readLen >= 0);
+ if (DecodeXLogRecord(state, *record, errormsg))
+ return XLREAD_SUCCESS;
+
+ *record = NULL;
+ return XLREAD_FAIL;
err:
/*
- * Invalidate the read state. We might read from a different source after
+ * Invalidate the read page. We might read from a different source after
* failure.
*/
XLogReaderInvalReadState(state);
@@ -572,7 +703,8 @@ err:
if (state->errormsg_buf[0] != '\0')
*errormsg = state->errormsg_buf;
- return NULL;
+ *record = NULL;
+ return XLREAD_FAIL;
}
/*
@@ -724,11 +856,12 @@ XLogReaderInvalReadState(XLogReaderState *state)
*
* This is just a convenience subroutine to avoid duplicated code in
* XLogReadRecord. It's not intended for use from anywhere else.
+ *
+ * If PrevRecPtr is valid, the xl_prev is is cross-checked with it.
*/
static bool
ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
- XLogRecPtr PrevRecPtr, XLogRecord *record,
- bool randAccess)
+ XLogRecPtr PrevRecPtr, XLogRecord *record)
{
if (record->xl_tot_len < SizeOfXLogRecord)
{
@@ -745,7 +878,7 @@ ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
record->xl_rmid, LSN_FORMAT_ARGS(RecPtr));
return false;
}
- if (randAccess)
+ if (PrevRecPtr == InvalidXLogRecPtr)
{
/*
* We can't exactly verify the prev-link, but surely it should be less
@@ -975,11 +1108,14 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
* XLogReadRecord() will read the next valid record.
*/
XLogRecPtr
-XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
+XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
+ XLogFindNextRecordCB read_page, void *private)
{
XLogRecPtr tmpRecPtr;
XLogRecPtr found = InvalidXLogRecPtr;
XLogPageHeader header;
+ XLogRecord *record;
+ XLogReadRecordResult result;
char *errormsg;
Assert(!XLogRecPtrIsInvalid(RecPtr));
@@ -1012,9 +1148,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
while(XLogNeedData(state, targetPagePtr, targetRecOff,
targetRecOff != 0))
{
- if (!state->routine.page_read(state, state->readPagePtr,
- state->readLen,
- state->ReadRecPtr, state->readBuf))
+ if (!read_page(state, private))
break;
}
@@ -1066,8 +1200,16 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
* or we just jumped over the remaining data of a continuation.
*/
XLogBeginRead(state, tmpRecPtr);
- while (XLogReadRecord(state, &errormsg) != NULL)
+ while ((result = XLogReadRecord(state, &record, &errormsg)) !=
+ XLREAD_FAIL)
{
+ if (result == XLREAD_NEED_DATA)
+ {
+ if (!read_page(state, private))
+ goto err;
+ continue;
+ }
+
/* past the record we've found, break out */
if (RecPtr <= state->ReadRecPtr)
{
@@ -1087,9 +1229,9 @@ err:
#endif /* FRONTEND */
/*
- * Helper function to ease writing of XLogRoutine->page_read callbacks.
- * If this function is used, caller must supply a segment_open callback in
- * 'state', as that is used here.
+ * Helper function to ease writing of page_read callback.
+ * If this function is used, caller must supply a segment_open callback and
+ * segment_close callback as that is used here.
*
* Read 'count' bytes into 'buf', starting at location 'startptr', from WAL
* fetched from timeline 'tli'.
@@ -1102,6 +1244,7 @@ err:
*/
bool
WALRead(XLogReaderState *state,
+ WALSegmentOpenCB segopenfn, WALSegmentCloseCB segclosefn,
char *buf, XLogRecPtr startptr, Size count, TimeLineID tli,
WALReadError *errinfo)
{
@@ -1133,10 +1276,10 @@ WALRead(XLogReaderState *state,
XLogSegNo nextSegNo;
if (state->seg.ws_file >= 0)
- state->routine.segment_close(state);
+ segclosefn(state);
XLByteToSeg(recptr, nextSegNo, state->segcxt.ws_segsize);
- state->routine.segment_open(state, nextSegNo, &tli);
+ segopenfn(state, nextSegNo, &tli);
/* This shouldn't happen -- indicates a bug in segment_open */
Assert(state->seg.ws_file >= 0);
diff --git a/src/backend/access/transam/xlogutils.c b/src/backend/access/transam/xlogutils.c
index 46eda33f25..b003990745 100644
--- a/src/backend/access/transam/xlogutils.c
+++ b/src/backend/access/transam/xlogutils.c
@@ -686,8 +686,7 @@ XLogTruncateRelation(RelFileNode rnode, ForkNumber forkNum,
void
XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wantLength)
{
- const XLogRecPtr lastReadPage = state->seg.ws_segno *
- state->segcxt.ws_segsize + state->readLen;
+ const XLogRecPtr lastReadPage = state->readPagePtr;
Assert(wantPage != InvalidXLogRecPtr && wantPage % XLOG_BLCKSZ == 0);
Assert(wantLength <= XLOG_BLCKSZ);
@@ -702,7 +701,7 @@ XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wa
* current TLI has since become historical.
*/
if (lastReadPage == wantPage &&
- state->readLen != 0 &&
+ state->page_verified &&
lastReadPage + state->readLen >= wantPage + Min(wantLength, XLOG_BLCKSZ - 1))
return;
@@ -788,6 +787,7 @@ wal_segment_open(XLogReaderState *state, XLogSegNo nextSegNo,
char path[MAXPGPATH];
XLogFilePath(path, tli, nextSegNo, state->segcxt.ws_segsize);
+ elog(LOG, "HOGE: %lu, %d => %s", nextSegNo, tli, path);
state->seg.ws_file = BasicOpenFile(path, O_RDONLY | PG_BINARY);
if (state->seg.ws_file >= 0)
return;
@@ -825,9 +825,11 @@ wal_segment_close(XLogReaderState *state)
* loop for now.
*/
bool
-read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *cur_page)
+read_local_xlog_page(XLogReaderState *state)
{
+ XLogRecPtr targetPagePtr = state->readPagePtr;
+ int reqLen = state->readLen;
+ char *cur_page = state->readBuf;
XLogRecPtr read_upto,
loc;
TimeLineID tli;
@@ -940,11 +942,12 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
* as 'count', read the whole page anyway. It's guaranteed to be
* zero-padded up to the page boundary if it's incomplete.
*/
- if (!WALRead(state, cur_page, targetPagePtr, XLOG_BLCKSZ, tli,
- &errinfo))
+ if (!WALRead(state, wal_segment_open, wal_segment_close,
+ cur_page, targetPagePtr, XLOG_BLCKSZ, tli, &errinfo))
WALReadRaiseError(&errinfo);
/* number of valid bytes in the buffer */
+ state->readPagePtr = targetPagePtr;
state->readLen = count;
return true;
}
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index 37b75deb72..9cbd8afa49 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -148,7 +148,8 @@ StartupDecodingContext(List *output_plugin_options,
TransactionId xmin_horizon,
bool need_full_snapshot,
bool fast_forward,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress)
@@ -198,11 +199,12 @@ StartupDecodingContext(List *output_plugin_options,
ctx->slot = slot;
- ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, xl_routine, ctx);
+ ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, cleanup_cb);
if (!ctx->reader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
errmsg("out of memory")));
+ ctx->page_read = page_read;
ctx->reorder = ReorderBufferAllocate();
ctx->snapshot_builder =
@@ -319,7 +321,8 @@ CreateInitDecodingContext(const char *plugin,
List *output_plugin_options,
bool need_full_snapshot,
XLogRecPtr restart_lsn,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress)
@@ -422,7 +425,7 @@ CreateInitDecodingContext(const char *plugin,
ctx = StartupDecodingContext(NIL, restart_lsn, xmin_horizon,
need_full_snapshot, false,
- xl_routine, prepare_write, do_write,
+ page_read, cleanup_cb, prepare_write, do_write,
update_progress);
/* call output plugin initialization callback */
@@ -476,7 +479,8 @@ LogicalDecodingContext *
CreateDecodingContext(XLogRecPtr start_lsn,
List *output_plugin_options,
bool fast_forward,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress)
@@ -528,8 +532,8 @@ CreateDecodingContext(XLogRecPtr start_lsn,
ctx = StartupDecodingContext(output_plugin_options,
start_lsn, InvalidTransactionId, false,
- fast_forward, xl_routine, prepare_write,
- do_write, update_progress);
+ fast_forward, page_read, cleanup_cb,
+ prepare_write, do_write, update_progress);
/* call output plugin initialization callback */
old_context = MemoryContextSwitchTo(ctx->context);
@@ -585,7 +589,13 @@ DecodingContextFindStartpoint(LogicalDecodingContext *ctx)
char *err = NULL;
/* the read_page callback waits for new WAL */
- record = XLogReadRecord(ctx->reader, &err);
+ while (XLogReadRecord(ctx->reader, &record, &err) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!ctx->page_read(ctx->reader))
+ break;
+ }
+
if (err)
elog(ERROR, "%s", err);
if (!record)
diff --git a/src/backend/replication/logical/logicalfuncs.c b/src/backend/replication/logical/logicalfuncs.c
index f7e055874e..b69f6911c5 100644
--- a/src/backend/replication/logical/logicalfuncs.c
+++ b/src/backend/replication/logical/logicalfuncs.c
@@ -233,9 +233,8 @@ pg_logical_slot_get_changes_guts(FunctionCallInfo fcinfo, bool confirm, bool bin
ctx = CreateDecodingContext(InvalidXLogRecPtr,
options,
false,
- XL_ROUTINE(.page_read = read_local_xlog_page,
- .segment_open = wal_segment_open,
- .segment_close = wal_segment_close),
+ read_local_xlog_page,
+ wal_segment_close,
LogicalOutputPrepareWrite,
LogicalOutputWrite, NULL);
@@ -284,7 +283,13 @@ pg_logical_slot_get_changes_guts(FunctionCallInfo fcinfo, bool confirm, bool bin
XLogRecord *record;
char *errm = NULL;
- record = XLogReadRecord(ctx->reader, &errm);
+ while (XLogReadRecord(ctx->reader, &record, &errm) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!ctx->page_read(ctx->reader))
+ break;
+ }
+
if (errm)
elog(ERROR, "%s", errm);
diff --git a/src/backend/replication/slotfuncs.c b/src/backend/replication/slotfuncs.c
index 9817b44113..deb7cf15eb 100644
--- a/src/backend/replication/slotfuncs.c
+++ b/src/backend/replication/slotfuncs.c
@@ -153,9 +153,8 @@ create_logical_replication_slot(char *name, char *plugin,
ctx = CreateInitDecodingContext(plugin, NIL,
false, /* just catalogs is OK */
restart_lsn,
- XL_ROUTINE(.page_read = read_local_xlog_page,
- .segment_open = wal_segment_open,
- .segment_close = wal_segment_close),
+ read_local_xlog_page,
+ wal_segment_close,
NULL, NULL, NULL);
/*
@@ -512,9 +511,8 @@ pg_logical_replication_slot_advance(XLogRecPtr moveto)
ctx = CreateDecodingContext(InvalidXLogRecPtr,
NIL,
true, /* fast_forward */
- XL_ROUTINE(.page_read = read_local_xlog_page,
- .segment_open = wal_segment_open,
- .segment_close = wal_segment_close),
+ read_local_xlog_page,
+ wal_segment_close,
NULL, NULL, NULL);
/*
@@ -536,7 +534,13 @@ pg_logical_replication_slot_advance(XLogRecPtr moveto)
* Read records. No changes are generated in fast_forward mode,
* but snapbuilder/slot statuses are updated properly.
*/
- record = XLogReadRecord(ctx->reader, &errm);
+ while (XLogReadRecord(ctx->reader, &record, &errm) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!ctx->page_read(ctx->reader))
+ break;
+ }
+
if (errm)
elog(ERROR, "%s", errm);
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index c36b91ae5e..062093727d 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -580,10 +580,7 @@ StartReplication(StartReplicationCmd *cmd)
/* create xlogreader for physical replication */
xlogreader =
- XLogReaderAllocate(wal_segment_size, NULL,
- XL_ROUTINE(.segment_open = WalSndSegmentOpen,
- .segment_close = wal_segment_close),
- NULL);
+ XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
if (!xlogreader)
ereport(ERROR,
@@ -807,9 +804,11 @@ StartReplication(StartReplicationCmd *cmd)
* set every time WAL is flushed.
*/
static bool
-logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetRecPtr, char *cur_page)
+logical_read_xlog_page(XLogReaderState *state)
{
+ XLogRecPtr targetPagePtr = state->readPagePtr;
+ int reqLen = state->readLen;
+ char *cur_page = state->readBuf;
XLogRecPtr flushptr;
int count;
WALReadError errinfo;
@@ -837,7 +836,7 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
count = flushptr - targetPagePtr; /* part of the page available */
/* now actually read the data, we know it's there */
- if (!WALRead(state,
+ if (!WALRead(state, WalSndSegmentOpen, wal_segment_close,
cur_page,
targetPagePtr,
XLOG_BLCKSZ,
@@ -1011,9 +1010,8 @@ CreateReplicationSlot(CreateReplicationSlotCmd *cmd)
ctx = CreateInitDecodingContext(cmd->plugin, NIL, need_full_snapshot,
InvalidXLogRecPtr,
- XL_ROUTINE(.page_read = logical_read_xlog_page,
- .segment_open = WalSndSegmentOpen,
- .segment_close = wal_segment_close),
+ logical_read_xlog_page,
+ wal_segment_close,
WalSndPrepareWrite, WalSndWriteData,
WalSndUpdateProgress);
@@ -1171,9 +1169,8 @@ StartLogicalReplication(StartReplicationCmd *cmd)
*/
logical_decoding_ctx =
CreateDecodingContext(cmd->startpoint, cmd->options, false,
- XL_ROUTINE(.page_read = logical_read_xlog_page,
- .segment_open = WalSndSegmentOpen,
- .segment_close = wal_segment_close),
+ logical_read_xlog_page,
+ wal_segment_close,
WalSndPrepareWrite, WalSndWriteData,
WalSndUpdateProgress);
xlogreader = logical_decoding_ctx->reader;
@@ -2749,7 +2746,7 @@ XLogSendPhysical(void)
enlargeStringInfo(&output_message, nbytes);
retry:
- if (!WALRead(xlogreader,
+ if (!WALRead(xlogreader, WalSndSegmentOpen, wal_segment_close,
&output_message.data[output_message.len],
startptr,
nbytes,
@@ -2847,7 +2844,12 @@ XLogSendLogical(void)
*/
WalSndCaughtUp = false;
- record = XLogReadRecord(logical_decoding_ctx->reader, &errm);
+ while (XLogReadRecord(logical_decoding_ctx->reader, &record, &errm) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!logical_decoding_ctx->page_read(logical_decoding_ctx->reader))
+ break;
+ }
/* xlog record was invalid */
if (errm != NULL)
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index cf119848b0..712c85281c 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -41,15 +41,9 @@ static int xlogreadfd = -1;
static XLogSegNo xlogreadsegno = -1;
static char xlogfpath[MAXPGPATH];
-typedef struct XLogPageReadPrivate
-{
- const char *restoreCommand;
- int tliIndex;
-} XLogPageReadPrivate;
-
-static bool SimpleXLogPageRead(XLogReaderState *xlogreader,
- XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
+static bool SimpleXLogPageRead(XLogReaderState *xlogreader,
+ const char *datadir, int *tliIndex,
+ const char *restoreCommand);
/*
* Read WAL from the datadir/pg_wal, starting from 'startpoint' on timeline
@@ -66,20 +60,22 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
XLogRecord *record;
XLogReaderState *xlogreader;
char *errormsg;
- XLogPageReadPrivate private;
- private.tliIndex = tliIndex;
- private.restoreCommand = restoreCommand;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir,
- XL_ROUTINE(.page_read = &SimpleXLogPageRead),
- &private);
+ xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+
if (xlogreader == NULL)
pg_fatal("out of memory");
XLogBeginRead(xlogreader, startpoint);
do
{
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!SimpleXLogPageRead(xlogreader, datadir,
+ &tliIndex, restoreCommand))
+ break;
+ }
if (record == NULL)
{
@@ -123,19 +119,19 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex,
XLogRecord *record;
XLogReaderState *xlogreader;
char *errormsg;
- XLogPageReadPrivate private;
XLogRecPtr endptr;
- private.tliIndex = tliIndex;
- private.restoreCommand = restoreCommand;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir,
- XL_ROUTINE(.page_read = &SimpleXLogPageRead),
- &private);
+ xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
XLogBeginRead(xlogreader, ptr);
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!SimpleXLogPageRead(xlogreader, datadir, &tliIndex, restoreCommand))
+ break;
+ }
if (record == NULL)
{
if (errormsg)
@@ -170,7 +166,6 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
XLogRecPtr searchptr;
XLogReaderState *xlogreader;
char *errormsg;
- XLogPageReadPrivate private;
/*
* The given fork pointer points to the end of the last common record,
@@ -186,11 +181,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
forkptr += SizeOfXLogShortPHD;
}
- private.tliIndex = tliIndex;
- private.restoreCommand = restoreCommand;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir,
- XL_ROUTINE(.page_read = &SimpleXLogPageRead),
- &private);
+ xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
@@ -200,7 +191,13 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
uint8 info;
XLogBeginRead(xlogreader, searchptr);
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!SimpleXLogPageRead(xlogreader, datadir,
+ &tliIndex, restoreCommand))
+ break;
+ }
if (record == NULL)
{
@@ -247,10 +244,11 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
/* XLogReader callback function, to read a WAL page */
static bool
-SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *readBuf)
+SimpleXLogPageRead(XLogReaderState *xlogreader, const char *datadir,
+ int *tliIndex, const char *restoreCommand)
{
- XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
+ XLogRecPtr targetPagePtr = xlogreader->readPagePtr;
+ char *readBuf = xlogreader->readBuf;
uint32 targetPageOff;
XLogRecPtr targetSegEnd;
XLogSegNo targetSegNo;
@@ -283,14 +281,14 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
* be done both forward and backward, consider also switching timeline
* accordingly.
*/
- while (private->tliIndex < targetNentries - 1 &&
- targetHistory[private->tliIndex].end < targetSegEnd)
- private->tliIndex++;
- while (private->tliIndex > 0 &&
- targetHistory[private->tliIndex].begin >= targetSegEnd)
- private->tliIndex--;
+ while (*tliIndex < targetNentries - 1 &&
+ targetHistory[*tliIndex].end < targetSegEnd)
+ (*tliIndex)++;
+ while (*tliIndex > 0 &&
+ targetHistory[*tliIndex].begin >= targetSegEnd)
+ (*tliIndex)--;
- XLogFileName(xlogfname, targetHistory[private->tliIndex].tli,
+ XLogFileName(xlogfname, targetHistory[*tliIndex].tli,
xlogreadsegno, WalSegSz);
snprintf(xlogfpath, MAXPGPATH, "%s/" XLOGDIR "/%s",
@@ -303,7 +301,7 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
/*
* If we have no restore_command to execute, then exit.
*/
- if (private->restoreCommand == NULL)
+ if (restoreCommand == NULL)
{
pg_log_error("could not open file \"%s\": %m", xlogfpath);
xlogreader->readLen = -1;
@@ -317,7 +315,7 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
xlogreadfd = RestoreArchivedFile(xlogreader->segcxt.ws_dir,
xlogfname,
WalSegSz,
- private->restoreCommand);
+ restoreCommand);
if (xlogreadfd < 0)
{
@@ -359,7 +357,7 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
Assert(targetSegNo == xlogreadsegno);
- xlogreader->seg.ws_tli = targetHistory[private->tliIndex].tli;
+ xlogreader->seg.ws_tli = targetHistory[*tliIndex].tli;
xlogreader->readLen = XLOG_BLCKSZ;
return true;
}
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index 0f0e0723a3..f514de6b13 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -330,12 +330,17 @@ WALDumpCloseSegment(XLogReaderState *state)
state->seg.ws_file = -1;
}
-/* pg_waldump's XLogReaderRoutine->page_read callback */
+/*
+ * pg_waldump's WAL page rader, also used as page_read callback for
+ * XLogFindNextRecord
+ */
static bool
-WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetPtr, char *readBuff)
+WALDumpReadPage(XLogReaderState *state, void *priv)
{
- XLogDumpPrivate *private = state->private_data;
+ XLogRecPtr targetPagePtr = state->readPagePtr;
+ int reqLen = state->readLen;
+ char *readBuff = state->readBuf;
+ XLogDumpPrivate *private = (XLogDumpPrivate *) priv;
int count = XLOG_BLCKSZ;
WALReadError errinfo;
@@ -353,8 +358,8 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
}
}
- if (!WALRead(state, readBuff, targetPagePtr, count, private->timeline,
- &errinfo))
+ if (!WALRead(state, WALDumpOpenSegment, WALDumpCloseSegment,
+ readBuff, targetPagePtr, count, private->timeline, &errinfo))
{
WALOpenSegment *seg = &errinfo.wre_seg;
char fname[MAXPGPATH];
@@ -374,6 +379,7 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
(Size) errinfo.wre_req);
}
+ Assert(count >= state->readLen);
state->readLen = count;
return true;
}
@@ -1035,16 +1041,14 @@ main(int argc, char **argv)
/* we have everything we need, start reading */
xlogreader_state =
- XLogReaderAllocate(WalSegSz, waldir,
- XL_ROUTINE(.page_read = WALDumpReadPage,
- .segment_open = WALDumpOpenSegment,
- .segment_close = WALDumpCloseSegment),
- &private);
+ XLogReaderAllocate(WalSegSz, waldir, WALDumpCloseSegment);
+
if (!xlogreader_state)
fatal_error("out of memory");
/* first find a valid recptr to start from */
- first_record = XLogFindNextRecord(xlogreader_state, private.startptr);
+ first_record = XLogFindNextRecord(xlogreader_state, private.startptr,
+ &WALDumpReadPage, (void*) &private);
if (first_record == InvalidXLogRecPtr)
fatal_error("could not find a valid record after %X/%X",
@@ -1067,7 +1071,13 @@ main(int argc, char **argv)
for (;;)
{
/* try to read the next record */
- record = XLogReadRecord(xlogreader_state, &errormsg);
+ while (XLogReadRecord(xlogreader_state, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!WALDumpReadPage(xlogreader_state, (void *) &private))
+ break;
+ }
+
if (!record)
{
if (!config.follow || private.endptr_reached)
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index 23d9e70a59..4e9268b553 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -57,64 +57,15 @@ typedef struct WALSegmentContext
typedef struct XLogReaderState XLogReaderState;
-/* Function type definition for the read_page callback */
-typedef bool (*XLogPageReadCB) (XLogReaderState *xlogreader,
- XLogRecPtr targetPagePtr,
- int reqLen,
- XLogRecPtr targetRecPtr,
- char *readBuf);
+/* Function type definition for the segment cleanup callback */
+typedef void (*WALSegmentCleanupCB) (XLogReaderState *xlogreader);
+
+/* Function type definition for the open/close callbacks for WALRead() */
typedef void (*WALSegmentOpenCB) (XLogReaderState *xlogreader,
XLogSegNo nextSegNo,
TimeLineID *tli_p);
typedef void (*WALSegmentCloseCB) (XLogReaderState *xlogreader);
-typedef struct XLogReaderRoutine
-{
- /*
- * Data input callback
- *
- * This callback shall read at least reqLen valid bytes of the xlog page
- * starting at targetPagePtr, and store them in readBuf. The callback
- * shall return the number of bytes read (never more than XLOG_BLCKSZ), or
- * -1 on failure. The callback shall sleep, if necessary, to wait for the
- * requested bytes to become available. The callback will not be invoked
- * again for the same page unless more than the returned number of bytes
- * are needed.
- *
- * targetRecPtr is the position of the WAL record we're reading. Usually
- * it is equal to targetPagePtr + reqLen, but sometimes xlogreader needs
- * to read and verify the page or segment header, before it reads the
- * actual WAL record it's interested in. In that case, targetRecPtr can
- * be used to determine which timeline to read the page from.
- *
- * The callback shall set ->seg.ws_tli to the TLI of the file the page was
- * read from.
- */
- XLogPageReadCB page_read;
-
- /*
- * Callback to open the specified WAL segment for reading. ->seg.ws_file
- * shall be set to the file descriptor of the opened segment. In case of
- * failure, an error shall be raised by the callback and it shall not
- * return.
- *
- * "nextSegNo" is the number of the segment to be opened.
- *
- * "tli_p" is an input/output argument. WALRead() uses it to pass the
- * timeline in which the new segment should be found, but the callback can
- * use it to return the TLI that it actually opened.
- */
- WALSegmentOpenCB segment_open;
-
- /*
- * WAL segment close callback. ->seg.ws_file shall be set to a negative
- * number.
- */
- WALSegmentCloseCB segment_close;
-} XLogReaderRoutine;
-
-#define XL_ROUTINE(...) &(XLogReaderRoutine){__VA_ARGS__}
-
typedef struct
{
/* Is this block ref in use? */
@@ -144,12 +95,35 @@ typedef struct
uint16 data_bufsz;
} DecodedBkpBlock;
+/* Return code from XLogReadRecord */
+typedef enum XLogReadRecordResult
+{
+ XLREAD_SUCCESS, /* record is successfully read */
+ XLREAD_NEED_DATA, /* need more data. see XLogReadRecord. */
+ XLREAD_FAIL /* failed during reading a record */
+} XLogReadRecordResult;
+
+/*
+ * internal state of XLogReadRecord
+ *
+ * XLogReadState runs a state machine while reading a record. Theses states
+ * are not seen outside the function. Each state may repeat several times
+ * exiting requesting caller for new data. See the comment of XLogReadRecrod
+ * for details.
+ */
+typedef enum XLogReadRecordState {
+ XLREAD_NEXT_RECORD,
+ XLREAD_TOT_LEN,
+ XLREAD_FIRST_FRAGMENT,
+ XLREAD_CONTINUATION
+} XLogReadRecordState;
+
struct XLogReaderState
{
/*
* Operational callbacks
*/
- XLogReaderRoutine routine;
+ WALSegmentCleanupCB cleanup_cb;
/* ----------------------------------------
* Public parameters
@@ -162,18 +136,14 @@ struct XLogReaderState
*/
uint64 system_identifier;
- /*
- * Opaque data for callbacks to use. Not used by XLogReader.
- */
- void *private_data;
-
/*
* Start and end point of last record read. EndRecPtr is also used as the
* position to read next. Calling XLogBeginRead() sets EndRecPtr to the
* starting position and ReadRecPtr to invalid.
*/
- XLogRecPtr ReadRecPtr; /* start of last record read */
+ XLogRecPtr ReadRecPtr; /* start of last record read or being read */
XLogRecPtr EndRecPtr; /* end+1 of last record read */
+ XLogRecPtr PrevRecPtr; /* start of previous record read */
/* ----------------------------------------
* Communication with page reader
@@ -187,7 +157,9 @@ struct XLogReaderState
* the request, or -1 on error */
TimeLineID readPageTLI; /* TLI for data currently in readBuf */
char *readBuf; /* buffer to store data */
- bool page_verified; /* is the page on the buffer verified? */
+ bool page_verified; /* is the page header on the buffer verified? */
+ bool record_verified;/* is the current record header verified? */
+
/* ----------------------------------------
@@ -229,8 +201,6 @@ struct XLogReaderState
XLogRecPtr latestPagePtr;
TimeLineID latestPageTLI;
- /* beginning of the WAL record being read. */
- XLogRecPtr currRecPtr;
/* timeline to read it from, 0 if a lookup is required */
TimeLineID currTLI;
@@ -257,6 +227,15 @@ struct XLogReaderState
char *readRecordBuf;
uint32 readRecordBufSize;
+ /*
+ * XLogReadRecord() state
+ */
+ XLogReadRecordState readRecordState;/* state machine state */
+ int recordGotLen; /* amount of current record that has
+ * already been read */
+ int recordRemainLen; /* length of current record that remains */
+ XLogRecPtr recordContRecPtr; /* where the current record continues */
+
/* Buffer to hold error message */
char *errormsg_buf;
};
@@ -264,9 +243,7 @@ struct XLogReaderState
/* Get a new XLogReader */
extern XLogReaderState *XLogReaderAllocate(int wal_segment_size,
const char *waldir,
- XLogReaderRoutine *routine,
- void *private_data);
-extern XLogReaderRoutine *LocalXLogReaderRoutine(void);
+ WALSegmentCleanupCB cleanup_cb);
/* Free an XLogReader */
extern void XLogReaderFree(XLogReaderState *state);
@@ -274,12 +251,17 @@ extern void XLogReaderFree(XLogReaderState *state);
/* Position the XLogReader to given record */
extern void XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr);
#ifdef FRONTEND
-extern XLogRecPtr XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr);
+/* Function type definition for the read_page callback */
+typedef bool (*XLogFindNextRecordCB) (XLogReaderState *xlogreader,
+ void *private);
+extern XLogRecPtr XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
+ XLogFindNextRecordCB read_page, void *private);
#endif /* FRONTEND */
/* Read the next XLog record. Returns NULL on end-of-WAL or failure */
-extern struct XLogRecord *XLogReadRecord(XLogReaderState *state,
- char **errormsg);
+extern XLogReadRecordResult XLogReadRecord(XLogReaderState *state,
+ XLogRecord **record,
+ char **errormsg);
/* Validate a page */
extern bool XLogReaderValidatePageHeader(XLogReaderState *state,
@@ -299,6 +281,7 @@ typedef struct WALReadError
} WALReadError;
extern bool WALRead(XLogReaderState *state,
+ WALSegmentOpenCB segopenfn, WALSegmentCloseCB sgclosefn,
char *buf, XLogRecPtr startptr, Size count,
TimeLineID tli, WALReadError *errinfo);
diff --git a/src/include/access/xlogutils.h b/src/include/access/xlogutils.h
index 364a21c4ea..397fb27fc2 100644
--- a/src/include/access/xlogutils.h
+++ b/src/include/access/xlogutils.h
@@ -47,9 +47,7 @@ extern Buffer XLogReadBufferExtended(RelFileNode rnode, ForkNumber forknum,
extern Relation CreateFakeRelcacheEntry(RelFileNode rnode);
extern void FreeFakeRelcacheEntry(Relation fakerel);
-extern bool read_local_xlog_page(XLogReaderState *state,
- XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetRecPtr, char *cur_page);
+extern bool read_local_xlog_page(XLogReaderState *state);
extern void wal_segment_open(XLogReaderState *state,
XLogSegNo nextSegNo,
TimeLineID *tli_p);
diff --git a/src/include/pg_config_manual.h b/src/include/pg_config_manual.h
index f10ad0acd6..94749609b4 100644
--- a/src/include/pg_config_manual.h
+++ b/src/include/pg_config_manual.h
@@ -377,7 +377,7 @@
* Enable debugging print statements for WAL-related operations; see
* also the wal_debug GUC var.
*/
-/* #define WAL_DEBUG */
+#define WAL_DEBUG
/*
* Enable tracing of resource consumption during sort operations;
diff --git a/src/include/replication/logical.h b/src/include/replication/logical.h
index c253403372..b936f31393 100644
--- a/src/include/replication/logical.h
+++ b/src/include/replication/logical.h
@@ -29,6 +29,10 @@ typedef void (*LogicalOutputPluginWriterUpdateProgress) (struct LogicalDecodingC
TransactionId xid
);
+typedef struct LogicalDecodingContext LogicalDecodingContext;
+
+typedef bool (*LogicalDecodingXLogPageReadCB)(XLogReaderState *ctx);
+
typedef struct LogicalDecodingContext
{
/* memory context this is all allocated in */
@@ -39,6 +43,7 @@ typedef struct LogicalDecodingContext
/* infrastructure pieces for decoding */
XLogReaderState *reader;
+ LogicalDecodingXLogPageReadCB page_read;
struct ReorderBuffer *reorder;
struct SnapBuild *snapshot_builder;
@@ -105,14 +110,16 @@ extern LogicalDecodingContext *CreateInitDecodingContext(const char *plugin,
List *output_plugin_options,
bool need_full_snapshot,
XLogRecPtr restart_lsn,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress);
extern LogicalDecodingContext *CreateDecodingContext(XLogRecPtr start_lsn,
List *output_plugin_options,
bool fast_forward,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress);
--
2.27.0
----Next_Part(Thu_Mar__4_11_28_53_2021_014)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
filename="v17-0003-Remove-globals-readOff-readLen-and-readSegNo.patch"
^ permalink raw reply [nested|flat] 70+ messages in thread
* [PATCH v15 2/4] Move page-reader out of XLogReadRecord
@ 2019-09-10 03:58 Kyotaro Horiguchi <[email protected]>
0 siblings, 0 replies; 70+ messages in thread
From: Kyotaro Horiguchi @ 2019-09-10 03:58 UTC (permalink / raw)
This is the second step of removing callbacks from WAL record reader.
Since it is essential to take in additional data while reading a
record, the function have to ask caller for new data while keeping
working state. Thus the function is turned into a state machine.
---
src/backend/access/transam/twophase.c | 15 +-
src/backend/access/transam/xlog.c | 58 +-
src/backend/access/transam/xlogreader.c | 700 +++++++++++-------
src/backend/access/transam/xlogutils.c | 17 +-
src/backend/replication/logical/logical.c | 26 +-
.../replication/logical/logicalfuncs.c | 13 +-
src/backend/replication/slotfuncs.c | 18 +-
src/backend/replication/walsender.c | 32 +-
src/bin/pg_rewind/parsexlog.c | 84 +--
src/bin/pg_waldump/pg_waldump.c | 36 +-
src/include/access/xlogreader.h | 121 ++-
src/include/access/xlogutils.h | 4 +-
src/include/replication/logical.h | 11 +-
13 files changed, 648 insertions(+), 487 deletions(-)
diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index 9b2e59bf0e..b0d60a0d0f 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -1330,11 +1330,8 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
XLogReaderState *xlogreader;
char *errormsg;
- xlogreader = XLogReaderAllocate(wal_segment_size, NULL,
- XL_ROUTINE(.page_read = &read_local_xlog_page,
- .segment_open = &wal_segment_open,
- .segment_close = &wal_segment_close),
- NULL);
+ xlogreader = XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+
if (!xlogreader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
@@ -1342,7 +1339,13 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
errdetail("Failed while allocating a WAL reading processor.")));
XLogBeginRead(xlogreader, lsn);
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!read_local_xlog_page(xlogreader))
+ break;
+ }
+
if (record == NULL)
ereport(ERROR,
(errcode_for_file_access(),
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index e570e56a24..f9b0108602 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -831,13 +831,6 @@ static XLogSource currentSource = XLOG_FROM_ANY;
static bool lastSourceFailed = false;
static bool pendingWalRcvRestart = false;
-typedef struct XLogPageReadPrivate
-{
- int emode;
- bool fetching_ckpt; /* are we fetching a checkpoint record? */
- bool randAccess;
-} XLogPageReadPrivate;
-
/*
* These variables track when we last obtained some WAL data to process,
* and where we got it from. (XLogReceiptSource is initially the same as
@@ -912,8 +905,8 @@ static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath,
static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli,
XLogSource source, bool notfoundOk);
static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, XLogSource source);
-static bool XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
+static bool XLogPageRead(XLogReaderState *xlogreader,
+ bool fetching_ckpt, int emode, bool randAccess);
static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
bool fetching_ckpt, XLogRecPtr tliRecPtr);
static int emode_for_corrupt_record(int emode, XLogRecPtr RecPtr);
@@ -1225,8 +1218,7 @@ XLogInsertRecord(XLogRecData *rdata,
appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len);
if (!debug_reader)
- debug_reader = XLogReaderAllocate(wal_segment_size, NULL,
- XL_ROUTINE(), NULL);
+ debug_reader = XLogReaderAllocate(wal_segment_size, NULL NULL);
if (!debug_reader)
{
@@ -4326,15 +4318,10 @@ CleanupBackupHistory(void)
* record is available.
*/
static XLogRecord *
-ReadRecord(XLogReaderState *xlogreader, int emode,
- bool fetching_ckpt)
+ReadRecord(XLogReaderState *xlogreader, int emode, bool fetching_ckpt)
{
XLogRecord *record;
- XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
-
- private->fetching_ckpt = fetching_ckpt;
- private->emode = emode;
- private->randAccess = (xlogreader->ReadRecPtr == InvalidXLogRecPtr);
+ bool randAccess = (xlogreader->ReadRecPtr == InvalidXLogRecPtr);
/* This is the first attempt to read this page. */
lastSourceFailed = false;
@@ -4342,8 +4329,16 @@ ReadRecord(XLogReaderState *xlogreader, int emode,
for (;;)
{
char *errormsg;
+ XLogReadRecordResult result;
+
+ while ((result = XLogReadRecord(xlogreader, &record, &errormsg))
+ == XLREAD_NEED_DATA)
+ {
+ if (!XLogPageRead(xlogreader, fetching_ckpt, emode, randAccess))
+ break;
+
+ }
- record = XLogReadRecord(xlogreader, &errormsg);
ReadRecPtr = xlogreader->ReadRecPtr;
EndRecPtr = xlogreader->EndRecPtr;
if (record == NULL)
@@ -6320,7 +6315,6 @@ StartupXLOG(void)
bool backupFromStandby = false;
DBState dbstate_at_startup;
XLogReaderState *xlogreader;
- XLogPageReadPrivate private;
bool fast_promoted = false;
struct stat st;
@@ -6480,13 +6474,9 @@ StartupXLOG(void)
OwnLatch(&XLogCtl->recoveryWakeupLatch);
/* Set up XLOG reader facility */
- MemSet(&private, 0, sizeof(XLogPageReadPrivate));
xlogreader =
- XLogReaderAllocate(wal_segment_size, NULL,
- XL_ROUTINE(.page_read = &XLogPageRead,
- .segment_open = NULL,
- .segment_close = wal_segment_close),
- &private);
+ XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+
if (!xlogreader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
@@ -11860,12 +11850,13 @@ CancelBackup(void)
* sleep and retry.
*/
static bool
-XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetRecPtr, char *readBuf)
+XLogPageRead(XLogReaderState *xlogreader,
+ bool fetching_ckpt, int emode, bool randAccess)
{
- XLogPageReadPrivate *private =
- (XLogPageReadPrivate *) xlogreader->private_data;
- int emode = private->emode;
+ char *readBuf = xlogreader->readBuf;
+ XLogRecPtr targetPagePtr = xlogreader->readPagePtr;
+ int reqLen = xlogreader->readLen;
+ XLogRecPtr targetRecPtr = xlogreader->ReadRecPtr;
uint32 targetPageOff;
XLogSegNo targetSegNo PG_USED_FOR_ASSERTS_ONLY;
int r;
@@ -11908,8 +11899,8 @@ retry:
flushedUpto < targetPagePtr + reqLen))
{
if (!WaitForWALToBecomeAvailable(targetPagePtr + reqLen,
- private->randAccess,
- private->fetching_ckpt,
+ randAccess,
+ fetching_ckpt,
targetRecPtr))
{
if (readFile >= 0)
@@ -12014,6 +12005,7 @@ retry:
goto next_record_is_invalid;
}
+ Assert(xlogreader->readPagePtr == targetPagePtr);
xlogreader->readLen = readLen;
return true;
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index 3d599325ee..9281b57379 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -40,7 +40,7 @@ static bool XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr,
int reqLen, bool header_inclusive);
static void XLogReaderInvalReadState(XLogReaderState *state);
static bool ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
- XLogRecPtr PrevRecPtr, XLogRecord *record, bool randAccess);
+ XLogRecPtr PrevRecPtr, XLogRecord *record);
static bool ValidXLogRecord(XLogReaderState *state, XLogRecord *record,
XLogRecPtr recptr);
static void ResetDecoder(XLogReaderState *state);
@@ -73,7 +73,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...)
*/
XLogReaderState *
XLogReaderAllocate(int wal_segment_size, const char *waldir,
- XLogReaderRoutine *routine, void *private_data)
+ WALSegmentCleanupCB cleanup_cb)
{
XLogReaderState *state;
@@ -84,7 +84,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
return NULL;
/* initialize caller-provided support functions */
- state->routine = *routine;
+ state->cleanup_cb = cleanup_cb;
state->max_block_id = -1;
@@ -107,8 +107,6 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
WALOpenSegmentInit(&state->seg, &state->segcxt, wal_segment_size,
waldir);
- /* system_identifier initialized to zeroes above */
- state->private_data = private_data;
/* ReadRecPtr, EndRecPtr and readLen initialized to zeroes above */
state->errormsg_buf = palloc_extended(MAX_ERRORMSG_LEN + 1,
MCXT_ALLOC_NO_OOM);
@@ -140,8 +138,8 @@ XLogReaderFree(XLogReaderState *state)
{
int block_id;
- if (state->seg.ws_file != -1)
- state->routine.segment_close(state);
+ if (state->seg.ws_file >= 0)
+ state->cleanup_cb(state);
for (block_id = 0; block_id <= XLR_MAX_BLOCK_ID; block_id++)
{
@@ -246,6 +244,7 @@ XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
/* Begin at the passed-in record pointer. */
state->EndRecPtr = RecPtr;
state->ReadRecPtr = InvalidXLogRecPtr;
+ state->readRecordState = XLREAD_NEXT_RECORD;
}
/*
@@ -254,317 +253,452 @@ XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
* XLogBeginRead() or XLogFindNextRecord() must be called before the first call
* to XLogReadRecord().
*
- * If the page_read callback fails to read the requested data, NULL is
- * returned. The callback is expected to have reported the error; errormsg
- * is set to NULL.
+ * This function may return XLREAD_NEED_DATA several times before returning a
+ * result record. The caller shall read in some new data then call this
+ * function again with the same parameters.
*
- * If the reading fails for some other reason, NULL is also returned, and
- * *errormsg is set to a string with details of the failure.
+ * When a record is successfully read, returns XLREAD_SUCCESS with result
+ * record being stored in *record. Otherwise *record is NULL.
*
- * The returned pointer (or *errormsg) points to an internal buffer that's
- * valid until the next call to XLogReadRecord.
+ * Returns XLREAD_NEED_DATA if more data is needed to finish reading the
+ * current record. In that case, state->readPagePtr and state->readLen inform
+ * the desired position and minimum length of data needed. The caller shall
+ * read in the requested data and set state->readBuf to point to a buffer
+ * containing it. The caller must also set state->readPageTLI and
+ * state->readLen to indicate the timeline that it was read from, and the
+ * length of data that is now available (which must be >= given readLen),
+ * respectively.
+ *
+ * If invalid data is encountered, returns XLREAD_FAIL with *record being set to
+ * NULL. *errormsg is set to a string with details of the failure.
+ * The returned pointer (or *errormsg) points to an internal buffer that's valid
+ * until the next call to XLogReadRecord.
+ *
+ *
+ * This function runs a state machine consists of the following states.
+ *
+ * XLREAD_NEXT_RECORD :
+ * The initial state, if called with valid RecPtr, try to read a record at
+ * that position. If invalid RecPtr is given try to read a record just after
+ * the last one previously read.
+ * This state ens after setting ReadRecPtr. Then goes to XLREAD_TOT_LEN.
+ *
+ * XLREAD_TOT_LEN:
+ * Examining record header. Ends after reading record total
+ * length. recordRemainLen and recordGotLen are initialized.
+ *
+ * XLREAD_FIRST_FRAGMENT:
+ * Reading the first fragment. Ends with finishing reading a single
+ * record. Goes to XLREAD_NEXT_RECORD if that's all or
+ * XLREAD_CONTINUATION if we have continuation.
+
+ * XLREAD_CONTINUATION:
+ * Reading continuation of record. Ends with finishing the whole record then
+ * goes to XLREAD_NEXT_RECORD. During this state, recordRemainLen indicates
+ * how much is left and readRecordBuf holds the partially assert
+ * record.recordContRecPtr points to the beginning of the next page where to
+ * continue.
+ *
+ * If wrong data found in any state, the state machine stays at the current
+ * state. This behavior allows to continue reading a reacord switching among
+ * different souces, while streaming replication.
*/
-XLogRecord *
-XLogReadRecord(XLogReaderState *state, char **errormsg)
+XLogReadRecordResult
+XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
{
- XLogRecPtr RecPtr;
- XLogRecord *record;
- XLogRecPtr targetPagePtr;
- bool randAccess;
- uint32 len,
- total_len;
- uint32 targetRecOff;
- uint32 pageHeaderSize;
- bool gotheader;
+ XLogRecord *prec;
- /*
- * randAccess indicates whether to verify the previous-record pointer of
- * the record we're reading. We only do this if we're reading
- * sequentially, which is what we initially assume.
- */
- randAccess = false;
+ *record = NULL;
/* reset error state */
*errormsg = NULL;
state->errormsg_buf[0] = '\0';
- ResetDecoder(state);
-
- RecPtr = state->EndRecPtr;
-
- if (state->ReadRecPtr != InvalidXLogRecPtr)
- {
- /* read the record after the one we just read */
-
- /*
- * EndRecPtr is pointing to end+1 of the previous WAL record. If
- * we're at a page boundary, no more records can fit on the current
- * page. We must skip over the page header, but we can't do that until
- * we've read in the page, since the header size is variable.
- */
- }
- else
- {
- /*
- * Caller supplied a position to start at.
- *
- * In this case, EndRecPtr should already be pointing to a valid
- * record starting position.
- */
- Assert(XRecOffIsValid(RecPtr));
- randAccess = true;
- }
-
- state->currRecPtr = RecPtr;
-
- targetPagePtr = RecPtr - (RecPtr % XLOG_BLCKSZ);
- targetRecOff = RecPtr % XLOG_BLCKSZ;
-
- /*
- * Read the page containing the record into state->readBuf. Request enough
- * byte to cover the whole record header, or at least the part of it that
- * fits on the same page.
- */
- while (XLogNeedData(state, targetPagePtr,
- Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
- targetRecOff != 0))
- {
- if (!state->routine.page_read(state, state->readPagePtr, state->readLen,
- RecPtr, state->readBuf))
- break;
- }
-
- if (!state->page_verified)
- goto err;
-
- /*
- * We have at least the page header, so we can examine it now.
- */
- pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
- if (targetRecOff == 0)
- {
- /*
- * At page start, so skip over page header.
- */
- RecPtr += pageHeaderSize;
- targetRecOff = pageHeaderSize;
- }
- else if (targetRecOff < pageHeaderSize)
- {
- report_invalid_record(state, "invalid record offset at %X/%X",
- (uint32) (RecPtr >> 32), (uint32) RecPtr);
- goto err;
- }
-
- if ((((XLogPageHeader) state->readBuf)->xlp_info & XLP_FIRST_IS_CONTRECORD) &&
- targetRecOff == pageHeaderSize)
- {
- report_invalid_record(state, "contrecord is requested by %X/%X",
- (uint32) (RecPtr >> 32), (uint32) RecPtr);
- goto err;
- }
-
- /* XLogNeedData has verified the page header */
- Assert(pageHeaderSize <= state->readLen);
-
- /*
- * Read the record length.
- *
- * NB: Even though we use an XLogRecord pointer here, the whole record
- * header might not fit on this page. xl_tot_len is the first field of the
- * struct, so it must be on this page (the records are MAXALIGNed), but we
- * cannot access any other fields until we've verified that we got the
- * whole header.
- */
- record = (XLogRecord *) (state->readBuf + RecPtr % XLOG_BLCKSZ);
- total_len = record->xl_tot_len;
-
- /*
- * If the whole record header is on this page, validate it immediately.
- * Otherwise do just a basic sanity check on xl_tot_len, and validate the
- * rest of the header after reading it from the next page. The xl_tot_len
- * check is necessary here to ensure that we enter the "Need to reassemble
- * record" code path below; otherwise we might fail to apply
- * ValidXLogRecordHeader at all.
- */
- if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
- {
- if (!ValidXLogRecordHeader(state, RecPtr, state->ReadRecPtr, record,
- randAccess))
- goto err;
- gotheader = true;
- }
- else
- {
- /* XXX: more validation should be done here */
- if (total_len < SizeOfXLogRecord)
- {
- report_invalid_record(state,
- "invalid record length at %X/%X: wanted %u, got %u",
- (uint32) (RecPtr >> 32), (uint32) RecPtr,
- (uint32) SizeOfXLogRecord, total_len);
- goto err;
- }
- gotheader = false;
- }
-
- len = XLOG_BLCKSZ - RecPtr % XLOG_BLCKSZ;
- if (total_len > len)
+ switch (state->readRecordState)
{
- /* Need to reassemble record */
- char *contdata;
- XLogPageHeader pageHeader;
- char *buffer;
- uint32 gotlen;
-
- /*
- * Enlarge readRecordBuf as needed.
- */
- if (total_len > state->readRecordBufSize &&
- !allocate_recordbuf(state, total_len))
- {
- /* We treat this as a "bogus data" condition */
- report_invalid_record(state, "record length %u at %X/%X too long",
- total_len,
- (uint32) (RecPtr >> 32), (uint32) RecPtr);
- goto err;
- }
+ case XLREAD_NEXT_RECORD:
+ ResetDecoder(state);
- /* Copy the first fragment of the record from the first page. */
- memcpy(state->readRecordBuf,
- state->readBuf + RecPtr % XLOG_BLCKSZ, len);
- buffer = state->readRecordBuf + len;
- gotlen = len;
-
- do
- {
- int rest_len = total_len - gotlen;
-
- /* Calculate pointer to beginning of next page */
- targetPagePtr += XLOG_BLCKSZ;
-
- /* Wait for the next page to become available */
- while (XLogNeedData(state, targetPagePtr,
- Min(rest_len, XLOG_BLCKSZ),
- false))
+ if (state->ReadRecPtr != InvalidXLogRecPtr)
{
- if (!state->routine.page_read(state, state->readPagePtr,
- state->readLen,
- state->ReadRecPtr,
- state->readBuf))
- break;
+ /* read the record after the one we just read */
+
+ /*
+ * EndRecPtr is pointing to end+1 of the previous WAL record.
+ * If we're at a page boundary, no more records can fit on the
+ * current page. We must skip over the page header, but we
+ * can't do that until we've read in the page, since the header
+ * size is variable.
+ */
+ state->PrevRecPtr = state->ReadRecPtr;
+ state->ReadRecPtr = state->EndRecPtr;
+ }
+ else
+ {
+ /*
+ * Caller supplied a position to start at.
+ *
+ * In this case, EndRecPtr should already be pointing to a
+ * valid record starting position.
+ */
+ Assert(XRecOffIsValid(state->EndRecPtr));
+ state->ReadRecPtr = state->EndRecPtr;
+
+ /*
+ * We cannot verify the previous-record pointer when we're
+ * seeking to a particular record. Reset PrevRecPtr so that we
+ * won't try doing that.
+ */
+ state->PrevRecPtr = InvalidXLogRecPtr;
+ state->EndRecPtr = InvalidXLogRecPtr; /* to be tidy */
}
+ state->record_verified = false;
+ state->readRecordState = XLREAD_TOT_LEN;
+ /* fall through */
+
+ case XLREAD_TOT_LEN:
+ {
+ uint32 total_len;
+ uint32 pageHeaderSize;
+ XLogRecPtr targetPagePtr;
+ uint32 targetRecOff;
+ XLogPageHeader pageHeader;
+
+ targetPagePtr =
+ state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
+ targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+
+ /*
+ * Check if we have enough data. For the first record in the page,
+ * the requesting length doesn't contain page header.
+ */
+ if (XLogNeedData(state, targetPagePtr,
+ Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+ targetRecOff != 0))
+ return XLREAD_NEED_DATA;
+
+ /* error out if caller supplied bogus page */
if (!state->page_verified)
goto err;
- Assert(SizeOfXLogShortPHD <= state->readLen);
+ /* examine page header now. */
+ pageHeaderSize =
+ XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+ if (targetRecOff == 0)
+ {
+ /* At page start, so skip over page header. */
+ state->ReadRecPtr += pageHeaderSize;
+ targetRecOff = pageHeaderSize;
+ }
+ else if (targetRecOff < pageHeaderSize)
+ {
+ report_invalid_record(state, "invalid record offset at %X/%X",
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr);
+ goto err;
+ }
- /* Check that the continuation on next page looks valid */
pageHeader = (XLogPageHeader) state->readBuf;
- if (!(pageHeader->xlp_info & XLP_FIRST_IS_CONTRECORD))
+ if ((pageHeader->xlp_info & XLP_FIRST_IS_CONTRECORD) &&
+ targetRecOff == pageHeaderSize)
{
- report_invalid_record(state,
- "there is no contrecord flag at %X/%X",
- (uint32) (RecPtr >> 32), (uint32) RecPtr);
+ report_invalid_record(state, "contrecord is requested by %X/%X",
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr);
goto err;
}
- /*
- * Cross-check that xlp_rem_len agrees with how much of the record
- * we expect there to be left.
- */
- if (pageHeader->xlp_rem_len == 0 ||
- total_len != (pageHeader->xlp_rem_len + gotlen))
- {
- report_invalid_record(state,
- "invalid contrecord length %u at %X/%X",
- pageHeader->xlp_rem_len,
- (uint32) (RecPtr >> 32), (uint32) RecPtr);
- goto err;
- }
-
- /* Append the continuation from this page to the buffer */
- pageHeaderSize = XLogPageHeaderSize(pageHeader);
-
+ /* XLogNeedData has verified the page header */
Assert(pageHeaderSize <= state->readLen);
- contdata = (char *) state->readBuf + pageHeaderSize;
- len = XLOG_BLCKSZ - pageHeaderSize;
- if (pageHeader->xlp_rem_len < len)
- len = pageHeader->xlp_rem_len;
+ /*
+ * Read the record length.
+ *
+ * NB: Even though we use an XLogRecord pointer here, the whole
+ * record header might not fit on this page. xl_tot_len is the first
+ * field of the struct, so it must be on this page (the records are
+ * MAXALIGNed), but we cannot access any other fields until we've
+ * verified that we got the whole header.
+ */
+ prec = (XLogRecord *) (state->readBuf +
+ state->ReadRecPtr % XLOG_BLCKSZ);
+ total_len = prec->xl_tot_len;
- Assert (pageHeaderSize + len <= state->readLen);
- memcpy(buffer, (char *) contdata, len);
- buffer += len;
- gotlen += len;
+ /*
+ * If the whole record header is on this page, validate it
+ * immediately. Otherwise do just a basic sanity check on
+ * xl_tot_len, and validate the rest of the header after reading it
+ * from the next page. The xl_tot_len check is necessary here to
+ * ensure that we enter the XLREAD_CONTINUATION state below;
+ * otherwise we might fail to apply ValidXLogRecordHeader at all.
+ */
+ if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
+ {
+ if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
+ state->PrevRecPtr, prec))
+ goto err;
- /* If we just reassembled the record header, validate it. */
- if (!gotheader)
+ state->record_verified = true;
+ }
+ else
{
- record = (XLogRecord *) state->readRecordBuf;
- if (!ValidXLogRecordHeader(state, RecPtr, state->ReadRecPtr,
- record, randAccess))
+ /* XXX: more validation should be done here */
+ if (total_len < SizeOfXLogRecord)
+ {
+ report_invalid_record(state,
+ "invalid record length at %X/%X: wanted %u, got %u",
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr,
+ (uint32) SizeOfXLogRecord, total_len);
goto err;
- gotheader = true;
+ }
}
- } while (gotlen < total_len);
-
- Assert(gotheader);
- record = (XLogRecord *) state->readRecordBuf;
- if (!ValidXLogRecord(state, record, RecPtr))
- goto err;
+ /*
+ * Wait for the rest of the record, or the part of the record that
+ * fit on the first page if crossed a page boundary, to become
+ * available.
+ */
+ state->recordGotLen = 0;
+ state->recordRemainLen = total_len;
+ state->readRecordState = XLREAD_FIRST_FRAGMENT;
+ }
+ /* fall through */
- pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
- state->ReadRecPtr = RecPtr;
- state->EndRecPtr = targetPagePtr + pageHeaderSize
- + MAXALIGN(pageHeader->xlp_rem_len);
- }
- else
- {
- /* Wait for the record data to become available */
- while (XLogNeedData(state, targetPagePtr,
- Min(targetRecOff + total_len, XLOG_BLCKSZ), true))
+ case XLREAD_FIRST_FRAGMENT:
{
- if (!state->routine.page_read(state, state->readPagePtr,
- state->readLen,
- state->ReadRecPtr, state->readBuf))
+ uint32 total_len = state->recordRemainLen;
+ uint32 request_len;
+ uint32 record_len;
+ XLogRecPtr targetPagePtr;
+ uint32 targetRecOff;
+
+ /*
+ * Wait for the rest of the record on the first page to become
+ * available
+ */
+ targetPagePtr =
+ state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
+ targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+
+ request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ);
+ record_len = request_len - targetRecOff;
+
+ /* ReadRecPtr contains page header */
+ Assert (targetRecOff != 0);
+ if (XLogNeedData(state, targetPagePtr, request_len, true))
+ return XLREAD_NEED_DATA;
+
+ /* error out if caller supplied bogus page */
+ if (!state->page_verified)
+ goto err;
+
+ prec = (XLogRecord *) (state->readBuf + targetRecOff);
+
+ /* validate record header if not yet */
+ if (!state->record_verified && record_len >= SizeOfXLogRecord)
+ {
+ if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
+ state->PrevRecPtr, prec))
+ goto err;
+
+ state->record_verified = true;
+ }
+
+
+ if (total_len == record_len)
+ {
+ /* Record does not cross a page boundary */
+ Assert(state->record_verified);
+
+ if (!ValidXLogRecord(state, prec, state->ReadRecPtr))
+ goto err;
+
+ state->record_verified = true; /* to be tidy */
+
+ /* We already checked the header earlier */
+ state->EndRecPtr = state->ReadRecPtr + MAXALIGN(record_len);
+
+ *record = prec;
+ state->readRecordState = XLREAD_NEXT_RECORD;
break;
+ }
+
+ /*
+ * The record continues on the next page. Need to reassemble
+ * record
+ */
+ Assert(total_len > record_len);
+
+ /* Enlarge readRecordBuf as needed. */
+ if (total_len > state->readRecordBufSize &&
+ !allocate_recordbuf(state, total_len))
+ {
+ /* We treat this as a "bogus data" condition */
+ report_invalid_record(state,
+ "record length %u at %X/%X too long",
+ total_len,
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr);
+ goto err;
+ }
+
+ /* Copy the first fragment of the record from the first page. */
+ memcpy(state->readRecordBuf, state->readBuf + targetRecOff,
+ record_len);
+ state->recordGotLen += record_len;
+ state->recordRemainLen -= record_len;
+
+ /* Calculate pointer to beginning of next page */
+ state->recordContRecPtr = state->ReadRecPtr + record_len;
+ Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0);
+
+ state->readRecordState = XLREAD_CONTINUATION;
}
+ /* fall through */
+
+ case XLREAD_CONTINUATION:
+ {
+ XLogPageHeader pageHeader;
+ uint32 pageHeaderSize;
+ XLogRecPtr targetPagePtr;
+
+ /* we enter this state only if we haven't read the whole record. */
+ Assert (state->recordRemainLen > 0);
+
+ while(state->recordRemainLen > 0)
+ {
+ char *contdata;
+ uint32 request_len;
+ uint32 record_len;
+
+ /* Wait for the next page to become available */
+ targetPagePtr = state->recordContRecPtr;
+
+ /* this request contains page header */
+ Assert (targetPagePtr != 0);
+ if (XLogNeedData(state, targetPagePtr,
+ Min(state->recordRemainLen, XLOG_BLCKSZ),
+ false))
+ return XLREAD_NEED_DATA;
+
+ if (!state->page_verified)
+ goto err;
+
+ Assert(SizeOfXLogShortPHD <= state->readLen);
- if (!state->page_verified)
- goto err;
+ /* Check that the continuation on next page looks valid */
+ pageHeader = (XLogPageHeader) state->readBuf;
+ if (!(pageHeader->xlp_info & XLP_FIRST_IS_CONTRECORD))
+ {
+ report_invalid_record(
+ state,
+ "there is no contrecord flag at %X/%X reading %X/%X",
+ (uint32) (state->recordContRecPtr >> 32),
+ (uint32) state->recordContRecPtr,
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr);
+ goto err;
+ }
- /* Record does not cross a page boundary */
- if (!ValidXLogRecord(state, record, RecPtr))
- goto err;
+ /*
+ * Cross-check that xlp_rem_len agrees with how much of the
+ * record we expect there to be left.
+ */
+ if (pageHeader->xlp_rem_len == 0 ||
+ pageHeader->xlp_rem_len != state->recordRemainLen)
+ {
+ report_invalid_record(
+ state,
+ "invalid contrecord length %u at %X/%X reading %X/%X, expected %u",
+ pageHeader->xlp_rem_len,
+ (uint32) (state->recordContRecPtr >> 32),
+ (uint32) state->recordContRecPtr,
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr,
+ state->recordRemainLen);
+ goto err;
+ }
- state->EndRecPtr = RecPtr + MAXALIGN(total_len);
+ /* Append the continuation from this page to the buffer */
+ pageHeaderSize = XLogPageHeaderSize(pageHeader);
- state->ReadRecPtr = RecPtr;
+ /*
+ * XLogNeedData should have ensured that the whole page header
+ * was read
+ */
+ Assert(state->readLen >= pageHeaderSize);
+
+ contdata = (char *) state->readBuf + pageHeaderSize;
+ record_len = XLOG_BLCKSZ - pageHeaderSize;
+ if (pageHeader->xlp_rem_len < record_len)
+ record_len = pageHeader->xlp_rem_len;
+
+ request_len = record_len + pageHeaderSize;
+
+ /* XLogNeedData should have ensured all needed data was read */
+ Assert (state->readLen >= request_len);
+
+ memcpy(state->readRecordBuf + state->recordGotLen,
+ (char *) contdata, record_len);
+ state->recordGotLen += record_len;
+ state->recordRemainLen -= record_len;
+
+ /* If we just reassembled the record header, validate it. */
+ if (!state->record_verified)
+ {
+ Assert(state->recordGotLen >= SizeOfXLogRecord);
+ if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
+ state->PrevRecPtr,
+ (XLogRecord *) state->readRecordBuf))
+ goto err;
+
+ state->record_verified = true;
+ }
+
+ /* Calculate pointer to beginning of next page, and continue */
+ state->recordContRecPtr += XLOG_BLCKSZ;
+ }
+
+ /* targetPagePtr is pointing the last-read page here */
+ prec = (XLogRecord *) state->readRecordBuf;
+ if (!ValidXLogRecord(state, prec, state->ReadRecPtr))
+ goto err;
+
+ pageHeaderSize =
+ XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+ state->EndRecPtr = targetPagePtr + pageHeaderSize
+ + MAXALIGN(pageHeader->xlp_rem_len);
+
+ *record = prec;
+ state->readRecordState = XLREAD_NEXT_RECORD;
+ break;
+ }
}
/*
* Special processing if it's an XLOG SWITCH record
*/
- if (record->xl_rmid == RM_XLOG_ID &&
- (record->xl_info & ~XLR_INFO_MASK) == XLOG_SWITCH)
+ if ((*record)->xl_rmid == RM_XLOG_ID &&
+ ((*record)->xl_info & ~XLR_INFO_MASK) == XLOG_SWITCH)
{
/* Pretend it extends to end of segment */
state->EndRecPtr += state->segcxt.ws_segsize - 1;
state->EndRecPtr -= XLogSegmentOffset(state->EndRecPtr, state->segcxt.ws_segsize);
}
- if (DecodeXLogRecord(state, record, errormsg))
- return record;
- else
- return NULL;
+ Assert (!*record || state->readLen >= 0);
+ if (DecodeXLogRecord(state, *record, errormsg))
+ return XLREAD_SUCCESS;
+
+ *record = NULL;
+ return XLREAD_FAIL;
err:
/*
- * Invalidate the read state. We might read from a different source after
+ * Invalidate the read page. We might read from a different source after
* failure.
*/
XLogReaderInvalReadState(state);
@@ -572,7 +706,8 @@ err:
if (state->errormsg_buf[0] != '\0')
*errormsg = state->errormsg_buf;
- return NULL;
+ *record = NULL;
+ return XLREAD_FAIL;
}
/*
@@ -724,11 +859,12 @@ XLogReaderInvalReadState(XLogReaderState *state)
*
* This is just a convenience subroutine to avoid duplicated code in
* XLogReadRecord. It's not intended for use from anywhere else.
+ *
+ * If PrevRecPtr is valid, the xl_prev is is cross-checked with it.
*/
static bool
ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
- XLogRecPtr PrevRecPtr, XLogRecord *record,
- bool randAccess)
+ XLogRecPtr PrevRecPtr, XLogRecord *record)
{
if (record->xl_tot_len < SizeOfXLogRecord)
{
@@ -746,7 +882,7 @@ ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
(uint32) RecPtr);
return false;
}
- if (randAccess)
+ if (PrevRecPtr == InvalidXLogRecPtr)
{
/*
* We can't exactly verify the prev-link, but surely it should be less
@@ -978,11 +1114,14 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
* XLogReadRecord() will read the next valid record.
*/
XLogRecPtr
-XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
+XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
+ XLogFindNextRecordCB read_page, void *private)
{
XLogRecPtr tmpRecPtr;
XLogRecPtr found = InvalidXLogRecPtr;
XLogPageHeader header;
+ XLogRecord *record;
+ XLogReadRecordResult result;
char *errormsg;
Assert(!XLogRecPtrIsInvalid(RecPtr));
@@ -1015,9 +1154,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
while(XLogNeedData(state, targetPagePtr, targetRecOff,
targetRecOff != 0))
{
- if (!state->routine.page_read(state, state->readPagePtr,
- state->readLen,
- state->ReadRecPtr, state->readBuf))
+ if (!read_page(state, private))
break;
}
@@ -1069,8 +1206,16 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
* or we just jumped over the remaining data of a continuation.
*/
XLogBeginRead(state, tmpRecPtr);
- while (XLogReadRecord(state, &errormsg) != NULL)
+ while ((result = XLogReadRecord(state, &record, &errormsg)) !=
+ XLREAD_FAIL)
{
+ if (result == XLREAD_NEED_DATA)
+ {
+ if (!read_page(state, private))
+ goto err;
+ continue;
+ }
+
/* past the record we've found, break out */
if (RecPtr <= state->ReadRecPtr)
{
@@ -1090,9 +1235,9 @@ err:
#endif /* FRONTEND */
/*
- * Helper function to ease writing of XLogRoutine->page_read callbacks.
- * If this function is used, caller must supply a segment_open callback in
- * 'state', as that is used here.
+ * Helper function to ease writing of page_read callback.
+ * If this function is used, caller must supply a segment_open callback and
+ * segment_close callback as that is used here.
*
* Read 'count' bytes into 'buf', starting at location 'startptr', from WAL
* fetched from timeline 'tli'.
@@ -1105,6 +1250,7 @@ err:
*/
bool
WALRead(XLogReaderState *state,
+ WALSegmentOpenCB segopenfn, WALSegmentCloseCB segclosefn,
char *buf, XLogRecPtr startptr, Size count, TimeLineID tli,
WALReadError *errinfo)
{
@@ -1136,10 +1282,10 @@ WALRead(XLogReaderState *state,
XLogSegNo nextSegNo;
if (state->seg.ws_file >= 0)
- state->routine.segment_close(state);
+ segclosefn(state);
XLByteToSeg(recptr, nextSegNo, state->segcxt.ws_segsize);
- state->routine.segment_open(state, nextSegNo, &tli);
+ segopenfn(state, nextSegNo, &tli);
/* This shouldn't happen -- indicates a bug in segment_open */
Assert(state->seg.ws_file >= 0);
diff --git a/src/backend/access/transam/xlogutils.c b/src/backend/access/transam/xlogutils.c
index 1b10b6df20..303333571f 100644
--- a/src/backend/access/transam/xlogutils.c
+++ b/src/backend/access/transam/xlogutils.c
@@ -689,8 +689,7 @@ XLogTruncateRelation(RelFileNode rnode, ForkNumber forkNum,
void
XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wantLength)
{
- const XLogRecPtr lastReadPage = state->seg.ws_segno *
- state->segcxt.ws_segsize + state->readLen;
+ const XLogRecPtr lastReadPage = state->readPagePtr;
Assert(wantPage != InvalidXLogRecPtr && wantPage % XLOG_BLCKSZ == 0);
Assert(wantLength <= XLOG_BLCKSZ);
@@ -705,7 +704,7 @@ XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wa
* current TLI has since become historical.
*/
if (lastReadPage == wantPage &&
- state->readLen != 0 &&
+ state->page_verified &&
lastReadPage + state->readLen >= wantPage + Min(wantLength, XLOG_BLCKSZ - 1))
return;
@@ -792,6 +791,7 @@ wal_segment_open(XLogReaderState *state, XLogSegNo nextSegNo,
char path[MAXPGPATH];
XLogFilePath(path, tli, nextSegNo, state->segcxt.ws_segsize);
+ elog(LOG, "HOGE: %lu, %d => %s", nextSegNo, tli, path);
state->seg.ws_file = BasicOpenFile(path, O_RDONLY | PG_BINARY);
if (state->seg.ws_file >= 0)
return;
@@ -829,9 +829,11 @@ wal_segment_close(XLogReaderState *state)
* loop for now.
*/
bool
-read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *cur_page)
+read_local_xlog_page(XLogReaderState *state)
{
+ XLogRecPtr targetPagePtr = state->readPagePtr;
+ int reqLen = state->readLen;
+ char *cur_page = state->readBuf;
XLogRecPtr read_upto,
loc;
TimeLineID tli;
@@ -944,11 +946,12 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
* as 'count', read the whole page anyway. It's guaranteed to be
* zero-padded up to the page boundary if it's incomplete.
*/
- if (!WALRead(state, cur_page, targetPagePtr, XLOG_BLCKSZ, tli,
- &errinfo))
+ if (!WALRead(state, wal_segment_open, wal_segment_close,
+ cur_page, targetPagePtr, XLOG_BLCKSZ, tli, &errinfo))
WALReadRaiseError(&errinfo);
/* number of valid bytes in the buffer */
+ state->readPagePtr = targetPagePtr;
state->readLen = count;
return true;
}
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index 61902be3b0..a15b0b3355 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -120,7 +120,8 @@ StartupDecodingContext(List *output_plugin_options,
TransactionId xmin_horizon,
bool need_full_snapshot,
bool fast_forward,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress)
@@ -169,11 +170,12 @@ StartupDecodingContext(List *output_plugin_options,
ctx->slot = slot;
- ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, xl_routine, ctx);
+ ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, cleanup_cb);
if (!ctx->reader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
errmsg("out of memory")));
+ ctx->page_read = page_read;
ctx->reorder = ReorderBufferAllocate();
ctx->snapshot_builder =
@@ -231,7 +233,8 @@ CreateInitDecodingContext(char *plugin,
List *output_plugin_options,
bool need_full_snapshot,
XLogRecPtr restart_lsn,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress)
@@ -328,7 +331,7 @@ CreateInitDecodingContext(char *plugin,
ctx = StartupDecodingContext(NIL, restart_lsn, xmin_horizon,
need_full_snapshot, false,
- xl_routine, prepare_write, do_write,
+ page_read, cleanup_cb, prepare_write, do_write,
update_progress);
/* call output plugin initialization callback */
@@ -376,7 +379,8 @@ LogicalDecodingContext *
CreateDecodingContext(XLogRecPtr start_lsn,
List *output_plugin_options,
bool fast_forward,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress)
@@ -429,8 +433,8 @@ CreateDecodingContext(XLogRecPtr start_lsn,
ctx = StartupDecodingContext(output_plugin_options,
start_lsn, InvalidTransactionId, false,
- fast_forward, xl_routine, prepare_write,
- do_write, update_progress);
+ fast_forward, page_read, cleanup_cb,
+ prepare_write, do_write, update_progress);
/* call output plugin initialization callback */
old_context = MemoryContextSwitchTo(ctx->context);
@@ -483,7 +487,13 @@ DecodingContextFindStartpoint(LogicalDecodingContext *ctx)
char *err = NULL;
/* the read_page callback waits for new WAL */
- record = XLogReadRecord(ctx->reader, &err);
+ while (XLogReadRecord(ctx->reader, &record, &err) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!ctx->page_read(ctx->reader))
+ break;
+ }
+
if (err)
elog(ERROR, "%s", err);
if (!record)
diff --git a/src/backend/replication/logical/logicalfuncs.c b/src/backend/replication/logical/logicalfuncs.c
index b99c94e848..fcc81b358e 100644
--- a/src/backend/replication/logical/logicalfuncs.c
+++ b/src/backend/replication/logical/logicalfuncs.c
@@ -233,9 +233,8 @@ pg_logical_slot_get_changes_guts(FunctionCallInfo fcinfo, bool confirm, bool bin
ctx = CreateDecodingContext(InvalidXLogRecPtr,
options,
false,
- XL_ROUTINE(.page_read = read_local_xlog_page,
- .segment_open = wal_segment_open,
- .segment_close = wal_segment_close),
+ read_local_xlog_page,
+ wal_segment_close,
LogicalOutputPrepareWrite,
LogicalOutputWrite, NULL);
@@ -284,7 +283,13 @@ pg_logical_slot_get_changes_guts(FunctionCallInfo fcinfo, bool confirm, bool bin
XLogRecord *record;
char *errm = NULL;
- record = XLogReadRecord(ctx->reader, &errm);
+ while (XLogReadRecord(ctx->reader, &record, &errm) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!ctx->page_read(ctx->reader))
+ break;
+ }
+
if (errm)
elog(ERROR, "%s", errm);
diff --git a/src/backend/replication/slotfuncs.c b/src/backend/replication/slotfuncs.c
index 88033a79b2..8a678074d5 100644
--- a/src/backend/replication/slotfuncs.c
+++ b/src/backend/replication/slotfuncs.c
@@ -152,9 +152,8 @@ create_logical_replication_slot(char *name, char *plugin,
ctx = CreateInitDecodingContext(plugin, NIL,
false, /* just catalogs is OK */
restart_lsn,
- XL_ROUTINE(.page_read = read_local_xlog_page,
- .segment_open = wal_segment_open,
- .segment_close = wal_segment_close),
+ read_local_xlog_page,
+ wal_segment_close,
NULL, NULL, NULL);
/*
@@ -486,9 +485,8 @@ pg_logical_replication_slot_advance(XLogRecPtr moveto)
ctx = CreateDecodingContext(InvalidXLogRecPtr,
NIL,
true, /* fast_forward */
- XL_ROUTINE(.page_read = read_local_xlog_page,
- .segment_open = wal_segment_open,
- .segment_close = wal_segment_close),
+ read_local_xlog_page,
+ wal_segment_close,
NULL, NULL, NULL);
/*
@@ -513,7 +511,13 @@ pg_logical_replication_slot_advance(XLogRecPtr moveto)
* Read records. No changes are generated in fast_forward mode,
* but snapbuilder/slot statuses are updated properly.
*/
- record = XLogReadRecord(ctx->reader, &errm);
+ while (XLogReadRecord(ctx->reader, &record, &errm) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!ctx->page_read(ctx->reader))
+ break;
+ }
+
if (errm)
elog(ERROR, "%s", errm);
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index bf2711bddd..2fd24a9b14 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -580,10 +580,7 @@ StartReplication(StartReplicationCmd *cmd)
/* create xlogreader for physical replication */
xlogreader =
- XLogReaderAllocate(wal_segment_size, NULL,
- XL_ROUTINE(.segment_open = WalSndSegmentOpen,
- .segment_close = wal_segment_close),
- NULL);
+ XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
if (!xlogreader)
ereport(ERROR,
@@ -812,9 +809,11 @@ StartReplication(StartReplicationCmd *cmd)
* set every time WAL is flushed.
*/
static bool
-logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetRecPtr, char *cur_page)
+logical_read_xlog_page(XLogReaderState *state)
{
+ XLogRecPtr targetPagePtr = state->readPagePtr;
+ int reqLen = state->readLen;
+ char *cur_page = state->readBuf;
XLogRecPtr flushptr;
int count;
WALReadError errinfo;
@@ -842,7 +841,7 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
count = flushptr - targetPagePtr; /* part of the page available */
/* now actually read the data, we know it's there */
- if (!WALRead(state,
+ if (!WALRead(state, WalSndSegmentOpen, wal_segment_close,
cur_page,
targetPagePtr,
XLOG_BLCKSZ,
@@ -1014,9 +1013,8 @@ CreateReplicationSlot(CreateReplicationSlotCmd *cmd)
ctx = CreateInitDecodingContext(cmd->plugin, NIL, need_full_snapshot,
InvalidXLogRecPtr,
- XL_ROUTINE(.page_read = logical_read_xlog_page,
- .segment_open = WalSndSegmentOpen,
- .segment_close = wal_segment_close),
+ logical_read_xlog_page,
+ wal_segment_close,
WalSndPrepareWrite, WalSndWriteData,
WalSndUpdateProgress);
@@ -1179,9 +1177,8 @@ StartLogicalReplication(StartReplicationCmd *cmd)
*/
logical_decoding_ctx =
CreateDecodingContext(cmd->startpoint, cmd->options, false,
- XL_ROUTINE(.page_read = logical_read_xlog_page,
- .segment_open = WalSndSegmentOpen,
- .segment_close = wal_segment_close),
+ logical_read_xlog_page,
+ wal_segment_close,
WalSndPrepareWrite, WalSndWriteData,
WalSndUpdateProgress);
xlogreader = logical_decoding_ctx->reader;
@@ -2761,7 +2758,7 @@ XLogSendPhysical(void)
enlargeStringInfo(&output_message, nbytes);
retry:
- if (!WALRead(xlogreader,
+ if (!WALRead(xlogreader, WalSndSegmentOpen, wal_segment_close,
&output_message.data[output_message.len],
startptr,
nbytes,
@@ -2859,7 +2856,12 @@ XLogSendLogical(void)
*/
WalSndCaughtUp = false;
- record = XLogReadRecord(logical_decoding_ctx->reader, &errm);
+ while (XLogReadRecord(logical_decoding_ctx->reader, &record, &errm) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!logical_decoding_ctx->page_read(logical_decoding_ctx->reader))
+ break;
+ }
/* xlog record was invalid */
if (errm != NULL)
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index 0b7e73ae79..3dd6df4be6 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -40,15 +40,9 @@ static int xlogreadfd = -1;
static XLogSegNo xlogreadsegno = -1;
static char xlogfpath[MAXPGPATH];
-typedef struct XLogPageReadPrivate
-{
- const char *restoreCommand;
- int tliIndex;
-} XLogPageReadPrivate;
-
-static bool SimpleXLogPageRead(XLogReaderState *xlogreader,
- XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
+static bool SimpleXLogPageRead(XLogReaderState *xlogreader,
+ const char *datadir, int *tliIndex,
+ const char *restoreCommand);
/*
* Read WAL from the datadir/pg_wal, starting from 'startpoint' on timeline
@@ -62,20 +56,22 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
XLogRecord *record;
XLogReaderState *xlogreader;
char *errormsg;
- XLogPageReadPrivate private;
- private.tliIndex = tliIndex;
- private.restoreCommand = restoreCommand;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir,
- XL_ROUTINE(.page_read = &SimpleXLogPageRead),
- &private);
+ xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+
if (xlogreader == NULL)
pg_fatal("out of memory");
XLogBeginRead(xlogreader, startpoint);
do
{
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!SimpleXLogPageRead(xlogreader, datadir,
+ &tliIndex, restoreCommand))
+ break;
+ }
if (record == NULL)
{
@@ -113,19 +109,19 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex,
XLogRecord *record;
XLogReaderState *xlogreader;
char *errormsg;
- XLogPageReadPrivate private;
XLogRecPtr endptr;
- private.tliIndex = tliIndex;
- private.restoreCommand = restoreCommand;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir,
- XL_ROUTINE(.page_read = &SimpleXLogPageRead),
- &private);
+ xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
XLogBeginRead(xlogreader, ptr);
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!SimpleXLogPageRead(xlogreader, datadir, &tliIndex, restoreCommand))
+ break;
+ }
if (record == NULL)
{
if (errormsg)
@@ -160,7 +156,6 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
XLogRecPtr searchptr;
XLogReaderState *xlogreader;
char *errormsg;
- XLogPageReadPrivate private;
/*
* The given fork pointer points to the end of the last common record,
@@ -176,11 +171,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
forkptr += SizeOfXLogShortPHD;
}
- private.tliIndex = tliIndex;
- private.restoreCommand = restoreCommand;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir,
- XL_ROUTINE(.page_read = &SimpleXLogPageRead),
- &private);
+ xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
@@ -190,7 +181,13 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
uint8 info;
XLogBeginRead(xlogreader, searchptr);
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!SimpleXLogPageRead(xlogreader, datadir,
+ &tliIndex, restoreCommand))
+ break;
+ }
if (record == NULL)
{
@@ -237,10 +234,11 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
/* XLogReader callback function, to read a WAL page */
static bool
-SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *readBuf)
+SimpleXLogPageRead(XLogReaderState *xlogreader, const char *datadir,
+ int *tliIndex, const char *restoreCommand)
{
- XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
+ XLogRecPtr targetPagePtr = xlogreader->readPagePtr;
+ char *readBuf = xlogreader->readBuf;
uint32 targetPageOff;
XLogRecPtr targetSegEnd;
XLogSegNo targetSegNo;
@@ -273,14 +271,14 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
* be done both forward and backward, consider also switching timeline
* accordingly.
*/
- while (private->tliIndex < targetNentries - 1 &&
- targetHistory[private->tliIndex].end < targetSegEnd)
- private->tliIndex++;
- while (private->tliIndex > 0 &&
- targetHistory[private->tliIndex].begin >= targetSegEnd)
- private->tliIndex--;
+ while (*tliIndex < targetNentries - 1 &&
+ targetHistory[*tliIndex].end < targetSegEnd)
+ (*tliIndex)++;
+ while (*tliIndex > 0 &&
+ targetHistory[*tliIndex].begin >= targetSegEnd)
+ (*tliIndex)--;
- XLogFileName(xlogfname, targetHistory[private->tliIndex].tli,
+ XLogFileName(xlogfname, targetHistory[*tliIndex].tli,
xlogreadsegno, WalSegSz);
snprintf(xlogfpath, MAXPGPATH, "%s/" XLOGDIR "/%s",
@@ -293,7 +291,7 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
/*
* If we have no restore_command to execute, then exit.
*/
- if (private->restoreCommand == NULL)
+ if (restoreCommand == NULL)
{
pg_log_error("could not open file \"%s\": %m", xlogfpath);
xlogreader->readLen = -1;
@@ -307,7 +305,7 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
xlogreadfd = RestoreArchivedFile(xlogreader->segcxt.ws_dir,
xlogfname,
WalSegSz,
- private->restoreCommand);
+ restoreCommand);
if (xlogreadfd < 0)
{
@@ -349,7 +347,7 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
Assert(targetSegNo == xlogreadsegno);
- xlogreader->seg.ws_tli = targetHistory[private->tliIndex].tli;
+ xlogreader->seg.ws_tli = targetHistory[*tliIndex].tli;
xlogreader->readLen = XLOG_BLCKSZ;
return true;
}
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index ab0c28d9b7..02e71138cc 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -330,12 +330,17 @@ WALDumpCloseSegment(XLogReaderState *state)
state->seg.ws_file = -1;
}
-/* pg_waldump's XLogReaderRoutine->page_read callback */
+/*
+ * pg_waldump's WAL page rader, also used as page_read callback for
+ * XLogFindNextRecord
+ */
static bool
-WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetPtr, char *readBuff)
+WALDumpReadPage(XLogReaderState *state, void *priv)
{
- XLogDumpPrivate *private = state->private_data;
+ XLogRecPtr targetPagePtr = state->readPagePtr;
+ int reqLen = state->readLen;
+ char *readBuff = state->readBuf;
+ XLogDumpPrivate *private = (XLogDumpPrivate *) priv;
int count = XLOG_BLCKSZ;
WALReadError errinfo;
@@ -353,8 +358,8 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
}
}
- if (!WALRead(state, readBuff, targetPagePtr, count, private->timeline,
- &errinfo))
+ if (!WALRead(state, WALDumpOpenSegment, WALDumpCloseSegment,
+ readBuff, targetPagePtr, count, private->timeline, &errinfo))
{
WALOpenSegment *seg = &errinfo.wre_seg;
char fname[MAXPGPATH];
@@ -374,6 +379,7 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
(Size) errinfo.wre_req);
}
+ Assert(count >= state->readLen);
state->readLen = count;
return true;
}
@@ -1042,16 +1048,14 @@ main(int argc, char **argv)
/* we have everything we need, start reading */
xlogreader_state =
- XLogReaderAllocate(WalSegSz, waldir,
- XL_ROUTINE(.page_read = WALDumpReadPage,
- .segment_open = WALDumpOpenSegment,
- .segment_close = WALDumpCloseSegment),
- &private);
+ XLogReaderAllocate(WalSegSz, waldir, WALDumpCloseSegment);
+
if (!xlogreader_state)
fatal_error("out of memory");
/* first find a valid recptr to start from */
- first_record = XLogFindNextRecord(xlogreader_state, private.startptr);
+ first_record = XLogFindNextRecord(xlogreader_state, private.startptr,
+ &WALDumpReadPage, (void*) &private);
if (first_record == InvalidXLogRecPtr)
fatal_error("could not find a valid record after %X/%X",
@@ -1075,7 +1079,13 @@ main(int argc, char **argv)
for (;;)
{
/* try to read the next record */
- record = XLogReadRecord(xlogreader_state, &errormsg);
+ while (XLogReadRecord(xlogreader_state, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!WALDumpReadPage(xlogreader_state, (void *) &private))
+ break;
+ }
+
if (!record)
{
if (!config.follow || private.endptr_reached)
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index 6c8848e14f..6a7fe140cb 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -57,64 +57,15 @@ typedef struct WALSegmentContext
typedef struct XLogReaderState XLogReaderState;
-/* Function type definition for the read_page callback */
-typedef bool (*XLogPageReadCB) (XLogReaderState *xlogreader,
- XLogRecPtr targetPagePtr,
- int reqLen,
- XLogRecPtr targetRecPtr,
- char *readBuf);
+/* Function type definition for the segment cleanup callback */
+typedef void (*WALSegmentCleanupCB) (XLogReaderState *xlogreader);
+
+/* Function type definition for the open/close callbacks for WALRead() */
typedef void (*WALSegmentOpenCB) (XLogReaderState *xlogreader,
XLogSegNo nextSegNo,
TimeLineID *tli_p);
typedef void (*WALSegmentCloseCB) (XLogReaderState *xlogreader);
-typedef struct XLogReaderRoutine
-{
- /*
- * Data input callback
- *
- * This callback shall read at least reqLen valid bytes of the xlog page
- * starting at targetPagePtr, and store them in readBuf. The callback
- * shall return the number of bytes read (never more than XLOG_BLCKSZ), or
- * -1 on failure. The callback shall sleep, if necessary, to wait for the
- * requested bytes to become available. The callback will not be invoked
- * again for the same page unless more than the returned number of bytes
- * are needed.
- *
- * targetRecPtr is the position of the WAL record we're reading. Usually
- * it is equal to targetPagePtr + reqLen, but sometimes xlogreader needs
- * to read and verify the page or segment header, before it reads the
- * actual WAL record it's interested in. In that case, targetRecPtr can
- * be used to determine which timeline to read the page from.
- *
- * The callback shall set ->seg.ws_tli to the TLI of the file the page was
- * read from.
- */
- XLogPageReadCB page_read;
-
- /*
- * Callback to open the specified WAL segment for reading. ->seg.ws_file
- * shall be set to the file descriptor of the opened segment. In case of
- * failure, an error shall be raised by the callback and it shall not
- * return.
- *
- * "nextSegNo" is the number of the segment to be opened.
- *
- * "tli_p" is an input/output argument. WALRead() uses it to pass the
- * timeline in which the new segment should be found, but the callback can
- * use it to return the TLI that it actually opened.
- */
- WALSegmentOpenCB segment_open;
-
- /*
- * WAL segment close callback. ->seg.ws_file shall be set to a negative
- * number.
- */
- WALSegmentCloseCB segment_close;
-} XLogReaderRoutine;
-
-#define XL_ROUTINE(...) &(XLogReaderRoutine){__VA_ARGS__}
-
typedef struct
{
/* Is this block ref in use? */
@@ -144,12 +95,35 @@ typedef struct
uint16 data_bufsz;
} DecodedBkpBlock;
+/* Return code from XLogReadRecord */
+typedef enum XLogReadRecordResult
+{
+ XLREAD_SUCCESS, /* record is successfully read */
+ XLREAD_NEED_DATA, /* need more data. see XLogReadRecord. */
+ XLREAD_FAIL /* failed during reading a record */
+} XLogReadRecordResult;
+
+/*
+ * internal state of XLogReadRecord
+ *
+ * XLogReadState runs a state machine while reading a record. Theses states
+ * are not seen outside the function. Each state may repeat several times
+ * exiting requesting caller for new data. See the comment of XLogReadRecrod
+ * for details.
+ */
+typedef enum XLogReadRecordState {
+ XLREAD_NEXT_RECORD,
+ XLREAD_TOT_LEN,
+ XLREAD_FIRST_FRAGMENT,
+ XLREAD_CONTINUATION
+} XLogReadRecordState;
+
struct XLogReaderState
{
/*
* Operational callbacks
*/
- XLogReaderRoutine routine;
+ WALSegmentCleanupCB cleanup_cb;
/* ----------------------------------------
* Public parameters
@@ -162,18 +136,14 @@ struct XLogReaderState
*/
uint64 system_identifier;
- /*
- * Opaque data for callbacks to use. Not used by XLogReader.
- */
- void *private_data;
-
/*
* Start and end point of last record read. EndRecPtr is also used as the
* position to read next. Calling XLogBeginRead() sets EndRecPtr to the
* starting position and ReadRecPtr to invalid.
*/
- XLogRecPtr ReadRecPtr; /* start of last record read */
+ XLogRecPtr ReadRecPtr; /* start of last record read or being read */
XLogRecPtr EndRecPtr; /* end+1 of last record read */
+ XLogRecPtr PrevRecPtr; /* start of previous record read */
/* ----------------------------------------
* Communication with page reader
@@ -187,7 +157,9 @@ struct XLogReaderState
* the request, or -1 on error */
TimeLineID readPageTLI; /* TLI for data currently in readBuf */
char *readBuf; /* buffer to store data */
- bool page_verified; /* is the page on the buffer verified? */
+ bool page_verified; /* is the page header on the buffer verified? */
+ bool record_verified;/* is the current record header verified? */
+
/* ----------------------------------------
@@ -227,8 +199,6 @@ struct XLogReaderState
XLogRecPtr latestPagePtr;
TimeLineID latestPageTLI;
- /* beginning of the WAL record being read. */
- XLogRecPtr currRecPtr;
/* timeline to read it from, 0 if a lookup is required */
TimeLineID currTLI;
@@ -255,6 +225,15 @@ struct XLogReaderState
char *readRecordBuf;
uint32 readRecordBufSize;
+ /*
+ * XLogReadRecord() state
+ */
+ XLogReadRecordState readRecordState;/* state machine state */
+ int recordGotLen; /* amount of current record that has
+ * already been read */
+ int recordRemainLen; /* length of current record that remains */
+ XLogRecPtr recordContRecPtr; /* where the current record continues */
+
/* Buffer to hold error message */
char *errormsg_buf;
};
@@ -262,9 +241,7 @@ struct XLogReaderState
/* Get a new XLogReader */
extern XLogReaderState *XLogReaderAllocate(int wal_segment_size,
const char *waldir,
- XLogReaderRoutine *routine,
- void *private_data);
-extern XLogReaderRoutine *LocalXLogReaderRoutine(void);
+ WALSegmentCleanupCB cleanup_cb);
/* Free an XLogReader */
extern void XLogReaderFree(XLogReaderState *state);
@@ -272,12 +249,17 @@ extern void XLogReaderFree(XLogReaderState *state);
/* Position the XLogReader to given record */
extern void XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr);
#ifdef FRONTEND
-extern XLogRecPtr XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr);
+/* Function type definition for the read_page callback */
+typedef bool (*XLogFindNextRecordCB) (XLogReaderState *xlogreader,
+ void *private);
+extern XLogRecPtr XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
+ XLogFindNextRecordCB read_page, void *private);
#endif /* FRONTEND */
/* Read the next XLog record. Returns NULL on end-of-WAL or failure */
-extern struct XLogRecord *XLogReadRecord(XLogReaderState *state,
- char **errormsg);
+extern XLogReadRecordResult XLogReadRecord(XLogReaderState *state,
+ XLogRecord **record,
+ char **errormsg);
/* Validate a page */
extern bool XLogReaderValidatePageHeader(XLogReaderState *state,
@@ -297,6 +279,7 @@ typedef struct WALReadError
} WALReadError;
extern bool WALRead(XLogReaderState *state,
+ WALSegmentOpenCB segopenfn, WALSegmentCloseCB sgclosefn,
char *buf, XLogRecPtr startptr, Size count,
TimeLineID tli, WALReadError *errinfo);
diff --git a/src/include/access/xlogutils.h b/src/include/access/xlogutils.h
index c287edf206..482e4ced69 100644
--- a/src/include/access/xlogutils.h
+++ b/src/include/access/xlogutils.h
@@ -47,9 +47,7 @@ extern Buffer XLogReadBufferExtended(RelFileNode rnode, ForkNumber forknum,
extern Relation CreateFakeRelcacheEntry(RelFileNode rnode);
extern void FreeFakeRelcacheEntry(Relation fakerel);
-extern bool read_local_xlog_page(XLogReaderState *state,
- XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetRecPtr, char *cur_page);
+extern bool read_local_xlog_page(XLogReaderState *state);
extern void wal_segment_open(XLogReaderState *state,
XLogSegNo nextSegNo,
TimeLineID *tli_p);
diff --git a/src/include/replication/logical.h b/src/include/replication/logical.h
index c2f2475e5d..a743db49cf 100644
--- a/src/include/replication/logical.h
+++ b/src/include/replication/logical.h
@@ -29,6 +29,10 @@ typedef void (*LogicalOutputPluginWriterUpdateProgress) (struct LogicalDecodingC
TransactionId xid
);
+typedef struct LogicalDecodingContext LogicalDecodingContext;
+
+typedef bool (*LogicalDecodingXLogPageReadCB)(XLogReaderState *ctx);
+
typedef struct LogicalDecodingContext
{
/* memory context this is all allocated in */
@@ -39,6 +43,7 @@ typedef struct LogicalDecodingContext
/* infrastructure pieces for decoding */
XLogReaderState *reader;
+ LogicalDecodingXLogPageReadCB page_read;
struct ReorderBuffer *reorder;
struct SnapBuild *snapshot_builder;
@@ -95,14 +100,16 @@ extern LogicalDecodingContext *CreateInitDecodingContext(char *plugin,
List *output_plugin_options,
bool need_full_snapshot,
XLogRecPtr restart_lsn,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress);
extern LogicalDecodingContext *CreateDecodingContext(XLogRecPtr start_lsn,
List *output_plugin_options,
bool fast_forward,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress);
--
2.18.4
----Next_Part(Thu_Jul__2_13_53_30_2020_072)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
filename="v15-0003-Remove-globals-readOff-readLen-and-readSegNo.patch"
^ permalink raw reply [nested|flat] 70+ messages in thread
* [PATCH v17 2/4] Move page-reader out of XLogReadRecord
@ 2019-09-10 03:58 Kyotaro Horiguchi <[email protected]>
0 siblings, 0 replies; 70+ messages in thread
From: Kyotaro Horiguchi @ 2019-09-10 03:58 UTC (permalink / raw)
This is the second step of removing callbacks from WAL record reader.
Since it is essential to take in additional data while reading a
record, the function have to ask caller for new data while keeping
working state. Thus the function is turned into a state machine.
---
src/backend/access/transam/twophase.c | 15 +-
src/backend/access/transam/xlog.c | 58 +-
src/backend/access/transam/xlogreader.c | 697 +++++++++++-------
src/backend/access/transam/xlogutils.c | 17 +-
src/backend/replication/logical/logical.c | 26 +-
.../replication/logical/logicalfuncs.c | 13 +-
src/backend/replication/slotfuncs.c | 18 +-
src/backend/replication/walsender.c | 32 +-
src/bin/pg_rewind/parsexlog.c | 84 ++-
src/bin/pg_waldump/pg_waldump.c | 36 +-
src/include/access/xlogreader.h | 121 ++-
src/include/access/xlogutils.h | 4 +-
src/include/pg_config_manual.h | 2 +-
src/include/replication/logical.h | 11 +-
14 files changed, 646 insertions(+), 488 deletions(-)
diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index 80d2d20d6c..7d5a96a5f5 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -1325,11 +1325,8 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
XLogReaderState *xlogreader;
char *errormsg;
- xlogreader = XLogReaderAllocate(wal_segment_size, NULL,
- XL_ROUTINE(.page_read = &read_local_xlog_page,
- .segment_open = &wal_segment_open,
- .segment_close = &wal_segment_close),
- NULL);
+ xlogreader = XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+
if (!xlogreader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
@@ -1337,7 +1334,13 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
errdetail("Failed while allocating a WAL reading processor.")));
XLogBeginRead(xlogreader, lsn);
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!read_local_xlog_page(xlogreader))
+ break;
+ }
+
if (record == NULL)
ereport(ERROR,
(errcode_for_file_access(),
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 606328a65a..670996ae67 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -836,13 +836,6 @@ static XLogSource currentSource = XLOG_FROM_ANY;
static bool lastSourceFailed = false;
static bool pendingWalRcvRestart = false;
-typedef struct XLogPageReadPrivate
-{
- int emode;
- bool fetching_ckpt; /* are we fetching a checkpoint record? */
- bool randAccess;
-} XLogPageReadPrivate;
-
/*
* These variables track when we last obtained some WAL data to process,
* and where we got it from. (XLogReceiptSource is initially the same as
@@ -917,8 +910,8 @@ static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath,
static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli,
XLogSource source, bool notfoundOk);
static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, XLogSource source);
-static bool XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
+static bool XLogPageRead(XLogReaderState *xlogreader,
+ bool fetching_ckpt, int emode, bool randAccess);
static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
bool fetching_ckpt, XLogRecPtr tliRecPtr);
static int emode_for_corrupt_record(int emode, XLogRecPtr RecPtr);
@@ -1231,8 +1224,7 @@ XLogInsertRecord(XLogRecData *rdata,
appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len);
if (!debug_reader)
- debug_reader = XLogReaderAllocate(wal_segment_size, NULL,
- XL_ROUTINE(), NULL);
+ debug_reader = XLogReaderAllocate(wal_segment_size, NULL, NULL);
if (!debug_reader)
{
@@ -4344,15 +4336,10 @@ CleanupBackupHistory(void)
* record is available.
*/
static XLogRecord *
-ReadRecord(XLogReaderState *xlogreader, int emode,
- bool fetching_ckpt)
+ReadRecord(XLogReaderState *xlogreader, int emode, bool fetching_ckpt)
{
XLogRecord *record;
- XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
-
- private->fetching_ckpt = fetching_ckpt;
- private->emode = emode;
- private->randAccess = (xlogreader->ReadRecPtr == InvalidXLogRecPtr);
+ bool randAccess = (xlogreader->ReadRecPtr == InvalidXLogRecPtr);
/* This is the first attempt to read this page. */
lastSourceFailed = false;
@@ -4360,8 +4347,16 @@ ReadRecord(XLogReaderState *xlogreader, int emode,
for (;;)
{
char *errormsg;
+ XLogReadRecordResult result;
+
+ while ((result = XLogReadRecord(xlogreader, &record, &errormsg))
+ == XLREAD_NEED_DATA)
+ {
+ if (!XLogPageRead(xlogreader, fetching_ckpt, emode, randAccess))
+ break;
+
+ }
- record = XLogReadRecord(xlogreader, &errormsg);
ReadRecPtr = xlogreader->ReadRecPtr;
EndRecPtr = xlogreader->EndRecPtr;
if (record == NULL)
@@ -6380,7 +6375,6 @@ StartupXLOG(void)
bool backupFromStandby = false;
DBState dbstate_at_startup;
XLogReaderState *xlogreader;
- XLogPageReadPrivate private;
bool promoted = false;
struct stat st;
@@ -6539,13 +6533,9 @@ StartupXLOG(void)
OwnLatch(&XLogCtl->recoveryWakeupLatch);
/* Set up XLOG reader facility */
- MemSet(&private, 0, sizeof(XLogPageReadPrivate));
xlogreader =
- XLogReaderAllocate(wal_segment_size, NULL,
- XL_ROUTINE(.page_read = &XLogPageRead,
- .segment_open = NULL,
- .segment_close = wal_segment_close),
- &private);
+ XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+
if (!xlogreader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
@@ -11987,12 +11977,13 @@ CancelBackup(void)
* sleep and retry.
*/
static bool
-XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetRecPtr, char *readBuf)
+XLogPageRead(XLogReaderState *xlogreader,
+ bool fetching_ckpt, int emode, bool randAccess)
{
- XLogPageReadPrivate *private =
- (XLogPageReadPrivate *) xlogreader->private_data;
- int emode = private->emode;
+ char *readBuf = xlogreader->readBuf;
+ XLogRecPtr targetPagePtr = xlogreader->readPagePtr;
+ int reqLen = xlogreader->readLen;
+ XLogRecPtr targetRecPtr = xlogreader->ReadRecPtr;
uint32 targetPageOff;
XLogSegNo targetSegNo PG_USED_FOR_ASSERTS_ONLY;
int r;
@@ -12035,8 +12026,8 @@ retry:
flushedUpto < targetPagePtr + reqLen))
{
if (!WaitForWALToBecomeAvailable(targetPagePtr + reqLen,
- private->randAccess,
- private->fetching_ckpt,
+ randAccess,
+ fetching_ckpt,
targetRecPtr))
{
if (readFile >= 0)
@@ -12141,6 +12132,7 @@ retry:
goto next_record_is_invalid;
}
+ Assert(xlogreader->readPagePtr == targetPagePtr);
xlogreader->readLen = readLen;
return true;
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index 7d04995b7b..ba3e4868f7 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -40,7 +40,7 @@ static bool XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr,
int reqLen, bool header_inclusive);
static void XLogReaderInvalReadState(XLogReaderState *state);
static bool ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
- XLogRecPtr PrevRecPtr, XLogRecord *record, bool randAccess);
+ XLogRecPtr PrevRecPtr, XLogRecord *record);
static bool ValidXLogRecord(XLogReaderState *state, XLogRecord *record,
XLogRecPtr recptr);
static void ResetDecoder(XLogReaderState *state);
@@ -73,7 +73,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...)
*/
XLogReaderState *
XLogReaderAllocate(int wal_segment_size, const char *waldir,
- XLogReaderRoutine *routine, void *private_data)
+ WALSegmentCleanupCB cleanup_cb)
{
XLogReaderState *state;
@@ -84,7 +84,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
return NULL;
/* initialize caller-provided support functions */
- state->routine = *routine;
+ state->cleanup_cb = cleanup_cb;
state->max_block_id = -1;
@@ -107,8 +107,6 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
WALOpenSegmentInit(&state->seg, &state->segcxt, wal_segment_size,
waldir);
- /* system_identifier initialized to zeroes above */
- state->private_data = private_data;
/* ReadRecPtr, EndRecPtr and readLen initialized to zeroes above */
state->errormsg_buf = palloc_extended(MAX_ERRORMSG_LEN + 1,
MCXT_ALLOC_NO_OOM);
@@ -140,8 +138,8 @@ XLogReaderFree(XLogReaderState *state)
{
int block_id;
- if (state->seg.ws_file != -1)
- state->routine.segment_close(state);
+ if (state->seg.ws_file >= 0)
+ state->cleanup_cb(state);
for (block_id = 0; block_id <= XLR_MAX_BLOCK_ID; block_id++)
{
@@ -246,6 +244,7 @@ XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
/* Begin at the passed-in record pointer. */
state->EndRecPtr = RecPtr;
state->ReadRecPtr = InvalidXLogRecPtr;
+ state->readRecordState = XLREAD_NEXT_RECORD;
}
/*
@@ -254,317 +253,449 @@ XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
* XLogBeginRead() or XLogFindNextRecord() must be called before the first call
* to XLogReadRecord().
*
- * If the page_read callback fails to read the requested data, NULL is
- * returned. The callback is expected to have reported the error; errormsg
- * is set to NULL.
+ * This function may return XLREAD_NEED_DATA several times before returning a
+ * result record. The caller shall read in some new data then call this
+ * function again with the same parameters.
*
- * If the reading fails for some other reason, NULL is also returned, and
- * *errormsg is set to a string with details of the failure.
+ * When a record is successfully read, returns XLREAD_SUCCESS with result
+ * record being stored in *record. Otherwise *record is NULL.
*
- * The returned pointer (or *errormsg) points to an internal buffer that's
- * valid until the next call to XLogReadRecord.
+ * Returns XLREAD_NEED_DATA if more data is needed to finish reading the
+ * current record. In that case, state->readPagePtr and state->readLen inform
+ * the desired position and minimum length of data needed. The caller shall
+ * read in the requested data and set state->readBuf to point to a buffer
+ * containing it. The caller must also set state->readPageTLI and
+ * state->readLen to indicate the timeline that it was read from, and the
+ * length of data that is now available (which must be >= given readLen),
+ * respectively.
+ *
+ * If invalid data is encountered, returns XLREAD_FAIL with *record being set to
+ * NULL. *errormsg is set to a string with details of the failure.
+ * The returned pointer (or *errormsg) points to an internal buffer that's valid
+ * until the next call to XLogReadRecord.
+ *
+ *
+ * This function runs a state machine consists of the following states.
+ *
+ * XLREAD_NEXT_RECORD :
+ * The initial state, if called with valid RecPtr, try to read a record at
+ * that position. If invalid RecPtr is given try to read a record just after
+ * the last one previously read.
+ * This state ens after setting ReadRecPtr. Then goes to XLREAD_TOT_LEN.
+ *
+ * XLREAD_TOT_LEN:
+ * Examining record header. Ends after reading record total
+ * length. recordRemainLen and recordGotLen are initialized.
+ *
+ * XLREAD_FIRST_FRAGMENT:
+ * Reading the first fragment. Ends with finishing reading a single
+ * record. Goes to XLREAD_NEXT_RECORD if that's all or
+ * XLREAD_CONTINUATION if we have continuation.
+
+ * XLREAD_CONTINUATION:
+ * Reading continuation of record. Ends with finishing the whole record then
+ * goes to XLREAD_NEXT_RECORD. During this state, recordRemainLen indicates
+ * how much is left and readRecordBuf holds the partially assert
+ * record.recordContRecPtr points to the beginning of the next page where to
+ * continue.
+ *
+ * If wrong data found in any state, the state machine stays at the current
+ * state. This behavior allows to continue reading a reacord switching among
+ * different souces, while streaming replication.
*/
-XLogRecord *
-XLogReadRecord(XLogReaderState *state, char **errormsg)
+XLogReadRecordResult
+XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
{
- XLogRecPtr RecPtr;
- XLogRecord *record;
- XLogRecPtr targetPagePtr;
- bool randAccess;
- uint32 len,
- total_len;
- uint32 targetRecOff;
- uint32 pageHeaderSize;
- bool gotheader;
+ XLogRecord *prec;
- /*
- * randAccess indicates whether to verify the previous-record pointer of
- * the record we're reading. We only do this if we're reading
- * sequentially, which is what we initially assume.
- */
- randAccess = false;
+ *record = NULL;
/* reset error state */
*errormsg = NULL;
state->errormsg_buf[0] = '\0';
- ResetDecoder(state);
-
- RecPtr = state->EndRecPtr;
-
- if (state->ReadRecPtr != InvalidXLogRecPtr)
- {
- /* read the record after the one we just read */
-
- /*
- * EndRecPtr is pointing to end+1 of the previous WAL record. If
- * we're at a page boundary, no more records can fit on the current
- * page. We must skip over the page header, but we can't do that until
- * we've read in the page, since the header size is variable.
- */
- }
- else
- {
- /*
- * Caller supplied a position to start at.
- *
- * In this case, EndRecPtr should already be pointing to a valid
- * record starting position.
- */
- Assert(XRecOffIsValid(RecPtr));
- randAccess = true;
- }
-
- state->currRecPtr = RecPtr;
-
- targetPagePtr = RecPtr - (RecPtr % XLOG_BLCKSZ);
- targetRecOff = RecPtr % XLOG_BLCKSZ;
-
- /*
- * Read the page containing the record into state->readBuf. Request enough
- * byte to cover the whole record header, or at least the part of it that
- * fits on the same page.
- */
- while (XLogNeedData(state, targetPagePtr,
- Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
- targetRecOff != 0))
- {
- if (!state->routine.page_read(state, state->readPagePtr, state->readLen,
- RecPtr, state->readBuf))
- break;
- }
-
- if (!state->page_verified)
- goto err;
-
- /*
- * We have at least the page header, so we can examine it now.
- */
- pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
- if (targetRecOff == 0)
- {
- /*
- * At page start, so skip over page header.
- */
- RecPtr += pageHeaderSize;
- targetRecOff = pageHeaderSize;
- }
- else if (targetRecOff < pageHeaderSize)
- {
- report_invalid_record(state, "invalid record offset at %X/%X",
- LSN_FORMAT_ARGS(RecPtr));
- goto err;
- }
-
- if ((((XLogPageHeader) state->readBuf)->xlp_info & XLP_FIRST_IS_CONTRECORD) &&
- targetRecOff == pageHeaderSize)
- {
- report_invalid_record(state, "contrecord is requested by %X/%X",
- LSN_FORMAT_ARGS(RecPtr));
- goto err;
- }
-
- /* XLogNeedData has verified the page header */
- Assert(pageHeaderSize <= state->readLen);
-
- /*
- * Read the record length.
- *
- * NB: Even though we use an XLogRecord pointer here, the whole record
- * header might not fit on this page. xl_tot_len is the first field of the
- * struct, so it must be on this page (the records are MAXALIGNed), but we
- * cannot access any other fields until we've verified that we got the
- * whole header.
- */
- record = (XLogRecord *) (state->readBuf + RecPtr % XLOG_BLCKSZ);
- total_len = record->xl_tot_len;
-
- /*
- * If the whole record header is on this page, validate it immediately.
- * Otherwise do just a basic sanity check on xl_tot_len, and validate the
- * rest of the header after reading it from the next page. The xl_tot_len
- * check is necessary here to ensure that we enter the "Need to reassemble
- * record" code path below; otherwise we might fail to apply
- * ValidXLogRecordHeader at all.
- */
- if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
- {
- if (!ValidXLogRecordHeader(state, RecPtr, state->ReadRecPtr, record,
- randAccess))
- goto err;
- gotheader = true;
- }
- else
- {
- /* XXX: more validation should be done here */
- if (total_len < SizeOfXLogRecord)
- {
- report_invalid_record(state,
- "invalid record length at %X/%X: wanted %u, got %u",
- LSN_FORMAT_ARGS(RecPtr),
- (uint32) SizeOfXLogRecord, total_len);
- goto err;
- }
- gotheader = false;
- }
-
- len = XLOG_BLCKSZ - RecPtr % XLOG_BLCKSZ;
- if (total_len > len)
+ switch (state->readRecordState)
{
- /* Need to reassemble record */
- char *contdata;
- XLogPageHeader pageHeader;
- char *buffer;
- uint32 gotlen;
-
- /*
- * Enlarge readRecordBuf as needed.
- */
- if (total_len > state->readRecordBufSize &&
- !allocate_recordbuf(state, total_len))
- {
- /* We treat this as a "bogus data" condition */
- report_invalid_record(state, "record length %u at %X/%X too long",
- total_len, LSN_FORMAT_ARGS(RecPtr));
- goto err;
- }
+ case XLREAD_NEXT_RECORD:
+ ResetDecoder(state);
- /* Copy the first fragment of the record from the first page. */
- memcpy(state->readRecordBuf,
- state->readBuf + RecPtr % XLOG_BLCKSZ, len);
- buffer = state->readRecordBuf + len;
- gotlen = len;
-
- do
- {
- int rest_len = total_len - gotlen;
-
- /* Calculate pointer to beginning of next page */
- targetPagePtr += XLOG_BLCKSZ;
-
- /* Wait for the next page to become available */
- while (XLogNeedData(state, targetPagePtr,
- Min(rest_len, XLOG_BLCKSZ),
- false))
+ if (state->ReadRecPtr != InvalidXLogRecPtr)
{
- if (!state->routine.page_read(state, state->readPagePtr,
- state->readLen,
- state->ReadRecPtr,
- state->readBuf))
- break;
+ /* read the record after the one we just read */
+
+ /*
+ * EndRecPtr is pointing to end+1 of the previous WAL record.
+ * If we're at a page boundary, no more records can fit on the
+ * current page. We must skip over the page header, but we
+ * can't do that until we've read in the page, since the header
+ * size is variable.
+ */
+ state->PrevRecPtr = state->ReadRecPtr;
+ state->ReadRecPtr = state->EndRecPtr;
+ }
+ else
+ {
+ /*
+ * Caller supplied a position to start at.
+ *
+ * In this case, EndRecPtr should already be pointing to a
+ * valid record starting position.
+ */
+ Assert(XRecOffIsValid(state->EndRecPtr));
+ state->ReadRecPtr = state->EndRecPtr;
+
+ /*
+ * We cannot verify the previous-record pointer when we're
+ * seeking to a particular record. Reset PrevRecPtr so that we
+ * won't try doing that.
+ */
+ state->PrevRecPtr = InvalidXLogRecPtr;
+ state->EndRecPtr = InvalidXLogRecPtr; /* to be tidy */
}
+ state->record_verified = false;
+ state->readRecordState = XLREAD_TOT_LEN;
+ /* fall through */
+
+ case XLREAD_TOT_LEN:
+ {
+ uint32 total_len;
+ uint32 pageHeaderSize;
+ XLogRecPtr targetPagePtr;
+ uint32 targetRecOff;
+ XLogPageHeader pageHeader;
+
+ targetPagePtr =
+ state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
+ targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+
+ /*
+ * Check if we have enough data. For the first record in the page,
+ * the requesting length doesn't contain page header.
+ */
+ if (XLogNeedData(state, targetPagePtr,
+ Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+ targetRecOff != 0))
+ return XLREAD_NEED_DATA;
+
+ /* error out if caller supplied bogus page */
if (!state->page_verified)
goto err;
- Assert(SizeOfXLogShortPHD <= state->readLen);
+ /* examine page header now. */
+ pageHeaderSize =
+ XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+ if (targetRecOff == 0)
+ {
+ /* At page start, so skip over page header. */
+ state->ReadRecPtr += pageHeaderSize;
+ targetRecOff = pageHeaderSize;
+ }
+ else if (targetRecOff < pageHeaderSize)
+ {
+ report_invalid_record(state, "invalid record offset at %X/%X",
+ LSN_FORMAT_ARGS(state->ReadRecPtr));
+ goto err;
+ }
- /* Check that the continuation on next page looks valid */
pageHeader = (XLogPageHeader) state->readBuf;
- if (!(pageHeader->xlp_info & XLP_FIRST_IS_CONTRECORD))
+ if ((pageHeader->xlp_info & XLP_FIRST_IS_CONTRECORD) &&
+ targetRecOff == pageHeaderSize)
{
- report_invalid_record(state,
- "there is no contrecord flag at %X/%X",
- LSN_FORMAT_ARGS(RecPtr));
+ report_invalid_record(state, "contrecord is requested by %X/%X",
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr);
goto err;
}
- /*
- * Cross-check that xlp_rem_len agrees with how much of the record
- * we expect there to be left.
- */
- if (pageHeader->xlp_rem_len == 0 ||
- total_len != (pageHeader->xlp_rem_len + gotlen))
- {
- report_invalid_record(state,
- "invalid contrecord length %u (expected %lld) at %X/%X",
- pageHeader->xlp_rem_len,
- ((long long) total_len) - gotlen,
- LSN_FORMAT_ARGS(RecPtr));
- goto err;
- }
-
- /* Append the continuation from this page to the buffer */
- pageHeaderSize = XLogPageHeaderSize(pageHeader);
-
+ /* XLogNeedData has verified the page header */
Assert(pageHeaderSize <= state->readLen);
- contdata = (char *) state->readBuf + pageHeaderSize;
- len = XLOG_BLCKSZ - pageHeaderSize;
- if (pageHeader->xlp_rem_len < len)
- len = pageHeader->xlp_rem_len;
+ /*
+ * Read the record length.
+ *
+ * NB: Even though we use an XLogRecord pointer here, the whole
+ * record header might not fit on this page. xl_tot_len is the first
+ * field of the struct, so it must be on this page (the records are
+ * MAXALIGNed), but we cannot access any other fields until we've
+ * verified that we got the whole header.
+ */
+ prec = (XLogRecord *) (state->readBuf +
+ state->ReadRecPtr % XLOG_BLCKSZ);
+ total_len = prec->xl_tot_len;
- Assert (pageHeaderSize + len <= state->readLen);
- memcpy(buffer, (char *) contdata, len);
- buffer += len;
- gotlen += len;
+ /*
+ * If the whole record header is on this page, validate it
+ * immediately. Otherwise do just a basic sanity check on
+ * xl_tot_len, and validate the rest of the header after reading it
+ * from the next page. The xl_tot_len check is necessary here to
+ * ensure that we enter the XLREAD_CONTINUATION state below;
+ * otherwise we might fail to apply ValidXLogRecordHeader at all.
+ */
+ if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
+ {
+ if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
+ state->PrevRecPtr, prec))
+ goto err;
- /* If we just reassembled the record header, validate it. */
- if (!gotheader)
+ state->record_verified = true;
+ }
+ else
{
- record = (XLogRecord *) state->readRecordBuf;
- if (!ValidXLogRecordHeader(state, RecPtr, state->ReadRecPtr,
- record, randAccess))
+ /* XXX: more validation should be done here */
+ if (total_len < SizeOfXLogRecord)
+ {
+ report_invalid_record(state,
+ "invalid record length at %X/%X: wanted %u, got %u",
+ LSN_FORMAT_ARGS(state->ReadRecPtr),
+ (uint32) SizeOfXLogRecord, total_len);
goto err;
- gotheader = true;
+ }
}
- } while (gotlen < total_len);
-
- Assert(gotheader);
- record = (XLogRecord *) state->readRecordBuf;
- if (!ValidXLogRecord(state, record, RecPtr))
- goto err;
+ /*
+ * Wait for the rest of the record, or the part of the record that
+ * fit on the first page if crossed a page boundary, to become
+ * available.
+ */
+ state->recordGotLen = 0;
+ state->recordRemainLen = total_len;
+ state->readRecordState = XLREAD_FIRST_FRAGMENT;
+ }
+ /* fall through */
- pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
- state->ReadRecPtr = RecPtr;
- state->EndRecPtr = targetPagePtr + pageHeaderSize
- + MAXALIGN(pageHeader->xlp_rem_len);
- }
- else
- {
- /* Wait for the record data to become available */
- while (XLogNeedData(state, targetPagePtr,
- Min(targetRecOff + total_len, XLOG_BLCKSZ), true))
+ case XLREAD_FIRST_FRAGMENT:
{
- if (!state->routine.page_read(state, state->readPagePtr,
- state->readLen,
- state->ReadRecPtr, state->readBuf))
+ uint32 total_len = state->recordRemainLen;
+ uint32 request_len;
+ uint32 record_len;
+ XLogRecPtr targetPagePtr;
+ uint32 targetRecOff;
+
+ /*
+ * Wait for the rest of the record on the first page to become
+ * available
+ */
+ targetPagePtr =
+ state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
+ targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+
+ request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ);
+ record_len = request_len - targetRecOff;
+
+ /* ReadRecPtr contains page header */
+ Assert (targetRecOff != 0);
+ if (XLogNeedData(state, targetPagePtr, request_len, true))
+ return XLREAD_NEED_DATA;
+
+ /* error out if caller supplied bogus page */
+ if (!state->page_verified)
+ goto err;
+
+ prec = (XLogRecord *) (state->readBuf + targetRecOff);
+
+ /* validate record header if not yet */
+ if (!state->record_verified && record_len >= SizeOfXLogRecord)
+ {
+ if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
+ state->PrevRecPtr, prec))
+ goto err;
+
+ state->record_verified = true;
+ }
+
+
+ if (total_len == record_len)
+ {
+ /* Record does not cross a page boundary */
+ Assert(state->record_verified);
+
+ if (!ValidXLogRecord(state, prec, state->ReadRecPtr))
+ goto err;
+
+ state->record_verified = true; /* to be tidy */
+
+ /* We already checked the header earlier */
+ state->EndRecPtr = state->ReadRecPtr + MAXALIGN(record_len);
+
+ *record = prec;
+ state->readRecordState = XLREAD_NEXT_RECORD;
break;
+ }
+
+ /*
+ * The record continues on the next page. Need to reassemble
+ * record
+ */
+ Assert(total_len > record_len);
+
+ /* Enlarge readRecordBuf as needed. */
+ if (total_len > state->readRecordBufSize &&
+ !allocate_recordbuf(state, total_len))
+ {
+ /* We treat this as a "bogus data" condition */
+ report_invalid_record(state,
+ "record length %u at %X/%X too long",
+ total_len,
+ LSN_FORMAT_ARGS(state->ReadRecPtr));
+ goto err;
+ }
+
+ /* Copy the first fragment of the record from the first page. */
+ memcpy(state->readRecordBuf, state->readBuf + targetRecOff,
+ record_len);
+ state->recordGotLen += record_len;
+ state->recordRemainLen -= record_len;
+
+ /* Calculate pointer to beginning of next page */
+ state->recordContRecPtr = state->ReadRecPtr + record_len;
+ Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0);
+
+ state->readRecordState = XLREAD_CONTINUATION;
}
+ /* fall through */
+
+ case XLREAD_CONTINUATION:
+ {
+ XLogPageHeader pageHeader;
+ uint32 pageHeaderSize;
+ XLogRecPtr targetPagePtr;
+
+ /* we enter this state only if we haven't read the whole record. */
+ Assert (state->recordRemainLen > 0);
+
+ while(state->recordRemainLen > 0)
+ {
+ char *contdata;
+ uint32 request_len;
+ uint32 record_len;
+
+ /* Wait for the next page to become available */
+ targetPagePtr = state->recordContRecPtr;
+
+ /* this request contains page header */
+ Assert (targetPagePtr != 0);
+ if (XLogNeedData(state, targetPagePtr,
+ Min(state->recordRemainLen, XLOG_BLCKSZ),
+ false))
+ return XLREAD_NEED_DATA;
+
+ if (!state->page_verified)
+ goto err;
+
+ Assert(SizeOfXLogShortPHD <= state->readLen);
- if (!state->page_verified)
- goto err;
+ /* Check that the continuation on next page looks valid */
+ pageHeader = (XLogPageHeader) state->readBuf;
+ if (!(pageHeader->xlp_info & XLP_FIRST_IS_CONTRECORD))
+ {
+ report_invalid_record(
+ state,
+ "there is no contrecord flag at %X/%X reading %X/%X",
+ (uint32) (state->recordContRecPtr >> 32),
+ (uint32) state->recordContRecPtr,
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr);
+ goto err;
+ }
- /* Record does not cross a page boundary */
- if (!ValidXLogRecord(state, record, RecPtr))
- goto err;
+ /*
+ * Cross-check that xlp_rem_len agrees with how much of the
+ * record we expect there to be left.
+ */
+ if (pageHeader->xlp_rem_len == 0 ||
+ pageHeader->xlp_rem_len != state->recordRemainLen)
+ {
+ report_invalid_record(
+ state,
+ "invalid contrecord length %u at %X/%X reading %X/%X, expected %u",
+ pageHeader->xlp_rem_len,
+ (uint32) (state->recordContRecPtr >> 32),
+ (uint32) state->recordContRecPtr,
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr,
+ state->recordRemainLen);
+ goto err;
+ }
- state->EndRecPtr = RecPtr + MAXALIGN(total_len);
+ /* Append the continuation from this page to the buffer */
+ pageHeaderSize = XLogPageHeaderSize(pageHeader);
- state->ReadRecPtr = RecPtr;
+ /*
+ * XLogNeedData should have ensured that the whole page header
+ * was read
+ */
+ Assert(state->readLen >= pageHeaderSize);
+
+ contdata = (char *) state->readBuf + pageHeaderSize;
+ record_len = XLOG_BLCKSZ - pageHeaderSize;
+ if (pageHeader->xlp_rem_len < record_len)
+ record_len = pageHeader->xlp_rem_len;
+
+ request_len = record_len + pageHeaderSize;
+
+ /* XLogNeedData should have ensured all needed data was read */
+ Assert (state->readLen >= request_len);
+
+ memcpy(state->readRecordBuf + state->recordGotLen,
+ (char *) contdata, record_len);
+ state->recordGotLen += record_len;
+ state->recordRemainLen -= record_len;
+
+ /* If we just reassembled the record header, validate it. */
+ if (!state->record_verified)
+ {
+ Assert(state->recordGotLen >= SizeOfXLogRecord);
+ if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
+ state->PrevRecPtr,
+ (XLogRecord *) state->readRecordBuf))
+ goto err;
+
+ state->record_verified = true;
+ }
+
+ /* Calculate pointer to beginning of next page, and continue */
+ state->recordContRecPtr += XLOG_BLCKSZ;
+ }
+
+ /* targetPagePtr is pointing the last-read page here */
+ prec = (XLogRecord *) state->readRecordBuf;
+ if (!ValidXLogRecord(state, prec, state->ReadRecPtr))
+ goto err;
+
+ pageHeaderSize =
+ XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+ state->EndRecPtr = targetPagePtr + pageHeaderSize
+ + MAXALIGN(pageHeader->xlp_rem_len);
+
+ *record = prec;
+ state->readRecordState = XLREAD_NEXT_RECORD;
+ break;
+ }
}
/*
* Special processing if it's an XLOG SWITCH record
*/
- if (record->xl_rmid == RM_XLOG_ID &&
- (record->xl_info & ~XLR_INFO_MASK) == XLOG_SWITCH)
+ if ((*record)->xl_rmid == RM_XLOG_ID &&
+ ((*record)->xl_info & ~XLR_INFO_MASK) == XLOG_SWITCH)
{
/* Pretend it extends to end of segment */
state->EndRecPtr += state->segcxt.ws_segsize - 1;
state->EndRecPtr -= XLogSegmentOffset(state->EndRecPtr, state->segcxt.ws_segsize);
}
- if (DecodeXLogRecord(state, record, errormsg))
- return record;
- else
- return NULL;
+ Assert (!*record || state->readLen >= 0);
+ if (DecodeXLogRecord(state, *record, errormsg))
+ return XLREAD_SUCCESS;
+
+ *record = NULL;
+ return XLREAD_FAIL;
err:
/*
- * Invalidate the read state. We might read from a different source after
+ * Invalidate the read page. We might read from a different source after
* failure.
*/
XLogReaderInvalReadState(state);
@@ -572,7 +703,8 @@ err:
if (state->errormsg_buf[0] != '\0')
*errormsg = state->errormsg_buf;
- return NULL;
+ *record = NULL;
+ return XLREAD_FAIL;
}
/*
@@ -724,11 +856,12 @@ XLogReaderInvalReadState(XLogReaderState *state)
*
* This is just a convenience subroutine to avoid duplicated code in
* XLogReadRecord. It's not intended for use from anywhere else.
+ *
+ * If PrevRecPtr is valid, the xl_prev is is cross-checked with it.
*/
static bool
ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
- XLogRecPtr PrevRecPtr, XLogRecord *record,
- bool randAccess)
+ XLogRecPtr PrevRecPtr, XLogRecord *record)
{
if (record->xl_tot_len < SizeOfXLogRecord)
{
@@ -745,7 +878,7 @@ ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
record->xl_rmid, LSN_FORMAT_ARGS(RecPtr));
return false;
}
- if (randAccess)
+ if (PrevRecPtr == InvalidXLogRecPtr)
{
/*
* We can't exactly verify the prev-link, but surely it should be less
@@ -975,11 +1108,14 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
* XLogReadRecord() will read the next valid record.
*/
XLogRecPtr
-XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
+XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
+ XLogFindNextRecordCB read_page, void *private)
{
XLogRecPtr tmpRecPtr;
XLogRecPtr found = InvalidXLogRecPtr;
XLogPageHeader header;
+ XLogRecord *record;
+ XLogReadRecordResult result;
char *errormsg;
Assert(!XLogRecPtrIsInvalid(RecPtr));
@@ -1012,9 +1148,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
while(XLogNeedData(state, targetPagePtr, targetRecOff,
targetRecOff != 0))
{
- if (!state->routine.page_read(state, state->readPagePtr,
- state->readLen,
- state->ReadRecPtr, state->readBuf))
+ if (!read_page(state, private))
break;
}
@@ -1066,8 +1200,16 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
* or we just jumped over the remaining data of a continuation.
*/
XLogBeginRead(state, tmpRecPtr);
- while (XLogReadRecord(state, &errormsg) != NULL)
+ while ((result = XLogReadRecord(state, &record, &errormsg)) !=
+ XLREAD_FAIL)
{
+ if (result == XLREAD_NEED_DATA)
+ {
+ if (!read_page(state, private))
+ goto err;
+ continue;
+ }
+
/* past the record we've found, break out */
if (RecPtr <= state->ReadRecPtr)
{
@@ -1087,9 +1229,9 @@ err:
#endif /* FRONTEND */
/*
- * Helper function to ease writing of XLogRoutine->page_read callbacks.
- * If this function is used, caller must supply a segment_open callback in
- * 'state', as that is used here.
+ * Helper function to ease writing of page_read callback.
+ * If this function is used, caller must supply a segment_open callback and
+ * segment_close callback as that is used here.
*
* Read 'count' bytes into 'buf', starting at location 'startptr', from WAL
* fetched from timeline 'tli'.
@@ -1102,6 +1244,7 @@ err:
*/
bool
WALRead(XLogReaderState *state,
+ WALSegmentOpenCB segopenfn, WALSegmentCloseCB segclosefn,
char *buf, XLogRecPtr startptr, Size count, TimeLineID tli,
WALReadError *errinfo)
{
@@ -1133,10 +1276,10 @@ WALRead(XLogReaderState *state,
XLogSegNo nextSegNo;
if (state->seg.ws_file >= 0)
- state->routine.segment_close(state);
+ segclosefn(state);
XLByteToSeg(recptr, nextSegNo, state->segcxt.ws_segsize);
- state->routine.segment_open(state, nextSegNo, &tli);
+ segopenfn(state, nextSegNo, &tli);
/* This shouldn't happen -- indicates a bug in segment_open */
Assert(state->seg.ws_file >= 0);
diff --git a/src/backend/access/transam/xlogutils.c b/src/backend/access/transam/xlogutils.c
index 46eda33f25..b003990745 100644
--- a/src/backend/access/transam/xlogutils.c
+++ b/src/backend/access/transam/xlogutils.c
@@ -686,8 +686,7 @@ XLogTruncateRelation(RelFileNode rnode, ForkNumber forkNum,
void
XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wantLength)
{
- const XLogRecPtr lastReadPage = state->seg.ws_segno *
- state->segcxt.ws_segsize + state->readLen;
+ const XLogRecPtr lastReadPage = state->readPagePtr;
Assert(wantPage != InvalidXLogRecPtr && wantPage % XLOG_BLCKSZ == 0);
Assert(wantLength <= XLOG_BLCKSZ);
@@ -702,7 +701,7 @@ XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wa
* current TLI has since become historical.
*/
if (lastReadPage == wantPage &&
- state->readLen != 0 &&
+ state->page_verified &&
lastReadPage + state->readLen >= wantPage + Min(wantLength, XLOG_BLCKSZ - 1))
return;
@@ -788,6 +787,7 @@ wal_segment_open(XLogReaderState *state, XLogSegNo nextSegNo,
char path[MAXPGPATH];
XLogFilePath(path, tli, nextSegNo, state->segcxt.ws_segsize);
+ elog(LOG, "HOGE: %lu, %d => %s", nextSegNo, tli, path);
state->seg.ws_file = BasicOpenFile(path, O_RDONLY | PG_BINARY);
if (state->seg.ws_file >= 0)
return;
@@ -825,9 +825,11 @@ wal_segment_close(XLogReaderState *state)
* loop for now.
*/
bool
-read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *cur_page)
+read_local_xlog_page(XLogReaderState *state)
{
+ XLogRecPtr targetPagePtr = state->readPagePtr;
+ int reqLen = state->readLen;
+ char *cur_page = state->readBuf;
XLogRecPtr read_upto,
loc;
TimeLineID tli;
@@ -940,11 +942,12 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
* as 'count', read the whole page anyway. It's guaranteed to be
* zero-padded up to the page boundary if it's incomplete.
*/
- if (!WALRead(state, cur_page, targetPagePtr, XLOG_BLCKSZ, tli,
- &errinfo))
+ if (!WALRead(state, wal_segment_open, wal_segment_close,
+ cur_page, targetPagePtr, XLOG_BLCKSZ, tli, &errinfo))
WALReadRaiseError(&errinfo);
/* number of valid bytes in the buffer */
+ state->readPagePtr = targetPagePtr;
state->readLen = count;
return true;
}
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index 37b75deb72..9cbd8afa49 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -148,7 +148,8 @@ StartupDecodingContext(List *output_plugin_options,
TransactionId xmin_horizon,
bool need_full_snapshot,
bool fast_forward,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress)
@@ -198,11 +199,12 @@ StartupDecodingContext(List *output_plugin_options,
ctx->slot = slot;
- ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, xl_routine, ctx);
+ ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, cleanup_cb);
if (!ctx->reader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
errmsg("out of memory")));
+ ctx->page_read = page_read;
ctx->reorder = ReorderBufferAllocate();
ctx->snapshot_builder =
@@ -319,7 +321,8 @@ CreateInitDecodingContext(const char *plugin,
List *output_plugin_options,
bool need_full_snapshot,
XLogRecPtr restart_lsn,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress)
@@ -422,7 +425,7 @@ CreateInitDecodingContext(const char *plugin,
ctx = StartupDecodingContext(NIL, restart_lsn, xmin_horizon,
need_full_snapshot, false,
- xl_routine, prepare_write, do_write,
+ page_read, cleanup_cb, prepare_write, do_write,
update_progress);
/* call output plugin initialization callback */
@@ -476,7 +479,8 @@ LogicalDecodingContext *
CreateDecodingContext(XLogRecPtr start_lsn,
List *output_plugin_options,
bool fast_forward,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress)
@@ -528,8 +532,8 @@ CreateDecodingContext(XLogRecPtr start_lsn,
ctx = StartupDecodingContext(output_plugin_options,
start_lsn, InvalidTransactionId, false,
- fast_forward, xl_routine, prepare_write,
- do_write, update_progress);
+ fast_forward, page_read, cleanup_cb,
+ prepare_write, do_write, update_progress);
/* call output plugin initialization callback */
old_context = MemoryContextSwitchTo(ctx->context);
@@ -585,7 +589,13 @@ DecodingContextFindStartpoint(LogicalDecodingContext *ctx)
char *err = NULL;
/* the read_page callback waits for new WAL */
- record = XLogReadRecord(ctx->reader, &err);
+ while (XLogReadRecord(ctx->reader, &record, &err) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!ctx->page_read(ctx->reader))
+ break;
+ }
+
if (err)
elog(ERROR, "%s", err);
if (!record)
diff --git a/src/backend/replication/logical/logicalfuncs.c b/src/backend/replication/logical/logicalfuncs.c
index f7e055874e..b69f6911c5 100644
--- a/src/backend/replication/logical/logicalfuncs.c
+++ b/src/backend/replication/logical/logicalfuncs.c
@@ -233,9 +233,8 @@ pg_logical_slot_get_changes_guts(FunctionCallInfo fcinfo, bool confirm, bool bin
ctx = CreateDecodingContext(InvalidXLogRecPtr,
options,
false,
- XL_ROUTINE(.page_read = read_local_xlog_page,
- .segment_open = wal_segment_open,
- .segment_close = wal_segment_close),
+ read_local_xlog_page,
+ wal_segment_close,
LogicalOutputPrepareWrite,
LogicalOutputWrite, NULL);
@@ -284,7 +283,13 @@ pg_logical_slot_get_changes_guts(FunctionCallInfo fcinfo, bool confirm, bool bin
XLogRecord *record;
char *errm = NULL;
- record = XLogReadRecord(ctx->reader, &errm);
+ while (XLogReadRecord(ctx->reader, &record, &errm) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!ctx->page_read(ctx->reader))
+ break;
+ }
+
if (errm)
elog(ERROR, "%s", errm);
diff --git a/src/backend/replication/slotfuncs.c b/src/backend/replication/slotfuncs.c
index 9817b44113..deb7cf15eb 100644
--- a/src/backend/replication/slotfuncs.c
+++ b/src/backend/replication/slotfuncs.c
@@ -153,9 +153,8 @@ create_logical_replication_slot(char *name, char *plugin,
ctx = CreateInitDecodingContext(plugin, NIL,
false, /* just catalogs is OK */
restart_lsn,
- XL_ROUTINE(.page_read = read_local_xlog_page,
- .segment_open = wal_segment_open,
- .segment_close = wal_segment_close),
+ read_local_xlog_page,
+ wal_segment_close,
NULL, NULL, NULL);
/*
@@ -512,9 +511,8 @@ pg_logical_replication_slot_advance(XLogRecPtr moveto)
ctx = CreateDecodingContext(InvalidXLogRecPtr,
NIL,
true, /* fast_forward */
- XL_ROUTINE(.page_read = read_local_xlog_page,
- .segment_open = wal_segment_open,
- .segment_close = wal_segment_close),
+ read_local_xlog_page,
+ wal_segment_close,
NULL, NULL, NULL);
/*
@@ -536,7 +534,13 @@ pg_logical_replication_slot_advance(XLogRecPtr moveto)
* Read records. No changes are generated in fast_forward mode,
* but snapbuilder/slot statuses are updated properly.
*/
- record = XLogReadRecord(ctx->reader, &errm);
+ while (XLogReadRecord(ctx->reader, &record, &errm) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!ctx->page_read(ctx->reader))
+ break;
+ }
+
if (errm)
elog(ERROR, "%s", errm);
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index c36b91ae5e..062093727d 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -580,10 +580,7 @@ StartReplication(StartReplicationCmd *cmd)
/* create xlogreader for physical replication */
xlogreader =
- XLogReaderAllocate(wal_segment_size, NULL,
- XL_ROUTINE(.segment_open = WalSndSegmentOpen,
- .segment_close = wal_segment_close),
- NULL);
+ XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
if (!xlogreader)
ereport(ERROR,
@@ -807,9 +804,11 @@ StartReplication(StartReplicationCmd *cmd)
* set every time WAL is flushed.
*/
static bool
-logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetRecPtr, char *cur_page)
+logical_read_xlog_page(XLogReaderState *state)
{
+ XLogRecPtr targetPagePtr = state->readPagePtr;
+ int reqLen = state->readLen;
+ char *cur_page = state->readBuf;
XLogRecPtr flushptr;
int count;
WALReadError errinfo;
@@ -837,7 +836,7 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
count = flushptr - targetPagePtr; /* part of the page available */
/* now actually read the data, we know it's there */
- if (!WALRead(state,
+ if (!WALRead(state, WalSndSegmentOpen, wal_segment_close,
cur_page,
targetPagePtr,
XLOG_BLCKSZ,
@@ -1011,9 +1010,8 @@ CreateReplicationSlot(CreateReplicationSlotCmd *cmd)
ctx = CreateInitDecodingContext(cmd->plugin, NIL, need_full_snapshot,
InvalidXLogRecPtr,
- XL_ROUTINE(.page_read = logical_read_xlog_page,
- .segment_open = WalSndSegmentOpen,
- .segment_close = wal_segment_close),
+ logical_read_xlog_page,
+ wal_segment_close,
WalSndPrepareWrite, WalSndWriteData,
WalSndUpdateProgress);
@@ -1171,9 +1169,8 @@ StartLogicalReplication(StartReplicationCmd *cmd)
*/
logical_decoding_ctx =
CreateDecodingContext(cmd->startpoint, cmd->options, false,
- XL_ROUTINE(.page_read = logical_read_xlog_page,
- .segment_open = WalSndSegmentOpen,
- .segment_close = wal_segment_close),
+ logical_read_xlog_page,
+ wal_segment_close,
WalSndPrepareWrite, WalSndWriteData,
WalSndUpdateProgress);
xlogreader = logical_decoding_ctx->reader;
@@ -2749,7 +2746,7 @@ XLogSendPhysical(void)
enlargeStringInfo(&output_message, nbytes);
retry:
- if (!WALRead(xlogreader,
+ if (!WALRead(xlogreader, WalSndSegmentOpen, wal_segment_close,
&output_message.data[output_message.len],
startptr,
nbytes,
@@ -2847,7 +2844,12 @@ XLogSendLogical(void)
*/
WalSndCaughtUp = false;
- record = XLogReadRecord(logical_decoding_ctx->reader, &errm);
+ while (XLogReadRecord(logical_decoding_ctx->reader, &record, &errm) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!logical_decoding_ctx->page_read(logical_decoding_ctx->reader))
+ break;
+ }
/* xlog record was invalid */
if (errm != NULL)
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index cf119848b0..712c85281c 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -41,15 +41,9 @@ static int xlogreadfd = -1;
static XLogSegNo xlogreadsegno = -1;
static char xlogfpath[MAXPGPATH];
-typedef struct XLogPageReadPrivate
-{
- const char *restoreCommand;
- int tliIndex;
-} XLogPageReadPrivate;
-
-static bool SimpleXLogPageRead(XLogReaderState *xlogreader,
- XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
+static bool SimpleXLogPageRead(XLogReaderState *xlogreader,
+ const char *datadir, int *tliIndex,
+ const char *restoreCommand);
/*
* Read WAL from the datadir/pg_wal, starting from 'startpoint' on timeline
@@ -66,20 +60,22 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
XLogRecord *record;
XLogReaderState *xlogreader;
char *errormsg;
- XLogPageReadPrivate private;
- private.tliIndex = tliIndex;
- private.restoreCommand = restoreCommand;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir,
- XL_ROUTINE(.page_read = &SimpleXLogPageRead),
- &private);
+ xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+
if (xlogreader == NULL)
pg_fatal("out of memory");
XLogBeginRead(xlogreader, startpoint);
do
{
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!SimpleXLogPageRead(xlogreader, datadir,
+ &tliIndex, restoreCommand))
+ break;
+ }
if (record == NULL)
{
@@ -123,19 +119,19 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex,
XLogRecord *record;
XLogReaderState *xlogreader;
char *errormsg;
- XLogPageReadPrivate private;
XLogRecPtr endptr;
- private.tliIndex = tliIndex;
- private.restoreCommand = restoreCommand;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir,
- XL_ROUTINE(.page_read = &SimpleXLogPageRead),
- &private);
+ xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
XLogBeginRead(xlogreader, ptr);
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!SimpleXLogPageRead(xlogreader, datadir, &tliIndex, restoreCommand))
+ break;
+ }
if (record == NULL)
{
if (errormsg)
@@ -170,7 +166,6 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
XLogRecPtr searchptr;
XLogReaderState *xlogreader;
char *errormsg;
- XLogPageReadPrivate private;
/*
* The given fork pointer points to the end of the last common record,
@@ -186,11 +181,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
forkptr += SizeOfXLogShortPHD;
}
- private.tliIndex = tliIndex;
- private.restoreCommand = restoreCommand;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir,
- XL_ROUTINE(.page_read = &SimpleXLogPageRead),
- &private);
+ xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
@@ -200,7 +191,13 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
uint8 info;
XLogBeginRead(xlogreader, searchptr);
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!SimpleXLogPageRead(xlogreader, datadir,
+ &tliIndex, restoreCommand))
+ break;
+ }
if (record == NULL)
{
@@ -247,10 +244,11 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
/* XLogReader callback function, to read a WAL page */
static bool
-SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *readBuf)
+SimpleXLogPageRead(XLogReaderState *xlogreader, const char *datadir,
+ int *tliIndex, const char *restoreCommand)
{
- XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
+ XLogRecPtr targetPagePtr = xlogreader->readPagePtr;
+ char *readBuf = xlogreader->readBuf;
uint32 targetPageOff;
XLogRecPtr targetSegEnd;
XLogSegNo targetSegNo;
@@ -283,14 +281,14 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
* be done both forward and backward, consider also switching timeline
* accordingly.
*/
- while (private->tliIndex < targetNentries - 1 &&
- targetHistory[private->tliIndex].end < targetSegEnd)
- private->tliIndex++;
- while (private->tliIndex > 0 &&
- targetHistory[private->tliIndex].begin >= targetSegEnd)
- private->tliIndex--;
+ while (*tliIndex < targetNentries - 1 &&
+ targetHistory[*tliIndex].end < targetSegEnd)
+ (*tliIndex)++;
+ while (*tliIndex > 0 &&
+ targetHistory[*tliIndex].begin >= targetSegEnd)
+ (*tliIndex)--;
- XLogFileName(xlogfname, targetHistory[private->tliIndex].tli,
+ XLogFileName(xlogfname, targetHistory[*tliIndex].tli,
xlogreadsegno, WalSegSz);
snprintf(xlogfpath, MAXPGPATH, "%s/" XLOGDIR "/%s",
@@ -303,7 +301,7 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
/*
* If we have no restore_command to execute, then exit.
*/
- if (private->restoreCommand == NULL)
+ if (restoreCommand == NULL)
{
pg_log_error("could not open file \"%s\": %m", xlogfpath);
xlogreader->readLen = -1;
@@ -317,7 +315,7 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
xlogreadfd = RestoreArchivedFile(xlogreader->segcxt.ws_dir,
xlogfname,
WalSegSz,
- private->restoreCommand);
+ restoreCommand);
if (xlogreadfd < 0)
{
@@ -359,7 +357,7 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
Assert(targetSegNo == xlogreadsegno);
- xlogreader->seg.ws_tli = targetHistory[private->tliIndex].tli;
+ xlogreader->seg.ws_tli = targetHistory[*tliIndex].tli;
xlogreader->readLen = XLOG_BLCKSZ;
return true;
}
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index 0f0e0723a3..f514de6b13 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -330,12 +330,17 @@ WALDumpCloseSegment(XLogReaderState *state)
state->seg.ws_file = -1;
}
-/* pg_waldump's XLogReaderRoutine->page_read callback */
+/*
+ * pg_waldump's WAL page rader, also used as page_read callback for
+ * XLogFindNextRecord
+ */
static bool
-WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetPtr, char *readBuff)
+WALDumpReadPage(XLogReaderState *state, void *priv)
{
- XLogDumpPrivate *private = state->private_data;
+ XLogRecPtr targetPagePtr = state->readPagePtr;
+ int reqLen = state->readLen;
+ char *readBuff = state->readBuf;
+ XLogDumpPrivate *private = (XLogDumpPrivate *) priv;
int count = XLOG_BLCKSZ;
WALReadError errinfo;
@@ -353,8 +358,8 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
}
}
- if (!WALRead(state, readBuff, targetPagePtr, count, private->timeline,
- &errinfo))
+ if (!WALRead(state, WALDumpOpenSegment, WALDumpCloseSegment,
+ readBuff, targetPagePtr, count, private->timeline, &errinfo))
{
WALOpenSegment *seg = &errinfo.wre_seg;
char fname[MAXPGPATH];
@@ -374,6 +379,7 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
(Size) errinfo.wre_req);
}
+ Assert(count >= state->readLen);
state->readLen = count;
return true;
}
@@ -1035,16 +1041,14 @@ main(int argc, char **argv)
/* we have everything we need, start reading */
xlogreader_state =
- XLogReaderAllocate(WalSegSz, waldir,
- XL_ROUTINE(.page_read = WALDumpReadPage,
- .segment_open = WALDumpOpenSegment,
- .segment_close = WALDumpCloseSegment),
- &private);
+ XLogReaderAllocate(WalSegSz, waldir, WALDumpCloseSegment);
+
if (!xlogreader_state)
fatal_error("out of memory");
/* first find a valid recptr to start from */
- first_record = XLogFindNextRecord(xlogreader_state, private.startptr);
+ first_record = XLogFindNextRecord(xlogreader_state, private.startptr,
+ &WALDumpReadPage, (void*) &private);
if (first_record == InvalidXLogRecPtr)
fatal_error("could not find a valid record after %X/%X",
@@ -1067,7 +1071,13 @@ main(int argc, char **argv)
for (;;)
{
/* try to read the next record */
- record = XLogReadRecord(xlogreader_state, &errormsg);
+ while (XLogReadRecord(xlogreader_state, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!WALDumpReadPage(xlogreader_state, (void *) &private))
+ break;
+ }
+
if (!record)
{
if (!config.follow || private.endptr_reached)
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index 23d9e70a59..4e9268b553 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -57,64 +57,15 @@ typedef struct WALSegmentContext
typedef struct XLogReaderState XLogReaderState;
-/* Function type definition for the read_page callback */
-typedef bool (*XLogPageReadCB) (XLogReaderState *xlogreader,
- XLogRecPtr targetPagePtr,
- int reqLen,
- XLogRecPtr targetRecPtr,
- char *readBuf);
+/* Function type definition for the segment cleanup callback */
+typedef void (*WALSegmentCleanupCB) (XLogReaderState *xlogreader);
+
+/* Function type definition for the open/close callbacks for WALRead() */
typedef void (*WALSegmentOpenCB) (XLogReaderState *xlogreader,
XLogSegNo nextSegNo,
TimeLineID *tli_p);
typedef void (*WALSegmentCloseCB) (XLogReaderState *xlogreader);
-typedef struct XLogReaderRoutine
-{
- /*
- * Data input callback
- *
- * This callback shall read at least reqLen valid bytes of the xlog page
- * starting at targetPagePtr, and store them in readBuf. The callback
- * shall return the number of bytes read (never more than XLOG_BLCKSZ), or
- * -1 on failure. The callback shall sleep, if necessary, to wait for the
- * requested bytes to become available. The callback will not be invoked
- * again for the same page unless more than the returned number of bytes
- * are needed.
- *
- * targetRecPtr is the position of the WAL record we're reading. Usually
- * it is equal to targetPagePtr + reqLen, but sometimes xlogreader needs
- * to read and verify the page or segment header, before it reads the
- * actual WAL record it's interested in. In that case, targetRecPtr can
- * be used to determine which timeline to read the page from.
- *
- * The callback shall set ->seg.ws_tli to the TLI of the file the page was
- * read from.
- */
- XLogPageReadCB page_read;
-
- /*
- * Callback to open the specified WAL segment for reading. ->seg.ws_file
- * shall be set to the file descriptor of the opened segment. In case of
- * failure, an error shall be raised by the callback and it shall not
- * return.
- *
- * "nextSegNo" is the number of the segment to be opened.
- *
- * "tli_p" is an input/output argument. WALRead() uses it to pass the
- * timeline in which the new segment should be found, but the callback can
- * use it to return the TLI that it actually opened.
- */
- WALSegmentOpenCB segment_open;
-
- /*
- * WAL segment close callback. ->seg.ws_file shall be set to a negative
- * number.
- */
- WALSegmentCloseCB segment_close;
-} XLogReaderRoutine;
-
-#define XL_ROUTINE(...) &(XLogReaderRoutine){__VA_ARGS__}
-
typedef struct
{
/* Is this block ref in use? */
@@ -144,12 +95,35 @@ typedef struct
uint16 data_bufsz;
} DecodedBkpBlock;
+/* Return code from XLogReadRecord */
+typedef enum XLogReadRecordResult
+{
+ XLREAD_SUCCESS, /* record is successfully read */
+ XLREAD_NEED_DATA, /* need more data. see XLogReadRecord. */
+ XLREAD_FAIL /* failed during reading a record */
+} XLogReadRecordResult;
+
+/*
+ * internal state of XLogReadRecord
+ *
+ * XLogReadState runs a state machine while reading a record. Theses states
+ * are not seen outside the function. Each state may repeat several times
+ * exiting requesting caller for new data. See the comment of XLogReadRecrod
+ * for details.
+ */
+typedef enum XLogReadRecordState {
+ XLREAD_NEXT_RECORD,
+ XLREAD_TOT_LEN,
+ XLREAD_FIRST_FRAGMENT,
+ XLREAD_CONTINUATION
+} XLogReadRecordState;
+
struct XLogReaderState
{
/*
* Operational callbacks
*/
- XLogReaderRoutine routine;
+ WALSegmentCleanupCB cleanup_cb;
/* ----------------------------------------
* Public parameters
@@ -162,18 +136,14 @@ struct XLogReaderState
*/
uint64 system_identifier;
- /*
- * Opaque data for callbacks to use. Not used by XLogReader.
- */
- void *private_data;
-
/*
* Start and end point of last record read. EndRecPtr is also used as the
* position to read next. Calling XLogBeginRead() sets EndRecPtr to the
* starting position and ReadRecPtr to invalid.
*/
- XLogRecPtr ReadRecPtr; /* start of last record read */
+ XLogRecPtr ReadRecPtr; /* start of last record read or being read */
XLogRecPtr EndRecPtr; /* end+1 of last record read */
+ XLogRecPtr PrevRecPtr; /* start of previous record read */
/* ----------------------------------------
* Communication with page reader
@@ -187,7 +157,9 @@ struct XLogReaderState
* the request, or -1 on error */
TimeLineID readPageTLI; /* TLI for data currently in readBuf */
char *readBuf; /* buffer to store data */
- bool page_verified; /* is the page on the buffer verified? */
+ bool page_verified; /* is the page header on the buffer verified? */
+ bool record_verified;/* is the current record header verified? */
+
/* ----------------------------------------
@@ -229,8 +201,6 @@ struct XLogReaderState
XLogRecPtr latestPagePtr;
TimeLineID latestPageTLI;
- /* beginning of the WAL record being read. */
- XLogRecPtr currRecPtr;
/* timeline to read it from, 0 if a lookup is required */
TimeLineID currTLI;
@@ -257,6 +227,15 @@ struct XLogReaderState
char *readRecordBuf;
uint32 readRecordBufSize;
+ /*
+ * XLogReadRecord() state
+ */
+ XLogReadRecordState readRecordState;/* state machine state */
+ int recordGotLen; /* amount of current record that has
+ * already been read */
+ int recordRemainLen; /* length of current record that remains */
+ XLogRecPtr recordContRecPtr; /* where the current record continues */
+
/* Buffer to hold error message */
char *errormsg_buf;
};
@@ -264,9 +243,7 @@ struct XLogReaderState
/* Get a new XLogReader */
extern XLogReaderState *XLogReaderAllocate(int wal_segment_size,
const char *waldir,
- XLogReaderRoutine *routine,
- void *private_data);
-extern XLogReaderRoutine *LocalXLogReaderRoutine(void);
+ WALSegmentCleanupCB cleanup_cb);
/* Free an XLogReader */
extern void XLogReaderFree(XLogReaderState *state);
@@ -274,12 +251,17 @@ extern void XLogReaderFree(XLogReaderState *state);
/* Position the XLogReader to given record */
extern void XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr);
#ifdef FRONTEND
-extern XLogRecPtr XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr);
+/* Function type definition for the read_page callback */
+typedef bool (*XLogFindNextRecordCB) (XLogReaderState *xlogreader,
+ void *private);
+extern XLogRecPtr XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
+ XLogFindNextRecordCB read_page, void *private);
#endif /* FRONTEND */
/* Read the next XLog record. Returns NULL on end-of-WAL or failure */
-extern struct XLogRecord *XLogReadRecord(XLogReaderState *state,
- char **errormsg);
+extern XLogReadRecordResult XLogReadRecord(XLogReaderState *state,
+ XLogRecord **record,
+ char **errormsg);
/* Validate a page */
extern bool XLogReaderValidatePageHeader(XLogReaderState *state,
@@ -299,6 +281,7 @@ typedef struct WALReadError
} WALReadError;
extern bool WALRead(XLogReaderState *state,
+ WALSegmentOpenCB segopenfn, WALSegmentCloseCB sgclosefn,
char *buf, XLogRecPtr startptr, Size count,
TimeLineID tli, WALReadError *errinfo);
diff --git a/src/include/access/xlogutils.h b/src/include/access/xlogutils.h
index 364a21c4ea..397fb27fc2 100644
--- a/src/include/access/xlogutils.h
+++ b/src/include/access/xlogutils.h
@@ -47,9 +47,7 @@ extern Buffer XLogReadBufferExtended(RelFileNode rnode, ForkNumber forknum,
extern Relation CreateFakeRelcacheEntry(RelFileNode rnode);
extern void FreeFakeRelcacheEntry(Relation fakerel);
-extern bool read_local_xlog_page(XLogReaderState *state,
- XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetRecPtr, char *cur_page);
+extern bool read_local_xlog_page(XLogReaderState *state);
extern void wal_segment_open(XLogReaderState *state,
XLogSegNo nextSegNo,
TimeLineID *tli_p);
diff --git a/src/include/pg_config_manual.h b/src/include/pg_config_manual.h
index f10ad0acd6..94749609b4 100644
--- a/src/include/pg_config_manual.h
+++ b/src/include/pg_config_manual.h
@@ -377,7 +377,7 @@
* Enable debugging print statements for WAL-related operations; see
* also the wal_debug GUC var.
*/
-/* #define WAL_DEBUG */
+#define WAL_DEBUG
/*
* Enable tracing of resource consumption during sort operations;
diff --git a/src/include/replication/logical.h b/src/include/replication/logical.h
index c253403372..b936f31393 100644
--- a/src/include/replication/logical.h
+++ b/src/include/replication/logical.h
@@ -29,6 +29,10 @@ typedef void (*LogicalOutputPluginWriterUpdateProgress) (struct LogicalDecodingC
TransactionId xid
);
+typedef struct LogicalDecodingContext LogicalDecodingContext;
+
+typedef bool (*LogicalDecodingXLogPageReadCB)(XLogReaderState *ctx);
+
typedef struct LogicalDecodingContext
{
/* memory context this is all allocated in */
@@ -39,6 +43,7 @@ typedef struct LogicalDecodingContext
/* infrastructure pieces for decoding */
XLogReaderState *reader;
+ LogicalDecodingXLogPageReadCB page_read;
struct ReorderBuffer *reorder;
struct SnapBuild *snapshot_builder;
@@ -105,14 +110,16 @@ extern LogicalDecodingContext *CreateInitDecodingContext(const char *plugin,
List *output_plugin_options,
bool need_full_snapshot,
XLogRecPtr restart_lsn,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress);
extern LogicalDecodingContext *CreateDecodingContext(XLogRecPtr start_lsn,
List *output_plugin_options,
bool fast_forward,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress);
--
2.27.0
----Next_Part(Thu_Mar__4_11_28_53_2021_014)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
filename="v17-0003-Remove-globals-readOff-readLen-and-readSegNo.patch"
^ permalink raw reply [nested|flat] 70+ messages in thread
* [PATCH v19 2/5] Move page-reader out of XLogReadRecord().
@ 2019-09-10 03:58 Kyotaro Horiguchi <[email protected]>
0 siblings, 0 replies; 70+ messages in thread
From: Kyotaro Horiguchi @ 2019-09-10 03:58 UTC (permalink / raw)
This is the second step of removing callbacks from the WAL decoder.
XLogReadRecord() return XLREAD_NEED_DATA to indicate that the caller
should supply new data, and the decoder works as a state machine.
Author: Kyotaro HORIGUCHI <[email protected]>
Author: Heikki Linnakangas <[email protected]>
Reviewed-by: Antonin Houska <[email protected]>
Reviewed-by: Alvaro Herrera <[email protected]>
Reviewed-by: Takashi Menjo <[email protected]>
Reviewed-by: Thomas Munro <[email protected]>
Discussion: https://postgr.es/m/20190418.210257.43726183.horiguchi.kyotaro%40lab.ntt.co.jp
---
src/backend/access/transam/twophase.c | 14 +-
src/backend/access/transam/xlog.c | 58 +-
src/backend/access/transam/xlogreader.c | 690 ++++++++++--------
src/backend/access/transam/xlogutils.c | 17 +-
src/backend/replication/logical/logical.c | 26 +-
.../replication/logical/logicalfuncs.c | 13 +-
src/backend/replication/slotfuncs.c | 18 +-
src/backend/replication/walsender.c | 32 +-
src/bin/pg_rewind/parsexlog.c | 84 ++-
src/bin/pg_waldump/pg_waldump.c | 36 +-
src/include/access/xlogreader.h | 122 ++--
src/include/access/xlogutils.h | 4 +-
src/include/pg_config_manual.h | 2 +-
src/include/replication/logical.h | 11 +-
14 files changed, 628 insertions(+), 499 deletions(-)
diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index 89335b64a2..3137cb3ecc 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -1330,11 +1330,8 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
char *errormsg;
TimeLineID save_currtli = ThisTimeLineID;
- xlogreader = XLogReaderAllocate(wal_segment_size, NULL,
- XL_ROUTINE(.page_read = &read_local_xlog_page,
- .segment_open = &wal_segment_open,
- .segment_close = &wal_segment_close),
- NULL);
+ xlogreader = XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+
if (!xlogreader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
@@ -1342,7 +1339,12 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
errdetail("Failed while allocating a WAL reading processor.")));
XLogBeginRead(xlogreader, lsn);
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!read_local_xlog_page(xlogreader))
+ break;
+ }
/*
* Restore immediately the timeline where it was previously, as
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 8085ca1117..b7d7e6d31b 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -838,13 +838,6 @@ static XLogSource currentSource = XLOG_FROM_ANY;
static bool lastSourceFailed = false;
static bool pendingWalRcvRestart = false;
-typedef struct XLogPageReadPrivate
-{
- int emode;
- bool fetching_ckpt; /* are we fetching a checkpoint record? */
- bool randAccess;
-} XLogPageReadPrivate;
-
/*
* These variables track when we last obtained some WAL data to process,
* and where we got it from. (XLogReceiptSource is initially the same as
@@ -920,8 +913,8 @@ static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath,
static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli,
XLogSource source, bool notfoundOk);
static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, XLogSource source);
-static bool XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
+static bool XLogPageRead(XLogReaderState *xlogreader,
+ bool fetching_ckpt, int emode, bool randAccess);
static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
bool fetching_ckpt, XLogRecPtr tliRecPtr);
static int emode_for_corrupt_record(int emode, XLogRecPtr RecPtr);
@@ -1234,8 +1227,7 @@ XLogInsertRecord(XLogRecData *rdata,
appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len);
if (!debug_reader)
- debug_reader = XLogReaderAllocate(wal_segment_size, NULL,
- XL_ROUTINE(), NULL);
+ debug_reader = XLogReaderAllocate(wal_segment_size, NULL, NULL);
if (!debug_reader)
{
@@ -4369,15 +4361,10 @@ CleanupBackupHistory(void)
* record is available.
*/
static XLogRecord *
-ReadRecord(XLogReaderState *xlogreader, int emode,
- bool fetching_ckpt)
+ReadRecord(XLogReaderState *xlogreader, int emode, bool fetching_ckpt)
{
XLogRecord *record;
- XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
-
- private->fetching_ckpt = fetching_ckpt;
- private->emode = emode;
- private->randAccess = (xlogreader->ReadRecPtr == InvalidXLogRecPtr);
+ bool randAccess = (xlogreader->ReadRecPtr == InvalidXLogRecPtr);
/* This is the first attempt to read this page. */
lastSourceFailed = false;
@@ -4385,8 +4372,16 @@ ReadRecord(XLogReaderState *xlogreader, int emode,
for (;;)
{
char *errormsg;
+ XLogReadRecordResult result;
+
+ while ((result = XLogReadRecord(xlogreader, &record, &errormsg))
+ == XLREAD_NEED_DATA)
+ {
+ if (!XLogPageRead(xlogreader, fetching_ckpt, emode, randAccess))
+ break;
+
+ }
- record = XLogReadRecord(xlogreader, &errormsg);
ReadRecPtr = xlogreader->ReadRecPtr;
EndRecPtr = xlogreader->EndRecPtr;
if (record == NULL)
@@ -6456,7 +6451,6 @@ StartupXLOG(void)
bool backupFromStandby = false;
DBState dbstate_at_startup;
XLogReaderState *xlogreader;
- XLogPageReadPrivate private;
bool promoted = false;
struct stat st;
@@ -6615,13 +6609,9 @@ StartupXLOG(void)
OwnLatch(&XLogCtl->recoveryWakeupLatch);
/* Set up XLOG reader facility */
- MemSet(&private, 0, sizeof(XLogPageReadPrivate));
xlogreader =
- XLogReaderAllocate(wal_segment_size, NULL,
- XL_ROUTINE(.page_read = &XLogPageRead,
- .segment_open = NULL,
- .segment_close = wal_segment_close),
- &private);
+ XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+
if (!xlogreader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
@@ -12107,12 +12097,13 @@ CancelBackup(void)
* sleep and retry.
*/
static bool
-XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetRecPtr, char *readBuf)
+XLogPageRead(XLogReaderState *xlogreader,
+ bool fetching_ckpt, int emode, bool randAccess)
{
- XLogPageReadPrivate *private =
- (XLogPageReadPrivate *) xlogreader->private_data;
- int emode = private->emode;
+ char *readBuf = xlogreader->readBuf;
+ XLogRecPtr targetPagePtr = xlogreader->readPagePtr;
+ int reqLen = xlogreader->readLen;
+ XLogRecPtr targetRecPtr = xlogreader->ReadRecPtr;
uint32 targetPageOff;
XLogSegNo targetSegNo PG_USED_FOR_ASSERTS_ONLY;
int r;
@@ -12155,8 +12146,8 @@ retry:
flushedUpto < targetPagePtr + reqLen))
{
if (!WaitForWALToBecomeAvailable(targetPagePtr + reqLen,
- private->randAccess,
- private->fetching_ckpt,
+ randAccess,
+ fetching_ckpt,
targetRecPtr))
{
if (readFile >= 0)
@@ -12261,6 +12252,7 @@ retry:
goto next_record_is_invalid;
}
+ Assert(xlogreader->readPagePtr == targetPagePtr);
xlogreader->readLen = readLen;
return true;
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index f2345ab09e..661863e94b 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -40,7 +40,7 @@ static bool XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr,
int reqLen, bool header_inclusive);
static void XLogReaderInvalReadState(XLogReaderState *state);
static bool ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
- XLogRecPtr PrevRecPtr, XLogRecord *record, bool randAccess);
+ XLogRecPtr PrevRecPtr, XLogRecord *record);
static bool ValidXLogRecord(XLogReaderState *state, XLogRecord *record,
XLogRecPtr recptr);
static void ResetDecoder(XLogReaderState *state);
@@ -73,7 +73,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...)
*/
XLogReaderState *
XLogReaderAllocate(int wal_segment_size, const char *waldir,
- XLogReaderRoutine *routine, void *private_data)
+ WALSegmentCleanupCB cleanup_cb)
{
XLogReaderState *state;
@@ -84,7 +84,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
return NULL;
/* initialize caller-provided support functions */
- state->routine = *routine;
+ state->cleanup_cb = cleanup_cb;
state->max_block_id = -1;
@@ -107,8 +107,6 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
WALOpenSegmentInit(&state->seg, &state->segcxt, wal_segment_size,
waldir);
- /* system_identifier initialized to zeroes above */
- state->private_data = private_data;
/* ReadRecPtr, EndRecPtr and readLen initialized to zeroes above */
state->errormsg_buf = palloc_extended(MAX_ERRORMSG_LEN + 1,
MCXT_ALLOC_NO_OOM);
@@ -140,8 +138,8 @@ XLogReaderFree(XLogReaderState *state)
{
int block_id;
- if (state->seg.ws_file != -1)
- state->routine.segment_close(state);
+ if (state->seg.ws_file >= 0)
+ state->cleanup_cb(state);
for (block_id = 0; block_id <= XLR_MAX_BLOCK_ID; block_id++)
{
@@ -246,6 +244,7 @@ XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
/* Begin at the passed-in record pointer. */
state->EndRecPtr = RecPtr;
state->ReadRecPtr = InvalidXLogRecPtr;
+ state->readRecordState = XLREAD_NEXT_RECORD;
}
/*
@@ -254,12 +253,12 @@ XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
* XLogBeginRead() or XLogFindNextRecord() must be called before the first call
* to XLogReadRecord().
*
- * If the page_read callback fails to read the requested data, NULL is
- * returned. The callback is expected to have reported the error; errormsg
- * is set to NULL.
+ * This function may return XLREAD_NEED_DATA several times before returning a
+ * result record. The caller shall read in some new data then call this
+ * function again with the same parameters.
*
- * If the reading fails for some other reason, NULL is also returned, and
- * *errormsg is set to a string with details of the failure.
+ * When a record is successfully read, returns XLREAD_SUCCESS with result
+ * record being stored in *record. Otherwise *record is NULL.
*
* Returns XLREAD_NEED_DATA if more data is needed to finish reading the
* current record. In that case, state->readPagePtr and state->readLen inform
@@ -304,307 +303,410 @@ XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
* state. This behavior allows to continue reading a reacord switching among
* different souces, while streaming replication.
*/
-XLogRecord *
-XLogReadRecord(XLogReaderState *state, char **errormsg)
+XLogReadRecordResult
+XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
{
- XLogRecPtr RecPtr;
- XLogRecord *record;
- XLogRecPtr targetPagePtr;
- bool randAccess;
- uint32 len,
- total_len;
- uint32 targetRecOff;
- uint32 pageHeaderSize;
- bool gotheader;
+ XLogRecord *prec;
- /*
- * randAccess indicates whether to verify the previous-record pointer of
- * the record we're reading. We only do this if we're reading
- * sequentially, which is what we initially assume.
- */
- randAccess = false;
+ *record = NULL;
/* reset error state */
*errormsg = NULL;
state->errormsg_buf[0] = '\0';
- ResetDecoder(state);
-
- RecPtr = state->EndRecPtr;
-
- if (state->ReadRecPtr != InvalidXLogRecPtr)
- {
- /* read the record after the one we just read */
-
- /*
- * EndRecPtr is pointing to end+1 of the previous WAL record. If
- * we're at a page boundary, no more records can fit on the current
- * page. We must skip over the page header, but we can't do that until
- * we've read in the page, since the header size is variable.
- */
- }
- else
- {
- /*
- * Caller supplied a position to start at.
- *
- * In this case, EndRecPtr should already be pointing to a valid
- * record starting position.
- */
- Assert(XRecOffIsValid(RecPtr));
- randAccess = true;
- }
-
- state->currRecPtr = RecPtr;
-
- targetPagePtr = RecPtr - (RecPtr % XLOG_BLCKSZ);
- targetRecOff = RecPtr % XLOG_BLCKSZ;
-
- /*
- * Read the page containing the record into state->readBuf. Request enough
- * byte to cover the whole record header, or at least the part of it that
- * fits on the same page.
- */
- while (XLogNeedData(state, targetPagePtr,
- Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
- targetRecOff != 0))
+ switch (state->readRecordState)
{
- if (!state->routine.page_read(state, state->readPagePtr, state->readLen,
- RecPtr, state->readBuf))
- break;
- }
-
- if (!state->page_verified)
- goto err;
-
- /*
- * We have at least the page header, so we can examine it now.
- */
- pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
- if (targetRecOff == 0)
- {
- /*
- * At page start, so skip over page header.
- */
- RecPtr += pageHeaderSize;
- targetRecOff = pageHeaderSize;
- }
- else if (targetRecOff < pageHeaderSize)
- {
- report_invalid_record(state, "invalid record offset at %X/%X",
- LSN_FORMAT_ARGS(RecPtr));
- goto err;
- }
+ case XLREAD_NEXT_RECORD:
+ ResetDecoder(state);
- if ((((XLogPageHeader) state->readBuf)->xlp_info & XLP_FIRST_IS_CONTRECORD) &&
- targetRecOff == pageHeaderSize)
- {
- report_invalid_record(state, "contrecord is requested by %X/%X",
- LSN_FORMAT_ARGS(RecPtr));
- goto err;
- }
-
- /* XLogNeedData has verified the page header */
- Assert(pageHeaderSize <= state->readLen);
-
- /*
- * Read the record length.
- *
- * NB: Even though we use an XLogRecord pointer here, the whole record
- * header might not fit on this page. xl_tot_len is the first field of the
- * struct, so it must be on this page (the records are MAXALIGNed), but we
- * cannot access any other fields until we've verified that we got the
- * whole header.
- */
- record = (XLogRecord *) (state->readBuf + RecPtr % XLOG_BLCKSZ);
- total_len = record->xl_tot_len;
-
- /*
- * If the whole record header is on this page, validate it immediately.
- * Otherwise do just a basic sanity check on xl_tot_len, and validate the
- * rest of the header after reading it from the next page. The xl_tot_len
- * check is necessary here to ensure that we enter the "Need to reassemble
- * record" code path below; otherwise we might fail to apply
- * ValidXLogRecordHeader at all.
- */
- if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
- {
- if (!ValidXLogRecordHeader(state, RecPtr, state->ReadRecPtr, record,
- randAccess))
- goto err;
- gotheader = true;
- }
- else
- {
- /* XXX: more validation should be done here */
- if (total_len < SizeOfXLogRecord)
- {
- report_invalid_record(state,
- "invalid record length at %X/%X: wanted %u, got %u",
- LSN_FORMAT_ARGS(RecPtr),
- (uint32) SizeOfXLogRecord, total_len);
- goto err;
- }
- gotheader = false;
- }
-
- len = XLOG_BLCKSZ - RecPtr % XLOG_BLCKSZ;
- if (total_len > len)
- {
- /* Need to reassemble record */
- char *contdata;
- XLogPageHeader pageHeader;
- char *buffer;
- uint32 gotlen;
-
- /*
- * Enlarge readRecordBuf as needed.
- */
- if (total_len > state->readRecordBufSize &&
- !allocate_recordbuf(state, total_len))
- {
- /* We treat this as a "bogus data" condition */
- report_invalid_record(state, "record length %u at %X/%X too long",
- total_len, LSN_FORMAT_ARGS(RecPtr));
- goto err;
- }
-
- /* Copy the first fragment of the record from the first page. */
- memcpy(state->readRecordBuf,
- state->readBuf + RecPtr % XLOG_BLCKSZ, len);
- buffer = state->readRecordBuf + len;
- gotlen = len;
-
- do
- {
- int rest_len = total_len - gotlen;
-
- /* Calculate pointer to beginning of next page */
- targetPagePtr += XLOG_BLCKSZ;
-
- /* Wait for the next page to become available */
- while (XLogNeedData(state, targetPagePtr,
- Min(rest_len, XLOG_BLCKSZ),
- false))
+ if (state->ReadRecPtr != InvalidXLogRecPtr)
+ {
+ /* read the record after the one we just read */
+
+ /*
+ * EndRecPtr is pointing to end+1 of the previous WAL record.
+ * If we're at a page boundary, no more records can fit on the
+ * current page. We must skip over the page header, but we
+ * can't do that until we've read in the page, since the
+ * header size is variable.
+ */
+ state->PrevRecPtr = state->ReadRecPtr;
+ state->ReadRecPtr = state->EndRecPtr;
+ }
+ else
+ {
+ /*
+ * Caller supplied a position to start at.
+ *
+ * In this case, EndRecPtr should already be pointing to a
+ * valid record starting position.
+ */
+ Assert(XRecOffIsValid(state->EndRecPtr));
+ state->ReadRecPtr = state->EndRecPtr;
+
+ /*
+ * We cannot verify the previous-record pointer when we're
+ * seeking to a particular record. Reset PrevRecPtr so that we
+ * won't try doing that.
+ */
+ state->PrevRecPtr = InvalidXLogRecPtr;
+ state->EndRecPtr = InvalidXLogRecPtr; /* to be tidy */
+ }
+
+ state->record_verified = false;
+ state->readRecordState = XLREAD_TOT_LEN;
+ /* fall through */
+
+ case XLREAD_TOT_LEN:
+ {
+ uint32 total_len;
+ uint32 pageHeaderSize;
+ XLogRecPtr targetPagePtr;
+ uint32 targetRecOff;
+ XLogPageHeader pageHeader;
+
+ targetPagePtr =
+ state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
+ targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+
+ /*
+ * Check if we have enough data. For the first record in the
+ * page, the requesting length doesn't contain page header.
+ */
+ if (XLogNeedData(state, targetPagePtr,
+ Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+ targetRecOff != 0))
+ return XLREAD_NEED_DATA;
+
+ /* error out if caller supplied bogus page */
+ if (!state->page_verified)
+ goto err;
+
+ /* examine page header now. */
+ pageHeaderSize =
+ XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+ if (targetRecOff == 0)
+ {
+ /* At page start, so skip over page header. */
+ state->ReadRecPtr += pageHeaderSize;
+ targetRecOff = pageHeaderSize;
+ }
+ else if (targetRecOff < pageHeaderSize)
+ {
+ report_invalid_record(state, "invalid record offset at %X/%X",
+ LSN_FORMAT_ARGS(state->ReadRecPtr));
+ goto err;
+ }
+
+ pageHeader = (XLogPageHeader) state->readBuf;
+ if ((pageHeader->xlp_info & XLP_FIRST_IS_CONTRECORD) &&
+ targetRecOff == pageHeaderSize)
+ {
+ report_invalid_record(state, "contrecord is requested by %X/%X",
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr);
+ goto err;
+ }
+
+ /* XLogNeedData has verified the page header */
+ Assert(pageHeaderSize <= state->readLen);
+
+ /*
+ * Read the record length.
+ *
+ * NB: Even though we use an XLogRecord pointer here, the
+ * whole record header might not fit on this page. xl_tot_len
+ * is the first field of the struct, so it must be on this
+ * page (the records are MAXALIGNed), but we cannot access any
+ * other fields until we've verified that we got the whole
+ * header.
+ */
+ prec = (XLogRecord *) (state->readBuf +
+ state->ReadRecPtr % XLOG_BLCKSZ);
+ total_len = prec->xl_tot_len;
+
+ /*
+ * If the whole record header is on this page, validate it
+ * immediately. Otherwise do just a basic sanity check on
+ * xl_tot_len, and validate the rest of the header after
+ * reading it from the next page. The xl_tot_len check is
+ * necessary here to ensure that we enter the
+ * XLREAD_CONTINUATION state below; otherwise we might fail to
+ * apply ValidXLogRecordHeader at all.
+ */
+ if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
+ {
+ if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
+ state->PrevRecPtr, prec))
+ goto err;
+
+ state->record_verified = true;
+ }
+ else
+ {
+ /* XXX: more validation should be done here */
+ if (total_len < SizeOfXLogRecord)
+ {
+ report_invalid_record(state,
+ "invalid record length at %X/%X: wanted %u, got %u",
+ LSN_FORMAT_ARGS(state->ReadRecPtr),
+ (uint32) SizeOfXLogRecord, total_len);
+ goto err;
+ }
+ }
+
+ /*
+ * Wait for the rest of the record, or the part of the record
+ * that fit on the first page if crossed a page boundary, to
+ * become available.
+ */
+ state->recordGotLen = 0;
+ state->recordRemainLen = total_len;
+ state->readRecordState = XLREAD_FIRST_FRAGMENT;
+ }
+ /* fall through */
+
+ case XLREAD_FIRST_FRAGMENT:
{
- if (!state->routine.page_read(state, state->readPagePtr,
- state->readLen,
- state->ReadRecPtr,
- state->readBuf))
+ uint32 total_len = state->recordRemainLen;
+ uint32 request_len;
+ uint32 record_len;
+ XLogRecPtr targetPagePtr;
+ uint32 targetRecOff;
+
+ /*
+ * Wait for the rest of the record on the first page to become
+ * available
+ */
+ targetPagePtr =
+ state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
+ targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+
+ request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ);
+ record_len = request_len - targetRecOff;
+
+ /* ReadRecPtr contains page header */
+ Assert(targetRecOff != 0);
+ if (XLogNeedData(state, targetPagePtr, request_len, true))
+ return XLREAD_NEED_DATA;
+
+ /* error out if caller supplied bogus page */
+ if (!state->page_verified)
+ goto err;
+
+ prec = (XLogRecord *) (state->readBuf + targetRecOff);
+
+ /* validate record header if not yet */
+ if (!state->record_verified && record_len >= SizeOfXLogRecord)
+ {
+ if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
+ state->PrevRecPtr, prec))
+ goto err;
+
+ state->record_verified = true;
+ }
+
+
+ if (total_len == record_len)
+ {
+ /* Record does not cross a page boundary */
+ Assert(state->record_verified);
+
+ if (!ValidXLogRecord(state, prec, state->ReadRecPtr))
+ goto err;
+
+ state->record_verified = true; /* to be tidy */
+
+ /* We already checked the header earlier */
+ state->EndRecPtr = state->ReadRecPtr + MAXALIGN(record_len);
+
+ *record = prec;
+ state->readRecordState = XLREAD_NEXT_RECORD;
break;
- }
+ }
- if (!state->page_verified)
- goto err;
+ /*
+ * The record continues on the next page. Need to reassemble
+ * record
+ */
+ Assert(total_len > record_len);
- Assert(SizeOfXLogShortPHD <= state->readLen);
+ /* Enlarge readRecordBuf as needed. */
+ if (total_len > state->readRecordBufSize &&
+ !allocate_recordbuf(state, total_len))
+ {
+ /* We treat this as a "bogus data" condition */
+ report_invalid_record(state,
+ "record length %u at %X/%X too long",
+ total_len,
+ LSN_FORMAT_ARGS(state->ReadRecPtr));
+ goto err;
+ }
- /* Check that the continuation on next page looks valid */
- pageHeader = (XLogPageHeader) state->readBuf;
- if (!(pageHeader->xlp_info & XLP_FIRST_IS_CONTRECORD))
- {
- report_invalid_record(state,
- "there is no contrecord flag at %X/%X",
- LSN_FORMAT_ARGS(RecPtr));
- goto err;
+ /* Copy the first fragment of the record from the first page. */
+ memcpy(state->readRecordBuf, state->readBuf + targetRecOff,
+ record_len);
+ state->recordGotLen += record_len;
+ state->recordRemainLen -= record_len;
+
+ /* Calculate pointer to beginning of next page */
+ state->recordContRecPtr = state->ReadRecPtr + record_len;
+ Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0);
+
+ state->readRecordState = XLREAD_CONTINUATION;
}
+ /* fall through */
- /*
- * Cross-check that xlp_rem_len agrees with how much of the record
- * we expect there to be left.
- */
- if (pageHeader->xlp_rem_len == 0 ||
- total_len != (pageHeader->xlp_rem_len + gotlen))
+ case XLREAD_CONTINUATION:
{
- report_invalid_record(state,
- "invalid contrecord length %u (expected %lld) at %X/%X",
- pageHeader->xlp_rem_len,
- ((long long) total_len) - gotlen,
- LSN_FORMAT_ARGS(RecPtr));
- goto err;
- }
+ XLogPageHeader pageHeader;
+ uint32 pageHeaderSize;
+ XLogRecPtr targetPagePtr;
- /* Append the continuation from this page to the buffer */
- pageHeaderSize = XLogPageHeaderSize(pageHeader);
+ /*
+ * we enter this state only if we haven't read the whole
+ * record.
+ */
+ Assert(state->recordRemainLen > 0);
- Assert(pageHeaderSize <= state->readLen);
+ while (state->recordRemainLen > 0)
+ {
+ char *contdata;
+ uint32 request_len;
+ uint32 record_len;
- contdata = (char *) state->readBuf + pageHeaderSize;
- len = XLOG_BLCKSZ - pageHeaderSize;
- if (pageHeader->xlp_rem_len < len)
- len = pageHeader->xlp_rem_len;
+ /* Wait for the next page to become available */
+ targetPagePtr = state->recordContRecPtr;
- Assert(pageHeaderSize + len <= state->readLen);
- memcpy(buffer, (char *) contdata, len);
- buffer += len;
- gotlen += len;
+ /* this request contains page header */
+ Assert(targetPagePtr != 0);
+ if (XLogNeedData(state, targetPagePtr,
+ Min(state->recordRemainLen, XLOG_BLCKSZ),
+ false))
+ return XLREAD_NEED_DATA;
- /* If we just reassembled the record header, validate it. */
- if (!gotheader)
- {
- record = (XLogRecord *) state->readRecordBuf;
- if (!ValidXLogRecordHeader(state, RecPtr, state->ReadRecPtr,
- record, randAccess))
+ if (!state->page_verified)
+ goto err;
+
+ Assert(SizeOfXLogShortPHD <= state->readLen);
+
+ /* Check that the continuation on next page looks valid */
+ pageHeader = (XLogPageHeader) state->readBuf;
+ if (!(pageHeader->xlp_info & XLP_FIRST_IS_CONTRECORD))
+ {
+ report_invalid_record(
+ state,
+ "there is no contrecord flag at %X/%X reading %X/%X",
+ (uint32) (state->recordContRecPtr >> 32),
+ (uint32) state->recordContRecPtr,
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr);
+ goto err;
+ }
+
+ /*
+ * Cross-check that xlp_rem_len agrees with how much of
+ * the record we expect there to be left.
+ */
+ if (pageHeader->xlp_rem_len == 0 ||
+ pageHeader->xlp_rem_len != state->recordRemainLen)
+ {
+ report_invalid_record(
+ state,
+ "invalid contrecord length %u at %X/%X reading %X/%X, expected %u",
+ pageHeader->xlp_rem_len,
+ (uint32) (state->recordContRecPtr >> 32),
+ (uint32) state->recordContRecPtr,
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr,
+ state->recordRemainLen);
+ goto err;
+ }
+
+ /* Append the continuation from this page to the buffer */
+ pageHeaderSize = XLogPageHeaderSize(pageHeader);
+
+ /*
+ * XLogNeedData should have ensured that the whole page
+ * header was read
+ */
+ Assert(state->readLen >= pageHeaderSize);
+
+ contdata = (char *) state->readBuf + pageHeaderSize;
+ record_len = XLOG_BLCKSZ - pageHeaderSize;
+ if (pageHeader->xlp_rem_len < record_len)
+ record_len = pageHeader->xlp_rem_len;
+
+ request_len = record_len + pageHeaderSize;
+
+ /*
+ * XLogNeedData should have ensured all needed data was
+ * read
+ */
+ Assert(state->readLen >= request_len);
+
+ memcpy(state->readRecordBuf + state->recordGotLen,
+ (char *) contdata, record_len);
+ state->recordGotLen += record_len;
+ state->recordRemainLen -= record_len;
+
+ /* If we just reassembled the record header, validate it. */
+ if (!state->record_verified)
+ {
+ Assert(state->recordGotLen >= SizeOfXLogRecord);
+ if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
+ state->PrevRecPtr,
+ (XLogRecord *) state->readRecordBuf))
+ goto err;
+
+ state->record_verified = true;
+ }
+
+ /*
+ * Calculate pointer to beginning of next page, and
+ * continue
+ */
+ state->recordContRecPtr += XLOG_BLCKSZ;
+ }
+
+ /* targetPagePtr is pointing the last-read page here */
+ prec = (XLogRecord *) state->readRecordBuf;
+ if (!ValidXLogRecord(state, prec, state->ReadRecPtr))
goto err;
- gotheader = true;
- }
- } while (gotlen < total_len);
-
- Assert(gotheader);
-
- record = (XLogRecord *) state->readRecordBuf;
- if (!ValidXLogRecord(state, record, RecPtr))
- goto err;
-
- pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
- state->ReadRecPtr = RecPtr;
- state->EndRecPtr = targetPagePtr + pageHeaderSize
- + MAXALIGN(pageHeader->xlp_rem_len);
- }
- else
- {
- /* Wait for the record data to become available */
- while (XLogNeedData(state, targetPagePtr,
- Min(targetRecOff + total_len, XLOG_BLCKSZ), true))
- {
- if (!state->routine.page_read(state, state->readPagePtr,
- state->readLen,
- state->ReadRecPtr, state->readBuf))
+
+ pageHeaderSize =
+ XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+ state->EndRecPtr = targetPagePtr + pageHeaderSize
+ + MAXALIGN(pageHeader->xlp_rem_len);
+
+ *record = prec;
+ state->readRecordState = XLREAD_NEXT_RECORD;
break;
- }
-
- if (!state->page_verified)
- goto err;
-
- /* Record does not cross a page boundary */
- if (!ValidXLogRecord(state, record, RecPtr))
- goto err;
-
- state->EndRecPtr = RecPtr + MAXALIGN(total_len);
-
- state->ReadRecPtr = RecPtr;
+ }
}
/*
* Special processing if it's an XLOG SWITCH record
*/
- if (record->xl_rmid == RM_XLOG_ID &&
- (record->xl_info & ~XLR_INFO_MASK) == XLOG_SWITCH)
+ if ((*record)->xl_rmid == RM_XLOG_ID &&
+ ((*record)->xl_info & ~XLR_INFO_MASK) == XLOG_SWITCH)
{
/* Pretend it extends to end of segment */
state->EndRecPtr += state->segcxt.ws_segsize - 1;
state->EndRecPtr -= XLogSegmentOffset(state->EndRecPtr, state->segcxt.ws_segsize);
}
- if (DecodeXLogRecord(state, record, errormsg))
- return record;
- else
- return NULL;
+ Assert(!*record || state->readLen >= 0);
+ if (DecodeXLogRecord(state, *record, errormsg))
+ return XLREAD_SUCCESS;
+
+ *record = NULL;
+ return XLREAD_FAIL;
err:
/*
- * Invalidate the read state. We might read from a different source after
+ * Invalidate the read page. We might read from a different source after
* failure.
*/
XLogReaderInvalReadState(state);
@@ -612,7 +714,8 @@ err:
if (state->errormsg_buf[0] != '\0')
*errormsg = state->errormsg_buf;
- return NULL;
+ *record = NULL;
+ return XLREAD_FAIL;
}
/*
@@ -764,11 +867,12 @@ XLogReaderInvalReadState(XLogReaderState *state)
*
* This is just a convenience subroutine to avoid duplicated code in
* XLogReadRecord. It's not intended for use from anywhere else.
+ *
+ * If PrevRecPtr is valid, the xl_prev is is cross-checked with it.
*/
static bool
ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
- XLogRecPtr PrevRecPtr, XLogRecord *record,
- bool randAccess)
+ XLogRecPtr PrevRecPtr, XLogRecord *record)
{
if (record->xl_tot_len < SizeOfXLogRecord)
{
@@ -785,7 +889,7 @@ ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
record->xl_rmid, LSN_FORMAT_ARGS(RecPtr));
return false;
}
- if (randAccess)
+ if (PrevRecPtr == InvalidXLogRecPtr)
{
/*
* We can't exactly verify the prev-link, but surely it should be less
@@ -1015,11 +1119,14 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
* XLogReadRecord() will read the next valid record.
*/
XLogRecPtr
-XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
+XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
+ XLogFindNextRecordCB read_page, void *private)
{
XLogRecPtr tmpRecPtr;
XLogRecPtr found = InvalidXLogRecPtr;
XLogPageHeader header;
+ XLogRecord *record;
+ XLogReadRecordResult result;
char *errormsg;
Assert(!XLogRecPtrIsInvalid(RecPtr));
@@ -1052,9 +1159,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
while (XLogNeedData(state, targetPagePtr, targetRecOff,
targetRecOff != 0))
{
- if (!state->routine.page_read(state, state->readPagePtr,
- state->readLen,
- state->ReadRecPtr, state->readBuf))
+ if (!read_page(state, private))
break;
}
@@ -1106,8 +1211,16 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
* or we just jumped over the remaining data of a continuation.
*/
XLogBeginRead(state, tmpRecPtr);
- while (XLogReadRecord(state, &errormsg) != NULL)
+ while ((result = XLogReadRecord(state, &record, &errormsg)) !=
+ XLREAD_FAIL)
{
+ if (result == XLREAD_NEED_DATA)
+ {
+ if (!read_page(state, private))
+ goto err;
+ continue;
+ }
+
/* past the record we've found, break out */
if (RecPtr <= state->ReadRecPtr)
{
@@ -1127,9 +1240,9 @@ err:
#endif /* FRONTEND */
/*
- * Helper function to ease writing of XLogRoutine->page_read callbacks.
- * If this function is used, caller must supply a segment_open callback in
- * 'state', as that is used here.
+ * Helper function to ease writing of page_read callback.
+ * If this function is used, caller must supply a segment_open callback and
+ * segment_close callback as that is used here.
*
* Read 'count' bytes into 'buf', starting at location 'startptr', from WAL
* fetched from timeline 'tli'.
@@ -1142,6 +1255,7 @@ err:
*/
bool
WALRead(XLogReaderState *state,
+ WALSegmentOpenCB segopenfn, WALSegmentCloseCB segclosefn,
char *buf, XLogRecPtr startptr, Size count, TimeLineID tli,
WALReadError *errinfo)
{
@@ -1173,10 +1287,10 @@ WALRead(XLogReaderState *state,
XLogSegNo nextSegNo;
if (state->seg.ws_file >= 0)
- state->routine.segment_close(state);
+ segclosefn(state);
XLByteToSeg(recptr, nextSegNo, state->segcxt.ws_segsize);
- state->routine.segment_open(state, nextSegNo, &tli);
+ segopenfn(state, nextSegNo, &tli);
/* This shouldn't happen -- indicates a bug in segment_open */
Assert(state->seg.ws_file >= 0);
diff --git a/src/backend/access/transam/xlogutils.c b/src/backend/access/transam/xlogutils.c
index 46eda33f25..b003990745 100644
--- a/src/backend/access/transam/xlogutils.c
+++ b/src/backend/access/transam/xlogutils.c
@@ -686,8 +686,7 @@ XLogTruncateRelation(RelFileNode rnode, ForkNumber forkNum,
void
XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wantLength)
{
- const XLogRecPtr lastReadPage = state->seg.ws_segno *
- state->segcxt.ws_segsize + state->readLen;
+ const XLogRecPtr lastReadPage = state->readPagePtr;
Assert(wantPage != InvalidXLogRecPtr && wantPage % XLOG_BLCKSZ == 0);
Assert(wantLength <= XLOG_BLCKSZ);
@@ -702,7 +701,7 @@ XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wa
* current TLI has since become historical.
*/
if (lastReadPage == wantPage &&
- state->readLen != 0 &&
+ state->page_verified &&
lastReadPage + state->readLen >= wantPage + Min(wantLength, XLOG_BLCKSZ - 1))
return;
@@ -788,6 +787,7 @@ wal_segment_open(XLogReaderState *state, XLogSegNo nextSegNo,
char path[MAXPGPATH];
XLogFilePath(path, tli, nextSegNo, state->segcxt.ws_segsize);
+ elog(LOG, "HOGE: %lu, %d => %s", nextSegNo, tli, path);
state->seg.ws_file = BasicOpenFile(path, O_RDONLY | PG_BINARY);
if (state->seg.ws_file >= 0)
return;
@@ -825,9 +825,11 @@ wal_segment_close(XLogReaderState *state)
* loop for now.
*/
bool
-read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *cur_page)
+read_local_xlog_page(XLogReaderState *state)
{
+ XLogRecPtr targetPagePtr = state->readPagePtr;
+ int reqLen = state->readLen;
+ char *cur_page = state->readBuf;
XLogRecPtr read_upto,
loc;
TimeLineID tli;
@@ -940,11 +942,12 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
* as 'count', read the whole page anyway. It's guaranteed to be
* zero-padded up to the page boundary if it's incomplete.
*/
- if (!WALRead(state, cur_page, targetPagePtr, XLOG_BLCKSZ, tli,
- &errinfo))
+ if (!WALRead(state, wal_segment_open, wal_segment_close,
+ cur_page, targetPagePtr, XLOG_BLCKSZ, tli, &errinfo))
WALReadRaiseError(&errinfo);
/* number of valid bytes in the buffer */
+ state->readPagePtr = targetPagePtr;
state->readLen = count;
return true;
}
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index 2f6803637b..4f6e87f18d 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -148,7 +148,8 @@ StartupDecodingContext(List *output_plugin_options,
TransactionId xmin_horizon,
bool need_full_snapshot,
bool fast_forward,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress)
@@ -198,11 +199,12 @@ StartupDecodingContext(List *output_plugin_options,
ctx->slot = slot;
- ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, xl_routine, ctx);
+ ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, cleanup_cb);
if (!ctx->reader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
errmsg("out of memory")));
+ ctx->page_read = page_read;
ctx->reorder = ReorderBufferAllocate();
ctx->snapshot_builder =
@@ -319,7 +321,8 @@ CreateInitDecodingContext(const char *plugin,
List *output_plugin_options,
bool need_full_snapshot,
XLogRecPtr restart_lsn,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress)
@@ -422,7 +425,7 @@ CreateInitDecodingContext(const char *plugin,
ctx = StartupDecodingContext(NIL, restart_lsn, xmin_horizon,
need_full_snapshot, false,
- xl_routine, prepare_write, do_write,
+ page_read, cleanup_cb, prepare_write, do_write,
update_progress);
/* call output plugin initialization callback */
@@ -476,7 +479,8 @@ LogicalDecodingContext *
CreateDecodingContext(XLogRecPtr start_lsn,
List *output_plugin_options,
bool fast_forward,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress)
@@ -528,8 +532,8 @@ CreateDecodingContext(XLogRecPtr start_lsn,
ctx = StartupDecodingContext(output_plugin_options,
start_lsn, InvalidTransactionId, false,
- fast_forward, xl_routine, prepare_write,
- do_write, update_progress);
+ fast_forward, page_read, cleanup_cb,
+ prepare_write, do_write, update_progress);
/* call output plugin initialization callback */
old_context = MemoryContextSwitchTo(ctx->context);
@@ -585,7 +589,13 @@ DecodingContextFindStartpoint(LogicalDecodingContext *ctx)
char *err = NULL;
/* the read_page callback waits for new WAL */
- record = XLogReadRecord(ctx->reader, &err);
+ while (XLogReadRecord(ctx->reader, &record, &err) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!ctx->page_read(ctx->reader))
+ break;
+ }
+
if (err)
elog(ERROR, "%s", err);
if (!record)
diff --git a/src/backend/replication/logical/logicalfuncs.c b/src/backend/replication/logical/logicalfuncs.c
index 01d354829b..8f8c129620 100644
--- a/src/backend/replication/logical/logicalfuncs.c
+++ b/src/backend/replication/logical/logicalfuncs.c
@@ -233,9 +233,8 @@ pg_logical_slot_get_changes_guts(FunctionCallInfo fcinfo, bool confirm, bool bin
ctx = CreateDecodingContext(InvalidXLogRecPtr,
options,
false,
- XL_ROUTINE(.page_read = read_local_xlog_page,
- .segment_open = wal_segment_open,
- .segment_close = wal_segment_close),
+ read_local_xlog_page,
+ wal_segment_close,
LogicalOutputPrepareWrite,
LogicalOutputWrite, NULL);
@@ -284,7 +283,13 @@ pg_logical_slot_get_changes_guts(FunctionCallInfo fcinfo, bool confirm, bool bin
XLogRecord *record;
char *errm = NULL;
- record = XLogReadRecord(ctx->reader, &errm);
+ while (XLogReadRecord(ctx->reader, &record, &errm) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!ctx->page_read(ctx->reader))
+ break;
+ }
+
if (errm)
elog(ERROR, "%s", errm);
diff --git a/src/backend/replication/slotfuncs.c b/src/backend/replication/slotfuncs.c
index d9d36879ed..7ab0b804e4 100644
--- a/src/backend/replication/slotfuncs.c
+++ b/src/backend/replication/slotfuncs.c
@@ -153,9 +153,8 @@ create_logical_replication_slot(char *name, char *plugin,
ctx = CreateInitDecodingContext(plugin, NIL,
false, /* just catalogs is OK */
restart_lsn,
- XL_ROUTINE(.page_read = read_local_xlog_page,
- .segment_open = wal_segment_open,
- .segment_close = wal_segment_close),
+ read_local_xlog_page,
+ wal_segment_close,
NULL, NULL, NULL);
/*
@@ -512,9 +511,8 @@ pg_logical_replication_slot_advance(XLogRecPtr moveto)
ctx = CreateDecodingContext(InvalidXLogRecPtr,
NIL,
true, /* fast_forward */
- XL_ROUTINE(.page_read = read_local_xlog_page,
- .segment_open = wal_segment_open,
- .segment_close = wal_segment_close),
+ read_local_xlog_page,
+ wal_segment_close,
NULL, NULL, NULL);
/*
@@ -536,7 +534,13 @@ pg_logical_replication_slot_advance(XLogRecPtr moveto)
* Read records. No changes are generated in fast_forward mode,
* but snapbuilder/slot statuses are updated properly.
*/
- record = XLogReadRecord(ctx->reader, &errm);
+ while (XLogReadRecord(ctx->reader, &record, &errm) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!ctx->page_read(ctx->reader))
+ break;
+ }
+
if (errm)
elog(ERROR, "%s", errm);
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index a4d6f30957..b024bbc3cd 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -580,10 +580,7 @@ StartReplication(StartReplicationCmd *cmd)
/* create xlogreader for physical replication */
xlogreader =
- XLogReaderAllocate(wal_segment_size, NULL,
- XL_ROUTINE(.segment_open = WalSndSegmentOpen,
- .segment_close = wal_segment_close),
- NULL);
+ XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
if (!xlogreader)
ereport(ERROR,
@@ -807,9 +804,11 @@ StartReplication(StartReplicationCmd *cmd)
* set every time WAL is flushed.
*/
static bool
-logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetRecPtr, char *cur_page)
+logical_read_xlog_page(XLogReaderState *state)
{
+ XLogRecPtr targetPagePtr = state->readPagePtr;
+ int reqLen = state->readLen;
+ char *cur_page = state->readBuf;
XLogRecPtr flushptr;
int count;
WALReadError errinfo;
@@ -837,7 +836,7 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
count = flushptr - targetPagePtr; /* part of the page available */
/* now actually read the data, we know it's there */
- if (!WALRead(state,
+ if (!WALRead(state, WalSndSegmentOpen, wal_segment_close,
cur_page,
targetPagePtr,
XLOG_BLCKSZ,
@@ -1011,9 +1010,8 @@ CreateReplicationSlot(CreateReplicationSlotCmd *cmd)
ctx = CreateInitDecodingContext(cmd->plugin, NIL, need_full_snapshot,
InvalidXLogRecPtr,
- XL_ROUTINE(.page_read = logical_read_xlog_page,
- .segment_open = WalSndSegmentOpen,
- .segment_close = wal_segment_close),
+ logical_read_xlog_page,
+ wal_segment_close,
WalSndPrepareWrite, WalSndWriteData,
WalSndUpdateProgress);
@@ -1171,9 +1169,8 @@ StartLogicalReplication(StartReplicationCmd *cmd)
*/
logical_decoding_ctx =
CreateDecodingContext(cmd->startpoint, cmd->options, false,
- XL_ROUTINE(.page_read = logical_read_xlog_page,
- .segment_open = WalSndSegmentOpen,
- .segment_close = wal_segment_close),
+ logical_read_xlog_page,
+ wal_segment_close,
WalSndPrepareWrite, WalSndWriteData,
WalSndUpdateProgress);
xlogreader = logical_decoding_ctx->reader;
@@ -2749,7 +2746,7 @@ XLogSendPhysical(void)
enlargeStringInfo(&output_message, nbytes);
retry:
- if (!WALRead(xlogreader,
+ if (!WALRead(xlogreader, WalSndSegmentOpen, wal_segment_close,
&output_message.data[output_message.len],
startptr,
nbytes,
@@ -2847,7 +2844,12 @@ XLogSendLogical(void)
*/
WalSndCaughtUp = false;
- record = XLogReadRecord(logical_decoding_ctx->reader, &errm);
+ while (XLogReadRecord(logical_decoding_ctx->reader, &record, &errm) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!logical_decoding_ctx->page_read(logical_decoding_ctx->reader))
+ break;
+ }
/* xlog record was invalid */
if (errm != NULL)
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index cf119848b0..712c85281c 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -41,15 +41,9 @@ static int xlogreadfd = -1;
static XLogSegNo xlogreadsegno = -1;
static char xlogfpath[MAXPGPATH];
-typedef struct XLogPageReadPrivate
-{
- const char *restoreCommand;
- int tliIndex;
-} XLogPageReadPrivate;
-
-static bool SimpleXLogPageRead(XLogReaderState *xlogreader,
- XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
+static bool SimpleXLogPageRead(XLogReaderState *xlogreader,
+ const char *datadir, int *tliIndex,
+ const char *restoreCommand);
/*
* Read WAL from the datadir/pg_wal, starting from 'startpoint' on timeline
@@ -66,20 +60,22 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
XLogRecord *record;
XLogReaderState *xlogreader;
char *errormsg;
- XLogPageReadPrivate private;
- private.tliIndex = tliIndex;
- private.restoreCommand = restoreCommand;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir,
- XL_ROUTINE(.page_read = &SimpleXLogPageRead),
- &private);
+ xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+
if (xlogreader == NULL)
pg_fatal("out of memory");
XLogBeginRead(xlogreader, startpoint);
do
{
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!SimpleXLogPageRead(xlogreader, datadir,
+ &tliIndex, restoreCommand))
+ break;
+ }
if (record == NULL)
{
@@ -123,19 +119,19 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex,
XLogRecord *record;
XLogReaderState *xlogreader;
char *errormsg;
- XLogPageReadPrivate private;
XLogRecPtr endptr;
- private.tliIndex = tliIndex;
- private.restoreCommand = restoreCommand;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir,
- XL_ROUTINE(.page_read = &SimpleXLogPageRead),
- &private);
+ xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
XLogBeginRead(xlogreader, ptr);
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!SimpleXLogPageRead(xlogreader, datadir, &tliIndex, restoreCommand))
+ break;
+ }
if (record == NULL)
{
if (errormsg)
@@ -170,7 +166,6 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
XLogRecPtr searchptr;
XLogReaderState *xlogreader;
char *errormsg;
- XLogPageReadPrivate private;
/*
* The given fork pointer points to the end of the last common record,
@@ -186,11 +181,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
forkptr += SizeOfXLogShortPHD;
}
- private.tliIndex = tliIndex;
- private.restoreCommand = restoreCommand;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir,
- XL_ROUTINE(.page_read = &SimpleXLogPageRead),
- &private);
+ xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
@@ -200,7 +191,13 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
uint8 info;
XLogBeginRead(xlogreader, searchptr);
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!SimpleXLogPageRead(xlogreader, datadir,
+ &tliIndex, restoreCommand))
+ break;
+ }
if (record == NULL)
{
@@ -247,10 +244,11 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
/* XLogReader callback function, to read a WAL page */
static bool
-SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *readBuf)
+SimpleXLogPageRead(XLogReaderState *xlogreader, const char *datadir,
+ int *tliIndex, const char *restoreCommand)
{
- XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
+ XLogRecPtr targetPagePtr = xlogreader->readPagePtr;
+ char *readBuf = xlogreader->readBuf;
uint32 targetPageOff;
XLogRecPtr targetSegEnd;
XLogSegNo targetSegNo;
@@ -283,14 +281,14 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
* be done both forward and backward, consider also switching timeline
* accordingly.
*/
- while (private->tliIndex < targetNentries - 1 &&
- targetHistory[private->tliIndex].end < targetSegEnd)
- private->tliIndex++;
- while (private->tliIndex > 0 &&
- targetHistory[private->tliIndex].begin >= targetSegEnd)
- private->tliIndex--;
+ while (*tliIndex < targetNentries - 1 &&
+ targetHistory[*tliIndex].end < targetSegEnd)
+ (*tliIndex)++;
+ while (*tliIndex > 0 &&
+ targetHistory[*tliIndex].begin >= targetSegEnd)
+ (*tliIndex)--;
- XLogFileName(xlogfname, targetHistory[private->tliIndex].tli,
+ XLogFileName(xlogfname, targetHistory[*tliIndex].tli,
xlogreadsegno, WalSegSz);
snprintf(xlogfpath, MAXPGPATH, "%s/" XLOGDIR "/%s",
@@ -303,7 +301,7 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
/*
* If we have no restore_command to execute, then exit.
*/
- if (private->restoreCommand == NULL)
+ if (restoreCommand == NULL)
{
pg_log_error("could not open file \"%s\": %m", xlogfpath);
xlogreader->readLen = -1;
@@ -317,7 +315,7 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
xlogreadfd = RestoreArchivedFile(xlogreader->segcxt.ws_dir,
xlogfname,
WalSegSz,
- private->restoreCommand);
+ restoreCommand);
if (xlogreadfd < 0)
{
@@ -359,7 +357,7 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
Assert(targetSegNo == xlogreadsegno);
- xlogreader->seg.ws_tli = targetHistory[private->tliIndex].tli;
+ xlogreader->seg.ws_tli = targetHistory[*tliIndex].tli;
xlogreader->readLen = XLOG_BLCKSZ;
return true;
}
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index 75ece5c658..c4047b92b5 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -330,12 +330,17 @@ WALDumpCloseSegment(XLogReaderState *state)
state->seg.ws_file = -1;
}
-/* pg_waldump's XLogReaderRoutine->page_read callback */
+/*
+ * pg_waldump's WAL page rader, also used as page_read callback for
+ * XLogFindNextRecord
+ */
static bool
-WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetPtr, char *readBuff)
+WALDumpReadPage(XLogReaderState *state, void *priv)
{
- XLogDumpPrivate *private = state->private_data;
+ XLogRecPtr targetPagePtr = state->readPagePtr;
+ int reqLen = state->readLen;
+ char *readBuff = state->readBuf;
+ XLogDumpPrivate *private = (XLogDumpPrivate *) priv;
int count = XLOG_BLCKSZ;
WALReadError errinfo;
@@ -353,8 +358,8 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
}
}
- if (!WALRead(state, readBuff, targetPagePtr, count, private->timeline,
- &errinfo))
+ if (!WALRead(state, WALDumpOpenSegment, WALDumpCloseSegment,
+ readBuff, targetPagePtr, count, private->timeline, &errinfo))
{
WALOpenSegment *seg = &errinfo.wre_seg;
char fname[MAXPGPATH];
@@ -374,6 +379,7 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
(Size) errinfo.wre_req);
}
+ Assert(count >= state->readLen);
state->readLen = count;
return true;
}
@@ -1044,16 +1050,14 @@ main(int argc, char **argv)
/* we have everything we need, start reading */
xlogreader_state =
- XLogReaderAllocate(WalSegSz, waldir,
- XL_ROUTINE(.page_read = WALDumpReadPage,
- .segment_open = WALDumpOpenSegment,
- .segment_close = WALDumpCloseSegment),
- &private);
+ XLogReaderAllocate(WalSegSz, waldir, WALDumpCloseSegment);
+
if (!xlogreader_state)
fatal_error("out of memory");
/* first find a valid recptr to start from */
- first_record = XLogFindNextRecord(xlogreader_state, private.startptr);
+ first_record = XLogFindNextRecord(xlogreader_state, private.startptr,
+ &WALDumpReadPage, (void*) &private);
if (first_record == InvalidXLogRecPtr)
fatal_error("could not find a valid record after %X/%X",
@@ -1076,7 +1080,13 @@ main(int argc, char **argv)
for (;;)
{
/* try to read the next record */
- record = XLogReadRecord(xlogreader_state, &errormsg);
+ while (XLogReadRecord(xlogreader_state, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!WALDumpReadPage(xlogreader_state, (void *) &private))
+ break;
+ }
+
if (!record)
{
if (!config.follow || private.endptr_reached)
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index 5d9e0d3292..1492f1992d 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -57,64 +57,15 @@ typedef struct WALSegmentContext
typedef struct XLogReaderState XLogReaderState;
-/* Function type definition for the read_page callback */
-typedef bool (*XLogPageReadCB) (XLogReaderState *xlogreader,
- XLogRecPtr targetPagePtr,
- int reqLen,
- XLogRecPtr targetRecPtr,
- char *readBuf);
+/* Function type definition for the segment cleanup callback */
+typedef void (*WALSegmentCleanupCB) (XLogReaderState *xlogreader);
+
+/* Function type definition for the open/close callbacks for WALRead() */
typedef void (*WALSegmentOpenCB) (XLogReaderState *xlogreader,
XLogSegNo nextSegNo,
TimeLineID *tli_p);
typedef void (*WALSegmentCloseCB) (XLogReaderState *xlogreader);
-typedef struct XLogReaderRoutine
-{
- /*
- * Data input callback
- *
- * This callback shall read at least reqLen valid bytes of the xlog page
- * starting at targetPagePtr, and store them in readBuf. The callback
- * shall return the number of bytes read (never more than XLOG_BLCKSZ), or
- * -1 on failure. The callback shall sleep, if necessary, to wait for the
- * requested bytes to become available. The callback will not be invoked
- * again for the same page unless more than the returned number of bytes
- * are needed.
- *
- * targetRecPtr is the position of the WAL record we're reading. Usually
- * it is equal to targetPagePtr + reqLen, but sometimes xlogreader needs
- * to read and verify the page or segment header, before it reads the
- * actual WAL record it's interested in. In that case, targetRecPtr can
- * be used to determine which timeline to read the page from.
- *
- * The callback shall set ->seg.ws_tli to the TLI of the file the page was
- * read from.
- */
- XLogPageReadCB page_read;
-
- /*
- * Callback to open the specified WAL segment for reading. ->seg.ws_file
- * shall be set to the file descriptor of the opened segment. In case of
- * failure, an error shall be raised by the callback and it shall not
- * return.
- *
- * "nextSegNo" is the number of the segment to be opened.
- *
- * "tli_p" is an input/output argument. WALRead() uses it to pass the
- * timeline in which the new segment should be found, but the callback can
- * use it to return the TLI that it actually opened.
- */
- WALSegmentOpenCB segment_open;
-
- /*
- * WAL segment close callback. ->seg.ws_file shall be set to a negative
- * number.
- */
- WALSegmentCloseCB segment_close;
-} XLogReaderRoutine;
-
-#define XL_ROUTINE(...) &(XLogReaderRoutine){__VA_ARGS__}
-
typedef struct
{
/* Is this block ref in use? */
@@ -144,12 +95,36 @@ typedef struct
uint16 data_bufsz;
} DecodedBkpBlock;
+/* Return code from XLogReadRecord */
+typedef enum XLogReadRecordResult
+{
+ XLREAD_SUCCESS, /* record is successfully read */
+ XLREAD_NEED_DATA, /* need more data. see XLogReadRecord. */
+ XLREAD_FAIL /* failed during reading a record */
+} XLogReadRecordResult;
+
+/*
+ * internal state of XLogReadRecord
+ *
+ * XLogReadState runs a state machine while reading a record. Theses states
+ * are not seen outside the function. Each state may repeat several times
+ * exiting requesting caller for new data. See the comment of XLogReadRecrod
+ * for details.
+ */
+typedef enum XLogReadRecordState
+{
+ XLREAD_NEXT_RECORD,
+ XLREAD_TOT_LEN,
+ XLREAD_FIRST_FRAGMENT,
+ XLREAD_CONTINUATION
+} XLogReadRecordState;
+
struct XLogReaderState
{
/*
* Operational callbacks
*/
- XLogReaderRoutine routine;
+ WALSegmentCleanupCB cleanup_cb;
/* ----------------------------------------
* Public parameters
@@ -162,18 +137,14 @@ struct XLogReaderState
*/
uint64 system_identifier;
- /*
- * Opaque data for callbacks to use. Not used by XLogReader.
- */
- void *private_data;
-
/*
* Start and end point of last record read. EndRecPtr is also used as the
* position to read next. Calling XLogBeginRead() sets EndRecPtr to the
* starting position and ReadRecPtr to invalid.
*/
- XLogRecPtr ReadRecPtr; /* start of last record read */
+ XLogRecPtr ReadRecPtr; /* start of last record read or being read */
XLogRecPtr EndRecPtr; /* end+1 of last record read */
+ XLogRecPtr PrevRecPtr; /* start of previous record read */
/* ----------------------------------------
* Communication with page reader
@@ -186,7 +157,9 @@ struct XLogReaderState
* read by reader, which must be larger than
* the request, or -1 on error */
char *readBuf; /* buffer to store data */
- bool page_verified; /* is the page on the buffer verified? */
+ bool page_verified; /* is the page header on the buffer verified? */
+ bool record_verified; /* is the current record header verified? */
+
/* ----------------------------------------
@@ -228,8 +201,6 @@ struct XLogReaderState
XLogRecPtr latestPagePtr;
TimeLineID latestPageTLI;
- /* beginning of the WAL record being read. */
- XLogRecPtr currRecPtr;
/* timeline to read it from, 0 if a lookup is required */
TimeLineID currTLI;
@@ -256,6 +227,15 @@ struct XLogReaderState
char *readRecordBuf;
uint32 readRecordBufSize;
+ /*
+ * XLogReadRecord() state
+ */
+ XLogReadRecordState readRecordState; /* state machine state */
+ int recordGotLen; /* amount of current record that has already
+ * been read */
+ int recordRemainLen; /* length of current record that remains */
+ XLogRecPtr recordContRecPtr; /* where the current record continues */
+
/* Buffer to hold error message */
char *errormsg_buf;
};
@@ -263,9 +243,7 @@ struct XLogReaderState
/* Get a new XLogReader */
extern XLogReaderState *XLogReaderAllocate(int wal_segment_size,
const char *waldir,
- XLogReaderRoutine *routine,
- void *private_data);
-extern XLogReaderRoutine *LocalXLogReaderRoutine(void);
+ WALSegmentCleanupCB cleanup_cb);
/* Free an XLogReader */
extern void XLogReaderFree(XLogReaderState *state);
@@ -273,12 +251,17 @@ extern void XLogReaderFree(XLogReaderState *state);
/* Position the XLogReader to given record */
extern void XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr);
#ifdef FRONTEND
-extern XLogRecPtr XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr);
+/* Function type definition for the read_page callback */
+typedef bool (*XLogFindNextRecordCB) (XLogReaderState *xlogreader,
+ void *private);
+extern XLogRecPtr XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
+ XLogFindNextRecordCB read_page, void *private);
#endif /* FRONTEND */
/* Read the next XLog record. Returns NULL on end-of-WAL or failure */
-extern struct XLogRecord *XLogReadRecord(XLogReaderState *state,
- char **errormsg);
+extern XLogReadRecordResult XLogReadRecord(XLogReaderState *state,
+ XLogRecord **record,
+ char **errormsg);
/* Validate a page */
extern bool XLogReaderValidatePageHeader(XLogReaderState *state,
@@ -298,6 +281,7 @@ typedef struct WALReadError
} WALReadError;
extern bool WALRead(XLogReaderState *state,
+ WALSegmentOpenCB segopenfn, WALSegmentCloseCB sgclosefn,
char *buf, XLogRecPtr startptr, Size count,
TimeLineID tli, WALReadError *errinfo);
diff --git a/src/include/access/xlogutils.h b/src/include/access/xlogutils.h
index 364a21c4ea..397fb27fc2 100644
--- a/src/include/access/xlogutils.h
+++ b/src/include/access/xlogutils.h
@@ -47,9 +47,7 @@ extern Buffer XLogReadBufferExtended(RelFileNode rnode, ForkNumber forknum,
extern Relation CreateFakeRelcacheEntry(RelFileNode rnode);
extern void FreeFakeRelcacheEntry(Relation fakerel);
-extern bool read_local_xlog_page(XLogReaderState *state,
- XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetRecPtr, char *cur_page);
+extern bool read_local_xlog_page(XLogReaderState *state);
extern void wal_segment_open(XLogReaderState *state,
XLogSegNo nextSegNo,
TimeLineID *tli_p);
diff --git a/src/include/pg_config_manual.h b/src/include/pg_config_manual.h
index e28c990382..bd9a5c6c2b 100644
--- a/src/include/pg_config_manual.h
+++ b/src/include/pg_config_manual.h
@@ -384,7 +384,7 @@
* Enable debugging print statements for WAL-related operations; see
* also the wal_debug GUC var.
*/
-/* #define WAL_DEBUG */
+#define WAL_DEBUG
/*
* Enable tracing of resource consumption during sort operations;
diff --git a/src/include/replication/logical.h b/src/include/replication/logical.h
index af551d6f4e..94e278ef81 100644
--- a/src/include/replication/logical.h
+++ b/src/include/replication/logical.h
@@ -29,6 +29,10 @@ typedef void (*LogicalOutputPluginWriterUpdateProgress) (struct LogicalDecodingC
TransactionId xid
);
+typedef struct LogicalDecodingContext LogicalDecodingContext;
+
+typedef bool (*LogicalDecodingXLogPageReadCB)(XLogReaderState *ctx);
+
typedef struct LogicalDecodingContext
{
/* memory context this is all allocated in */
@@ -39,6 +43,7 @@ typedef struct LogicalDecodingContext
/* infrastructure pieces for decoding */
XLogReaderState *reader;
+ LogicalDecodingXLogPageReadCB page_read;
struct ReorderBuffer *reorder;
struct SnapBuild *snapshot_builder;
@@ -105,14 +110,16 @@ extern LogicalDecodingContext *CreateInitDecodingContext(const char *plugin,
List *output_plugin_options,
bool need_full_snapshot,
XLogRecPtr restart_lsn,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress);
extern LogicalDecodingContext *CreateDecodingContext(XLogRecPtr start_lsn,
List *output_plugin_options,
bool fast_forward,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress);
--
2.27.0
----Next_Part(Wed_Apr__7_17_50_25_2021_942)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
filename="v19-0003-Remove-globals-readOff-readLen-and-readSegNo.patch"
^ permalink raw reply [nested|flat] 70+ messages in thread
* [PATCH v15 2/4] Move page-reader out of XLogReadRecord
@ 2019-09-10 03:58 Kyotaro Horiguchi <[email protected]>
0 siblings, 0 replies; 70+ messages in thread
From: Kyotaro Horiguchi @ 2019-09-10 03:58 UTC (permalink / raw)
This is the second step of removing callbacks from WAL record reader.
Since it is essential to take in additional data while reading a
record, the function have to ask caller for new data while keeping
working state. Thus the function is turned into a state machine.
---
src/backend/access/transam/twophase.c | 15 +-
src/backend/access/transam/xlog.c | 58 +-
src/backend/access/transam/xlogreader.c | 700 +++++++++++-------
src/backend/access/transam/xlogutils.c | 17 +-
src/backend/replication/logical/logical.c | 26 +-
.../replication/logical/logicalfuncs.c | 13 +-
src/backend/replication/slotfuncs.c | 18 +-
src/backend/replication/walsender.c | 32 +-
src/bin/pg_rewind/parsexlog.c | 84 +--
src/bin/pg_waldump/pg_waldump.c | 36 +-
src/include/access/xlogreader.h | 121 ++-
src/include/access/xlogutils.h | 4 +-
src/include/replication/logical.h | 11 +-
13 files changed, 648 insertions(+), 487 deletions(-)
diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index 9b2e59bf0e..b0d60a0d0f 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -1330,11 +1330,8 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
XLogReaderState *xlogreader;
char *errormsg;
- xlogreader = XLogReaderAllocate(wal_segment_size, NULL,
- XL_ROUTINE(.page_read = &read_local_xlog_page,
- .segment_open = &wal_segment_open,
- .segment_close = &wal_segment_close),
- NULL);
+ xlogreader = XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+
if (!xlogreader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
@@ -1342,7 +1339,13 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
errdetail("Failed while allocating a WAL reading processor.")));
XLogBeginRead(xlogreader, lsn);
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!read_local_xlog_page(xlogreader))
+ break;
+ }
+
if (record == NULL)
ereport(ERROR,
(errcode_for_file_access(),
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index e570e56a24..f9b0108602 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -831,13 +831,6 @@ static XLogSource currentSource = XLOG_FROM_ANY;
static bool lastSourceFailed = false;
static bool pendingWalRcvRestart = false;
-typedef struct XLogPageReadPrivate
-{
- int emode;
- bool fetching_ckpt; /* are we fetching a checkpoint record? */
- bool randAccess;
-} XLogPageReadPrivate;
-
/*
* These variables track when we last obtained some WAL data to process,
* and where we got it from. (XLogReceiptSource is initially the same as
@@ -912,8 +905,8 @@ static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath,
static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli,
XLogSource source, bool notfoundOk);
static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, XLogSource source);
-static bool XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
+static bool XLogPageRead(XLogReaderState *xlogreader,
+ bool fetching_ckpt, int emode, bool randAccess);
static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
bool fetching_ckpt, XLogRecPtr tliRecPtr);
static int emode_for_corrupt_record(int emode, XLogRecPtr RecPtr);
@@ -1225,8 +1218,7 @@ XLogInsertRecord(XLogRecData *rdata,
appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len);
if (!debug_reader)
- debug_reader = XLogReaderAllocate(wal_segment_size, NULL,
- XL_ROUTINE(), NULL);
+ debug_reader = XLogReaderAllocate(wal_segment_size, NULL NULL);
if (!debug_reader)
{
@@ -4326,15 +4318,10 @@ CleanupBackupHistory(void)
* record is available.
*/
static XLogRecord *
-ReadRecord(XLogReaderState *xlogreader, int emode,
- bool fetching_ckpt)
+ReadRecord(XLogReaderState *xlogreader, int emode, bool fetching_ckpt)
{
XLogRecord *record;
- XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
-
- private->fetching_ckpt = fetching_ckpt;
- private->emode = emode;
- private->randAccess = (xlogreader->ReadRecPtr == InvalidXLogRecPtr);
+ bool randAccess = (xlogreader->ReadRecPtr == InvalidXLogRecPtr);
/* This is the first attempt to read this page. */
lastSourceFailed = false;
@@ -4342,8 +4329,16 @@ ReadRecord(XLogReaderState *xlogreader, int emode,
for (;;)
{
char *errormsg;
+ XLogReadRecordResult result;
+
+ while ((result = XLogReadRecord(xlogreader, &record, &errormsg))
+ == XLREAD_NEED_DATA)
+ {
+ if (!XLogPageRead(xlogreader, fetching_ckpt, emode, randAccess))
+ break;
+
+ }
- record = XLogReadRecord(xlogreader, &errormsg);
ReadRecPtr = xlogreader->ReadRecPtr;
EndRecPtr = xlogreader->EndRecPtr;
if (record == NULL)
@@ -6320,7 +6315,6 @@ StartupXLOG(void)
bool backupFromStandby = false;
DBState dbstate_at_startup;
XLogReaderState *xlogreader;
- XLogPageReadPrivate private;
bool fast_promoted = false;
struct stat st;
@@ -6480,13 +6474,9 @@ StartupXLOG(void)
OwnLatch(&XLogCtl->recoveryWakeupLatch);
/* Set up XLOG reader facility */
- MemSet(&private, 0, sizeof(XLogPageReadPrivate));
xlogreader =
- XLogReaderAllocate(wal_segment_size, NULL,
- XL_ROUTINE(.page_read = &XLogPageRead,
- .segment_open = NULL,
- .segment_close = wal_segment_close),
- &private);
+ XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+
if (!xlogreader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
@@ -11860,12 +11850,13 @@ CancelBackup(void)
* sleep and retry.
*/
static bool
-XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetRecPtr, char *readBuf)
+XLogPageRead(XLogReaderState *xlogreader,
+ bool fetching_ckpt, int emode, bool randAccess)
{
- XLogPageReadPrivate *private =
- (XLogPageReadPrivate *) xlogreader->private_data;
- int emode = private->emode;
+ char *readBuf = xlogreader->readBuf;
+ XLogRecPtr targetPagePtr = xlogreader->readPagePtr;
+ int reqLen = xlogreader->readLen;
+ XLogRecPtr targetRecPtr = xlogreader->ReadRecPtr;
uint32 targetPageOff;
XLogSegNo targetSegNo PG_USED_FOR_ASSERTS_ONLY;
int r;
@@ -11908,8 +11899,8 @@ retry:
flushedUpto < targetPagePtr + reqLen))
{
if (!WaitForWALToBecomeAvailable(targetPagePtr + reqLen,
- private->randAccess,
- private->fetching_ckpt,
+ randAccess,
+ fetching_ckpt,
targetRecPtr))
{
if (readFile >= 0)
@@ -12014,6 +12005,7 @@ retry:
goto next_record_is_invalid;
}
+ Assert(xlogreader->readPagePtr == targetPagePtr);
xlogreader->readLen = readLen;
return true;
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index 3d599325ee..9281b57379 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -40,7 +40,7 @@ static bool XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr,
int reqLen, bool header_inclusive);
static void XLogReaderInvalReadState(XLogReaderState *state);
static bool ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
- XLogRecPtr PrevRecPtr, XLogRecord *record, bool randAccess);
+ XLogRecPtr PrevRecPtr, XLogRecord *record);
static bool ValidXLogRecord(XLogReaderState *state, XLogRecord *record,
XLogRecPtr recptr);
static void ResetDecoder(XLogReaderState *state);
@@ -73,7 +73,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...)
*/
XLogReaderState *
XLogReaderAllocate(int wal_segment_size, const char *waldir,
- XLogReaderRoutine *routine, void *private_data)
+ WALSegmentCleanupCB cleanup_cb)
{
XLogReaderState *state;
@@ -84,7 +84,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
return NULL;
/* initialize caller-provided support functions */
- state->routine = *routine;
+ state->cleanup_cb = cleanup_cb;
state->max_block_id = -1;
@@ -107,8 +107,6 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
WALOpenSegmentInit(&state->seg, &state->segcxt, wal_segment_size,
waldir);
- /* system_identifier initialized to zeroes above */
- state->private_data = private_data;
/* ReadRecPtr, EndRecPtr and readLen initialized to zeroes above */
state->errormsg_buf = palloc_extended(MAX_ERRORMSG_LEN + 1,
MCXT_ALLOC_NO_OOM);
@@ -140,8 +138,8 @@ XLogReaderFree(XLogReaderState *state)
{
int block_id;
- if (state->seg.ws_file != -1)
- state->routine.segment_close(state);
+ if (state->seg.ws_file >= 0)
+ state->cleanup_cb(state);
for (block_id = 0; block_id <= XLR_MAX_BLOCK_ID; block_id++)
{
@@ -246,6 +244,7 @@ XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
/* Begin at the passed-in record pointer. */
state->EndRecPtr = RecPtr;
state->ReadRecPtr = InvalidXLogRecPtr;
+ state->readRecordState = XLREAD_NEXT_RECORD;
}
/*
@@ -254,317 +253,452 @@ XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
* XLogBeginRead() or XLogFindNextRecord() must be called before the first call
* to XLogReadRecord().
*
- * If the page_read callback fails to read the requested data, NULL is
- * returned. The callback is expected to have reported the error; errormsg
- * is set to NULL.
+ * This function may return XLREAD_NEED_DATA several times before returning a
+ * result record. The caller shall read in some new data then call this
+ * function again with the same parameters.
*
- * If the reading fails for some other reason, NULL is also returned, and
- * *errormsg is set to a string with details of the failure.
+ * When a record is successfully read, returns XLREAD_SUCCESS with result
+ * record being stored in *record. Otherwise *record is NULL.
*
- * The returned pointer (or *errormsg) points to an internal buffer that's
- * valid until the next call to XLogReadRecord.
+ * Returns XLREAD_NEED_DATA if more data is needed to finish reading the
+ * current record. In that case, state->readPagePtr and state->readLen inform
+ * the desired position and minimum length of data needed. The caller shall
+ * read in the requested data and set state->readBuf to point to a buffer
+ * containing it. The caller must also set state->readPageTLI and
+ * state->readLen to indicate the timeline that it was read from, and the
+ * length of data that is now available (which must be >= given readLen),
+ * respectively.
+ *
+ * If invalid data is encountered, returns XLREAD_FAIL with *record being set to
+ * NULL. *errormsg is set to a string with details of the failure.
+ * The returned pointer (or *errormsg) points to an internal buffer that's valid
+ * until the next call to XLogReadRecord.
+ *
+ *
+ * This function runs a state machine consists of the following states.
+ *
+ * XLREAD_NEXT_RECORD :
+ * The initial state, if called with valid RecPtr, try to read a record at
+ * that position. If invalid RecPtr is given try to read a record just after
+ * the last one previously read.
+ * This state ens after setting ReadRecPtr. Then goes to XLREAD_TOT_LEN.
+ *
+ * XLREAD_TOT_LEN:
+ * Examining record header. Ends after reading record total
+ * length. recordRemainLen and recordGotLen are initialized.
+ *
+ * XLREAD_FIRST_FRAGMENT:
+ * Reading the first fragment. Ends with finishing reading a single
+ * record. Goes to XLREAD_NEXT_RECORD if that's all or
+ * XLREAD_CONTINUATION if we have continuation.
+
+ * XLREAD_CONTINUATION:
+ * Reading continuation of record. Ends with finishing the whole record then
+ * goes to XLREAD_NEXT_RECORD. During this state, recordRemainLen indicates
+ * how much is left and readRecordBuf holds the partially assert
+ * record.recordContRecPtr points to the beginning of the next page where to
+ * continue.
+ *
+ * If wrong data found in any state, the state machine stays at the current
+ * state. This behavior allows to continue reading a reacord switching among
+ * different souces, while streaming replication.
*/
-XLogRecord *
-XLogReadRecord(XLogReaderState *state, char **errormsg)
+XLogReadRecordResult
+XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
{
- XLogRecPtr RecPtr;
- XLogRecord *record;
- XLogRecPtr targetPagePtr;
- bool randAccess;
- uint32 len,
- total_len;
- uint32 targetRecOff;
- uint32 pageHeaderSize;
- bool gotheader;
+ XLogRecord *prec;
- /*
- * randAccess indicates whether to verify the previous-record pointer of
- * the record we're reading. We only do this if we're reading
- * sequentially, which is what we initially assume.
- */
- randAccess = false;
+ *record = NULL;
/* reset error state */
*errormsg = NULL;
state->errormsg_buf[0] = '\0';
- ResetDecoder(state);
-
- RecPtr = state->EndRecPtr;
-
- if (state->ReadRecPtr != InvalidXLogRecPtr)
- {
- /* read the record after the one we just read */
-
- /*
- * EndRecPtr is pointing to end+1 of the previous WAL record. If
- * we're at a page boundary, no more records can fit on the current
- * page. We must skip over the page header, but we can't do that until
- * we've read in the page, since the header size is variable.
- */
- }
- else
- {
- /*
- * Caller supplied a position to start at.
- *
- * In this case, EndRecPtr should already be pointing to a valid
- * record starting position.
- */
- Assert(XRecOffIsValid(RecPtr));
- randAccess = true;
- }
-
- state->currRecPtr = RecPtr;
-
- targetPagePtr = RecPtr - (RecPtr % XLOG_BLCKSZ);
- targetRecOff = RecPtr % XLOG_BLCKSZ;
-
- /*
- * Read the page containing the record into state->readBuf. Request enough
- * byte to cover the whole record header, or at least the part of it that
- * fits on the same page.
- */
- while (XLogNeedData(state, targetPagePtr,
- Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
- targetRecOff != 0))
- {
- if (!state->routine.page_read(state, state->readPagePtr, state->readLen,
- RecPtr, state->readBuf))
- break;
- }
-
- if (!state->page_verified)
- goto err;
-
- /*
- * We have at least the page header, so we can examine it now.
- */
- pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
- if (targetRecOff == 0)
- {
- /*
- * At page start, so skip over page header.
- */
- RecPtr += pageHeaderSize;
- targetRecOff = pageHeaderSize;
- }
- else if (targetRecOff < pageHeaderSize)
- {
- report_invalid_record(state, "invalid record offset at %X/%X",
- (uint32) (RecPtr >> 32), (uint32) RecPtr);
- goto err;
- }
-
- if ((((XLogPageHeader) state->readBuf)->xlp_info & XLP_FIRST_IS_CONTRECORD) &&
- targetRecOff == pageHeaderSize)
- {
- report_invalid_record(state, "contrecord is requested by %X/%X",
- (uint32) (RecPtr >> 32), (uint32) RecPtr);
- goto err;
- }
-
- /* XLogNeedData has verified the page header */
- Assert(pageHeaderSize <= state->readLen);
-
- /*
- * Read the record length.
- *
- * NB: Even though we use an XLogRecord pointer here, the whole record
- * header might not fit on this page. xl_tot_len is the first field of the
- * struct, so it must be on this page (the records are MAXALIGNed), but we
- * cannot access any other fields until we've verified that we got the
- * whole header.
- */
- record = (XLogRecord *) (state->readBuf + RecPtr % XLOG_BLCKSZ);
- total_len = record->xl_tot_len;
-
- /*
- * If the whole record header is on this page, validate it immediately.
- * Otherwise do just a basic sanity check on xl_tot_len, and validate the
- * rest of the header after reading it from the next page. The xl_tot_len
- * check is necessary here to ensure that we enter the "Need to reassemble
- * record" code path below; otherwise we might fail to apply
- * ValidXLogRecordHeader at all.
- */
- if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
- {
- if (!ValidXLogRecordHeader(state, RecPtr, state->ReadRecPtr, record,
- randAccess))
- goto err;
- gotheader = true;
- }
- else
- {
- /* XXX: more validation should be done here */
- if (total_len < SizeOfXLogRecord)
- {
- report_invalid_record(state,
- "invalid record length at %X/%X: wanted %u, got %u",
- (uint32) (RecPtr >> 32), (uint32) RecPtr,
- (uint32) SizeOfXLogRecord, total_len);
- goto err;
- }
- gotheader = false;
- }
-
- len = XLOG_BLCKSZ - RecPtr % XLOG_BLCKSZ;
- if (total_len > len)
+ switch (state->readRecordState)
{
- /* Need to reassemble record */
- char *contdata;
- XLogPageHeader pageHeader;
- char *buffer;
- uint32 gotlen;
-
- /*
- * Enlarge readRecordBuf as needed.
- */
- if (total_len > state->readRecordBufSize &&
- !allocate_recordbuf(state, total_len))
- {
- /* We treat this as a "bogus data" condition */
- report_invalid_record(state, "record length %u at %X/%X too long",
- total_len,
- (uint32) (RecPtr >> 32), (uint32) RecPtr);
- goto err;
- }
+ case XLREAD_NEXT_RECORD:
+ ResetDecoder(state);
- /* Copy the first fragment of the record from the first page. */
- memcpy(state->readRecordBuf,
- state->readBuf + RecPtr % XLOG_BLCKSZ, len);
- buffer = state->readRecordBuf + len;
- gotlen = len;
-
- do
- {
- int rest_len = total_len - gotlen;
-
- /* Calculate pointer to beginning of next page */
- targetPagePtr += XLOG_BLCKSZ;
-
- /* Wait for the next page to become available */
- while (XLogNeedData(state, targetPagePtr,
- Min(rest_len, XLOG_BLCKSZ),
- false))
+ if (state->ReadRecPtr != InvalidXLogRecPtr)
{
- if (!state->routine.page_read(state, state->readPagePtr,
- state->readLen,
- state->ReadRecPtr,
- state->readBuf))
- break;
+ /* read the record after the one we just read */
+
+ /*
+ * EndRecPtr is pointing to end+1 of the previous WAL record.
+ * If we're at a page boundary, no more records can fit on the
+ * current page. We must skip over the page header, but we
+ * can't do that until we've read in the page, since the header
+ * size is variable.
+ */
+ state->PrevRecPtr = state->ReadRecPtr;
+ state->ReadRecPtr = state->EndRecPtr;
+ }
+ else
+ {
+ /*
+ * Caller supplied a position to start at.
+ *
+ * In this case, EndRecPtr should already be pointing to a
+ * valid record starting position.
+ */
+ Assert(XRecOffIsValid(state->EndRecPtr));
+ state->ReadRecPtr = state->EndRecPtr;
+
+ /*
+ * We cannot verify the previous-record pointer when we're
+ * seeking to a particular record. Reset PrevRecPtr so that we
+ * won't try doing that.
+ */
+ state->PrevRecPtr = InvalidXLogRecPtr;
+ state->EndRecPtr = InvalidXLogRecPtr; /* to be tidy */
}
+ state->record_verified = false;
+ state->readRecordState = XLREAD_TOT_LEN;
+ /* fall through */
+
+ case XLREAD_TOT_LEN:
+ {
+ uint32 total_len;
+ uint32 pageHeaderSize;
+ XLogRecPtr targetPagePtr;
+ uint32 targetRecOff;
+ XLogPageHeader pageHeader;
+
+ targetPagePtr =
+ state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
+ targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+
+ /*
+ * Check if we have enough data. For the first record in the page,
+ * the requesting length doesn't contain page header.
+ */
+ if (XLogNeedData(state, targetPagePtr,
+ Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+ targetRecOff != 0))
+ return XLREAD_NEED_DATA;
+
+ /* error out if caller supplied bogus page */
if (!state->page_verified)
goto err;
- Assert(SizeOfXLogShortPHD <= state->readLen);
+ /* examine page header now. */
+ pageHeaderSize =
+ XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+ if (targetRecOff == 0)
+ {
+ /* At page start, so skip over page header. */
+ state->ReadRecPtr += pageHeaderSize;
+ targetRecOff = pageHeaderSize;
+ }
+ else if (targetRecOff < pageHeaderSize)
+ {
+ report_invalid_record(state, "invalid record offset at %X/%X",
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr);
+ goto err;
+ }
- /* Check that the continuation on next page looks valid */
pageHeader = (XLogPageHeader) state->readBuf;
- if (!(pageHeader->xlp_info & XLP_FIRST_IS_CONTRECORD))
+ if ((pageHeader->xlp_info & XLP_FIRST_IS_CONTRECORD) &&
+ targetRecOff == pageHeaderSize)
{
- report_invalid_record(state,
- "there is no contrecord flag at %X/%X",
- (uint32) (RecPtr >> 32), (uint32) RecPtr);
+ report_invalid_record(state, "contrecord is requested by %X/%X",
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr);
goto err;
}
- /*
- * Cross-check that xlp_rem_len agrees with how much of the record
- * we expect there to be left.
- */
- if (pageHeader->xlp_rem_len == 0 ||
- total_len != (pageHeader->xlp_rem_len + gotlen))
- {
- report_invalid_record(state,
- "invalid contrecord length %u at %X/%X",
- pageHeader->xlp_rem_len,
- (uint32) (RecPtr >> 32), (uint32) RecPtr);
- goto err;
- }
-
- /* Append the continuation from this page to the buffer */
- pageHeaderSize = XLogPageHeaderSize(pageHeader);
-
+ /* XLogNeedData has verified the page header */
Assert(pageHeaderSize <= state->readLen);
- contdata = (char *) state->readBuf + pageHeaderSize;
- len = XLOG_BLCKSZ - pageHeaderSize;
- if (pageHeader->xlp_rem_len < len)
- len = pageHeader->xlp_rem_len;
+ /*
+ * Read the record length.
+ *
+ * NB: Even though we use an XLogRecord pointer here, the whole
+ * record header might not fit on this page. xl_tot_len is the first
+ * field of the struct, so it must be on this page (the records are
+ * MAXALIGNed), but we cannot access any other fields until we've
+ * verified that we got the whole header.
+ */
+ prec = (XLogRecord *) (state->readBuf +
+ state->ReadRecPtr % XLOG_BLCKSZ);
+ total_len = prec->xl_tot_len;
- Assert (pageHeaderSize + len <= state->readLen);
- memcpy(buffer, (char *) contdata, len);
- buffer += len;
- gotlen += len;
+ /*
+ * If the whole record header is on this page, validate it
+ * immediately. Otherwise do just a basic sanity check on
+ * xl_tot_len, and validate the rest of the header after reading it
+ * from the next page. The xl_tot_len check is necessary here to
+ * ensure that we enter the XLREAD_CONTINUATION state below;
+ * otherwise we might fail to apply ValidXLogRecordHeader at all.
+ */
+ if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
+ {
+ if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
+ state->PrevRecPtr, prec))
+ goto err;
- /* If we just reassembled the record header, validate it. */
- if (!gotheader)
+ state->record_verified = true;
+ }
+ else
{
- record = (XLogRecord *) state->readRecordBuf;
- if (!ValidXLogRecordHeader(state, RecPtr, state->ReadRecPtr,
- record, randAccess))
+ /* XXX: more validation should be done here */
+ if (total_len < SizeOfXLogRecord)
+ {
+ report_invalid_record(state,
+ "invalid record length at %X/%X: wanted %u, got %u",
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr,
+ (uint32) SizeOfXLogRecord, total_len);
goto err;
- gotheader = true;
+ }
}
- } while (gotlen < total_len);
-
- Assert(gotheader);
- record = (XLogRecord *) state->readRecordBuf;
- if (!ValidXLogRecord(state, record, RecPtr))
- goto err;
+ /*
+ * Wait for the rest of the record, or the part of the record that
+ * fit on the first page if crossed a page boundary, to become
+ * available.
+ */
+ state->recordGotLen = 0;
+ state->recordRemainLen = total_len;
+ state->readRecordState = XLREAD_FIRST_FRAGMENT;
+ }
+ /* fall through */
- pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
- state->ReadRecPtr = RecPtr;
- state->EndRecPtr = targetPagePtr + pageHeaderSize
- + MAXALIGN(pageHeader->xlp_rem_len);
- }
- else
- {
- /* Wait for the record data to become available */
- while (XLogNeedData(state, targetPagePtr,
- Min(targetRecOff + total_len, XLOG_BLCKSZ), true))
+ case XLREAD_FIRST_FRAGMENT:
{
- if (!state->routine.page_read(state, state->readPagePtr,
- state->readLen,
- state->ReadRecPtr, state->readBuf))
+ uint32 total_len = state->recordRemainLen;
+ uint32 request_len;
+ uint32 record_len;
+ XLogRecPtr targetPagePtr;
+ uint32 targetRecOff;
+
+ /*
+ * Wait for the rest of the record on the first page to become
+ * available
+ */
+ targetPagePtr =
+ state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
+ targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+
+ request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ);
+ record_len = request_len - targetRecOff;
+
+ /* ReadRecPtr contains page header */
+ Assert (targetRecOff != 0);
+ if (XLogNeedData(state, targetPagePtr, request_len, true))
+ return XLREAD_NEED_DATA;
+
+ /* error out if caller supplied bogus page */
+ if (!state->page_verified)
+ goto err;
+
+ prec = (XLogRecord *) (state->readBuf + targetRecOff);
+
+ /* validate record header if not yet */
+ if (!state->record_verified && record_len >= SizeOfXLogRecord)
+ {
+ if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
+ state->PrevRecPtr, prec))
+ goto err;
+
+ state->record_verified = true;
+ }
+
+
+ if (total_len == record_len)
+ {
+ /* Record does not cross a page boundary */
+ Assert(state->record_verified);
+
+ if (!ValidXLogRecord(state, prec, state->ReadRecPtr))
+ goto err;
+
+ state->record_verified = true; /* to be tidy */
+
+ /* We already checked the header earlier */
+ state->EndRecPtr = state->ReadRecPtr + MAXALIGN(record_len);
+
+ *record = prec;
+ state->readRecordState = XLREAD_NEXT_RECORD;
break;
+ }
+
+ /*
+ * The record continues on the next page. Need to reassemble
+ * record
+ */
+ Assert(total_len > record_len);
+
+ /* Enlarge readRecordBuf as needed. */
+ if (total_len > state->readRecordBufSize &&
+ !allocate_recordbuf(state, total_len))
+ {
+ /* We treat this as a "bogus data" condition */
+ report_invalid_record(state,
+ "record length %u at %X/%X too long",
+ total_len,
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr);
+ goto err;
+ }
+
+ /* Copy the first fragment of the record from the first page. */
+ memcpy(state->readRecordBuf, state->readBuf + targetRecOff,
+ record_len);
+ state->recordGotLen += record_len;
+ state->recordRemainLen -= record_len;
+
+ /* Calculate pointer to beginning of next page */
+ state->recordContRecPtr = state->ReadRecPtr + record_len;
+ Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0);
+
+ state->readRecordState = XLREAD_CONTINUATION;
}
+ /* fall through */
+
+ case XLREAD_CONTINUATION:
+ {
+ XLogPageHeader pageHeader;
+ uint32 pageHeaderSize;
+ XLogRecPtr targetPagePtr;
+
+ /* we enter this state only if we haven't read the whole record. */
+ Assert (state->recordRemainLen > 0);
+
+ while(state->recordRemainLen > 0)
+ {
+ char *contdata;
+ uint32 request_len;
+ uint32 record_len;
+
+ /* Wait for the next page to become available */
+ targetPagePtr = state->recordContRecPtr;
+
+ /* this request contains page header */
+ Assert (targetPagePtr != 0);
+ if (XLogNeedData(state, targetPagePtr,
+ Min(state->recordRemainLen, XLOG_BLCKSZ),
+ false))
+ return XLREAD_NEED_DATA;
+
+ if (!state->page_verified)
+ goto err;
+
+ Assert(SizeOfXLogShortPHD <= state->readLen);
- if (!state->page_verified)
- goto err;
+ /* Check that the continuation on next page looks valid */
+ pageHeader = (XLogPageHeader) state->readBuf;
+ if (!(pageHeader->xlp_info & XLP_FIRST_IS_CONTRECORD))
+ {
+ report_invalid_record(
+ state,
+ "there is no contrecord flag at %X/%X reading %X/%X",
+ (uint32) (state->recordContRecPtr >> 32),
+ (uint32) state->recordContRecPtr,
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr);
+ goto err;
+ }
- /* Record does not cross a page boundary */
- if (!ValidXLogRecord(state, record, RecPtr))
- goto err;
+ /*
+ * Cross-check that xlp_rem_len agrees with how much of the
+ * record we expect there to be left.
+ */
+ if (pageHeader->xlp_rem_len == 0 ||
+ pageHeader->xlp_rem_len != state->recordRemainLen)
+ {
+ report_invalid_record(
+ state,
+ "invalid contrecord length %u at %X/%X reading %X/%X, expected %u",
+ pageHeader->xlp_rem_len,
+ (uint32) (state->recordContRecPtr >> 32),
+ (uint32) state->recordContRecPtr,
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr,
+ state->recordRemainLen);
+ goto err;
+ }
- state->EndRecPtr = RecPtr + MAXALIGN(total_len);
+ /* Append the continuation from this page to the buffer */
+ pageHeaderSize = XLogPageHeaderSize(pageHeader);
- state->ReadRecPtr = RecPtr;
+ /*
+ * XLogNeedData should have ensured that the whole page header
+ * was read
+ */
+ Assert(state->readLen >= pageHeaderSize);
+
+ contdata = (char *) state->readBuf + pageHeaderSize;
+ record_len = XLOG_BLCKSZ - pageHeaderSize;
+ if (pageHeader->xlp_rem_len < record_len)
+ record_len = pageHeader->xlp_rem_len;
+
+ request_len = record_len + pageHeaderSize;
+
+ /* XLogNeedData should have ensured all needed data was read */
+ Assert (state->readLen >= request_len);
+
+ memcpy(state->readRecordBuf + state->recordGotLen,
+ (char *) contdata, record_len);
+ state->recordGotLen += record_len;
+ state->recordRemainLen -= record_len;
+
+ /* If we just reassembled the record header, validate it. */
+ if (!state->record_verified)
+ {
+ Assert(state->recordGotLen >= SizeOfXLogRecord);
+ if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
+ state->PrevRecPtr,
+ (XLogRecord *) state->readRecordBuf))
+ goto err;
+
+ state->record_verified = true;
+ }
+
+ /* Calculate pointer to beginning of next page, and continue */
+ state->recordContRecPtr += XLOG_BLCKSZ;
+ }
+
+ /* targetPagePtr is pointing the last-read page here */
+ prec = (XLogRecord *) state->readRecordBuf;
+ if (!ValidXLogRecord(state, prec, state->ReadRecPtr))
+ goto err;
+
+ pageHeaderSize =
+ XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+ state->EndRecPtr = targetPagePtr + pageHeaderSize
+ + MAXALIGN(pageHeader->xlp_rem_len);
+
+ *record = prec;
+ state->readRecordState = XLREAD_NEXT_RECORD;
+ break;
+ }
}
/*
* Special processing if it's an XLOG SWITCH record
*/
- if (record->xl_rmid == RM_XLOG_ID &&
- (record->xl_info & ~XLR_INFO_MASK) == XLOG_SWITCH)
+ if ((*record)->xl_rmid == RM_XLOG_ID &&
+ ((*record)->xl_info & ~XLR_INFO_MASK) == XLOG_SWITCH)
{
/* Pretend it extends to end of segment */
state->EndRecPtr += state->segcxt.ws_segsize - 1;
state->EndRecPtr -= XLogSegmentOffset(state->EndRecPtr, state->segcxt.ws_segsize);
}
- if (DecodeXLogRecord(state, record, errormsg))
- return record;
- else
- return NULL;
+ Assert (!*record || state->readLen >= 0);
+ if (DecodeXLogRecord(state, *record, errormsg))
+ return XLREAD_SUCCESS;
+
+ *record = NULL;
+ return XLREAD_FAIL;
err:
/*
- * Invalidate the read state. We might read from a different source after
+ * Invalidate the read page. We might read from a different source after
* failure.
*/
XLogReaderInvalReadState(state);
@@ -572,7 +706,8 @@ err:
if (state->errormsg_buf[0] != '\0')
*errormsg = state->errormsg_buf;
- return NULL;
+ *record = NULL;
+ return XLREAD_FAIL;
}
/*
@@ -724,11 +859,12 @@ XLogReaderInvalReadState(XLogReaderState *state)
*
* This is just a convenience subroutine to avoid duplicated code in
* XLogReadRecord. It's not intended for use from anywhere else.
+ *
+ * If PrevRecPtr is valid, the xl_prev is is cross-checked with it.
*/
static bool
ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
- XLogRecPtr PrevRecPtr, XLogRecord *record,
- bool randAccess)
+ XLogRecPtr PrevRecPtr, XLogRecord *record)
{
if (record->xl_tot_len < SizeOfXLogRecord)
{
@@ -746,7 +882,7 @@ ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
(uint32) RecPtr);
return false;
}
- if (randAccess)
+ if (PrevRecPtr == InvalidXLogRecPtr)
{
/*
* We can't exactly verify the prev-link, but surely it should be less
@@ -978,11 +1114,14 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
* XLogReadRecord() will read the next valid record.
*/
XLogRecPtr
-XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
+XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
+ XLogFindNextRecordCB read_page, void *private)
{
XLogRecPtr tmpRecPtr;
XLogRecPtr found = InvalidXLogRecPtr;
XLogPageHeader header;
+ XLogRecord *record;
+ XLogReadRecordResult result;
char *errormsg;
Assert(!XLogRecPtrIsInvalid(RecPtr));
@@ -1015,9 +1154,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
while(XLogNeedData(state, targetPagePtr, targetRecOff,
targetRecOff != 0))
{
- if (!state->routine.page_read(state, state->readPagePtr,
- state->readLen,
- state->ReadRecPtr, state->readBuf))
+ if (!read_page(state, private))
break;
}
@@ -1069,8 +1206,16 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
* or we just jumped over the remaining data of a continuation.
*/
XLogBeginRead(state, tmpRecPtr);
- while (XLogReadRecord(state, &errormsg) != NULL)
+ while ((result = XLogReadRecord(state, &record, &errormsg)) !=
+ XLREAD_FAIL)
{
+ if (result == XLREAD_NEED_DATA)
+ {
+ if (!read_page(state, private))
+ goto err;
+ continue;
+ }
+
/* past the record we've found, break out */
if (RecPtr <= state->ReadRecPtr)
{
@@ -1090,9 +1235,9 @@ err:
#endif /* FRONTEND */
/*
- * Helper function to ease writing of XLogRoutine->page_read callbacks.
- * If this function is used, caller must supply a segment_open callback in
- * 'state', as that is used here.
+ * Helper function to ease writing of page_read callback.
+ * If this function is used, caller must supply a segment_open callback and
+ * segment_close callback as that is used here.
*
* Read 'count' bytes into 'buf', starting at location 'startptr', from WAL
* fetched from timeline 'tli'.
@@ -1105,6 +1250,7 @@ err:
*/
bool
WALRead(XLogReaderState *state,
+ WALSegmentOpenCB segopenfn, WALSegmentCloseCB segclosefn,
char *buf, XLogRecPtr startptr, Size count, TimeLineID tli,
WALReadError *errinfo)
{
@@ -1136,10 +1282,10 @@ WALRead(XLogReaderState *state,
XLogSegNo nextSegNo;
if (state->seg.ws_file >= 0)
- state->routine.segment_close(state);
+ segclosefn(state);
XLByteToSeg(recptr, nextSegNo, state->segcxt.ws_segsize);
- state->routine.segment_open(state, nextSegNo, &tli);
+ segopenfn(state, nextSegNo, &tli);
/* This shouldn't happen -- indicates a bug in segment_open */
Assert(state->seg.ws_file >= 0);
diff --git a/src/backend/access/transam/xlogutils.c b/src/backend/access/transam/xlogutils.c
index 1b10b6df20..303333571f 100644
--- a/src/backend/access/transam/xlogutils.c
+++ b/src/backend/access/transam/xlogutils.c
@@ -689,8 +689,7 @@ XLogTruncateRelation(RelFileNode rnode, ForkNumber forkNum,
void
XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wantLength)
{
- const XLogRecPtr lastReadPage = state->seg.ws_segno *
- state->segcxt.ws_segsize + state->readLen;
+ const XLogRecPtr lastReadPage = state->readPagePtr;
Assert(wantPage != InvalidXLogRecPtr && wantPage % XLOG_BLCKSZ == 0);
Assert(wantLength <= XLOG_BLCKSZ);
@@ -705,7 +704,7 @@ XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wa
* current TLI has since become historical.
*/
if (lastReadPage == wantPage &&
- state->readLen != 0 &&
+ state->page_verified &&
lastReadPage + state->readLen >= wantPage + Min(wantLength, XLOG_BLCKSZ - 1))
return;
@@ -792,6 +791,7 @@ wal_segment_open(XLogReaderState *state, XLogSegNo nextSegNo,
char path[MAXPGPATH];
XLogFilePath(path, tli, nextSegNo, state->segcxt.ws_segsize);
+ elog(LOG, "HOGE: %lu, %d => %s", nextSegNo, tli, path);
state->seg.ws_file = BasicOpenFile(path, O_RDONLY | PG_BINARY);
if (state->seg.ws_file >= 0)
return;
@@ -829,9 +829,11 @@ wal_segment_close(XLogReaderState *state)
* loop for now.
*/
bool
-read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *cur_page)
+read_local_xlog_page(XLogReaderState *state)
{
+ XLogRecPtr targetPagePtr = state->readPagePtr;
+ int reqLen = state->readLen;
+ char *cur_page = state->readBuf;
XLogRecPtr read_upto,
loc;
TimeLineID tli;
@@ -944,11 +946,12 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
* as 'count', read the whole page anyway. It's guaranteed to be
* zero-padded up to the page boundary if it's incomplete.
*/
- if (!WALRead(state, cur_page, targetPagePtr, XLOG_BLCKSZ, tli,
- &errinfo))
+ if (!WALRead(state, wal_segment_open, wal_segment_close,
+ cur_page, targetPagePtr, XLOG_BLCKSZ, tli, &errinfo))
WALReadRaiseError(&errinfo);
/* number of valid bytes in the buffer */
+ state->readPagePtr = targetPagePtr;
state->readLen = count;
return true;
}
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index 61902be3b0..a15b0b3355 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -120,7 +120,8 @@ StartupDecodingContext(List *output_plugin_options,
TransactionId xmin_horizon,
bool need_full_snapshot,
bool fast_forward,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress)
@@ -169,11 +170,12 @@ StartupDecodingContext(List *output_plugin_options,
ctx->slot = slot;
- ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, xl_routine, ctx);
+ ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, cleanup_cb);
if (!ctx->reader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
errmsg("out of memory")));
+ ctx->page_read = page_read;
ctx->reorder = ReorderBufferAllocate();
ctx->snapshot_builder =
@@ -231,7 +233,8 @@ CreateInitDecodingContext(char *plugin,
List *output_plugin_options,
bool need_full_snapshot,
XLogRecPtr restart_lsn,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress)
@@ -328,7 +331,7 @@ CreateInitDecodingContext(char *plugin,
ctx = StartupDecodingContext(NIL, restart_lsn, xmin_horizon,
need_full_snapshot, false,
- xl_routine, prepare_write, do_write,
+ page_read, cleanup_cb, prepare_write, do_write,
update_progress);
/* call output plugin initialization callback */
@@ -376,7 +379,8 @@ LogicalDecodingContext *
CreateDecodingContext(XLogRecPtr start_lsn,
List *output_plugin_options,
bool fast_forward,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress)
@@ -429,8 +433,8 @@ CreateDecodingContext(XLogRecPtr start_lsn,
ctx = StartupDecodingContext(output_plugin_options,
start_lsn, InvalidTransactionId, false,
- fast_forward, xl_routine, prepare_write,
- do_write, update_progress);
+ fast_forward, page_read, cleanup_cb,
+ prepare_write, do_write, update_progress);
/* call output plugin initialization callback */
old_context = MemoryContextSwitchTo(ctx->context);
@@ -483,7 +487,13 @@ DecodingContextFindStartpoint(LogicalDecodingContext *ctx)
char *err = NULL;
/* the read_page callback waits for new WAL */
- record = XLogReadRecord(ctx->reader, &err);
+ while (XLogReadRecord(ctx->reader, &record, &err) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!ctx->page_read(ctx->reader))
+ break;
+ }
+
if (err)
elog(ERROR, "%s", err);
if (!record)
diff --git a/src/backend/replication/logical/logicalfuncs.c b/src/backend/replication/logical/logicalfuncs.c
index b99c94e848..fcc81b358e 100644
--- a/src/backend/replication/logical/logicalfuncs.c
+++ b/src/backend/replication/logical/logicalfuncs.c
@@ -233,9 +233,8 @@ pg_logical_slot_get_changes_guts(FunctionCallInfo fcinfo, bool confirm, bool bin
ctx = CreateDecodingContext(InvalidXLogRecPtr,
options,
false,
- XL_ROUTINE(.page_read = read_local_xlog_page,
- .segment_open = wal_segment_open,
- .segment_close = wal_segment_close),
+ read_local_xlog_page,
+ wal_segment_close,
LogicalOutputPrepareWrite,
LogicalOutputWrite, NULL);
@@ -284,7 +283,13 @@ pg_logical_slot_get_changes_guts(FunctionCallInfo fcinfo, bool confirm, bool bin
XLogRecord *record;
char *errm = NULL;
- record = XLogReadRecord(ctx->reader, &errm);
+ while (XLogReadRecord(ctx->reader, &record, &errm) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!ctx->page_read(ctx->reader))
+ break;
+ }
+
if (errm)
elog(ERROR, "%s", errm);
diff --git a/src/backend/replication/slotfuncs.c b/src/backend/replication/slotfuncs.c
index 88033a79b2..8a678074d5 100644
--- a/src/backend/replication/slotfuncs.c
+++ b/src/backend/replication/slotfuncs.c
@@ -152,9 +152,8 @@ create_logical_replication_slot(char *name, char *plugin,
ctx = CreateInitDecodingContext(plugin, NIL,
false, /* just catalogs is OK */
restart_lsn,
- XL_ROUTINE(.page_read = read_local_xlog_page,
- .segment_open = wal_segment_open,
- .segment_close = wal_segment_close),
+ read_local_xlog_page,
+ wal_segment_close,
NULL, NULL, NULL);
/*
@@ -486,9 +485,8 @@ pg_logical_replication_slot_advance(XLogRecPtr moveto)
ctx = CreateDecodingContext(InvalidXLogRecPtr,
NIL,
true, /* fast_forward */
- XL_ROUTINE(.page_read = read_local_xlog_page,
- .segment_open = wal_segment_open,
- .segment_close = wal_segment_close),
+ read_local_xlog_page,
+ wal_segment_close,
NULL, NULL, NULL);
/*
@@ -513,7 +511,13 @@ pg_logical_replication_slot_advance(XLogRecPtr moveto)
* Read records. No changes are generated in fast_forward mode,
* but snapbuilder/slot statuses are updated properly.
*/
- record = XLogReadRecord(ctx->reader, &errm);
+ while (XLogReadRecord(ctx->reader, &record, &errm) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!ctx->page_read(ctx->reader))
+ break;
+ }
+
if (errm)
elog(ERROR, "%s", errm);
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index bf2711bddd..2fd24a9b14 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -580,10 +580,7 @@ StartReplication(StartReplicationCmd *cmd)
/* create xlogreader for physical replication */
xlogreader =
- XLogReaderAllocate(wal_segment_size, NULL,
- XL_ROUTINE(.segment_open = WalSndSegmentOpen,
- .segment_close = wal_segment_close),
- NULL);
+ XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
if (!xlogreader)
ereport(ERROR,
@@ -812,9 +809,11 @@ StartReplication(StartReplicationCmd *cmd)
* set every time WAL is flushed.
*/
static bool
-logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetRecPtr, char *cur_page)
+logical_read_xlog_page(XLogReaderState *state)
{
+ XLogRecPtr targetPagePtr = state->readPagePtr;
+ int reqLen = state->readLen;
+ char *cur_page = state->readBuf;
XLogRecPtr flushptr;
int count;
WALReadError errinfo;
@@ -842,7 +841,7 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
count = flushptr - targetPagePtr; /* part of the page available */
/* now actually read the data, we know it's there */
- if (!WALRead(state,
+ if (!WALRead(state, WalSndSegmentOpen, wal_segment_close,
cur_page,
targetPagePtr,
XLOG_BLCKSZ,
@@ -1014,9 +1013,8 @@ CreateReplicationSlot(CreateReplicationSlotCmd *cmd)
ctx = CreateInitDecodingContext(cmd->plugin, NIL, need_full_snapshot,
InvalidXLogRecPtr,
- XL_ROUTINE(.page_read = logical_read_xlog_page,
- .segment_open = WalSndSegmentOpen,
- .segment_close = wal_segment_close),
+ logical_read_xlog_page,
+ wal_segment_close,
WalSndPrepareWrite, WalSndWriteData,
WalSndUpdateProgress);
@@ -1179,9 +1177,8 @@ StartLogicalReplication(StartReplicationCmd *cmd)
*/
logical_decoding_ctx =
CreateDecodingContext(cmd->startpoint, cmd->options, false,
- XL_ROUTINE(.page_read = logical_read_xlog_page,
- .segment_open = WalSndSegmentOpen,
- .segment_close = wal_segment_close),
+ logical_read_xlog_page,
+ wal_segment_close,
WalSndPrepareWrite, WalSndWriteData,
WalSndUpdateProgress);
xlogreader = logical_decoding_ctx->reader;
@@ -2761,7 +2758,7 @@ XLogSendPhysical(void)
enlargeStringInfo(&output_message, nbytes);
retry:
- if (!WALRead(xlogreader,
+ if (!WALRead(xlogreader, WalSndSegmentOpen, wal_segment_close,
&output_message.data[output_message.len],
startptr,
nbytes,
@@ -2859,7 +2856,12 @@ XLogSendLogical(void)
*/
WalSndCaughtUp = false;
- record = XLogReadRecord(logical_decoding_ctx->reader, &errm);
+ while (XLogReadRecord(logical_decoding_ctx->reader, &record, &errm) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!logical_decoding_ctx->page_read(logical_decoding_ctx->reader))
+ break;
+ }
/* xlog record was invalid */
if (errm != NULL)
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index 0b7e73ae79..3dd6df4be6 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -40,15 +40,9 @@ static int xlogreadfd = -1;
static XLogSegNo xlogreadsegno = -1;
static char xlogfpath[MAXPGPATH];
-typedef struct XLogPageReadPrivate
-{
- const char *restoreCommand;
- int tliIndex;
-} XLogPageReadPrivate;
-
-static bool SimpleXLogPageRead(XLogReaderState *xlogreader,
- XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
+static bool SimpleXLogPageRead(XLogReaderState *xlogreader,
+ const char *datadir, int *tliIndex,
+ const char *restoreCommand);
/*
* Read WAL from the datadir/pg_wal, starting from 'startpoint' on timeline
@@ -62,20 +56,22 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
XLogRecord *record;
XLogReaderState *xlogreader;
char *errormsg;
- XLogPageReadPrivate private;
- private.tliIndex = tliIndex;
- private.restoreCommand = restoreCommand;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir,
- XL_ROUTINE(.page_read = &SimpleXLogPageRead),
- &private);
+ xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+
if (xlogreader == NULL)
pg_fatal("out of memory");
XLogBeginRead(xlogreader, startpoint);
do
{
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!SimpleXLogPageRead(xlogreader, datadir,
+ &tliIndex, restoreCommand))
+ break;
+ }
if (record == NULL)
{
@@ -113,19 +109,19 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex,
XLogRecord *record;
XLogReaderState *xlogreader;
char *errormsg;
- XLogPageReadPrivate private;
XLogRecPtr endptr;
- private.tliIndex = tliIndex;
- private.restoreCommand = restoreCommand;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir,
- XL_ROUTINE(.page_read = &SimpleXLogPageRead),
- &private);
+ xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
XLogBeginRead(xlogreader, ptr);
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!SimpleXLogPageRead(xlogreader, datadir, &tliIndex, restoreCommand))
+ break;
+ }
if (record == NULL)
{
if (errormsg)
@@ -160,7 +156,6 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
XLogRecPtr searchptr;
XLogReaderState *xlogreader;
char *errormsg;
- XLogPageReadPrivate private;
/*
* The given fork pointer points to the end of the last common record,
@@ -176,11 +171,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
forkptr += SizeOfXLogShortPHD;
}
- private.tliIndex = tliIndex;
- private.restoreCommand = restoreCommand;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir,
- XL_ROUTINE(.page_read = &SimpleXLogPageRead),
- &private);
+ xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
@@ -190,7 +181,13 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
uint8 info;
XLogBeginRead(xlogreader, searchptr);
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!SimpleXLogPageRead(xlogreader, datadir,
+ &tliIndex, restoreCommand))
+ break;
+ }
if (record == NULL)
{
@@ -237,10 +234,11 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
/* XLogReader callback function, to read a WAL page */
static bool
-SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *readBuf)
+SimpleXLogPageRead(XLogReaderState *xlogreader, const char *datadir,
+ int *tliIndex, const char *restoreCommand)
{
- XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
+ XLogRecPtr targetPagePtr = xlogreader->readPagePtr;
+ char *readBuf = xlogreader->readBuf;
uint32 targetPageOff;
XLogRecPtr targetSegEnd;
XLogSegNo targetSegNo;
@@ -273,14 +271,14 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
* be done both forward and backward, consider also switching timeline
* accordingly.
*/
- while (private->tliIndex < targetNentries - 1 &&
- targetHistory[private->tliIndex].end < targetSegEnd)
- private->tliIndex++;
- while (private->tliIndex > 0 &&
- targetHistory[private->tliIndex].begin >= targetSegEnd)
- private->tliIndex--;
+ while (*tliIndex < targetNentries - 1 &&
+ targetHistory[*tliIndex].end < targetSegEnd)
+ (*tliIndex)++;
+ while (*tliIndex > 0 &&
+ targetHistory[*tliIndex].begin >= targetSegEnd)
+ (*tliIndex)--;
- XLogFileName(xlogfname, targetHistory[private->tliIndex].tli,
+ XLogFileName(xlogfname, targetHistory[*tliIndex].tli,
xlogreadsegno, WalSegSz);
snprintf(xlogfpath, MAXPGPATH, "%s/" XLOGDIR "/%s",
@@ -293,7 +291,7 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
/*
* If we have no restore_command to execute, then exit.
*/
- if (private->restoreCommand == NULL)
+ if (restoreCommand == NULL)
{
pg_log_error("could not open file \"%s\": %m", xlogfpath);
xlogreader->readLen = -1;
@@ -307,7 +305,7 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
xlogreadfd = RestoreArchivedFile(xlogreader->segcxt.ws_dir,
xlogfname,
WalSegSz,
- private->restoreCommand);
+ restoreCommand);
if (xlogreadfd < 0)
{
@@ -349,7 +347,7 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
Assert(targetSegNo == xlogreadsegno);
- xlogreader->seg.ws_tli = targetHistory[private->tliIndex].tli;
+ xlogreader->seg.ws_tli = targetHistory[*tliIndex].tli;
xlogreader->readLen = XLOG_BLCKSZ;
return true;
}
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index ab0c28d9b7..02e71138cc 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -330,12 +330,17 @@ WALDumpCloseSegment(XLogReaderState *state)
state->seg.ws_file = -1;
}
-/* pg_waldump's XLogReaderRoutine->page_read callback */
+/*
+ * pg_waldump's WAL page rader, also used as page_read callback for
+ * XLogFindNextRecord
+ */
static bool
-WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetPtr, char *readBuff)
+WALDumpReadPage(XLogReaderState *state, void *priv)
{
- XLogDumpPrivate *private = state->private_data;
+ XLogRecPtr targetPagePtr = state->readPagePtr;
+ int reqLen = state->readLen;
+ char *readBuff = state->readBuf;
+ XLogDumpPrivate *private = (XLogDumpPrivate *) priv;
int count = XLOG_BLCKSZ;
WALReadError errinfo;
@@ -353,8 +358,8 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
}
}
- if (!WALRead(state, readBuff, targetPagePtr, count, private->timeline,
- &errinfo))
+ if (!WALRead(state, WALDumpOpenSegment, WALDumpCloseSegment,
+ readBuff, targetPagePtr, count, private->timeline, &errinfo))
{
WALOpenSegment *seg = &errinfo.wre_seg;
char fname[MAXPGPATH];
@@ -374,6 +379,7 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
(Size) errinfo.wre_req);
}
+ Assert(count >= state->readLen);
state->readLen = count;
return true;
}
@@ -1042,16 +1048,14 @@ main(int argc, char **argv)
/* we have everything we need, start reading */
xlogreader_state =
- XLogReaderAllocate(WalSegSz, waldir,
- XL_ROUTINE(.page_read = WALDumpReadPage,
- .segment_open = WALDumpOpenSegment,
- .segment_close = WALDumpCloseSegment),
- &private);
+ XLogReaderAllocate(WalSegSz, waldir, WALDumpCloseSegment);
+
if (!xlogreader_state)
fatal_error("out of memory");
/* first find a valid recptr to start from */
- first_record = XLogFindNextRecord(xlogreader_state, private.startptr);
+ first_record = XLogFindNextRecord(xlogreader_state, private.startptr,
+ &WALDumpReadPage, (void*) &private);
if (first_record == InvalidXLogRecPtr)
fatal_error("could not find a valid record after %X/%X",
@@ -1075,7 +1079,13 @@ main(int argc, char **argv)
for (;;)
{
/* try to read the next record */
- record = XLogReadRecord(xlogreader_state, &errormsg);
+ while (XLogReadRecord(xlogreader_state, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!WALDumpReadPage(xlogreader_state, (void *) &private))
+ break;
+ }
+
if (!record)
{
if (!config.follow || private.endptr_reached)
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index 6c8848e14f..6a7fe140cb 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -57,64 +57,15 @@ typedef struct WALSegmentContext
typedef struct XLogReaderState XLogReaderState;
-/* Function type definition for the read_page callback */
-typedef bool (*XLogPageReadCB) (XLogReaderState *xlogreader,
- XLogRecPtr targetPagePtr,
- int reqLen,
- XLogRecPtr targetRecPtr,
- char *readBuf);
+/* Function type definition for the segment cleanup callback */
+typedef void (*WALSegmentCleanupCB) (XLogReaderState *xlogreader);
+
+/* Function type definition for the open/close callbacks for WALRead() */
typedef void (*WALSegmentOpenCB) (XLogReaderState *xlogreader,
XLogSegNo nextSegNo,
TimeLineID *tli_p);
typedef void (*WALSegmentCloseCB) (XLogReaderState *xlogreader);
-typedef struct XLogReaderRoutine
-{
- /*
- * Data input callback
- *
- * This callback shall read at least reqLen valid bytes of the xlog page
- * starting at targetPagePtr, and store them in readBuf. The callback
- * shall return the number of bytes read (never more than XLOG_BLCKSZ), or
- * -1 on failure. The callback shall sleep, if necessary, to wait for the
- * requested bytes to become available. The callback will not be invoked
- * again for the same page unless more than the returned number of bytes
- * are needed.
- *
- * targetRecPtr is the position of the WAL record we're reading. Usually
- * it is equal to targetPagePtr + reqLen, but sometimes xlogreader needs
- * to read and verify the page or segment header, before it reads the
- * actual WAL record it's interested in. In that case, targetRecPtr can
- * be used to determine which timeline to read the page from.
- *
- * The callback shall set ->seg.ws_tli to the TLI of the file the page was
- * read from.
- */
- XLogPageReadCB page_read;
-
- /*
- * Callback to open the specified WAL segment for reading. ->seg.ws_file
- * shall be set to the file descriptor of the opened segment. In case of
- * failure, an error shall be raised by the callback and it shall not
- * return.
- *
- * "nextSegNo" is the number of the segment to be opened.
- *
- * "tli_p" is an input/output argument. WALRead() uses it to pass the
- * timeline in which the new segment should be found, but the callback can
- * use it to return the TLI that it actually opened.
- */
- WALSegmentOpenCB segment_open;
-
- /*
- * WAL segment close callback. ->seg.ws_file shall be set to a negative
- * number.
- */
- WALSegmentCloseCB segment_close;
-} XLogReaderRoutine;
-
-#define XL_ROUTINE(...) &(XLogReaderRoutine){__VA_ARGS__}
-
typedef struct
{
/* Is this block ref in use? */
@@ -144,12 +95,35 @@ typedef struct
uint16 data_bufsz;
} DecodedBkpBlock;
+/* Return code from XLogReadRecord */
+typedef enum XLogReadRecordResult
+{
+ XLREAD_SUCCESS, /* record is successfully read */
+ XLREAD_NEED_DATA, /* need more data. see XLogReadRecord. */
+ XLREAD_FAIL /* failed during reading a record */
+} XLogReadRecordResult;
+
+/*
+ * internal state of XLogReadRecord
+ *
+ * XLogReadState runs a state machine while reading a record. Theses states
+ * are not seen outside the function. Each state may repeat several times
+ * exiting requesting caller for new data. See the comment of XLogReadRecrod
+ * for details.
+ */
+typedef enum XLogReadRecordState {
+ XLREAD_NEXT_RECORD,
+ XLREAD_TOT_LEN,
+ XLREAD_FIRST_FRAGMENT,
+ XLREAD_CONTINUATION
+} XLogReadRecordState;
+
struct XLogReaderState
{
/*
* Operational callbacks
*/
- XLogReaderRoutine routine;
+ WALSegmentCleanupCB cleanup_cb;
/* ----------------------------------------
* Public parameters
@@ -162,18 +136,14 @@ struct XLogReaderState
*/
uint64 system_identifier;
- /*
- * Opaque data for callbacks to use. Not used by XLogReader.
- */
- void *private_data;
-
/*
* Start and end point of last record read. EndRecPtr is also used as the
* position to read next. Calling XLogBeginRead() sets EndRecPtr to the
* starting position and ReadRecPtr to invalid.
*/
- XLogRecPtr ReadRecPtr; /* start of last record read */
+ XLogRecPtr ReadRecPtr; /* start of last record read or being read */
XLogRecPtr EndRecPtr; /* end+1 of last record read */
+ XLogRecPtr PrevRecPtr; /* start of previous record read */
/* ----------------------------------------
* Communication with page reader
@@ -187,7 +157,9 @@ struct XLogReaderState
* the request, or -1 on error */
TimeLineID readPageTLI; /* TLI for data currently in readBuf */
char *readBuf; /* buffer to store data */
- bool page_verified; /* is the page on the buffer verified? */
+ bool page_verified; /* is the page header on the buffer verified? */
+ bool record_verified;/* is the current record header verified? */
+
/* ----------------------------------------
@@ -227,8 +199,6 @@ struct XLogReaderState
XLogRecPtr latestPagePtr;
TimeLineID latestPageTLI;
- /* beginning of the WAL record being read. */
- XLogRecPtr currRecPtr;
/* timeline to read it from, 0 if a lookup is required */
TimeLineID currTLI;
@@ -255,6 +225,15 @@ struct XLogReaderState
char *readRecordBuf;
uint32 readRecordBufSize;
+ /*
+ * XLogReadRecord() state
+ */
+ XLogReadRecordState readRecordState;/* state machine state */
+ int recordGotLen; /* amount of current record that has
+ * already been read */
+ int recordRemainLen; /* length of current record that remains */
+ XLogRecPtr recordContRecPtr; /* where the current record continues */
+
/* Buffer to hold error message */
char *errormsg_buf;
};
@@ -262,9 +241,7 @@ struct XLogReaderState
/* Get a new XLogReader */
extern XLogReaderState *XLogReaderAllocate(int wal_segment_size,
const char *waldir,
- XLogReaderRoutine *routine,
- void *private_data);
-extern XLogReaderRoutine *LocalXLogReaderRoutine(void);
+ WALSegmentCleanupCB cleanup_cb);
/* Free an XLogReader */
extern void XLogReaderFree(XLogReaderState *state);
@@ -272,12 +249,17 @@ extern void XLogReaderFree(XLogReaderState *state);
/* Position the XLogReader to given record */
extern void XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr);
#ifdef FRONTEND
-extern XLogRecPtr XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr);
+/* Function type definition for the read_page callback */
+typedef bool (*XLogFindNextRecordCB) (XLogReaderState *xlogreader,
+ void *private);
+extern XLogRecPtr XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
+ XLogFindNextRecordCB read_page, void *private);
#endif /* FRONTEND */
/* Read the next XLog record. Returns NULL on end-of-WAL or failure */
-extern struct XLogRecord *XLogReadRecord(XLogReaderState *state,
- char **errormsg);
+extern XLogReadRecordResult XLogReadRecord(XLogReaderState *state,
+ XLogRecord **record,
+ char **errormsg);
/* Validate a page */
extern bool XLogReaderValidatePageHeader(XLogReaderState *state,
@@ -297,6 +279,7 @@ typedef struct WALReadError
} WALReadError;
extern bool WALRead(XLogReaderState *state,
+ WALSegmentOpenCB segopenfn, WALSegmentCloseCB sgclosefn,
char *buf, XLogRecPtr startptr, Size count,
TimeLineID tli, WALReadError *errinfo);
diff --git a/src/include/access/xlogutils.h b/src/include/access/xlogutils.h
index c287edf206..482e4ced69 100644
--- a/src/include/access/xlogutils.h
+++ b/src/include/access/xlogutils.h
@@ -47,9 +47,7 @@ extern Buffer XLogReadBufferExtended(RelFileNode rnode, ForkNumber forknum,
extern Relation CreateFakeRelcacheEntry(RelFileNode rnode);
extern void FreeFakeRelcacheEntry(Relation fakerel);
-extern bool read_local_xlog_page(XLogReaderState *state,
- XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetRecPtr, char *cur_page);
+extern bool read_local_xlog_page(XLogReaderState *state);
extern void wal_segment_open(XLogReaderState *state,
XLogSegNo nextSegNo,
TimeLineID *tli_p);
diff --git a/src/include/replication/logical.h b/src/include/replication/logical.h
index c2f2475e5d..a743db49cf 100644
--- a/src/include/replication/logical.h
+++ b/src/include/replication/logical.h
@@ -29,6 +29,10 @@ typedef void (*LogicalOutputPluginWriterUpdateProgress) (struct LogicalDecodingC
TransactionId xid
);
+typedef struct LogicalDecodingContext LogicalDecodingContext;
+
+typedef bool (*LogicalDecodingXLogPageReadCB)(XLogReaderState *ctx);
+
typedef struct LogicalDecodingContext
{
/* memory context this is all allocated in */
@@ -39,6 +43,7 @@ typedef struct LogicalDecodingContext
/* infrastructure pieces for decoding */
XLogReaderState *reader;
+ LogicalDecodingXLogPageReadCB page_read;
struct ReorderBuffer *reorder;
struct SnapBuild *snapshot_builder;
@@ -95,14 +100,16 @@ extern LogicalDecodingContext *CreateInitDecodingContext(char *plugin,
List *output_plugin_options,
bool need_full_snapshot,
XLogRecPtr restart_lsn,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress);
extern LogicalDecodingContext *CreateDecodingContext(XLogRecPtr start_lsn,
List *output_plugin_options,
bool fast_forward,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress);
--
2.18.4
----Next_Part(Thu_Jul__2_13_53_30_2020_072)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
filename="v15-0003-Remove-globals-readOff-readLen-and-readSegNo.patch"
^ permalink raw reply [nested|flat] 70+ messages in thread
* [PATCH v18 2/5] Move page-reader out of XLogReadRecord().
@ 2021-09-30 03:04 Kyotaro Horiguchi <[email protected]>
0 siblings, 0 replies; 70+ messages in thread
From: Kyotaro Horiguchi @ 2021-09-30 03:04 UTC (permalink / raw)
This is the second step of removing callbacks from the WAL decoder.
XLogReadRecord() return XLREAD_NEED_DATA to indicate that the caller
should supply new data, and the decoder works as a state machine.
Author: Kyotaro HORIGUCHI <[email protected]>
Author: Heikki Linnakangas <[email protected]>
Reviewed-by: Antonin Houska <[email protected]>
Reviewed-by: Alvaro Herrera <[email protected]>
Reviewed-by: Takashi Menjo <[email protected]>
Reviewed-by: Thomas Munro <[email protected]>
Discussion: https://postgr.es/m/20190418.210257.43726183.horiguchi.kyotaro%40lab.ntt.co.jp
---
src/backend/access/transam/twophase.c | 16 +-
src/backend/access/transam/xlog.c | 60 +-
src/backend/access/transam/xlogreader.c | 760 ++++++++++--------
src/backend/access/transam/xlogutils.c | 17 +-
src/backend/replication/logical/logical.c | 29 +-
.../replication/logical/logicalfuncs.c | 16 +-
src/backend/replication/slotfuncs.c | 21 +-
src/backend/replication/walsender.c | 35 +-
src/bin/pg_rewind/parsexlog.c | 93 ++-
src/bin/pg_waldump/pg_waldump.c | 39 +-
src/include/access/xlogreader.h | 124 ++-
src/include/access/xlogutils.h | 4 +-
src/include/pg_config_manual.h | 2 +-
src/include/replication/logical.h | 11 +-
14 files changed, 703 insertions(+), 524 deletions(-)
diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index 2156de187c..9b42a935a7 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -1330,11 +1330,8 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
char *errormsg;
TimeLineID save_currtli = ThisTimeLineID;
- xlogreader = XLogReaderAllocate(wal_segment_size, NULL,
- XL_ROUTINE(.page_read = &read_local_xlog_page,
- .segment_open = &wal_segment_open,
- .segment_close = &wal_segment_close),
- NULL);
+ xlogreader = XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+
if (!xlogreader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
@@ -1342,7 +1339,14 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
errdetail("Failed while allocating a WAL reading processor.")));
XLogBeginRead(xlogreader, lsn);
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) == XLREAD_NEED_DATA)
+ {
+ if (!read_local_xlog_page(xlogreader))
+ {
+ XLogTerminateRead(xlogreader);
+ break;
+ }
+ }
/*
* Restore immediately the timeline where it was previously, as
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 1557ceb8c1..316ba256f6 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -836,13 +836,6 @@ static XLogSource currentSource = XLOG_FROM_ANY;
static bool lastSourceFailed = false;
static bool pendingWalRcvRestart = false;
-typedef struct XLogPageReadPrivate
-{
- int emode;
- bool fetching_ckpt; /* are we fetching a checkpoint record? */
- bool randAccess;
-} XLogPageReadPrivate;
-
/*
* These variables track when we last obtained some WAL data to process,
* and where we got it from. (XLogReceiptSource is initially the same as
@@ -918,8 +911,8 @@ static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath,
static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli,
XLogSource source, bool notfoundOk);
static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, XLogSource source);
-static bool XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
+static bool XLogPageRead(XLogReaderState *xlogreader,
+ bool fetching_ckpt, int emode, bool randAccess);
static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
bool fetching_ckpt, XLogRecPtr tliRecPtr);
static void XLogShutdownWalRcv(void);
@@ -1233,8 +1226,7 @@ XLogInsertRecord(XLogRecData *rdata,
appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len);
if (!debug_reader)
- debug_reader = XLogReaderAllocate(wal_segment_size, NULL,
- XL_ROUTINE(), NULL);
+ debug_reader = XLogReaderAllocate(wal_segment_size, NULL, NULL);
if (!debug_reader)
{
@@ -4395,15 +4387,10 @@ CleanupBackupHistory(void)
* record is available.
*/
static XLogRecord *
-ReadRecord(XLogReaderState *xlogreader, int emode,
- bool fetching_ckpt)
+ReadRecord(XLogReaderState *xlogreader, int emode, bool fetching_ckpt)
{
XLogRecord *record;
- XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
-
- private->fetching_ckpt = fetching_ckpt;
- private->emode = emode;
- private->randAccess = (xlogreader->ReadRecPtr == InvalidXLogRecPtr);
+ bool randAccess = (xlogreader->ReadRecPtr == InvalidXLogRecPtr);
/* This is the first attempt to read this page. */
lastSourceFailed = false;
@@ -4411,8 +4398,18 @@ ReadRecord(XLogReaderState *xlogreader, int emode,
for (;;)
{
char *errormsg;
+ XLogReadRecordResult result;
+
+ while ((result = XLogReadRecord(xlogreader, &record, &errormsg))
+ == XLREAD_NEED_DATA)
+ {
+ if (!XLogPageRead(xlogreader, fetching_ckpt, emode, randAccess))
+ {
+ XLogTerminateRead(xlogreader);
+ break;
+ }
+ }
- record = XLogReadRecord(xlogreader, &errormsg);
ReadRecPtr = xlogreader->ReadRecPtr;
EndRecPtr = xlogreader->EndRecPtr;
if (record == NULL)
@@ -6547,7 +6544,6 @@ StartupXLOG(void)
bool backupFromStandby = false;
DBState dbstate_at_startup;
XLogReaderState *xlogreader;
- XLogPageReadPrivate private;
bool promoted = false;
struct stat st;
@@ -6706,13 +6702,9 @@ StartupXLOG(void)
OwnLatch(&XLogCtl->recoveryWakeupLatch);
/* Set up XLOG reader facility */
- MemSet(&private, 0, sizeof(XLogPageReadPrivate));
xlogreader =
- XLogReaderAllocate(wal_segment_size, NULL,
- XL_ROUTINE(.page_read = &XLogPageRead,
- .segment_open = NULL,
- .segment_close = wal_segment_close),
- &private);
+ XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+
if (!xlogreader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
@@ -12300,12 +12292,13 @@ CancelBackup(void)
* sleep and retry.
*/
static bool
-XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetRecPtr, char *readBuf)
+XLogPageRead(XLogReaderState *xlogreader,
+ bool fetching_ckpt, int emode, bool randAccess)
{
- XLogPageReadPrivate *private =
- (XLogPageReadPrivate *) xlogreader->private_data;
- int emode = private->emode;
+ char *readBuf = xlogreader->readBuf;
+ XLogRecPtr targetPagePtr = xlogreader->readPagePtr;
+ int reqLen = xlogreader->readLen;
+ XLogRecPtr targetRecPtr = xlogreader->ReadRecPtr;
uint32 targetPageOff;
XLogSegNo targetSegNo PG_USED_FOR_ASSERTS_ONLY;
int r;
@@ -12348,8 +12341,8 @@ retry:
flushedUpto < targetPagePtr + reqLen))
{
if (!WaitForWALToBecomeAvailable(targetPagePtr + reqLen,
- private->randAccess,
- private->fetching_ckpt,
+ randAccess,
+ fetching_ckpt,
targetRecPtr))
{
if (readFile >= 0)
@@ -12469,6 +12462,7 @@ retry:
goto next_record_is_invalid;
}
+ Assert(xlogreader->readPagePtr == targetPagePtr);
xlogreader->readLen = readLen;
return true;
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index 1d9976ecf4..4501a09245 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -43,7 +43,7 @@ static bool XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr,
int reqLen, bool header_inclusive);
static void XLogReaderInvalReadState(XLogReaderState *state);
static bool ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
- XLogRecPtr PrevRecPtr, XLogRecord *record, bool randAccess);
+ XLogRecPtr PrevRecPtr, XLogRecord *record);
static bool ValidXLogRecord(XLogReaderState *state, XLogRecord *record,
XLogRecPtr recptr);
static void ResetDecoder(XLogReaderState *state);
@@ -76,7 +76,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...)
*/
XLogReaderState *
XLogReaderAllocate(int wal_segment_size, const char *waldir,
- XLogReaderRoutine *routine, void *private_data)
+ WALSegmentCleanupCB cleanup_cb)
{
XLogReaderState *state;
@@ -87,7 +87,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
return NULL;
/* initialize caller-provided support functions */
- state->routine = *routine;
+ state->cleanup_cb = cleanup_cb;
state->max_block_id = -1;
@@ -110,8 +110,6 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
WALOpenSegmentInit(&state->seg, &state->segcxt, wal_segment_size,
waldir);
- /* system_identifier initialized to zeroes above */
- state->private_data = private_data;
/* ReadRecPtr, EndRecPtr and readLen initialized to zeroes above */
state->errormsg_buf = palloc_extended(MAX_ERRORMSG_LEN + 1,
MCXT_ALLOC_NO_OOM);
@@ -143,8 +141,8 @@ XLogReaderFree(XLogReaderState *state)
{
int block_id;
- if (state->seg.ws_file != -1)
- state->routine.segment_close(state);
+ if (state->seg.ws_file >= 0)
+ state->cleanup_cb(state);
for (block_id = 0; block_id <= XLR_MAX_BLOCK_ID; block_id++)
{
@@ -249,6 +247,7 @@ XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
/* Begin at the passed-in record pointer. */
state->EndRecPtr = RecPtr;
state->ReadRecPtr = InvalidXLogRecPtr;
+ state->readRecordState = XLREAD_NEXT_RECORD;
}
/*
@@ -257,12 +256,12 @@ XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
* XLogBeginRead() or XLogFindNextRecord() must be called before the first call
* to XLogReadRecord().
*
- * If the page_read callback fails to read the requested data, NULL is
- * returned. The callback is expected to have reported the error; errormsg
- * is set to NULL.
+ * This function may return XLREAD_NEED_DATA several times before returning a
+ * result record. The caller shall read in some new data then call this
+ * function again with the same parameters.
*
- * If the reading fails for some other reason, NULL is also returned, and
- * *errormsg is set to a string with details of the failure.
+ * When a record is successfully read, returns XLREAD_SUCCESS with result
+ * record being stored in *record. Otherwise *record is NULL.
*
* Returns XLREAD_NEED_DATA if more data is needed to finish reading the
* current record. In that case, state->readPagePtr and state->readLen inform
@@ -307,329 +306,458 @@ XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
* state. This behavior allows to continue reading a reacord switching among
* different souces, while streaming replication.
*/
-XLogRecord *
-XLogReadRecord(XLogReaderState *state, char **errormsg)
+XLogReadRecordResult
+XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
{
- XLogRecPtr RecPtr;
- XLogRecord *record;
- XLogRecPtr targetPagePtr;
- bool randAccess;
- uint32 len,
- total_len;
- uint32 targetRecOff;
- uint32 pageHeaderSize;
- bool assembled;
- bool gotheader;
+ XLogRecord *prec;
- /*
- * randAccess indicates whether to verify the previous-record pointer of
- * the record we're reading. We only do this if we're reading
- * sequentially, which is what we initially assume.
- */
- randAccess = false;
+ *record = NULL;
/* reset error state */
*errormsg = NULL;
state->errormsg_buf[0] = '\0';
- ResetDecoder(state);
state->abortedRecPtr = InvalidXLogRecPtr;
state->missingContrecPtr = InvalidXLogRecPtr;
- RecPtr = state->EndRecPtr;
-
- if (state->ReadRecPtr != InvalidXLogRecPtr)
- {
- /* read the record after the one we just read */
-
- /*
- * EndRecPtr is pointing to end+1 of the previous WAL record. If
- * we're at a page boundary, no more records can fit on the current
- * page. We must skip over the page header, but we can't do that until
- * we've read in the page, since the header size is variable.
- */
- }
- else
- {
- /*
- * Caller supplied a position to start at.
- *
- * In this case, EndRecPtr should already be pointing to a valid
- * record starting position.
- */
- Assert(XRecOffIsValid(RecPtr));
- randAccess = true;
- }
-
restart:
- state->currRecPtr = RecPtr;
- assembled = false;
-
- targetPagePtr = RecPtr - (RecPtr % XLOG_BLCKSZ);
- targetRecOff = RecPtr % XLOG_BLCKSZ;
-
- /*
- * Read the page containing the record into state->readBuf. Request enough
- * byte to cover the whole record header, or at least the part of it that
- * fits on the same page.
- */
- while (XLogNeedData(state, targetPagePtr,
- Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
- targetRecOff != 0))
+ switch (state->readRecordState)
{
- if (!state->routine.page_read(state, state->readPagePtr, state->readLen,
- RecPtr, state->readBuf))
- break;
- }
-
- if (!state->page_verified)
- goto err;
-
- /*
- * We have at least the page header, so we can examine it now.
- */
- pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
- if (targetRecOff == 0)
- {
- /*
- * At page start, so skip over page header.
- */
- RecPtr += pageHeaderSize;
- targetRecOff = pageHeaderSize;
- }
- else if (targetRecOff < pageHeaderSize)
- {
- report_invalid_record(state, "invalid record offset at %X/%X",
- LSN_FORMAT_ARGS(RecPtr));
- goto err;
- }
+ case XLREAD_NEXT_RECORD:
+ ResetDecoder(state);
- if ((((XLogPageHeader) state->readBuf)->xlp_info & XLP_FIRST_IS_CONTRECORD) &&
- targetRecOff == pageHeaderSize)
- {
- report_invalid_record(state, "contrecord is requested by %X/%X",
- LSN_FORMAT_ARGS(RecPtr));
- goto err;
- }
-
- /* XLogNeedData has verified the page header */
- Assert(pageHeaderSize <= state->readLen);
-
- /*
- * Read the record length.
- *
- * NB: Even though we use an XLogRecord pointer here, the whole record
- * header might not fit on this page. xl_tot_len is the first field of the
- * struct, so it must be on this page (the records are MAXALIGNed), but we
- * cannot access any other fields until we've verified that we got the
- * whole header.
- */
- record = (XLogRecord *) (state->readBuf + RecPtr % XLOG_BLCKSZ);
- total_len = record->xl_tot_len;
-
- /*
- * If the whole record header is on this page, validate it immediately.
- * Otherwise do just a basic sanity check on xl_tot_len, and validate the
- * rest of the header after reading it from the next page. The xl_tot_len
- * check is necessary here to ensure that we enter the "Need to reassemble
- * record" code path below; otherwise we might fail to apply
- * ValidXLogRecordHeader at all.
- */
- if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
- {
- if (!ValidXLogRecordHeader(state, RecPtr, state->ReadRecPtr, record,
- randAccess))
- goto err;
- gotheader = true;
- }
- else
- {
- /* XXX: more validation should be done here */
- if (total_len < SizeOfXLogRecord)
- {
- report_invalid_record(state,
- "invalid record length at %X/%X: wanted %u, got %u",
- LSN_FORMAT_ARGS(RecPtr),
- (uint32) SizeOfXLogRecord, total_len);
- goto err;
- }
- gotheader = false;
- }
-
- len = XLOG_BLCKSZ - RecPtr % XLOG_BLCKSZ;
- if (total_len > len)
- {
- /* Need to reassemble record */
- char *contdata;
- XLogPageHeader pageHeader;
- char *buffer;
- uint32 gotlen;
-
- assembled = true;
-
- /*
- * Enlarge readRecordBuf as needed.
- */
- if (total_len > state->readRecordBufSize &&
- !allocate_recordbuf(state, total_len))
- {
- /* We treat this as a "bogus data" condition */
- report_invalid_record(state, "record length %u at %X/%X too long",
- total_len, LSN_FORMAT_ARGS(RecPtr));
- goto err;
- }
-
- /* Copy the first fragment of the record from the first page. */
- memcpy(state->readRecordBuf,
- state->readBuf + RecPtr % XLOG_BLCKSZ, len);
- buffer = state->readRecordBuf + len;
- gotlen = len;
-
- do
- {
- int rest_len = total_len - gotlen;
-
- /* Calculate pointer to beginning of next page */
- targetPagePtr += XLOG_BLCKSZ;
-
- /* Wait for the next page to become available */
- while (XLogNeedData(state, targetPagePtr,
- Min(rest_len, XLOG_BLCKSZ),
- false))
+ if (state->ReadRecPtr != InvalidXLogRecPtr)
+ {
+ /* read the record after the one we just read */
+
+ /*
+ * EndRecPtr is pointing to end+1 of the previous WAL record.
+ * If we're at a page boundary, no more records can fit on the
+ * current page. We must skip over the page header, but we
+ * can't do that until we've read in the page, since the
+ * header size is variable.
+ */
+ state->PrevRecPtr = state->ReadRecPtr;
+ state->ReadRecPtr = state->EndRecPtr;
+ }
+ else
{
- if (!state->routine.page_read(state, state->readPagePtr,
- state->readLen,
- state->ReadRecPtr,
- state->readBuf))
+ /*
+ * Caller supplied a position to start at.
+ *
+ * In this case, EndRecPtr should already be pointing to a
+ * valid record starting position.
+ */
+ Assert(XRecOffIsValid(state->EndRecPtr));
+ state->ReadRecPtr = state->EndRecPtr;
+
+ /*
+ * We cannot verify the previous-record pointer when we're
+ * seeking to a particular record. Reset PrevRecPtr so that we
+ * won't try doing that.
+ */
+ state->PrevRecPtr = InvalidXLogRecPtr;
+ state->EndRecPtr = InvalidXLogRecPtr; /* to be tidy */
+ }
+
+ state->record_verified = false;
+ state->readRecordState = XLREAD_TOT_LEN;
+ /* fall through */
+
+ case XLREAD_TOT_LEN:
+ {
+ uint32 total_len;
+ uint32 pageHeaderSize;
+ XLogRecPtr targetPagePtr;
+ uint32 targetRecOff;
+ XLogPageHeader pageHeader;
+
+ targetPagePtr =
+ state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
+ targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+
+ /*
+ * Check if we have enough data. For the first record in the
+ * page, the requesting length doesn't contain page header.
+ */
+ if (XLogNeedData(state, targetPagePtr,
+ Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+ targetRecOff != 0))
+ return XLREAD_NEED_DATA;
+
+ /* error out if caller supplied bogus page */
+ if (!state->page_verified)
+ goto err;
+
+ /* examine page header now. */
+ pageHeaderSize =
+ XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+ if (targetRecOff == 0)
+ {
+ /* At page start, so skip over page header. */
+ state->ReadRecPtr += pageHeaderSize;
+ targetRecOff = pageHeaderSize;
+ }
+ else if (targetRecOff < pageHeaderSize)
+ {
+ report_invalid_record(state, "invalid record offset at %X/%X",
+ LSN_FORMAT_ARGS(state->ReadRecPtr));
+ goto err;
+ }
+
+ pageHeader = (XLogPageHeader) state->readBuf;
+ if ((pageHeader->xlp_info & XLP_FIRST_IS_CONTRECORD) &&
+ targetRecOff == pageHeaderSize)
+ {
+ report_invalid_record(state, "contrecord is requested by %X/%X",
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr);
+ goto err;
+ }
+
+ /* XLogNeedData has verified the page header */
+ Assert(pageHeaderSize <= state->readLen);
+
+ /*
+ * Read the record length.
+ *
+ * NB: Even though we use an XLogRecord pointer here, the
+ * whole record header might not fit on this page. xl_tot_len
+ * is the first field of the struct, so it must be on this
+ * page (the records are MAXALIGNed), but we cannot access any
+ * other fields until we've verified that we got the whole
+ * header.
+ */
+ prec = (XLogRecord *) (state->readBuf +
+ state->ReadRecPtr % XLOG_BLCKSZ);
+ total_len = prec->xl_tot_len;
+
+ /*
+ * If the whole record header is on this page, validate it
+ * immediately. Otherwise do just a basic sanity check on
+ * xl_tot_len, and validate the rest of the header after
+ * reading it from the next page. The xl_tot_len check is
+ * necessary here to ensure that we enter the
+ * XLREAD_CONTINUATION state below; otherwise we might fail to
+ * apply ValidXLogRecordHeader at all.
+ */
+ if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
+ {
+ if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
+ state->PrevRecPtr, prec))
+ goto err;
+
+ state->record_verified = true;
+ }
+ else
+ {
+ /* XXX: more validation should be done here */
+ if (total_len < SizeOfXLogRecord)
+ {
+ report_invalid_record(state,
+ "invalid record length at %X/%X: wanted %u, got %u",
+ LSN_FORMAT_ARGS(state->ReadRecPtr),
+ (uint32) SizeOfXLogRecord, total_len);
+ goto err;
+ }
+ }
+
+ /*
+ * Wait for the rest of the record, or the part of the record
+ * that fit on the first page if crossed a page boundary, to
+ * become available.
+ */
+ state->recordGotLen = 0;
+ state->recordRemainLen = total_len;
+ state->readRecordState = XLREAD_FIRST_FRAGMENT;
+ }
+ /* fall through */
+
+ case XLREAD_FIRST_FRAGMENT:
+ {
+ uint32 total_len = state->recordRemainLen;
+ uint32 request_len;
+ uint32 record_len;
+ XLogRecPtr targetPagePtr;
+ uint32 targetRecOff;
+
+ /*
+ * Wait for the rest of the record on the first page to become
+ * available
+ */
+ targetPagePtr =
+ state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
+ targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+
+ request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ);
+ record_len = request_len - targetRecOff;
+
+ /* ReadRecPtr contains page header */
+ Assert(targetRecOff != 0);
+ if (XLogNeedData(state, targetPagePtr, request_len, true))
+ return XLREAD_NEED_DATA;
+
+ /* error out if caller supplied bogus page */
+ if (!state->page_verified)
+ goto err;
+
+ prec = (XLogRecord *) (state->readBuf + targetRecOff);
+
+ /* validate record header if not yet */
+ if (!state->record_verified && record_len >= SizeOfXLogRecord)
+ {
+ if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
+ state->PrevRecPtr, prec))
+ goto err;
+
+ state->record_verified = true;
+ }
+
+
+ if (total_len == record_len)
+ {
+ /* Record does not cross a page boundary */
+ Assert(state->record_verified);
+
+ if (!ValidXLogRecord(state, prec, state->ReadRecPtr))
+ goto err;
+
+ state->record_verified = true; /* to be tidy */
+
+ /* We already checked the header earlier */
+ state->EndRecPtr = state->ReadRecPtr + MAXALIGN(record_len);
+
+ *record = prec;
+ state->readRecordState = XLREAD_NEXT_RECORD;
break;
- }
+ }
- if (!state->page_verified)
- goto err;
+ /*
+ * The record continues on the next page. Need to reassemble
+ * record
+ */
+ Assert(total_len > record_len);
- Assert(SizeOfXLogShortPHD <= state->readLen);
+ /* Enlarge readRecordBuf as needed. */
+ if (total_len > state->readRecordBufSize &&
+ !allocate_recordbuf(state, total_len))
+ {
+ /* We treat this as a "bogus data" condition */
+ report_invalid_record(state,
+ "record length %u at %X/%X too long",
+ total_len,
+ LSN_FORMAT_ARGS(state->ReadRecPtr));
+ goto err;
+ }
- pageHeader = (XLogPageHeader) state->readBuf;
+ /* Copy the first fragment of the record from the first page. */
+ memcpy(state->readRecordBuf, state->readBuf + targetRecOff,
+ record_len);
+ state->recordGotLen += record_len;
+ state->recordRemainLen -= record_len;
- /*
- * If we were expecting a continuation record and got an
- * "overwrite contrecord" flag, that means the continuation record
- * was overwritten with a different record. Restart the read by
- * assuming the address to read is the location where we found
- * this flag; but keep track of the LSN of the record we were
- * reading, for later verification.
- */
- if (pageHeader->xlp_info & XLP_FIRST_IS_OVERWRITE_CONTRECORD)
- {
- state->overwrittenRecPtr = state->currRecPtr;
- ResetDecoder(state);
- RecPtr = targetPagePtr;
- goto restart;
- }
+ /* Calculate pointer to beginning of next page */
+ state->recordContRecPtr = state->ReadRecPtr + record_len;
+ Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0);
- /* Check that the continuation on next page looks valid */
- if (!(pageHeader->xlp_info & XLP_FIRST_IS_CONTRECORD))
- {
- report_invalid_record(state,
- "there is no contrecord flag at %X/%X",
- LSN_FORMAT_ARGS(RecPtr));
- goto err;
+ state->readRecordState = XLREAD_CONTINUATION;
}
+ /* fall through */
- /*
- * Cross-check that xlp_rem_len agrees with how much of the record
- * we expect there to be left.
- */
- if (pageHeader->xlp_rem_len == 0 ||
- total_len != (pageHeader->xlp_rem_len + gotlen))
+ case XLREAD_CONTINUATION:
{
- report_invalid_record(state,
- "invalid contrecord length %u (expected %lld) at %X/%X",
- pageHeader->xlp_rem_len,
- ((long long) total_len) - gotlen,
- LSN_FORMAT_ARGS(RecPtr));
- goto err;
- }
+ XLogPageHeader pageHeader;
+ uint32 pageHeaderSize;
+ XLogRecPtr targetPagePtr;
- /* Append the continuation from this page to the buffer */
- pageHeaderSize = XLogPageHeaderSize(pageHeader);
+ /*
+ * we enter this state only if we haven't read the whole
+ * record.
+ */
+ Assert(state->recordRemainLen > 0);
- Assert(pageHeaderSize <= state->readLen);
+ while (state->recordRemainLen > 0)
+ {
+ char *contdata;
+ uint32 request_len;
+ uint32 record_len;
- contdata = (char *) state->readBuf + pageHeaderSize;
- len = XLOG_BLCKSZ - pageHeaderSize;
- if (pageHeader->xlp_rem_len < len)
- len = pageHeader->xlp_rem_len;
+ /* Wait for the next page to become available */
+ targetPagePtr = state->recordContRecPtr;
- Assert(pageHeaderSize + len <= state->readLen);
- memcpy(buffer, (char *) contdata, len);
- buffer += len;
- gotlen += len;
+ /* this request contains page header */
+ Assert(targetPagePtr != 0);
+ if (XLogNeedData(state, targetPagePtr,
+ Min(state->recordRemainLen, XLOG_BLCKSZ),
+ false))
+ return XLREAD_NEED_DATA;
- /* If we just reassembled the record header, validate it. */
- if (!gotheader)
- {
- record = (XLogRecord *) state->readRecordBuf;
- if (!ValidXLogRecordHeader(state, RecPtr, state->ReadRecPtr,
- record, randAccess))
+ if (!state->page_verified)
+ goto err;
+
+ Assert(SizeOfXLogShortPHD <= state->readLen);
+
+ /* Check that the continuation on next page looks valid */
+ pageHeader = (XLogPageHeader) state->readBuf;
+
+ /*
+ * If we were expecting a continuation record and got an
+ * "overwrite contrecord" flag, that means the continuation
+ * record was overwritten with a different record. Restart
+ * the read by assuming the address to read is the location
+ * where we found this flag; but keep track of the LSN of
+ * the record we were reading, for later verification.
+ */
+ if (pageHeader->xlp_info &
+ XLP_FIRST_IS_OVERWRITE_CONTRECORD)
+ {
+ state->overwrittenRecPtr = state->ReadRecPtr;
+ state->EndRecPtr = targetPagePtr;
+ state->readRecordState = XLREAD_NEXT_RECORD;
+
+ /*
+ * ReadRecPtr is the PrevRecPtr of the next
+ * record. Keep the LSN at the retry.
+ */
+ state->ReadRecPtr = state->PrevRecPtr;
+
+ goto restart;
+ }
+
+ if (!(pageHeader->xlp_info & XLP_FIRST_IS_CONTRECORD))
+ {
+ report_invalid_record(
+ state,
+ "there is no contrecord flag at %X/%X reading %X/%X",
+ (uint32) (state->recordContRecPtr >> 32),
+ (uint32) state->recordContRecPtr,
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr);
+ goto err;
+ }
+
+ /*
+ * Cross-check that xlp_rem_len agrees with how much of
+ * the record we expect there to be left.
+ */
+ if (pageHeader->xlp_rem_len == 0 ||
+ pageHeader->xlp_rem_len != state->recordRemainLen)
+ {
+ report_invalid_record(
+ state,
+ "invalid contrecord length %u at %X/%X reading %X/%X, expected %u",
+ pageHeader->xlp_rem_len,
+ (uint32) (state->recordContRecPtr >> 32),
+ (uint32) state->recordContRecPtr,
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr,
+ state->recordRemainLen);
+ goto err;
+ }
+
+ /* Append the continuation from this page to the buffer */
+ pageHeaderSize = XLogPageHeaderSize(pageHeader);
+
+ /*
+ * XLogNeedData should have ensured that the whole page
+ * header was read
+ */
+ Assert(state->readLen >= pageHeaderSize);
+
+ contdata = (char *) state->readBuf + pageHeaderSize;
+ record_len = XLOG_BLCKSZ - pageHeaderSize;
+ if (pageHeader->xlp_rem_len < record_len)
+ record_len = pageHeader->xlp_rem_len;
+
+ request_len = record_len + pageHeaderSize;
+
+ /*
+ * XLogNeedData should have ensured all needed data was
+ * read
+ */
+ Assert(state->readLen >= request_len);
+
+ memcpy(state->readRecordBuf + state->recordGotLen,
+ (char *) contdata, record_len);
+ state->recordGotLen += record_len;
+ state->recordRemainLen -= record_len;
+
+ /* If we just reassembled the record header, validate it. */
+ if (!state->record_verified)
+ {
+ Assert(state->recordGotLen >= SizeOfXLogRecord);
+ if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
+ state->PrevRecPtr,
+ (XLogRecord *) state->readRecordBuf))
+ goto err;
+
+ state->record_verified = true;
+ }
+
+ /*
+ * Calculate pointer to beginning of next page, and
+ * continue
+ */
+ state->recordContRecPtr += XLOG_BLCKSZ;
+ }
+
+ /* targetPagePtr is pointing the last-read page here */
+ prec = (XLogRecord *) state->readRecordBuf;
+ if (!ValidXLogRecord(state, prec, state->ReadRecPtr))
goto err;
- gotheader = true;
- }
- } while (gotlen < total_len);
-
- Assert(gotheader);
-
- record = (XLogRecord *) state->readRecordBuf;
- if (!ValidXLogRecord(state, record, RecPtr))
- goto err;
-
- pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
- state->ReadRecPtr = RecPtr;
- state->EndRecPtr = targetPagePtr + pageHeaderSize
- + MAXALIGN(pageHeader->xlp_rem_len);
- }
- else
- {
- /* Wait for the record data to become available */
- while (XLogNeedData(state, targetPagePtr,
- Min(targetRecOff + total_len, XLOG_BLCKSZ), true))
- {
- if (!state->routine.page_read(state, state->readPagePtr,
- state->readLen,
- state->ReadRecPtr, state->readBuf))
+
+ pageHeaderSize =
+ XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+ state->EndRecPtr = targetPagePtr + pageHeaderSize
+ + MAXALIGN(pageHeader->xlp_rem_len);
+
+ *record = prec;
+ state->readRecordState = XLREAD_NEXT_RECORD;
break;
- }
-
- if (!state->page_verified)
- goto err;
-
- /* Record does not cross a page boundary */
- if (!ValidXLogRecord(state, record, RecPtr))
- goto err;
-
- state->EndRecPtr = RecPtr + MAXALIGN(total_len);
-
- state->ReadRecPtr = RecPtr;
+ }
}
/*
* Special processing if it's an XLOG SWITCH record
*/
- if (record->xl_rmid == RM_XLOG_ID &&
- (record->xl_info & ~XLR_INFO_MASK) == XLOG_SWITCH)
+ if ((*record)->xl_rmid == RM_XLOG_ID &&
+ ((*record)->xl_info & ~XLR_INFO_MASK) == XLOG_SWITCH)
{
/* Pretend it extends to end of segment */
state->EndRecPtr += state->segcxt.ws_segsize - 1;
state->EndRecPtr -= XLogSegmentOffset(state->EndRecPtr, state->segcxt.ws_segsize);
}
- if (DecodeXLogRecord(state, record, errormsg))
- return record;
- else
- return NULL;
+ Assert(!*record || state->readLen >= 0);
+ if (DecodeXLogRecord(state, *record, errormsg))
+ return XLREAD_SUCCESS;
+
+ *record = NULL;
+ return XLREAD_FAIL;
err:
- if (assembled)
+ Assert(state->readRecordState != XLREAD_NEXT_RECORD);
+ XLogTerminateRead(state);
+
+ if (state->errormsg_buf[0] != '\0')
+ *errormsg = state->errormsg_buf;
+
+ *record = NULL;
+
+ return XLREAD_FAIL;
+}
+
+/*
+ * Terminate read WAL.
+ *
+ * When the caller failed to read the data requested from XLogReadRecord, it is
+ * supposed to call this function to set the correct reader state to reflect
+ * the failure.
+ */
+void
+XLogTerminateRead(XLogReaderState *state)
+{
+ if (state->readRecordState == XLREAD_CONTINUATION)
{
/*
* We get here when a record that spans multiple pages needs to be
@@ -640,20 +768,15 @@ err:
* in turn signal downstream WAL consumers that the broken WAL record
* is to be ignored.
*/
- state->abortedRecPtr = RecPtr;
- state->missingContrecPtr = targetPagePtr;
+ state->abortedRecPtr = state->ReadRecPtr;
+ state->missingContrecPtr = state->recordContRecPtr;
}
/*
- * Invalidate the read state. We might read from a different source after
+ * Invalidate the read page. We might read from a different source after
* failure.
*/
XLogReaderInvalReadState(state);
-
- if (state->errormsg_buf[0] != '\0')
- *errormsg = state->errormsg_buf;
-
- return NULL;
}
/*
@@ -805,11 +928,12 @@ XLogReaderInvalReadState(XLogReaderState *state)
*
* This is just a convenience subroutine to avoid duplicated code in
* XLogReadRecord. It's not intended for use from anywhere else.
+ *
+ * If PrevRecPtr is valid, the xl_prev is is cross-checked with it.
*/
static bool
ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
- XLogRecPtr PrevRecPtr, XLogRecord *record,
- bool randAccess)
+ XLogRecPtr PrevRecPtr, XLogRecord *record)
{
if (record->xl_tot_len < SizeOfXLogRecord)
{
@@ -826,7 +950,7 @@ ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
record->xl_rmid, LSN_FORMAT_ARGS(RecPtr));
return false;
}
- if (randAccess)
+ if (PrevRecPtr == InvalidXLogRecPtr)
{
/*
* We can't exactly verify the prev-link, but surely it should be less
@@ -1056,11 +1180,14 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
* XLogReadRecord() will read the next valid record.
*/
XLogRecPtr
-XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
+XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
+ XLogFindNextRecordCB read_page, void *private)
{
XLogRecPtr tmpRecPtr;
XLogRecPtr found = InvalidXLogRecPtr;
XLogPageHeader header;
+ XLogRecord *record;
+ XLogReadRecordResult result;
char *errormsg;
Assert(!XLogRecPtrIsInvalid(RecPtr));
@@ -1093,9 +1220,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
while (XLogNeedData(state, targetPagePtr, targetRecOff,
targetRecOff != 0))
{
- if (!state->routine.page_read(state, state->readPagePtr,
- state->readLen,
- state->ReadRecPtr, state->readBuf))
+ if (!read_page(state, private))
break;
}
@@ -1147,8 +1272,16 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
* or we just jumped over the remaining data of a continuation.
*/
XLogBeginRead(state, tmpRecPtr);
- while (XLogReadRecord(state, &errormsg) != NULL)
+ while ((result = XLogReadRecord(state, &record, &errormsg)) !=
+ XLREAD_FAIL)
{
+ if (result == XLREAD_NEED_DATA)
+ {
+ if (!read_page(state, private))
+ goto err;
+ continue;
+ }
+
/* past the record we've found, break out */
if (RecPtr <= state->ReadRecPtr)
{
@@ -1168,9 +1301,9 @@ err:
#endif /* FRONTEND */
/*
- * Helper function to ease writing of XLogRoutine->page_read callbacks.
- * If this function is used, caller must supply a segment_open callback in
- * 'state', as that is used here.
+ * Helper function to ease writing of page_read callback.
+ * If this function is used, caller must supply a segment_open callback and
+ * segment_close callback as that is used here.
*
* Read 'count' bytes into 'buf', starting at location 'startptr', from WAL
* fetched from timeline 'tli'.
@@ -1183,6 +1316,7 @@ err:
*/
bool
WALRead(XLogReaderState *state,
+ WALSegmentOpenCB segopenfn, WALSegmentCloseCB segclosefn,
char *buf, XLogRecPtr startptr, Size count, TimeLineID tli,
WALReadError *errinfo)
{
@@ -1214,10 +1348,10 @@ WALRead(XLogReaderState *state,
XLogSegNo nextSegNo;
if (state->seg.ws_file >= 0)
- state->routine.segment_close(state);
+ segclosefn(state);
XLByteToSeg(recptr, nextSegNo, state->segcxt.ws_segsize);
- state->routine.segment_open(state, nextSegNo, &tli);
+ segopenfn(state, nextSegNo, &tli);
/* This shouldn't happen -- indicates a bug in segment_open */
Assert(state->seg.ws_file >= 0);
diff --git a/src/backend/access/transam/xlogutils.c b/src/backend/access/transam/xlogutils.c
index 421040f391..fc09a72b8b 100644
--- a/src/backend/access/transam/xlogutils.c
+++ b/src/backend/access/transam/xlogutils.c
@@ -706,8 +706,7 @@ XLogTruncateRelation(RelFileNode rnode, ForkNumber forkNum,
void
XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wantLength)
{
- const XLogRecPtr lastReadPage = state->seg.ws_segno *
- state->segcxt.ws_segsize + state->readLen;
+ const XLogRecPtr lastReadPage = state->readPagePtr;
Assert(wantPage != InvalidXLogRecPtr && wantPage % XLOG_BLCKSZ == 0);
Assert(wantLength <= XLOG_BLCKSZ);
@@ -722,7 +721,7 @@ XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wa
* current TLI has since become historical.
*/
if (lastReadPage == wantPage &&
- state->readLen != 0 &&
+ state->page_verified &&
lastReadPage + state->readLen >= wantPage + Min(wantLength, XLOG_BLCKSZ - 1))
return;
@@ -808,6 +807,7 @@ wal_segment_open(XLogReaderState *state, XLogSegNo nextSegNo,
char path[MAXPGPATH];
XLogFilePath(path, tli, nextSegNo, state->segcxt.ws_segsize);
+ elog(LOG, "HOGE: %lu, %d => %s", nextSegNo, tli, path);
state->seg.ws_file = BasicOpenFile(path, O_RDONLY | PG_BINARY);
if (state->seg.ws_file >= 0)
return;
@@ -845,9 +845,11 @@ wal_segment_close(XLogReaderState *state)
* loop for now.
*/
bool
-read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *cur_page)
+read_local_xlog_page(XLogReaderState *state)
{
+ XLogRecPtr targetPagePtr = state->readPagePtr;
+ int reqLen = state->readLen;
+ char *cur_page = state->readBuf;
XLogRecPtr read_upto,
loc;
TimeLineID tli;
@@ -960,11 +962,12 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
* as 'count', read the whole page anyway. It's guaranteed to be
* zero-padded up to the page boundary if it's incomplete.
*/
- if (!WALRead(state, cur_page, targetPagePtr, XLOG_BLCKSZ, tli,
- &errinfo))
+ if (!WALRead(state, wal_segment_open, wal_segment_close,
+ cur_page, targetPagePtr, XLOG_BLCKSZ, tli, &errinfo))
WALReadRaiseError(&errinfo);
/* number of valid bytes in the buffer */
+ state->readPagePtr = targetPagePtr;
state->readLen = count;
return true;
}
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index aae0ae5b8a..2a88aa063a 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -148,7 +148,8 @@ StartupDecodingContext(List *output_plugin_options,
TransactionId xmin_horizon,
bool need_full_snapshot,
bool fast_forward,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress)
@@ -198,11 +199,12 @@ StartupDecodingContext(List *output_plugin_options,
ctx->slot = slot;
- ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, xl_routine, ctx);
+ ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, cleanup_cb);
if (!ctx->reader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
errmsg("out of memory")));
+ ctx->page_read = page_read;
ctx->reorder = ReorderBufferAllocate();
ctx->snapshot_builder =
@@ -319,7 +321,8 @@ CreateInitDecodingContext(const char *plugin,
List *output_plugin_options,
bool need_full_snapshot,
XLogRecPtr restart_lsn,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress)
@@ -422,7 +425,7 @@ CreateInitDecodingContext(const char *plugin,
ctx = StartupDecodingContext(NIL, restart_lsn, xmin_horizon,
need_full_snapshot, false,
- xl_routine, prepare_write, do_write,
+ page_read, cleanup_cb, prepare_write, do_write,
update_progress);
/* call output plugin initialization callback */
@@ -478,7 +481,8 @@ LogicalDecodingContext *
CreateDecodingContext(XLogRecPtr start_lsn,
List *output_plugin_options,
bool fast_forward,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress)
@@ -534,8 +538,8 @@ CreateDecodingContext(XLogRecPtr start_lsn,
ctx = StartupDecodingContext(output_plugin_options,
start_lsn, InvalidTransactionId, false,
- fast_forward, xl_routine, prepare_write,
- do_write, update_progress);
+ fast_forward, page_read, cleanup_cb,
+ prepare_write, do_write, update_progress);
/* call output plugin initialization callback */
old_context = MemoryContextSwitchTo(ctx->context);
@@ -603,7 +607,16 @@ DecodingContextFindStartpoint(LogicalDecodingContext *ctx)
char *err = NULL;
/* the read_page callback waits for new WAL */
- record = XLogReadRecord(ctx->reader, &err);
+ while (XLogReadRecord(ctx->reader, &record, &err) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!ctx->page_read(ctx->reader))
+ {
+ XLogTerminateRead(ctx->reader);
+ break;
+ }
+ }
+
if (err)
elog(ERROR, "%s", err);
if (!record)
diff --git a/src/backend/replication/logical/logicalfuncs.c b/src/backend/replication/logical/logicalfuncs.c
index e59939aad1..f68d08fdee 100644
--- a/src/backend/replication/logical/logicalfuncs.c
+++ b/src/backend/replication/logical/logicalfuncs.c
@@ -224,9 +224,8 @@ pg_logical_slot_get_changes_guts(FunctionCallInfo fcinfo, bool confirm, bool bin
ctx = CreateDecodingContext(InvalidXLogRecPtr,
options,
false,
- XL_ROUTINE(.page_read = read_local_xlog_page,
- .segment_open = wal_segment_open,
- .segment_close = wal_segment_close),
+ read_local_xlog_page,
+ wal_segment_close,
LogicalOutputPrepareWrite,
LogicalOutputWrite, NULL);
@@ -275,7 +274,16 @@ pg_logical_slot_get_changes_guts(FunctionCallInfo fcinfo, bool confirm, bool bin
XLogRecord *record;
char *errm = NULL;
- record = XLogReadRecord(ctx->reader, &errm);
+ while (XLogReadRecord(ctx->reader, &record, &errm) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!ctx->page_read(ctx->reader))
+ {
+ XLogTerminateRead(ctx->reader);
+ break;
+ }
+ }
+
if (errm)
elog(ERROR, "%s", errm);
diff --git a/src/backend/replication/slotfuncs.c b/src/backend/replication/slotfuncs.c
index 17df99c2ac..57426aa9d7 100644
--- a/src/backend/replication/slotfuncs.c
+++ b/src/backend/replication/slotfuncs.c
@@ -144,9 +144,8 @@ create_logical_replication_slot(char *name, char *plugin,
ctx = CreateInitDecodingContext(plugin, NIL,
false, /* just catalogs is OK */
restart_lsn,
- XL_ROUTINE(.page_read = read_local_xlog_page,
- .segment_open = wal_segment_open,
- .segment_close = wal_segment_close),
+ read_local_xlog_page,
+ wal_segment_close,
NULL, NULL, NULL);
/*
@@ -503,9 +502,8 @@ pg_logical_replication_slot_advance(XLogRecPtr moveto)
ctx = CreateDecodingContext(InvalidXLogRecPtr,
NIL,
true, /* fast_forward */
- XL_ROUTINE(.page_read = read_local_xlog_page,
- .segment_open = wal_segment_open,
- .segment_close = wal_segment_close),
+ read_local_xlog_page,
+ wal_segment_close,
NULL, NULL, NULL);
/*
@@ -527,7 +525,16 @@ pg_logical_replication_slot_advance(XLogRecPtr moveto)
* Read records. No changes are generated in fast_forward mode,
* but snapbuilder/slot statuses are updated properly.
*/
- record = XLogReadRecord(ctx->reader, &errm);
+ while (XLogReadRecord(ctx->reader, &record, &errm) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!ctx->page_read(ctx->reader))
+ {
+ XLogTerminateRead(ctx->reader);
+ break;
+ }
+ }
+
if (errm)
elog(ERROR, "%s", errm);
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index ddba340653..78d805ab80 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -575,10 +575,7 @@ StartReplication(StartReplicationCmd *cmd)
/* create xlogreader for physical replication */
xlogreader =
- XLogReaderAllocate(wal_segment_size, NULL,
- XL_ROUTINE(.segment_open = WalSndSegmentOpen,
- .segment_close = wal_segment_close),
- NULL);
+ XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
if (!xlogreader)
ereport(ERROR,
@@ -803,9 +800,11 @@ StartReplication(StartReplicationCmd *cmd)
* set every time WAL is flushed.
*/
static bool
-logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetRecPtr, char *cur_page)
+logical_read_xlog_page(XLogReaderState *state)
{
+ XLogRecPtr targetPagePtr = state->readPagePtr;
+ int reqLen = state->readLen;
+ char *cur_page = state->readBuf;
XLogRecPtr flushptr;
int count;
WALReadError errinfo;
@@ -833,7 +832,7 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
count = flushptr - targetPagePtr; /* part of the page available */
/* now actually read the data, we know it's there */
- if (!WALRead(state,
+ if (!WALRead(state, WalSndSegmentOpen, wal_segment_close,
cur_page,
targetPagePtr,
XLOG_BLCKSZ,
@@ -1023,9 +1022,8 @@ CreateReplicationSlot(CreateReplicationSlotCmd *cmd)
ctx = CreateInitDecodingContext(cmd->plugin, NIL, need_full_snapshot,
InvalidXLogRecPtr,
- XL_ROUTINE(.page_read = logical_read_xlog_page,
- .segment_open = WalSndSegmentOpen,
- .segment_close = wal_segment_close),
+ logical_read_xlog_page,
+ wal_segment_close,
WalSndPrepareWrite, WalSndWriteData,
WalSndUpdateProgress);
@@ -1183,9 +1181,8 @@ StartLogicalReplication(StartReplicationCmd *cmd)
*/
logical_decoding_ctx =
CreateDecodingContext(cmd->startpoint, cmd->options, false,
- XL_ROUTINE(.page_read = logical_read_xlog_page,
- .segment_open = WalSndSegmentOpen,
- .segment_close = wal_segment_close),
+ logical_read_xlog_page,
+ wal_segment_close,
WalSndPrepareWrite, WalSndWriteData,
WalSndUpdateProgress);
xlogreader = logical_decoding_ctx->reader;
@@ -2778,7 +2775,7 @@ XLogSendPhysical(void)
enlargeStringInfo(&output_message, nbytes);
retry:
- if (!WALRead(xlogreader,
+ if (!WALRead(xlogreader, WalSndSegmentOpen, wal_segment_close,
&output_message.data[output_message.len],
startptr,
nbytes,
@@ -2876,7 +2873,15 @@ XLogSendLogical(void)
*/
WalSndCaughtUp = false;
- record = XLogReadRecord(logical_decoding_ctx->reader, &errm);
+ while (XLogReadRecord(logical_decoding_ctx->reader, &record, &errm) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!logical_decoding_ctx->page_read(logical_decoding_ctx->reader))
+ {
+ XLogTerminateRead(logical_decoding_ctx->reader);
+ break;
+ }
+ }
/* xlog record was invalid */
if (errm != NULL)
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index cf119848b0..da723e5340 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -41,15 +41,9 @@ static int xlogreadfd = -1;
static XLogSegNo xlogreadsegno = -1;
static char xlogfpath[MAXPGPATH];
-typedef struct XLogPageReadPrivate
-{
- const char *restoreCommand;
- int tliIndex;
-} XLogPageReadPrivate;
-
-static bool SimpleXLogPageRead(XLogReaderState *xlogreader,
- XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
+static bool SimpleXLogPageRead(XLogReaderState *xlogreader,
+ const char *datadir, int *tliIndex,
+ const char *restoreCommand);
/*
* Read WAL from the datadir/pg_wal, starting from 'startpoint' on timeline
@@ -66,20 +60,25 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
XLogRecord *record;
XLogReaderState *xlogreader;
char *errormsg;
- XLogPageReadPrivate private;
- private.tliIndex = tliIndex;
- private.restoreCommand = restoreCommand;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir,
- XL_ROUTINE(.page_read = &SimpleXLogPageRead),
- &private);
+ xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+
if (xlogreader == NULL)
pg_fatal("out of memory");
XLogBeginRead(xlogreader, startpoint);
do
{
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!SimpleXLogPageRead(xlogreader, datadir,
+ &tliIndex, restoreCommand))
+ {
+ XLogTerminateRead(xlogreader);
+ break;
+ }
+ }
if (record == NULL)
{
@@ -123,19 +122,22 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex,
XLogRecord *record;
XLogReaderState *xlogreader;
char *errormsg;
- XLogPageReadPrivate private;
XLogRecPtr endptr;
- private.tliIndex = tliIndex;
- private.restoreCommand = restoreCommand;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir,
- XL_ROUTINE(.page_read = &SimpleXLogPageRead),
- &private);
+ xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
XLogBeginRead(xlogreader, ptr);
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!SimpleXLogPageRead(xlogreader, datadir, &tliIndex, restoreCommand))
+ {
+ XLogTerminateRead(xlogreader);
+ break;
+ }
+ }
if (record == NULL)
{
if (errormsg)
@@ -170,7 +172,6 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
XLogRecPtr searchptr;
XLogReaderState *xlogreader;
char *errormsg;
- XLogPageReadPrivate private;
/*
* The given fork pointer points to the end of the last common record,
@@ -186,11 +187,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
forkptr += SizeOfXLogShortPHD;
}
- private.tliIndex = tliIndex;
- private.restoreCommand = restoreCommand;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir,
- XL_ROUTINE(.page_read = &SimpleXLogPageRead),
- &private);
+ xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
@@ -200,7 +197,16 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
uint8 info;
XLogBeginRead(xlogreader, searchptr);
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!SimpleXLogPageRead(xlogreader, datadir,
+ &tliIndex, restoreCommand))
+ {
+ XLogTerminateRead(xlogreader);
+ break;
+ }
+ }
if (record == NULL)
{
@@ -247,10 +253,11 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
/* XLogReader callback function, to read a WAL page */
static bool
-SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *readBuf)
+SimpleXLogPageRead(XLogReaderState *xlogreader, const char *datadir,
+ int *tliIndex, const char *restoreCommand)
{
- XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
+ XLogRecPtr targetPagePtr = xlogreader->readPagePtr;
+ char *readBuf = xlogreader->readBuf;
uint32 targetPageOff;
XLogRecPtr targetSegEnd;
XLogSegNo targetSegNo;
@@ -283,14 +290,14 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
* be done both forward and backward, consider also switching timeline
* accordingly.
*/
- while (private->tliIndex < targetNentries - 1 &&
- targetHistory[private->tliIndex].end < targetSegEnd)
- private->tliIndex++;
- while (private->tliIndex > 0 &&
- targetHistory[private->tliIndex].begin >= targetSegEnd)
- private->tliIndex--;
+ while (*tliIndex < targetNentries - 1 &&
+ targetHistory[*tliIndex].end < targetSegEnd)
+ (*tliIndex)++;
+ while (*tliIndex > 0 &&
+ targetHistory[*tliIndex].begin >= targetSegEnd)
+ (*tliIndex)--;
- XLogFileName(xlogfname, targetHistory[private->tliIndex].tli,
+ XLogFileName(xlogfname, targetHistory[*tliIndex].tli,
xlogreadsegno, WalSegSz);
snprintf(xlogfpath, MAXPGPATH, "%s/" XLOGDIR "/%s",
@@ -303,7 +310,7 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
/*
* If we have no restore_command to execute, then exit.
*/
- if (private->restoreCommand == NULL)
+ if (restoreCommand == NULL)
{
pg_log_error("could not open file \"%s\": %m", xlogfpath);
xlogreader->readLen = -1;
@@ -317,7 +324,7 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
xlogreadfd = RestoreArchivedFile(xlogreader->segcxt.ws_dir,
xlogfname,
WalSegSz,
- private->restoreCommand);
+ restoreCommand);
if (xlogreadfd < 0)
{
@@ -359,7 +366,7 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
Assert(targetSegNo == xlogreadsegno);
- xlogreader->seg.ws_tli = targetHistory[private->tliIndex].tli;
+ xlogreader->seg.ws_tli = targetHistory[*tliIndex].tli;
xlogreader->readLen = XLOG_BLCKSZ;
return true;
}
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index 833a64210b..80182621f8 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -331,12 +331,17 @@ WALDumpCloseSegment(XLogReaderState *state)
state->seg.ws_file = -1;
}
-/* pg_waldump's XLogReaderRoutine->page_read callback */
+/*
+ * pg_waldump's WAL page rader, also used as page_read callback for
+ * XLogFindNextRecord
+ */
static bool
-WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetPtr, char *readBuff)
+WALDumpReadPage(XLogReaderState *state, void *priv)
{
- XLogDumpPrivate *private = state->private_data;
+ XLogRecPtr targetPagePtr = state->readPagePtr;
+ int reqLen = state->readLen;
+ char *readBuff = state->readBuf;
+ XLogDumpPrivate *private = (XLogDumpPrivate *) priv;
int count = XLOG_BLCKSZ;
WALReadError errinfo;
@@ -354,8 +359,8 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
}
}
- if (!WALRead(state, readBuff, targetPagePtr, count, private->timeline,
- &errinfo))
+ if (!WALRead(state, WALDumpOpenSegment, WALDumpCloseSegment,
+ readBuff, targetPagePtr, count, private->timeline, &errinfo))
{
WALOpenSegment *seg = &errinfo.wre_seg;
char fname[MAXPGPATH];
@@ -375,6 +380,7 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
errinfo.wre_req);
}
+ Assert(count >= state->readLen);
state->readLen = count;
return true;
}
@@ -1057,16 +1063,14 @@ main(int argc, char **argv)
/* we have everything we need, start reading */
xlogreader_state =
- XLogReaderAllocate(WalSegSz, waldir,
- XL_ROUTINE(.page_read = WALDumpReadPage,
- .segment_open = WALDumpOpenSegment,
- .segment_close = WALDumpCloseSegment),
- &private);
+ XLogReaderAllocate(WalSegSz, waldir, WALDumpCloseSegment);
+
if (!xlogreader_state)
fatal_error("out of memory");
/* first find a valid recptr to start from */
- first_record = XLogFindNextRecord(xlogreader_state, private.startptr);
+ first_record = XLogFindNextRecord(xlogreader_state, private.startptr,
+ &WALDumpReadPage, (void*) &private);
if (first_record == InvalidXLogRecPtr)
fatal_error("could not find a valid record after %X/%X",
@@ -1089,7 +1093,16 @@ main(int argc, char **argv)
for (;;)
{
/* try to read the next record */
- record = XLogReadRecord(xlogreader_state, &errormsg);
+ while (XLogReadRecord(xlogreader_state, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!WALDumpReadPage(xlogreader_state, (void *) &private))
+ {
+ XLogTerminateRead(xlogreader_state);
+ break;
+ }
+ }
+
if (!record)
{
if (!config.follow || private.endptr_reached)
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index f3cf4f2f49..ff1aca719b 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -57,64 +57,15 @@ typedef struct WALSegmentContext
typedef struct XLogReaderState XLogReaderState;
-/* Function type definition for the read_page callback */
-typedef bool (*XLogPageReadCB) (XLogReaderState *xlogreader,
- XLogRecPtr targetPagePtr,
- int reqLen,
- XLogRecPtr targetRecPtr,
- char *readBuf);
+/* Function type definition for the segment cleanup callback */
+typedef void (*WALSegmentCleanupCB) (XLogReaderState *xlogreader);
+
+/* Function type definition for the open/close callbacks for WALRead() */
typedef void (*WALSegmentOpenCB) (XLogReaderState *xlogreader,
XLogSegNo nextSegNo,
TimeLineID *tli_p);
typedef void (*WALSegmentCloseCB) (XLogReaderState *xlogreader);
-typedef struct XLogReaderRoutine
-{
- /*
- * Data input callback
- *
- * This callback shall read at least reqLen valid bytes of the xlog page
- * starting at targetPagePtr, and store them in readBuf. The callback
- * shall return the number of bytes read (never more than XLOG_BLCKSZ), or
- * -1 on failure. The callback shall sleep, if necessary, to wait for the
- * requested bytes to become available. The callback will not be invoked
- * again for the same page unless more than the returned number of bytes
- * are needed.
- *
- * targetRecPtr is the position of the WAL record we're reading. Usually
- * it is equal to targetPagePtr + reqLen, but sometimes xlogreader needs
- * to read and verify the page or segment header, before it reads the
- * actual WAL record it's interested in. In that case, targetRecPtr can
- * be used to determine which timeline to read the page from.
- *
- * The callback shall set ->seg.ws_tli to the TLI of the file the page was
- * read from.
- */
- XLogPageReadCB page_read;
-
- /*
- * Callback to open the specified WAL segment for reading. ->seg.ws_file
- * shall be set to the file descriptor of the opened segment. In case of
- * failure, an error shall be raised by the callback and it shall not
- * return.
- *
- * "nextSegNo" is the number of the segment to be opened.
- *
- * "tli_p" is an input/output argument. WALRead() uses it to pass the
- * timeline in which the new segment should be found, but the callback can
- * use it to return the TLI that it actually opened.
- */
- WALSegmentOpenCB segment_open;
-
- /*
- * WAL segment close callback. ->seg.ws_file shall be set to a negative
- * number.
- */
- WALSegmentCloseCB segment_close;
-} XLogReaderRoutine;
-
-#define XL_ROUTINE(...) &(XLogReaderRoutine){__VA_ARGS__}
-
typedef struct
{
/* Is this block ref in use? */
@@ -144,12 +95,36 @@ typedef struct
uint16 data_bufsz;
} DecodedBkpBlock;
+/* Return code from XLogReadRecord */
+typedef enum XLogReadRecordResult
+{
+ XLREAD_SUCCESS, /* record is successfully read */
+ XLREAD_NEED_DATA, /* need more data. see XLogReadRecord. */
+ XLREAD_FAIL /* failed during reading a record */
+} XLogReadRecordResult;
+
+/*
+ * internal state of XLogReadRecord
+ *
+ * XLogReadState runs a state machine while reading a record. Theses states
+ * are not seen outside the function. Each state may repeat several times
+ * exiting requesting caller for new data. See the comment of XLogReadRecrod
+ * for details.
+ */
+typedef enum XLogReadRecordState
+{
+ XLREAD_NEXT_RECORD,
+ XLREAD_TOT_LEN,
+ XLREAD_FIRST_FRAGMENT,
+ XLREAD_CONTINUATION
+} XLogReadRecordState;
+
struct XLogReaderState
{
/*
* Operational callbacks
*/
- XLogReaderRoutine routine;
+ WALSegmentCleanupCB cleanup_cb;
/* ----------------------------------------
* Public parameters
@@ -162,18 +137,14 @@ struct XLogReaderState
*/
uint64 system_identifier;
- /*
- * Opaque data for callbacks to use. Not used by XLogReader.
- */
- void *private_data;
-
/*
* Start and end point of last record read. EndRecPtr is also used as the
* position to read next. Calling XLogBeginRead() sets EndRecPtr to the
* starting position and ReadRecPtr to invalid.
*/
- XLogRecPtr ReadRecPtr; /* start of last record read */
+ XLogRecPtr ReadRecPtr; /* start of last record read or being read */
XLogRecPtr EndRecPtr; /* end+1 of last record read */
+ XLogRecPtr PrevRecPtr; /* start of previous record read */
/*
* Set at the end of recovery: the start point of a partial record at the
@@ -196,7 +167,9 @@ struct XLogReaderState
* read by reader, which must be larger than
* the request, or -1 on error */
char *readBuf; /* buffer to store data */
- bool page_verified; /* is the page on the buffer verified? */
+ bool page_verified; /* is the page header on the buffer verified? */
+ bool record_verified; /* is the current record header verified? */
+
/* ----------------------------------------
* Decoded representation of current record
@@ -237,8 +210,6 @@ struct XLogReaderState
XLogRecPtr latestPagePtr;
TimeLineID latestPageTLI;
- /* beginning of the WAL record being read. */
- XLogRecPtr currRecPtr;
/* timeline to read it from, 0 if a lookup is required */
TimeLineID currTLI;
@@ -265,6 +236,15 @@ struct XLogReaderState
char *readRecordBuf;
uint32 readRecordBufSize;
+ /*
+ * XLogReadRecord() state
+ */
+ XLogReadRecordState readRecordState; /* state machine state */
+ int recordGotLen; /* amount of current record that has already
+ * been read */
+ int recordRemainLen; /* length of current record that remains */
+ XLogRecPtr recordContRecPtr; /* where the current record continues */
+
/* Buffer to hold error message */
char *errormsg_buf;
};
@@ -272,9 +252,7 @@ struct XLogReaderState
/* Get a new XLogReader */
extern XLogReaderState *XLogReaderAllocate(int wal_segment_size,
const char *waldir,
- XLogReaderRoutine *routine,
- void *private_data);
-extern XLogReaderRoutine *LocalXLogReaderRoutine(void);
+ WALSegmentCleanupCB cleanup_cb);
/* Free an XLogReader */
extern void XLogReaderFree(XLogReaderState *state);
@@ -282,12 +260,19 @@ extern void XLogReaderFree(XLogReaderState *state);
/* Position the XLogReader to given record */
extern void XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr);
#ifdef FRONTEND
-extern XLogRecPtr XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr);
+/* Function type definition for the read_page callback */
+typedef bool (*XLogFindNextRecordCB) (XLogReaderState *xlogreader,
+ void *private);
+extern XLogRecPtr XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
+ XLogFindNextRecordCB read_page, void *private);
#endif /* FRONTEND */
/* Read the next XLog record. Returns NULL on end-of-WAL or failure */
-extern struct XLogRecord *XLogReadRecord(XLogReaderState *state,
- char **errormsg);
+extern XLogReadRecordResult XLogReadRecord(XLogReaderState *state,
+ XLogRecord **record,
+ char **errormsg);
+/* Finalize function when the caller of XLogReadRecord failed */
+extern void XLogTerminateRead(XLogReaderState *state);
/* Validate a page */
extern bool XLogReaderValidatePageHeader(XLogReaderState *state,
@@ -307,6 +292,7 @@ typedef struct WALReadError
} WALReadError;
extern bool WALRead(XLogReaderState *state,
+ WALSegmentOpenCB segopenfn, WALSegmentCloseCB sgclosefn,
char *buf, XLogRecPtr startptr, Size count,
TimeLineID tli, WALReadError *errinfo);
diff --git a/src/include/access/xlogutils.h b/src/include/access/xlogutils.h
index 8669f7eeb3..e403d253d6 100644
--- a/src/include/access/xlogutils.h
+++ b/src/include/access/xlogutils.h
@@ -89,9 +89,7 @@ extern Buffer XLogReadBufferExtended(RelFileNode rnode, ForkNumber forknum,
extern Relation CreateFakeRelcacheEntry(RelFileNode rnode);
extern void FreeFakeRelcacheEntry(Relation fakerel);
-extern bool read_local_xlog_page(XLogReaderState *state,
- XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetRecPtr, char *cur_page);
+extern bool read_local_xlog_page(XLogReaderState *state);
extern void wal_segment_open(XLogReaderState *state,
XLogSegNo nextSegNo,
TimeLineID *tli_p);
diff --git a/src/include/pg_config_manual.h b/src/include/pg_config_manual.h
index 614035e215..fecffdb3f6 100644
--- a/src/include/pg_config_manual.h
+++ b/src/include/pg_config_manual.h
@@ -390,7 +390,7 @@
* Enable debugging print statements for WAL-related operations; see
* also the wal_debug GUC var.
*/
-/* #define WAL_DEBUG */
+#define WAL_DEBUG
/*
* Enable tracing of resource consumption during sort operations;
diff --git a/src/include/replication/logical.h b/src/include/replication/logical.h
index e0f513b773..3346188b48 100644
--- a/src/include/replication/logical.h
+++ b/src/include/replication/logical.h
@@ -29,6 +29,10 @@ typedef void (*LogicalOutputPluginWriterUpdateProgress) (struct LogicalDecodingC
TransactionId xid
);
+typedef struct LogicalDecodingContext LogicalDecodingContext;
+
+typedef bool (*LogicalDecodingXLogPageReadCB)(XLogReaderState *ctx);
+
typedef struct LogicalDecodingContext
{
/* memory context this is all allocated in */
@@ -39,6 +43,7 @@ typedef struct LogicalDecodingContext
/* infrastructure pieces for decoding */
XLogReaderState *reader;
+ LogicalDecodingXLogPageReadCB page_read;
struct ReorderBuffer *reorder;
struct SnapBuild *snapshot_builder;
@@ -115,14 +120,16 @@ extern LogicalDecodingContext *CreateInitDecodingContext(const char *plugin,
List *output_plugin_options,
bool need_full_snapshot,
XLogRecPtr restart_lsn,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress);
extern LogicalDecodingContext *CreateDecodingContext(XLogRecPtr start_lsn,
List *output_plugin_options,
bool fast_forward,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress);
--
2.27.0
----Next_Part(Thu_Oct__7_17_28_20_2021_531)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
filename="v18-0003-Remove-globals-readOff-readLen-and-readSegNo.patch"
^ permalink raw reply [nested|flat] 70+ messages in thread
* improve DEBUG1 logging of parallel workers for CREATE INDEX?
@ 2025-01-03 18:30 Sami Imseih <[email protected]>
2025-01-08 15:21 ` Re: improve DEBUG1 logging of parallel workers for CREATE INDEX? Bertrand Drouvot <[email protected]>
0 siblings, 1 reply; 70+ messages in thread
From: Sami Imseih @ 2025-01-03 18:30 UTC (permalink / raw)
To: pgsql-hackers
While reviewing patch [1], I realized that the DEBUG1 message
for CREATE INDEX could do better in providing information
about parallel workers launched. Currently, the message just
shows how many workers are planned, but a user may want
to ensure that they have the appropriate number of workers
launched as well when they enable DEBUG1 logging.
Therefore, I want to suggest that instead of:
postgres=# CREATE INDEX tbl_c1 ON tbl(c1);
DEBUG: building index "tbl_c1" on table "tbl" with request for 2
parallel workers
DEBUG: index "tbl_c1" can safely use deduplication
CREATE INDEX
to enhance the message to:
postgres=# create index tbl_c1 on tbl(c1);
DEBUG: building index "tbl_c1" on table "tbl"
DEBUG: launched 0 parallel vacuum workers for index vacuuming (planned: 2)
DEBUG: index "tbl_c1" can safely use deduplication
CREATE INDEX
postgres=#
I prepared a simple patch, attached, for this. The log message matches
the format
used in VACUUM VERBOSE ( for consistency sake ).
Also, I think the documentation in [2] should be updated to mention that DEBUG1
can be used to view parallel worker usage for the build, but I have
not included it in
the patch yet.
Any thoughts?
[1] https://www.postgresql.org/message-id/CAA5RZ0tS%3DFmgku9%3DqCp-U4EBC-dqVtj%2Bv_EGrKV_NA%3DTHnUDsQ%40...
[2] https://www.postgresql.org/docs/current/sql-createindex.html
Regards,
Sami Imseih
Amazon Web Services (AWS)
Attachments:
[application/octet-stream] 0001-improve-DEBUG1-logging-of-parallel-workers-for-CREAT.patch (2.5K, ../../CAA5RZ0trTUL6_vpvW79daGgkp7B-ZtWUc5yrPz5Sjm8Ns4KRgQ@mail.gmail.com/2-0001-improve-DEBUG1-logging-of-parallel-workers-for-CREAT.patch)
download | inline diff:
From 1cbe4d0aa375df0371da19ca88d2e0dda2708ea2 Mon Sep 17 00:00:00 2001
From: Sami Imseih <[email protected]>
Date: Fri, 3 Jan 2025 12:16:37 -0600
Subject: [PATCH 1/1] improve DEBUG1 logging of parallel workers for CREATE
INDEX
---
src/backend/access/brin/brin.c | 4 ++++
src/backend/access/nbtree/nbtsort.c | 4 ++++
src/backend/catalog/index.c | 5 ++---
3 files changed, 10 insertions(+), 3 deletions(-)
diff --git a/src/backend/access/brin/brin.c b/src/backend/access/brin/brin.c
index 9a98454757..f587e13135 100644
--- a/src/backend/access/brin/brin.c
+++ b/src/backend/access/brin/brin.c
@@ -2512,6 +2512,10 @@ _brin_begin_parallel(BrinBuildState *buildstate, Relation heap, Relation index,
brinleader->walusage = walusage;
brinleader->bufferusage = bufferusage;
+ ereport(DEBUG1,
+ (errmsg_internal("launched %d parallel vacuum workers for index vacuuming (planned: %d)",
+ pcxt->nworkers_launched, pcxt->nworkers_to_launch)));
+
/* If no workers were successfully launched, back out (do serial build) */
if (pcxt->nworkers_launched == 0)
{
diff --git a/src/backend/access/nbtree/nbtsort.c b/src/backend/access/nbtree/nbtsort.c
index 7aba852db9..b9f39aea73 100644
--- a/src/backend/access/nbtree/nbtsort.c
+++ b/src/backend/access/nbtree/nbtsort.c
@@ -1582,6 +1582,10 @@ _bt_begin_parallel(BTBuildState *buildstate, bool isconcurrent, int request)
btleader->walusage = walusage;
btleader->bufferusage = bufferusage;
+ ereport(DEBUG1,
+ (errmsg_internal("launched %d parallel vacuum workers for index vacuuming (planned: %d)",
+ pcxt->nworkers_launched, pcxt->nworkers_to_launch)));
+
/* If no workers were successfully launched, back out (do serial build) */
if (pcxt->nworkers_launched == 0)
{
diff --git a/src/backend/catalog/index.c b/src/backend/catalog/index.c
index 221fbb4e28..e7846fd9d4 100644
--- a/src/backend/catalog/index.c
+++ b/src/backend/catalog/index.c
@@ -3007,10 +3007,9 @@ index_build(Relation heapRelation,
RelationGetRelationName(heapRelation))));
else
ereport(DEBUG1,
- (errmsg_internal("building index \"%s\" on table \"%s\" with request for %d parallel workers",
+ (errmsg_internal("building index \"%s\" on table \"%s\"",
RelationGetRelationName(indexRelation),
- RelationGetRelationName(heapRelation),
- indexInfo->ii_ParallelWorkers)));
+ RelationGetRelationName(heapRelation))));
/*
* Switch to the table owner's userid, so that any index functions are run
--
2.39.5 (Apple Git-154)
^ permalink raw reply [nested|flat] 70+ messages in thread
* Re: improve DEBUG1 logging of parallel workers for CREATE INDEX?
2025-01-03 18:30 improve DEBUG1 logging of parallel workers for CREATE INDEX? Sami Imseih <[email protected]>
@ 2025-01-08 15:21 ` Bertrand Drouvot <[email protected]>
2025-01-08 17:36 ` Re: improve DEBUG1 logging of parallel workers for CREATE INDEX? Sami Imseih <[email protected]>
0 siblings, 1 reply; 70+ messages in thread
From: Bertrand Drouvot @ 2025-01-08 15:21 UTC (permalink / raw)
To: Sami Imseih <[email protected]>; +Cc: pgsql-hackers
Hi,
On Fri, Jan 03, 2025 at 12:30:14PM -0600, Sami Imseih wrote:
> While reviewing patch [1], I realized that the DEBUG1 message
> for CREATE INDEX could do better in providing information
> about parallel workers launched. Currently, the message just
> shows how many workers are planned, but a user may want
> to ensure that they have the appropriate number of workers
> launched as well when they enable DEBUG1 logging.
Yeah, one could see how many workers are currently running but would not be
able to figure out once the index is created. So adding the info in the log
makes sense to me.
> I prepared a simple patch, attached, for this. The log message matches
> the format
> used in VACUUM VERBOSE ( for consistency sake ).
A few random comments:
=== 1
s/parallel vacuum workers for index vacuuming/parallel workers for index creation/?
(2 times)
=== 2
- (errmsg_internal("building index \"%s\" on table \"%s\" with request for %d parallel workers",
+ (errmsg_internal("building index \"%s\" on table \"%s\"",
I'd add "in parallel" to match its counterpart "serially" above. That would
make it more clear in case one just look for "building index" in the log.
Regards,
--
Bertrand Drouvot
PostgreSQL Contributors Team
RDS Open Source Databases
Amazon Web Services: https://aws.amazon.com
^ permalink raw reply [nested|flat] 70+ messages in thread
* Re: improve DEBUG1 logging of parallel workers for CREATE INDEX?
2025-01-03 18:30 improve DEBUG1 logging of parallel workers for CREATE INDEX? Sami Imseih <[email protected]>
2025-01-08 15:21 ` Re: improve DEBUG1 logging of parallel workers for CREATE INDEX? Bertrand Drouvot <[email protected]>
@ 2025-01-08 17:36 ` Sami Imseih <[email protected]>
2025-01-08 20:21 ` Re: improve DEBUG1 logging of parallel workers for CREATE INDEX? Guillaume Lelarge <[email protected]>
0 siblings, 1 reply; 70+ messages in thread
From: Sami Imseih @ 2025-01-08 17:36 UTC (permalink / raw)
To: Bertrand Drouvot <[email protected]>; +Cc: pgsql-hackers
> s/parallel vacuum workers for index vacuuming/parallel workers for index creation/?
oops, that's my oversight from copying the message from vacuum. fixed.
> I'd add "in parallel" to match its counterpart "serially" above. That would
> make it more clear in case one just look for "building index" in the log.
good point.
Below is the new output with the attached v2.
postgres=# create index foo_idx1 on foo(id);
DEBUG: building index "foo_idx1" on table "foo" with parallel workers
DEBUG: launched 1 parallel workers for index creation (planned: 1)
DEBUG: index "foo_idx1" can safely use deduplication
CREATE INDEX
Regards,
Sami
Attachments:
[application/octet-stream] v2-0001-improve-DEBUG1-logging-of-parallel-workers-for-CR.patch (2.5K, ../../CAA5RZ0t2y1NVt3V+wVp=1cwi1oKKQif7+NrCVRoJg-oj_ScPfg@mail.gmail.com/2-v2-0001-improve-DEBUG1-logging-of-parallel-workers-for-CR.patch)
download | inline diff:
From d011edabb6cda6fce4b06c2b08e389ae11c7470c Mon Sep 17 00:00:00 2001
From: Sami Imseih <[email protected]>
Date: Wed, 8 Jan 2025 11:20:59 -0600
Subject: [PATCH v2 1/1] improve DEBUG1 logging of parallel workers for CREATE
INDEX
---
src/backend/access/brin/brin.c | 4 ++++
src/backend/access/nbtree/nbtsort.c | 4 ++++
src/backend/catalog/index.c | 5 ++---
3 files changed, 10 insertions(+), 3 deletions(-)
diff --git a/src/backend/access/brin/brin.c b/src/backend/access/brin/brin.c
index 9a98454757..7f72e5a4f9 100644
--- a/src/backend/access/brin/brin.c
+++ b/src/backend/access/brin/brin.c
@@ -2512,6 +2512,10 @@ _brin_begin_parallel(BrinBuildState *buildstate, Relation heap, Relation index,
brinleader->walusage = walusage;
brinleader->bufferusage = bufferusage;
+ ereport(DEBUG1,
+ (errmsg_internal("launched %d parallel workers for index creation (planned: %d)",
+ pcxt->nworkers_launched, pcxt->nworkers_to_launch)));
+
/* If no workers were successfully launched, back out (do serial build) */
if (pcxt->nworkers_launched == 0)
{
diff --git a/src/backend/access/nbtree/nbtsort.c b/src/backend/access/nbtree/nbtsort.c
index 7aba852db9..fe13f5fedf 100644
--- a/src/backend/access/nbtree/nbtsort.c
+++ b/src/backend/access/nbtree/nbtsort.c
@@ -1582,6 +1582,10 @@ _bt_begin_parallel(BTBuildState *buildstate, bool isconcurrent, int request)
btleader->walusage = walusage;
btleader->bufferusage = bufferusage;
+ ereport(DEBUG1,
+ (errmsg_internal("launched %d parallel workers for index creation (planned: %d)",
+ pcxt->nworkers_launched, pcxt->nworkers_to_launch)));
+
/* If no workers were successfully launched, back out (do serial build) */
if (pcxt->nworkers_launched == 0)
{
diff --git a/src/backend/catalog/index.c b/src/backend/catalog/index.c
index 221fbb4e28..40b97c3885 100644
--- a/src/backend/catalog/index.c
+++ b/src/backend/catalog/index.c
@@ -3007,10 +3007,9 @@ index_build(Relation heapRelation,
RelationGetRelationName(heapRelation))));
else
ereport(DEBUG1,
- (errmsg_internal("building index \"%s\" on table \"%s\" with request for %d parallel workers",
+ (errmsg_internal("building index \"%s\" on table \"%s\" with parallel workers",
RelationGetRelationName(indexRelation),
- RelationGetRelationName(heapRelation),
- indexInfo->ii_ParallelWorkers)));
+ RelationGetRelationName(heapRelation))));
/*
* Switch to the table owner's userid, so that any index functions are run
--
2.39.5 (Apple Git-154)
^ permalink raw reply [nested|flat] 70+ messages in thread
* Re: improve DEBUG1 logging of parallel workers for CREATE INDEX?
2025-01-03 18:30 improve DEBUG1 logging of parallel workers for CREATE INDEX? Sami Imseih <[email protected]>
2025-01-08 15:21 ` Re: improve DEBUG1 logging of parallel workers for CREATE INDEX? Bertrand Drouvot <[email protected]>
2025-01-08 17:36 ` Re: improve DEBUG1 logging of parallel workers for CREATE INDEX? Sami Imseih <[email protected]>
@ 2025-01-08 20:21 ` Guillaume Lelarge <[email protected]>
2025-01-08 20:35 ` Re: improve DEBUG1 logging of parallel workers for CREATE INDEX? Tom Lane <[email protected]>
0 siblings, 1 reply; 70+ messages in thread
From: Guillaume Lelarge @ 2025-01-08 20:21 UTC (permalink / raw)
To: Sami Imseih <[email protected]>; +Cc: Bertrand Drouvot <[email protected]>; pgsql-hackers
Hi,
Le mer. 8 janv. 2025 à 18:37, Sami Imseih <[email protected]> a écrit :
> > s/parallel vacuum workers for index vacuuming/parallel workers for index
> creation/?
>
> oops, that's my oversight from copying the message from vacuum. fixed.
>
> > I'd add "in parallel" to match its counterpart "serially" above. That
> would
> > make it more clear in case one just look for "building index" in the log.
>
> good point.
>
> Below is the new output with the attached v2.
>
> postgres=# create index foo_idx1 on foo(id);
> DEBUG: building index "foo_idx1" on table "foo" with parallel workers
> DEBUG: launched 1 parallel workers for index creation (planned: 1)
> DEBUG: index "foo_idx1" can safely use deduplication
> CREATE INDEX
>
>
I tend to agree it might be better than Benoit's patch on the index
messages, though I'm afraid that DEBUG1 level won't work for many users.
DEBUGx are for developers, not users. A better log level for me would be
LOG. For client messages, LOG messages won't be displayed by default. So
there's still a need for a "SET client_min_messages to LOG" but it won't be
necessary to go all the way to DEBUG1.
Regards.
--
Guillaume.
^ permalink raw reply [nested|flat] 70+ messages in thread
* Re: improve DEBUG1 logging of parallel workers for CREATE INDEX?
2025-01-03 18:30 improve DEBUG1 logging of parallel workers for CREATE INDEX? Sami Imseih <[email protected]>
2025-01-08 15:21 ` Re: improve DEBUG1 logging of parallel workers for CREATE INDEX? Bertrand Drouvot <[email protected]>
2025-01-08 17:36 ` Re: improve DEBUG1 logging of parallel workers for CREATE INDEX? Sami Imseih <[email protected]>
2025-01-08 20:21 ` Re: improve DEBUG1 logging of parallel workers for CREATE INDEX? Guillaume Lelarge <[email protected]>
@ 2025-01-08 20:35 ` Tom Lane <[email protected]>
2025-01-08 20:44 ` Re: improve DEBUG1 logging of parallel workers for CREATE INDEX? Sami Imseih <[email protected]>
2025-01-08 20:46 ` Re: improve DEBUG1 logging of parallel workers for CREATE INDEX? Guillaume Lelarge <[email protected]>
0 siblings, 2 replies; 70+ messages in thread
From: Tom Lane @ 2025-01-08 20:35 UTC (permalink / raw)
To: Guillaume Lelarge <[email protected]>; +Cc: Sami Imseih <[email protected]>; Bertrand Drouvot <[email protected]>; pgsql-hackers
Guillaume Lelarge <[email protected]> writes:
> I tend to agree it might be better than Benoit's patch on the index
> messages, though I'm afraid that DEBUG1 level won't work for many users.
> DEBUGx are for developers, not users. A better log level for me would be
> LOG.
Please, no. That would result in spamming the postmaster log
for perfectly ordinary actions, with no usable way to prevent it.
I think DEBUG is perfectly appropriate for this sort of message.
regards, tom lane
^ permalink raw reply [nested|flat] 70+ messages in thread
* Re: improve DEBUG1 logging of parallel workers for CREATE INDEX?
2025-01-03 18:30 improve DEBUG1 logging of parallel workers for CREATE INDEX? Sami Imseih <[email protected]>
2025-01-08 15:21 ` Re: improve DEBUG1 logging of parallel workers for CREATE INDEX? Bertrand Drouvot <[email protected]>
2025-01-08 17:36 ` Re: improve DEBUG1 logging of parallel workers for CREATE INDEX? Sami Imseih <[email protected]>
2025-01-08 20:21 ` Re: improve DEBUG1 logging of parallel workers for CREATE INDEX? Guillaume Lelarge <[email protected]>
2025-01-08 20:35 ` Re: improve DEBUG1 logging of parallel workers for CREATE INDEX? Tom Lane <[email protected]>
@ 2025-01-08 20:44 ` Sami Imseih <[email protected]>
2025-01-08 20:48 ` Re: improve DEBUG1 logging of parallel workers for CREATE INDEX? Guillaume Lelarge <[email protected]>
2025-01-08 20:58 ` Re: improve DEBUG1 logging of parallel workers for CREATE INDEX? Tom Lane <[email protected]>
1 sibling, 2 replies; 70+ messages in thread
From: Sami Imseih @ 2025-01-08 20:44 UTC (permalink / raw)
To: Tom Lane <[email protected]>; +Cc: Guillaume Lelarge <[email protected]>; Bertrand Drouvot <[email protected]>; pgsql-hackers
> I think DEBUG is perfectly appropriate for this sort of message.
I am curious what are the thoughts on introducing a
CREATE INDEX VERBOSE which can provide this info?
similar to users scripting VACUUM VERBOSE to log
more details about the vacuum operation including parallel
usage.
I have not convinced myself this is a good idea, but maybe
someone has an opinion. If not, we can just stick with a better
DEBUG1 message.
Regards,
Sami
^ permalink raw reply [nested|flat] 70+ messages in thread
* Re: improve DEBUG1 logging of parallel workers for CREATE INDEX?
2025-01-03 18:30 improve DEBUG1 logging of parallel workers for CREATE INDEX? Sami Imseih <[email protected]>
2025-01-08 15:21 ` Re: improve DEBUG1 logging of parallel workers for CREATE INDEX? Bertrand Drouvot <[email protected]>
2025-01-08 17:36 ` Re: improve DEBUG1 logging of parallel workers for CREATE INDEX? Sami Imseih <[email protected]>
2025-01-08 20:21 ` Re: improve DEBUG1 logging of parallel workers for CREATE INDEX? Guillaume Lelarge <[email protected]>
2025-01-08 20:35 ` Re: improve DEBUG1 logging of parallel workers for CREATE INDEX? Tom Lane <[email protected]>
2025-01-08 20:44 ` Re: improve DEBUG1 logging of parallel workers for CREATE INDEX? Sami Imseih <[email protected]>
@ 2025-01-08 20:48 ` Guillaume Lelarge <[email protected]>
1 sibling, 0 replies; 70+ messages in thread
From: Guillaume Lelarge @ 2025-01-08 20:48 UTC (permalink / raw)
To: Sami Imseih <[email protected]>; +Cc: Tom Lane <[email protected]>; Bertrand Drouvot <[email protected]>; pgsql-hackers
Le mer. 8 janv. 2025 à 21:44, Sami Imseih <[email protected]> a écrit :
> > I think DEBUG is perfectly appropriate for this sort of message.
>
> I am curious what are the thoughts on introducing a
> CREATE INDEX VERBOSE which can provide this info?
> similar to users scripting VACUUM VERBOSE to log
> more details about the vacuum operation including parallel
> usage.
>
> I have not convinced myself this is a good idea, but maybe
> someone has an opinion. If not, we can just stick with a better
> DEBUG1 message.
>
>
Sounds better to me than relying on DEBUG messages. So, yeah, VERBOSE has
my vote.
--
Guillaume.
^ permalink raw reply [nested|flat] 70+ messages in thread
* Re: improve DEBUG1 logging of parallel workers for CREATE INDEX?
2025-01-03 18:30 improve DEBUG1 logging of parallel workers for CREATE INDEX? Sami Imseih <[email protected]>
2025-01-08 15:21 ` Re: improve DEBUG1 logging of parallel workers for CREATE INDEX? Bertrand Drouvot <[email protected]>
2025-01-08 17:36 ` Re: improve DEBUG1 logging of parallel workers for CREATE INDEX? Sami Imseih <[email protected]>
2025-01-08 20:21 ` Re: improve DEBUG1 logging of parallel workers for CREATE INDEX? Guillaume Lelarge <[email protected]>
2025-01-08 20:35 ` Re: improve DEBUG1 logging of parallel workers for CREATE INDEX? Tom Lane <[email protected]>
2025-01-08 20:44 ` Re: improve DEBUG1 logging of parallel workers for CREATE INDEX? Sami Imseih <[email protected]>
@ 2025-01-08 20:58 ` Tom Lane <[email protected]>
2025-01-09 03:24 ` Re: improve DEBUG1 logging of parallel workers for CREATE INDEX? Sami Imseih <[email protected]>
1 sibling, 1 reply; 70+ messages in thread
From: Tom Lane @ 2025-01-08 20:58 UTC (permalink / raw)
To: Sami Imseih <[email protected]>; +Cc: Guillaume Lelarge <[email protected]>; Bertrand Drouvot <[email protected]>; pgsql-hackers
Sami Imseih <[email protected]> writes:
> I am curious what are the thoughts on introducing a
> CREATE INDEX VERBOSE which can provide this info?
> similar to users scripting VACUUM VERBOSE to log
> more details about the vacuum operation including parallel
> usage.
What I can recall being discussed in the past is to extend
EXPLAIN and/or EXPLAIN ANALYZE to cover utility statements
that have nontrivial execution complexity --- for example,
ALTER TABLE has a lot of machinery underneath, and people
often wish to know things like whether a particular ALTER
will cause a table rewrite or not.
Of course, a patch for that would be a few orders of magnitude
larger than what you've got here :-(. But if you're looking
for a framework for reporting these sorts of details, I'd
much rather go in that direction than follow the model of
VACUUM VERBOSE. VACUUM VERBOSE is a kluge with little to
recommend it other than having been easy to implement.
regards, tom lane
^ permalink raw reply [nested|flat] 70+ messages in thread
* Re: improve DEBUG1 logging of parallel workers for CREATE INDEX?
2025-01-03 18:30 improve DEBUG1 logging of parallel workers for CREATE INDEX? Sami Imseih <[email protected]>
2025-01-08 15:21 ` Re: improve DEBUG1 logging of parallel workers for CREATE INDEX? Bertrand Drouvot <[email protected]>
2025-01-08 17:36 ` Re: improve DEBUG1 logging of parallel workers for CREATE INDEX? Sami Imseih <[email protected]>
2025-01-08 20:21 ` Re: improve DEBUG1 logging of parallel workers for CREATE INDEX? Guillaume Lelarge <[email protected]>
2025-01-08 20:35 ` Re: improve DEBUG1 logging of parallel workers for CREATE INDEX? Tom Lane <[email protected]>
2025-01-08 20:44 ` Re: improve DEBUG1 logging of parallel workers for CREATE INDEX? Sami Imseih <[email protected]>
2025-01-08 20:58 ` Re: improve DEBUG1 logging of parallel workers for CREATE INDEX? Tom Lane <[email protected]>
@ 2025-01-09 03:24 ` Sami Imseih <[email protected]>
2025-01-11 10:29 ` Re: improve DEBUG1 logging of parallel workers for CREATE INDEX? Guillaume Lelarge <[email protected]>
0 siblings, 1 reply; 70+ messages in thread
From: Sami Imseih @ 2025-01-09 03:24 UTC (permalink / raw)
To: Tom Lane <[email protected]>; +Cc: Guillaume Lelarge <[email protected]>; Bertrand Drouvot <[email protected]>; pgsql-hackers
> Of course, a patch for that would be a few orders of magnitude
> larger than what you've got here :-(. But if you're looking
> for a framework for reporting these sorts of details, I'd
> much rather go in that direction than follow the model of
> VACUUM VERBOSE. VACUUM VERBOSE is a kluge with little to
> recommend it other than having been easy to implement.
To my surprise, REINDEX does have a VERBOSE option.
should have check this earlier :)
postgres=# reindex (verbose) index t_idx1;
INFO: index "t_idx1" was reindexed
DETAIL: CPU: user: 5.33 s, system: 0.48 s, elapsed: 6.26 s
REINDEX
Is there a reason not to do the same for CREATE INDEX?
Also, we can improve the REINDEX verbose message by
also providing the parallel usage.
Regards,
Sami
^ permalink raw reply [nested|flat] 70+ messages in thread
* Re: improve DEBUG1 logging of parallel workers for CREATE INDEX?
2025-01-03 18:30 improve DEBUG1 logging of parallel workers for CREATE INDEX? Sami Imseih <[email protected]>
2025-01-08 15:21 ` Re: improve DEBUG1 logging of parallel workers for CREATE INDEX? Bertrand Drouvot <[email protected]>
2025-01-08 17:36 ` Re: improve DEBUG1 logging of parallel workers for CREATE INDEX? Sami Imseih <[email protected]>
2025-01-08 20:21 ` Re: improve DEBUG1 logging of parallel workers for CREATE INDEX? Guillaume Lelarge <[email protected]>
2025-01-08 20:35 ` Re: improve DEBUG1 logging of parallel workers for CREATE INDEX? Tom Lane <[email protected]>
2025-01-08 20:44 ` Re: improve DEBUG1 logging of parallel workers for CREATE INDEX? Sami Imseih <[email protected]>
2025-01-08 20:58 ` Re: improve DEBUG1 logging of parallel workers for CREATE INDEX? Tom Lane <[email protected]>
2025-01-09 03:24 ` Re: improve DEBUG1 logging of parallel workers for CREATE INDEX? Sami Imseih <[email protected]>
@ 2025-01-11 10:29 ` Guillaume Lelarge <[email protected]>
2025-01-18 23:42 ` Re: improve DEBUG1 logging of parallel workers for CREATE INDEX? Sami Imseih <[email protected]>
0 siblings, 1 reply; 70+ messages in thread
From: Guillaume Lelarge @ 2025-01-11 10:29 UTC (permalink / raw)
To: Sami Imseih <[email protected]>; +Cc: Tom Lane <[email protected]>; Bertrand Drouvot <[email protected]>; pgsql-hackers
Le jeu. 9 janv. 2025 à 04:24, Sami Imseih <[email protected]> a écrit :
> > Of course, a patch for that would be a few orders of magnitude
> > larger than what you've got here :-(. But if you're looking
> > for a framework for reporting these sorts of details, I'd
> > much rather go in that direction than follow the model of
> > VACUUM VERBOSE. VACUUM VERBOSE is a kluge with little to
> > recommend it other than having been easy to implement.
>
> To my surprise, REINDEX does have a VERBOSE option.
> should have check this earlier :)
>
> postgres=# reindex (verbose) index t_idx1;
> INFO: index "t_idx1" was reindexed
> DETAIL: CPU: user: 5.33 s, system: 0.48 s, elapsed: 6.26 s
> REINDEX
>
> Is there a reason not to do the same for CREATE INDEX?
>
>
Sounds a good idea to me.
> Also, we can improve the REINDEX verbose message by
> also providing the parallel usage.
>
>
+1
--
Guillaume.
^ permalink raw reply [nested|flat] 70+ messages in thread
* Re: improve DEBUG1 logging of parallel workers for CREATE INDEX?
2025-01-03 18:30 improve DEBUG1 logging of parallel workers for CREATE INDEX? Sami Imseih <[email protected]>
2025-01-08 15:21 ` Re: improve DEBUG1 logging of parallel workers for CREATE INDEX? Bertrand Drouvot <[email protected]>
2025-01-08 17:36 ` Re: improve DEBUG1 logging of parallel workers for CREATE INDEX? Sami Imseih <[email protected]>
2025-01-08 20:21 ` Re: improve DEBUG1 logging of parallel workers for CREATE INDEX? Guillaume Lelarge <[email protected]>
2025-01-08 20:35 ` Re: improve DEBUG1 logging of parallel workers for CREATE INDEX? Tom Lane <[email protected]>
2025-01-08 20:44 ` Re: improve DEBUG1 logging of parallel workers for CREATE INDEX? Sami Imseih <[email protected]>
2025-01-08 20:58 ` Re: improve DEBUG1 logging of parallel workers for CREATE INDEX? Tom Lane <[email protected]>
2025-01-09 03:24 ` Re: improve DEBUG1 logging of parallel workers for CREATE INDEX? Sami Imseih <[email protected]>
2025-01-11 10:29 ` Re: improve DEBUG1 logging of parallel workers for CREATE INDEX? Guillaume Lelarge <[email protected]>
@ 2025-01-18 23:42 ` Sami Imseih <[email protected]>
2025-01-19 00:52 ` Re: improve DEBUG1 logging of parallel workers for CREATE INDEX? Michael Paquier <[email protected]>
0 siblings, 1 reply; 70+ messages in thread
From: Sami Imseih @ 2025-01-18 23:42 UTC (permalink / raw)
To: Guillaume Lelarge <[email protected]>; Bertrand Drouvot <[email protected]>; Tom Lane <[email protected]>; +Cc: pgsql-hackers
The opinion in this thread is leaning towards a VERBOSE
option and I agree with this as it provides the users with a
much more intuitive way to gather parallel information about
parallel index builds. Also, the current DEBUG1 output is
useless as it is now.
Here is a v1 that implements CREATE INDEX (VERBOSE)
to output the parallel workers planned and launched.
A few points about this patch:
1/ the (VERBOSE) option is added after the INDEX.
It seemed better than adding it after the CREATE keyword
to make the option specific to the "CREATE INDEX" operation
rather than the "CREATE" operation which can be many
things.
2/ The workers launched is determined in the index AM,
therefore a new field called "ii_verbose" is added to IndexInfo
so it can be available to the AM to determine if logging
is required. We currently handle the concurrently
option in the same manner.
3/ currently REINDEX (VERBOSE) does not show parallel
workers planned, but it will with this patch. Both
CREATE INDEX and REINDEX will show the same output.
4/ Updated auto-completed in psql to deal with (VERBOSE)
5/ Removed the existing DEBUG1 output.
6/ Updated the tests for create index/reinde
Here is sample output:
postgres=# CREATE INDEX (VERBOSE ON) ON t(c1);
INFO: building index "t_c1_idx" on table "t" with parallel workers
INFO: launched 1 parallel workers for index creation (planned: 1)
INFO: index "public.t" was created
DETAIL: CPU: user: 0.88 s, system: 0.01 s, elapsed: 0.89 s.
CREATE INDEX
postgres=#
postgres=# REINDEX (verbose) INDEX t_c1_idx ;
INFO: building index "t_c1_idx" on table "t" with parallel workers
INFO: launched 1 parallel workers for index creation (planned: 1)
INFO: index "t_c1_idx" was reindexed
DETAIL: CPU: user: 0.88 s, system: 0.00 s, elapsed: 0.89 s
REINDEX
Looking forward to feedback.
Regards,
Sami
Attachments:
[application/octet-stream] v1-0001-Improve-Parallel-worker-reporting-in-CREATE-INDEX.patch (30.9K, ../../CAA5RZ0v1DsqibU_am-CGM=_ve4_YMH+PXyfPL64rBzCgojpVMg@mail.gmail.com/2-v1-0001-Improve-Parallel-worker-reporting-in-CREATE-INDEX.patch)
download | inline diff:
From bca38498155eba925cd89829781ec0ab33d6cd1f Mon Sep 17 00:00:00 2001
From: Sami Imseih <[email protected]>
Date: Sat, 18 Jan 2025 16:43:14 -0600
Subject: [PATCH v1 1/1] Improve Parallel worker reporting in CREATE INDEX and
REINDEX
Prior to this patch, only DEBUG1 logging displayed the number
of parallel workers planned for a CREATE INDEX or REINDEX
operation. Furthermore, the DEBUG1 did not show the number
of workers launched. This patch improves the situation by
allowing for a CREATE INDEX with a VERBOSE option which will
show both the number of parallel workers planned and the number
launched at INFO level. REINDEX already has a VERBOSE option and
it will also show the same parallel information as CREATE INDEX.
In the future, VERBOSE can be expanded to track progress for other
steps being performed by CREATE INDEX or REINDEX.
As parallel workers launched can only be determined in the Index AM's,
a verbose field was added to IndexInfo to instruct the Index AM to print
the logging information.
---
doc/src/sgml/ref/create_index.sgml | 23 +++++++++-
src/backend/access/brin/brin.c | 11 +++--
src/backend/access/nbtree/nbtsort.c | 11 +++--
src/backend/bootstrap/bootparse.y | 2 +
src/backend/catalog/index.c | 39 +++++++++-------
src/backend/catalog/toasting.c | 1 +
src/backend/commands/indexcmds.c | 52 +++++++++++++++++++---
src/backend/commands/tablecmds.c | 7 +--
src/backend/nodes/makefuncs.c | 3 +-
src/backend/parser/gram.y | 52 +++++++++++++---------
src/backend/tcop/utility.c | 21 ++++++++-
src/bin/psql/tab-complete.in.c | 14 ++++++
src/include/catalog/index.h | 3 +-
src/include/commands/defrem.h | 3 +-
src/include/nodes/execnodes.h | 3 +-
src/include/nodes/makefuncs.h | 3 +-
src/include/nodes/parsenodes.h | 1 +
src/test/regress/expected/create_index.out | 19 ++++++++
src/test/regress/sql/create_index.sql | 12 +++++
19 files changed, 220 insertions(+), 60 deletions(-)
diff --git a/doc/src/sgml/ref/create_index.sgml b/doc/src/sgml/ref/create_index.sgml
index 208389e800..7b6f2eab34 100644
--- a/doc/src/sgml/ref/create_index.sgml
+++ b/doc/src/sgml/ref/create_index.sgml
@@ -21,7 +21,7 @@ PostgreSQL documentation
<refsynopsisdiv>
<synopsis>
-CREATE [ UNIQUE ] INDEX [ CONCURRENTLY ] [ [ IF NOT EXISTS ] <replaceable class="parameter">name</replaceable> ] ON [ ONLY ] <replaceable class="parameter">table_name</replaceable> [ USING <replaceable class="parameter">method</replaceable> ]
+CREATE [ UNIQUE ] INDEX [ ( VERBOSE <replaceable class="parameter">boolean</replaceable> ) ] [ CONCURRENTLY ] [ [ IF NOT EXISTS ] <replaceable class="parameter">name</replaceable> ] ON [ ONLY ] <replaceable class="parameter">table_name</replaceable> [ USING <replaceable class="parameter">method</replaceable> ]
( { <replaceable class="parameter">column_name</replaceable> | ( <replaceable class="parameter">expression</replaceable> ) } [ COLLATE <replaceable class="parameter">collation</replaceable> ] [ <replaceable class="parameter">opclass</replaceable> [ ( <replaceable class="parameter">opclass_parameter</replaceable> = <replaceable class="parameter">value</replaceable> [, ... ] ) ] ] [ ASC | DESC ] [ NULLS { FIRST | LAST } ] [, ...] )
[ INCLUDE ( <replaceable class="parameter">column_name</replaceable> [, ...] ) ]
[ NULLS [ NOT ] DISTINCT ]
@@ -149,6 +149,16 @@ CREATE [ UNIQUE ] INDEX [ CONCURRENTLY ] [ [ IF NOT EXISTS ] <replaceable class=
</listitem>
</varlistentry>
+ <varlistentry>
+ <term><literal>VERBOSE</literal></term>
+ <listitem>
+ <para>
+ Prints information about resource usage during the index creation.
+ at <literal>INFO</literal> level.
+ </para>
+ </listitem>
+ </varlistentry>
+
<varlistentry>
<term><literal>INCLUDE</literal></term>
<listitem>
@@ -704,6 +714,17 @@ Indexes:
</refsect2>
</refsect1>
+ <refsect1>
+ <title>Outputs</title>
+
+ <para>
+ When <literal>VERBOSE</literal> is specified, <command>CREATE INDEX</command>
+ reports resource usage by the operation, whether the index is being built
+ serially or with parallel workers, and, for parallel index builds, the number
+ of workers planned and the number of workers launched.
+ </para>
+ </refsect1>
+
<refsect1>
<title>Notes</title>
diff --git a/src/backend/access/brin/brin.c b/src/backend/access/brin/brin.c
index 4289142e20..d6e0aa6a34 100644
--- a/src/backend/access/brin/brin.c
+++ b/src/backend/access/brin/brin.c
@@ -229,7 +229,7 @@ static void brin_fill_empty_ranges(BrinBuildState *state,
/* parallel index builds */
static void _brin_begin_parallel(BrinBuildState *buildstate, Relation heap, Relation index,
- bool isconcurrent, int request);
+ bool isconcurrent, int request, bool verbose);
static void _brin_end_parallel(BrinLeader *brinleader, BrinBuildState *state);
static Size _brin_parallel_estimate_shared(Relation heap, Snapshot snapshot);
static double _brin_parallel_heapscan(BrinBuildState *state);
@@ -1161,7 +1161,7 @@ brinbuild(Relation heap, Relation index, IndexInfo *indexInfo)
*/
if (indexInfo->ii_ParallelWorkers > 0)
_brin_begin_parallel(state, heap, index, indexInfo->ii_Concurrent,
- indexInfo->ii_ParallelWorkers);
+ indexInfo->ii_ParallelWorkers, indexInfo->ii_verbose);
/*
* If parallel build requested and at least one worker process was
@@ -2353,7 +2353,7 @@ check_null_keys(BrinValues *bval, ScanKey *nullkeys, int nnullkeys)
*/
static void
_brin_begin_parallel(BrinBuildState *buildstate, Relation heap, Relation index,
- bool isconcurrent, int request)
+ bool isconcurrent, int request, bool verbose)
{
ParallelContext *pcxt;
int scantuplesortstates;
@@ -2512,6 +2512,11 @@ _brin_begin_parallel(BrinBuildState *buildstate, Relation heap, Relation index,
brinleader->walusage = walusage;
brinleader->bufferusage = bufferusage;
+ if (verbose)
+ ereport(INFO,
+ (errmsg_internal("launched %d parallel workers for index creation (planned: %d)",
+ pcxt->nworkers_launched, pcxt->nworkers_to_launch)));
+
/* If no workers were successfully launched, back out (do serial build) */
if (pcxt->nworkers_launched == 0)
{
diff --git a/src/backend/access/nbtree/nbtsort.c b/src/backend/access/nbtree/nbtsort.c
index 7aba852db9..f6ce33e17a 100644
--- a/src/backend/access/nbtree/nbtsort.c
+++ b/src/backend/access/nbtree/nbtsort.c
@@ -276,7 +276,7 @@ static void _bt_uppershutdown(BTWriteState *wstate, BTPageState *state);
static void _bt_load(BTWriteState *wstate,
BTSpool *btspool, BTSpool *btspool2);
static void _bt_begin_parallel(BTBuildState *buildstate, bool isconcurrent,
- int request);
+ int request, bool verbose);
static void _bt_end_parallel(BTLeader *btleader);
static Size _bt_parallel_estimate_shared(Relation heap, Snapshot snapshot);
static double _bt_parallel_heapscan(BTBuildState *buildstate,
@@ -390,7 +390,7 @@ _bt_spools_heapscan(Relation heap, Relation index, BTBuildState *buildstate,
/* Attempt to launch parallel worker scan when required */
if (indexInfo->ii_ParallelWorkers > 0)
_bt_begin_parallel(buildstate, indexInfo->ii_Concurrent,
- indexInfo->ii_ParallelWorkers);
+ indexInfo->ii_ParallelWorkers, indexInfo->ii_verbose);
/*
* If parallel build requested and at least one worker process was
@@ -1395,7 +1395,7 @@ _bt_load(BTWriteState *wstate, BTSpool *btspool, BTSpool *btspool2)
* never set, and caller should proceed with a serial index build.
*/
static void
-_bt_begin_parallel(BTBuildState *buildstate, bool isconcurrent, int request)
+_bt_begin_parallel(BTBuildState *buildstate, bool isconcurrent, int request, bool verbose)
{
ParallelContext *pcxt;
int scantuplesortstates;
@@ -1582,6 +1582,11 @@ _bt_begin_parallel(BTBuildState *buildstate, bool isconcurrent, int request)
btleader->walusage = walusage;
btleader->bufferusage = bufferusage;
+ if (verbose)
+ ereport(INFO,
+ (errmsg_internal("launched %d parallel workers for index creation (planned: %d)",
+ pcxt->nworkers_launched, pcxt->nworkers_to_launch)));
+
/* If no workers were successfully launched, back out (do serial build) */
if (pcxt->nworkers_launched == 0)
{
diff --git a/src/backend/bootstrap/bootparse.y b/src/backend/bootstrap/bootparse.y
index 9833f52c1b..0583bb3bcb 100644
--- a/src/backend/bootstrap/bootparse.y
+++ b/src/backend/bootstrap/bootparse.y
@@ -318,6 +318,7 @@ Boot_DeclareIndexStmt:
false,
false,
true, /* skip_build */
+ false,
false);
do_end();
}
@@ -371,6 +372,7 @@ Boot_DeclareUniqueIndexStmt:
false,
false,
true, /* skip_build */
+ false,
false);
do_end();
}
diff --git a/src/backend/catalog/index.c b/src/backend/catalog/index.c
index 7377912b41..2952b4f9e1 100644
--- a/src/backend/catalog/index.c
+++ b/src/backend/catalog/index.c
@@ -1396,7 +1396,8 @@ index_concurrently_create_copy(Relation heapRelation, Oid oldIndexId,
false, /* not ready for inserts */
true,
indexRelation->rd_indam->amsummarizing,
- oldInfo->ii_WithoutOverlaps);
+ oldInfo->ii_WithoutOverlaps,
+ false);
/*
* Extract the list of column names and the column numbers for the new
@@ -1482,7 +1483,8 @@ index_concurrently_create_copy(Relation heapRelation, Oid oldIndexId,
*/
void
index_concurrently_build(Oid heapRelationId,
- Oid indexRelationId)
+ Oid indexRelationId,
+ bool verbose)
{
Relation heapRel;
Oid save_userid;
@@ -1519,6 +1521,7 @@ index_concurrently_build(Oid heapRelationId,
Assert(!indexInfo->ii_ReadyForInserts);
indexInfo->ii_Concurrent = true;
indexInfo->ii_BrokenHotChain = false;
+ indexInfo->ii_verbose = verbose;
/* Now build the index */
index_build(heapRel, indexRelation, indexInfo, false, true);
@@ -2451,7 +2454,8 @@ BuildIndexInfo(Relation index)
indexStruct->indisready,
false,
index->rd_indam->amsummarizing,
- indexStruct->indisexclusion && indexStruct->indisunique);
+ indexStruct->indisexclusion && indexStruct->indisunique,
+ false);
/* fill in attribute numbers */
for (i = 0; i < numAtts; i++)
@@ -2511,7 +2515,8 @@ BuildDummyIndexInfo(Relation index)
indexStruct->indisready,
false,
index->rd_indam->amsummarizing,
- indexStruct->indisexclusion && indexStruct->indisunique);
+ indexStruct->indisexclusion && indexStruct->indisunique,
+ false);
/* fill in attribute numbers */
for (i = 0; i < numAtts; i++)
@@ -3001,17 +3006,19 @@ index_build(Relation heapRelation,
plan_create_index_workers(RelationGetRelid(heapRelation),
RelationGetRelid(indexRelation));
- if (indexInfo->ii_ParallelWorkers == 0)
- ereport(DEBUG1,
- (errmsg_internal("building index \"%s\" on table \"%s\" serially",
- RelationGetRelationName(indexRelation),
- RelationGetRelationName(heapRelation))));
- else
- ereport(DEBUG1,
- (errmsg_internal("building index \"%s\" on table \"%s\" with request for %d parallel workers",
- RelationGetRelationName(indexRelation),
- RelationGetRelationName(heapRelation),
- indexInfo->ii_ParallelWorkers)));
+ if (indexInfo->ii_verbose)
+ {
+ if (indexInfo->ii_ParallelWorkers == 0)
+ ereport(INFO,
+ (errmsg_internal("building index \"%s\" on table \"%s\" serially",
+ RelationGetRelationName(indexRelation),
+ RelationGetRelationName(heapRelation))));
+ else
+ ereport(INFO,
+ (errmsg_internal("building index \"%s\" on table \"%s\" with parallel workers",
+ RelationGetRelationName(indexRelation),
+ RelationGetRelationName(heapRelation))));
+ }
/*
* Switch to the table owner's userid, so that any index functions are run
@@ -3774,6 +3781,8 @@ reindex_index(const ReindexStmt *stmt, Oid indexId,
indexInfo->ii_ExclusionStrats = NULL;
}
+ indexInfo->ii_verbose = params->options & REINDEXOPT_VERBOSE;
+
/* Suppress use of the target index while rebuilding it */
SetReindexProcessing(heapId, indexId);
diff --git a/src/backend/catalog/toasting.c b/src/backend/catalog/toasting.c
index 874a8fc89a..57f9e9cbce 100644
--- a/src/backend/catalog/toasting.c
+++ b/src/backend/catalog/toasting.c
@@ -308,6 +308,7 @@ create_toast_table(Relation rel, Oid toastOid, Oid toastIndexOid,
indexInfo->ii_Am = BTREE_AM_OID;
indexInfo->ii_AmCache = NULL;
indexInfo->ii_Context = CurrentMemoryContext;
+ indexInfo->ii_verbose = false;
collationIds[0] = InvalidOid;
collationIds[1] = InvalidOid;
diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 59c836fc24..1257becea5 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -244,7 +244,8 @@ CheckIndexCompatible(Oid oldId,
*/
indexInfo = makeIndexInfo(numberOfAttributes, numberOfAttributes,
accessMethodId, NIL, NIL, false, false,
- false, false, amsummarizing, isWithoutOverlaps);
+ false, false, amsummarizing, isWithoutOverlaps,
+ false);
typeIds = palloc_array(Oid, numberOfAttributes);
collationIds = palloc_array(Oid, numberOfAttributes);
opclassIds = palloc_array(Oid, numberOfAttributes);
@@ -550,7 +551,8 @@ DefineIndex(Oid tableId,
bool check_rights,
bool check_not_in_use,
bool skip_build,
- bool quiet)
+ bool quiet,
+ bool verbose)
{
bool concurrent;
char *indexRelationName;
@@ -591,11 +593,14 @@ DefineIndex(Oid tableId,
Oid root_save_userid;
int root_save_sec_context;
int root_save_nestlevel;
+ PGRUsage ru0;
root_save_nestlevel = NewGUCNestLevel();
RestrictSearchPath();
+ pg_rusage_init(&ru0);
+
/*
* Some callers need us to run with an empty default_tablespace; this is a
* necessary hack to be able to reproduce catalog state accurately when
@@ -929,7 +934,8 @@ DefineIndex(Oid tableId,
!concurrent,
concurrent,
amissummarizing,
- stmt->iswithoutoverlaps);
+ stmt->iswithoutoverlaps,
+ verbose);
typeIds = palloc_array(Oid, numberOfAttributes);
collationIds = palloc_array(Oid, numberOfAttributes);
@@ -1246,7 +1252,16 @@ DefineIndex(Oid tableId,
/* If this is the top-level index, we're done */
if (!OidIsValid(parentIndexId))
+ {
+ /* Log what we did */
+ if (indexInfo->ii_verbose)
+ ereport(INFO,
+ (errmsg("index \"%s.%s\" was created",
+ get_namespace_name(RelationGetNamespace(rel)), RelationGetRelationName(rel)),
+ errdetail("%s.", pg_rusage_show(&ru0))));
+
pgstat_progress_end_command();
+ }
return address;
}
@@ -1502,7 +1517,7 @@ DefineIndex(Oid tableId,
-1,
is_alter_table, check_rights,
check_not_in_use,
- skip_build, quiet);
+ skip_build, quiet, false);
SetUserIdAndSecContext(child_save_userid,
child_save_sec_context);
@@ -1559,7 +1574,16 @@ DefineIndex(Oid tableId,
SetUserIdAndSecContext(root_save_userid, root_save_sec_context);
table_close(rel, NoLock);
if (!OidIsValid(parentIndexId))
+ {
+ /* Log what we did */
+ if (indexInfo->ii_verbose)
+ ereport(INFO,
+ (errmsg("index \"%s.%s\" was created",
+ get_namespace_name(RelationGetNamespace(rel)), RelationGetRelationName(rel)),
+ errdetail("%s.", pg_rusage_show(&ru0))));
+
pgstat_progress_end_command();
+ }
else
{
/* Update progress for an intermediate partitioned index itself */
@@ -1582,7 +1606,16 @@ DefineIndex(Oid tableId,
* otherwise, increment progress to report one child index is done.
*/
if (!OidIsValid(parentIndexId))
+ {
+ /* Log what we did */
+ if (indexInfo->ii_verbose)
+ ereport(INFO,
+ (errmsg("index \"%s.%s\" was created",
+ get_namespace_name(RelationGetNamespace(rel)), RelationGetRelationName(rel)),
+ errdetail("%s.", pg_rusage_show(&ru0))));
+
pgstat_progress_end_command();
+ }
else
pgstat_progress_incr_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, 1);
@@ -1682,7 +1715,7 @@ DefineIndex(Oid tableId,
PushActiveSnapshot(GetTransactionSnapshot());
/* Perform concurrent build of index */
- index_concurrently_build(tableId, indexRelationId);
+ index_concurrently_build(tableId, indexRelationId, verbose);
/* we can do away with our snapshot */
PopActiveSnapshot();
@@ -1798,6 +1831,13 @@ DefineIndex(Oid tableId,
*/
UnlockRelationIdForSession(&heaprelid, ShareUpdateExclusiveLock);
+ /* Log what we did */
+ if (indexInfo->ii_verbose)
+ ereport(INFO,
+ (errmsg("index \"%s.%s\" was created",
+ get_namespace_name(RelationGetNamespace(rel)), RelationGetRelationName(rel)),
+ errdetail("%s.", pg_rusage_show(&ru0))));
+
pgstat_progress_end_command();
return address;
@@ -4096,7 +4136,7 @@ ReindexRelationConcurrently(const ReindexStmt *stmt, Oid relationOid, const Rein
pgstat_progress_update_multi_param(4, progress_index, progress_vals);
/* Perform concurrent build of new index */
- index_concurrently_build(newidx->tableId, newidx->indexId);
+ index_concurrently_build(newidx->tableId, newidx->indexId, params->options & REINDEXOPT_VERBOSE);
PopActiveSnapshot();
CommitTransactionCommand();
diff --git a/src/backend/commands/tablecmds.c b/src/backend/commands/tablecmds.c
index d2420a9558..7b04656538 100644
--- a/src/backend/commands/tablecmds.c
+++ b/src/backend/commands/tablecmds.c
@@ -1275,7 +1275,7 @@ DefineRelation(CreateStmt *stmt, char relkind, Oid ownerId,
RelationGetRelid(idxRel),
constraintOid,
-1,
- false, false, false, false, false);
+ false, false, false, false, false, false);
index_close(idxRel, AccessShareLock);
}
@@ -9320,7 +9320,8 @@ ATExecAddIndex(AlteredTableInfo *tab, Relation rel,
check_rights,
false, /* check_not_in_use - we did it already */
skip_build,
- quiet);
+ quiet,
+ false);
/*
* If TryReuseIndex() stashed a relfilenumber for us, we used it for the
@@ -19624,7 +19625,7 @@ AttachPartitionEnsureIndexes(List **wqueue, Relation rel, Relation attachrel)
RelationGetRelid(idxRel),
conOid,
-1,
- true, false, false, false, false);
+ true, false, false, false, false, false);
}
index_close(idxRel, AccessShareLock);
diff --git a/src/backend/nodes/makefuncs.c b/src/backend/nodes/makefuncs.c
index 007612563c..60e3a67fa7 100644
--- a/src/backend/nodes/makefuncs.c
+++ b/src/backend/nodes/makefuncs.c
@@ -787,7 +787,7 @@ IndexInfo *
makeIndexInfo(int numattrs, int numkeyattrs, Oid amoid, List *expressions,
List *predicates, bool unique, bool nulls_not_distinct,
bool isready, bool concurrent, bool summarizing,
- bool withoutoverlaps)
+ bool withoutoverlaps, bool verbose)
{
IndexInfo *n = makeNode(IndexInfo);
@@ -803,6 +803,7 @@ makeIndexInfo(int numattrs, int numkeyattrs, Oid amoid, List *expressions,
n->ii_Concurrent = concurrent;
n->ii_Summarizing = summarizing;
n->ii_WithoutOverlaps = withoutoverlaps;
+ n->ii_verbose = verbose;
/* summarizing indexes cannot contain non-key attributes */
Assert(!summarizing || (numkeyattrs == numattrs));
diff --git a/src/backend/parser/gram.y b/src/backend/parser/gram.y
index d7f9c00c40..349db1854e 100644
--- a/src/backend/parser/gram.y
+++ b/src/backend/parser/gram.y
@@ -559,6 +559,8 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
%type <ival> reindex_target_relation reindex_target_all
%type <list> opt_reindex_option_list
+%type <list> opt_index_option_list
+
%type <node> copy_generic_opt_arg copy_generic_opt_arg_list_item
%type <defelt> copy_generic_opt_elem
%type <list> copy_generic_opt_list copy_generic_opt_arg_list
@@ -8158,23 +8160,24 @@ defacl_privilege_target:
* willing to make TABLESPACE a fully reserved word.
*****************************************************************************/
-IndexStmt: CREATE opt_unique INDEX opt_concurrently opt_single_name
+IndexStmt: CREATE opt_unique INDEX opt_index_option_list opt_concurrently opt_single_name
ON relation_expr access_method_clause '(' index_params ')'
opt_include opt_unique_null_treatment opt_reloptions OptTableSpace where_clause
{
IndexStmt *n = makeNode(IndexStmt);
n->unique = $2;
- n->concurrent = $4;
- n->idxname = $5;
- n->relation = $7;
- n->accessMethod = $8;
- n->indexParams = $10;
- n->indexIncludingParams = $12;
- n->nulls_not_distinct = !$13;
- n->options = $14;
- n->tableSpace = $15;
- n->whereClause = $16;
+ n->params = $4;
+ n->concurrent = $5;
+ n->idxname = $6;
+ n->relation = $8;
+ n->accessMethod = $9;
+ n->indexParams = $11;
+ n->indexIncludingParams = $13;
+ n->nulls_not_distinct = !$14;
+ n->options = $15;
+ n->tableSpace = $16;
+ n->whereClause = $17;
n->excludeOpNames = NIL;
n->idxcomment = NULL;
n->indexOid = InvalidOid;
@@ -8190,23 +8193,24 @@ IndexStmt: CREATE opt_unique INDEX opt_concurrently opt_single_name
n->reset_default_tblspc = false;
$$ = (Node *) n;
}
- | CREATE opt_unique INDEX opt_concurrently IF_P NOT EXISTS name
+ | CREATE opt_unique INDEX opt_index_option_list opt_concurrently IF_P NOT EXISTS name
ON relation_expr access_method_clause '(' index_params ')'
opt_include opt_unique_null_treatment opt_reloptions OptTableSpace where_clause
{
IndexStmt *n = makeNode(IndexStmt);
n->unique = $2;
- n->concurrent = $4;
- n->idxname = $8;
- n->relation = $10;
- n->accessMethod = $11;
- n->indexParams = $13;
- n->indexIncludingParams = $15;
- n->nulls_not_distinct = !$16;
- n->options = $17;
- n->tableSpace = $18;
- n->whereClause = $19;
+ n->params = $4;
+ n->concurrent = $5;
+ n->idxname = $9;
+ n->relation = $11;
+ n->accessMethod = $12;
+ n->indexParams = $14;
+ n->indexIncludingParams = $16;
+ n->nulls_not_distinct = !$17;
+ n->options = $18;
+ n->tableSpace = $19;
+ n->whereClause = $20;
n->excludeOpNames = NIL;
n->idxcomment = NULL;
n->indexOid = InvalidOid;
@@ -8238,6 +8242,10 @@ index_params: index_elem { $$ = list_make1($1); }
| index_params ',' index_elem { $$ = lappend($1, $3); }
;
+opt_index_option_list:
+ '(' utility_option_list ')' { $$ = $2; }
+ | /* EMPTY */ { $$ = NULL; }
+ ;
index_elem_options:
opt_collate opt_qualified_name opt_asc_desc opt_nulls_order
diff --git a/src/backend/tcop/utility.c b/src/backend/tcop/utility.c
index 25fe3d5801..c44e0b9ad7 100644
--- a/src/backend/tcop/utility.c
+++ b/src/backend/tcop/utility.c
@@ -1459,11 +1459,28 @@ ProcessUtilitySlow(ParseState *pstate,
LOCKMODE lockmode;
int nparts = -1;
bool is_alter_table;
+ bool verbose = false;
+ ListCell *lc;
if (stmt->concurrent)
PreventInTransactionBlock(isTopLevel,
"CREATE INDEX CONCURRENTLY");
+ /* Parse option list */
+ foreach(lc, stmt->params)
+ {
+ DefElem *opt = (DefElem *) lfirst(lc);
+
+ if (strcmp(opt->defname, "verbose") == 0)
+ verbose = defGetBoolean(opt);
+ else
+ ereport(ERROR,
+ (errcode(ERRCODE_SYNTAX_ERROR),
+ errmsg("unrecognized CREATE INDEX option \"%s\"",
+ opt->defname),
+ parser_errposition(pstate, opt->location)));
+ }
+
/*
* Look up the relation OID just once, right here at the
* beginning, so that we don't end up repeating the name
@@ -1495,7 +1512,6 @@ ProcessUtilitySlow(ParseState *pstate,
if (stmt->relation->inh &&
get_rel_relkind(relid) == RELKIND_PARTITIONED_TABLE)
{
- ListCell *lc;
List *inheritors = NIL;
inheritors = find_all_inheritors(relid, lockmode, NULL);
@@ -1552,7 +1568,8 @@ ProcessUtilitySlow(ParseState *pstate,
true, /* check_rights */
true, /* check_not_in_use */
false, /* skip_build */
- false); /* quiet */
+ false, /* quiet */
+ verbose); /* verbose */
/*
* Add the CREATE INDEX node itself to stash right away;
diff --git a/src/bin/psql/tab-complete.in.c b/src/bin/psql/tab-complete.in.c
index 81cbf10aa2..05ad4b8110 100644
--- a/src/bin/psql/tab-complete.in.c
+++ b/src/bin/psql/tab-complete.in.c
@@ -3373,6 +3373,20 @@ match_previous_words(int pattern_id,
else if (TailMatches("CREATE|UNIQUE", "INDEX", "CONCURRENTLY"))
COMPLETE_WITH_SCHEMA_QUERY_PLUS(Query_for_list_of_indexes,
"ON");
+
+ /* Complete CREATE|UNIQUE INDEX with "VERBOSE" */
+ else if (TailMatches("CREATE|UNIQUE", "INDEX", "(*") &&
+ !TailMatches("CREATE|UNIQUE", "INDEX", "(*)"))
+ {
+ /*
+ * This fires if we're in an unfinished parenthesized option list.
+ * get_previous_words treats a completed parenthesized option list as
+ * one word, so the above test is correct.
+ */
+ if (ends_with(prev_wd, '('))
+ COMPLETE_WITH( "VERBOSE");
+ }
+
/* Complete CREATE|UNIQUE INDEX [CONCURRENTLY] <sth> with "ON" */
else if (TailMatches("CREATE|UNIQUE", "INDEX", MatchAny) ||
TailMatches("CREATE|UNIQUE", "INDEX", "CONCURRENTLY", MatchAny))
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index 4daa8bef5e..d82026d7a4 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -101,7 +101,8 @@ extern Oid index_concurrently_create_copy(Relation heapRelation,
const char *newName);
extern void index_concurrently_build(Oid heapRelationId,
- Oid indexRelationId);
+ Oid indexRelationId,
+ bool verbose);
extern void index_concurrently_swap(Oid newIndexId,
Oid oldIndexId,
diff --git a/src/include/commands/defrem.h b/src/include/commands/defrem.h
index 6d9348bac8..1776e76250 100644
--- a/src/include/commands/defrem.h
+++ b/src/include/commands/defrem.h
@@ -35,7 +35,8 @@ extern ObjectAddress DefineIndex(Oid tableId,
bool check_rights,
bool check_not_in_use,
bool skip_build,
- bool quiet);
+ bool quiet,
+ bool verbose);
extern void ExecReindex(ParseState *pstate, const ReindexStmt *stmt, bool isTopLevel);
extern char *makeObjectName(const char *name1, const char *name2,
const char *label);
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index d0f2dca592..db75dbb2d6 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -185,7 +185,7 @@ typedef struct ExprState
* AmCache private cache area for index AM
* Context memory context holding this IndexInfo
*
- * ii_Concurrent, ii_BrokenHotChain, and ii_ParallelWorkers are used only
+ * ii_Concurrent, ii_BrokenHotChain, and ii_ParallelWorkers, ii_verbose are used only
* during index build; they're conventionally zeroed otherwise.
* ----------------
*/
@@ -218,6 +218,7 @@ typedef struct IndexInfo
Oid ii_Am;
void *ii_AmCache;
MemoryContext ii_Context;
+ bool ii_verbose;
} IndexInfo;
/* ----------------
diff --git a/src/include/nodes/makefuncs.h b/src/include/nodes/makefuncs.h
index 5473ce9a28..288cbe5035 100644
--- a/src/include/nodes/makefuncs.h
+++ b/src/include/nodes/makefuncs.h
@@ -99,7 +99,8 @@ extern IndexInfo *makeIndexInfo(int numattrs, int numkeyattrs, Oid amoid,
List *expressions, List *predicates,
bool unique, bool nulls_not_distinct,
bool isready, bool concurrent,
- bool summarizing, bool withoutoverlaps);
+ bool summarizing, bool withoutoverlaps,
+ bool verbose);
extern Node *makeStringConst(char *str, int location);
extern DefElem *makeDefElem(char *name, Node *arg, int location);
diff --git a/src/include/nodes/parsenodes.h b/src/include/nodes/parsenodes.h
index ffe155ee20..60d78acdad 100644
--- a/src/include/nodes/parsenodes.h
+++ b/src/include/nodes/parsenodes.h
@@ -3449,6 +3449,7 @@ typedef struct IndexStmt
bool if_not_exists; /* just do nothing if index already exists? */
bool reset_default_tblspc; /* reset default_tablespace prior to
* executing */
+ List *params; /* list of DefElem nodes */
} IndexStmt;
/* ----------------------
diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out
index 8011c141bf..c1b4052280 100644
--- a/src/test/regress/expected/create_index.out
+++ b/src/test/regress/expected/create_index.out
@@ -1449,6 +1449,19 @@ DROP FUNCTION predicate_stable();
BEGIN;
CREATE INDEX std_index on concur_heap(f2);
COMMIT;
+-- CREATE INDEX (VERBOSE)
+CREATE TABLE verbose_test (id int);
+\set VERBOSITY terse \\ -- suppress machine-dependent details
+CREATE INDEX (VERBOSE) verbose_test_id ON verbose_test(id);
+INFO: building index "verbose_test_id" on table "verbose_test" serially
+INFO: index "public.verbose_test" was created
+ALTER TABLE verbose_test SET (parallel_workers = 1);
+CREATE INDEX (VERBOSE) verbose_test_id2 ON verbose_test(id);
+INFO: building index "verbose_test_id2" on table "verbose_test" with parallel workers
+INFO: launched 1 parallel workers for index creation (planned: 1)
+INFO: index "public.verbose_test" was created
+\set VERBOSITY default
+DROP TABLE verbose_test;
-- Failed builds are left invalid by VACUUM FULL, fixed by REINDEX
VACUUM FULL concur_heap;
REINDEX TABLE concur_heap;
@@ -2515,6 +2528,12 @@ explain (costs off)
CREATE TABLE reindex_verbose(id integer primary key);
\set VERBOSITY terse \\ -- suppress machine-dependent details
REINDEX (VERBOSE) TABLE reindex_verbose;
+INFO: building index "reindex_verbose_pkey" on table "reindex_verbose" serially
+INFO: index "reindex_verbose_pkey" was reindexed
+ALTER TABLE reindex_verbose SET (parallel_workers = 1);
+REINDEX (VERBOSE) TABLE reindex_verbose;
+INFO: building index "reindex_verbose_pkey" on table "reindex_verbose" with parallel workers
+INFO: launched 1 parallel workers for index creation (planned: 1)
INFO: index "reindex_verbose_pkey" was reindexed
\set VERBOSITY default
DROP TABLE reindex_verbose;
diff --git a/src/test/regress/sql/create_index.sql b/src/test/regress/sql/create_index.sql
index 068c66b95a..fe7fc97f79 100644
--- a/src/test/regress/sql/create_index.sql
+++ b/src/test/regress/sql/create_index.sql
@@ -526,6 +526,16 @@ BEGIN;
CREATE INDEX std_index on concur_heap(f2);
COMMIT;
+-- CREATE INDEX (VERBOSE)
+CREATE TABLE verbose_test (id int);
+\set VERBOSITY terse \\ -- suppress machine-dependent details
+CREATE INDEX (VERBOSE) verbose_test_id ON verbose_test(id);
+ALTER TABLE verbose_test SET (parallel_workers = 1);
+CREATE INDEX (VERBOSE) verbose_test_id2 ON verbose_test(id);
+\set VERBOSITY default
+DROP TABLE verbose_test;
+
+
-- Failed builds are left invalid by VACUUM FULL, fixed by REINDEX
VACUUM FULL concur_heap;
REINDEX TABLE concur_heap;
@@ -968,6 +978,8 @@ explain (costs off)
CREATE TABLE reindex_verbose(id integer primary key);
\set VERBOSITY terse \\ -- suppress machine-dependent details
REINDEX (VERBOSE) TABLE reindex_verbose;
+ALTER TABLE reindex_verbose SET (parallel_workers = 1);
+REINDEX (VERBOSE) TABLE reindex_verbose;
\set VERBOSITY default
DROP TABLE reindex_verbose;
--
2.39.5 (Apple Git-154)
^ permalink raw reply [nested|flat] 70+ messages in thread
* Re: improve DEBUG1 logging of parallel workers for CREATE INDEX?
2025-01-03 18:30 improve DEBUG1 logging of parallel workers for CREATE INDEX? Sami Imseih <[email protected]>
2025-01-08 15:21 ` Re: improve DEBUG1 logging of parallel workers for CREATE INDEX? Bertrand Drouvot <[email protected]>
2025-01-08 17:36 ` Re: improve DEBUG1 logging of parallel workers for CREATE INDEX? Sami Imseih <[email protected]>
2025-01-08 20:21 ` Re: improve DEBUG1 logging of parallel workers for CREATE INDEX? Guillaume Lelarge <[email protected]>
2025-01-08 20:35 ` Re: improve DEBUG1 logging of parallel workers for CREATE INDEX? Tom Lane <[email protected]>
2025-01-08 20:44 ` Re: improve DEBUG1 logging of parallel workers for CREATE INDEX? Sami Imseih <[email protected]>
2025-01-08 20:58 ` Re: improve DEBUG1 logging of parallel workers for CREATE INDEX? Tom Lane <[email protected]>
2025-01-09 03:24 ` Re: improve DEBUG1 logging of parallel workers for CREATE INDEX? Sami Imseih <[email protected]>
2025-01-11 10:29 ` Re: improve DEBUG1 logging of parallel workers for CREATE INDEX? Guillaume Lelarge <[email protected]>
2025-01-18 23:42 ` Re: improve DEBUG1 logging of parallel workers for CREATE INDEX? Sami Imseih <[email protected]>
@ 2025-01-19 00:52 ` Michael Paquier <[email protected]>
2025-01-19 05:40 ` Re: improve DEBUG1 logging of parallel workers for CREATE INDEX? Sami Imseih <[email protected]>
0 siblings, 1 reply; 70+ messages in thread
From: Michael Paquier @ 2025-01-19 00:52 UTC (permalink / raw)
To: Sami Imseih <[email protected]>; +Cc: Guillaume Lelarge <[email protected]>; Bertrand Drouvot <[email protected]>; Tom Lane <[email protected]>; pgsql-hackers
On Sat, Jan 18, 2025 at 05:42:50PM -0600, Sami Imseih wrote:
> Here is a v1 that implements CREATE INDEX (VERBOSE)
> to output the parallel workers planned and launched.
Hmm. I am reading Tom's opinion that goes toward not going in this
direction for more commands, with the point to extend EXPLAIN to show
this kind of information:
https://www.postgresql.org/message-id/[email protected]
So do we really want to do what's proposed here? I'm +-0 about the
VERBOSE option attached to more commands, as it is a bit harder for
clients to catch the information wanted. So here comes my question:
how do we want to consume this information at the end from the
perspective of the client?
--
Michael
Attachments:
[application/pgp-signature] signature.asc (833B, ../../[email protected]/2-signature.asc)
download
^ permalink raw reply [nested|flat] 70+ messages in thread
* Re: improve DEBUG1 logging of parallel workers for CREATE INDEX?
2025-01-03 18:30 improve DEBUG1 logging of parallel workers for CREATE INDEX? Sami Imseih <[email protected]>
2025-01-08 15:21 ` Re: improve DEBUG1 logging of parallel workers for CREATE INDEX? Bertrand Drouvot <[email protected]>
2025-01-08 17:36 ` Re: improve DEBUG1 logging of parallel workers for CREATE INDEX? Sami Imseih <[email protected]>
2025-01-08 20:21 ` Re: improve DEBUG1 logging of parallel workers for CREATE INDEX? Guillaume Lelarge <[email protected]>
2025-01-08 20:35 ` Re: improve DEBUG1 logging of parallel workers for CREATE INDEX? Tom Lane <[email protected]>
2025-01-08 20:44 ` Re: improve DEBUG1 logging of parallel workers for CREATE INDEX? Sami Imseih <[email protected]>
2025-01-08 20:58 ` Re: improve DEBUG1 logging of parallel workers for CREATE INDEX? Tom Lane <[email protected]>
2025-01-09 03:24 ` Re: improve DEBUG1 logging of parallel workers for CREATE INDEX? Sami Imseih <[email protected]>
2025-01-11 10:29 ` Re: improve DEBUG1 logging of parallel workers for CREATE INDEX? Guillaume Lelarge <[email protected]>
2025-01-18 23:42 ` Re: improve DEBUG1 logging of parallel workers for CREATE INDEX? Sami Imseih <[email protected]>
2025-01-19 00:52 ` Re: improve DEBUG1 logging of parallel workers for CREATE INDEX? Michael Paquier <[email protected]>
@ 2025-01-19 05:40 ` Sami Imseih <[email protected]>
2025-01-19 10:30 ` Re: improve DEBUG1 logging of parallel workers for CREATE INDEX? Guillaume Lelarge <[email protected]>
0 siblings, 1 reply; 70+ messages in thread
From: Sami Imseih @ 2025-01-19 05:40 UTC (permalink / raw)
To: Michael Paquier <[email protected]>; +Cc: Guillaume Lelarge <[email protected]>; Bertrand Drouvot <[email protected]>; Tom Lane <[email protected]>; pgsql-hackers
> Hmm. I am reading Tom's opinion that goes toward not going in this
> direction for more commands, with the point to extend EXPLAIN to show
> this kind of information:
> https://www.postgresql.org/message-id/[email protected]
That sounds like the ability to do something like EXPLAIN CREATE INDEX ...
is that correct?
> So do we really want to do what's proposed here? I'm +-0 about the
> VERBOSE option attached to more commands, as it is a bit harder for
> clients to catch the information wanted. So here comes my question:
> how do we want to consume this information at the end from the
> perspective of the client?
For interactive usage in psql or pgadmin, it's trivial to capture this
information. The
information can also be written to the server log with log_min_messages=INFO
A bit more work is required to redirect messages to an out file from psql, as
you need to ensure that stderr is redirected to a file.
It's also a bit more work to capture this information from something like a
JDBC application.
IMO the interactive use-case is where this is the most useful as you can start a
CREATE INDEX (VERBOSE) and ensure that it's going to launch all the parallel
workers that it planned before letting it it continue; or control-c and figure
out why not all planned workers launched.
Regards,
Sami
^ permalink raw reply [nested|flat] 70+ messages in thread
* Re: improve DEBUG1 logging of parallel workers for CREATE INDEX?
2025-01-03 18:30 improve DEBUG1 logging of parallel workers for CREATE INDEX? Sami Imseih <[email protected]>
2025-01-08 15:21 ` Re: improve DEBUG1 logging of parallel workers for CREATE INDEX? Bertrand Drouvot <[email protected]>
2025-01-08 17:36 ` Re: improve DEBUG1 logging of parallel workers for CREATE INDEX? Sami Imseih <[email protected]>
2025-01-08 20:21 ` Re: improve DEBUG1 logging of parallel workers for CREATE INDEX? Guillaume Lelarge <[email protected]>
2025-01-08 20:35 ` Re: improve DEBUG1 logging of parallel workers for CREATE INDEX? Tom Lane <[email protected]>
2025-01-08 20:44 ` Re: improve DEBUG1 logging of parallel workers for CREATE INDEX? Sami Imseih <[email protected]>
2025-01-08 20:58 ` Re: improve DEBUG1 logging of parallel workers for CREATE INDEX? Tom Lane <[email protected]>
2025-01-09 03:24 ` Re: improve DEBUG1 logging of parallel workers for CREATE INDEX? Sami Imseih <[email protected]>
2025-01-11 10:29 ` Re: improve DEBUG1 logging of parallel workers for CREATE INDEX? Guillaume Lelarge <[email protected]>
2025-01-18 23:42 ` Re: improve DEBUG1 logging of parallel workers for CREATE INDEX? Sami Imseih <[email protected]>
2025-01-19 00:52 ` Re: improve DEBUG1 logging of parallel workers for CREATE INDEX? Michael Paquier <[email protected]>
2025-01-19 05:40 ` Re: improve DEBUG1 logging of parallel workers for CREATE INDEX? Sami Imseih <[email protected]>
@ 2025-01-19 10:30 ` Guillaume Lelarge <[email protected]>
2025-01-20 03:27 ` Re: improve DEBUG1 logging of parallel workers for CREATE INDEX? Sami Imseih <[email protected]>
0 siblings, 1 reply; 70+ messages in thread
From: Guillaume Lelarge @ 2025-01-19 10:30 UTC (permalink / raw)
To: Sami Imseih <[email protected]>; +Cc: Michael Paquier <[email protected]>; Bertrand Drouvot <[email protected]>; Tom Lane <[email protected]>; pgsql-hackers
Hi,
Le dim. 19 janv. 2025 à 06:40, Sami Imseih <[email protected]> a écrit :
> > Hmm. I am reading Tom's opinion that goes toward not going in this
> > direction for more commands, with the point to extend EXPLAIN to show
> > this kind of information:
> > https://www.postgresql.org/message-id/[email protected]
>
> That sounds like the ability to do something like EXPLAIN CREATE INDEX ...
> is that correct?
>
>
Yes, and I suppose this would be quite some work to do. Though I see how
interesting it would be, and a nice project to hack on.
> > So do we really want to do what's proposed here? I'm +-0 about the
> > VERBOSE option attached to more commands, as it is a bit harder for
> > clients to catch the information wanted.
It sounds to me a lot easier to know about a VERBOSE option that gives you
more information, than using a DEBUG level to get the same information.
DEBUG level messages aren't explained in the CREATE INDEX manpage, whereas
a VERBOSE option would be.
> So here comes my question:
> > how do we want to consume this information at the end from the
> > perspective of the client?
>
> For interactive usage in psql or pgadmin, it's trivial to capture this
> information.
+1
The information can also be written to the server log with
> log_min_messages=INFO
>
> A bit more work is required to redirect messages to an out file from psql,
> as
> you need to ensure that stderr is redirected to a file.
> It's also a bit more work to capture this information from something like a
> JDBC application.
>
> IMO the interactive use-case is where this is the most useful as you can
> start a
> CREATE INDEX (VERBOSE) and ensure that it's going to launch all the
> parallel
> workers that it planned before letting it it continue; or control-c and
> figure
> out why not all planned workers launched.
>
--
Guillaume.
^ permalink raw reply [nested|flat] 70+ messages in thread
* Re: improve DEBUG1 logging of parallel workers for CREATE INDEX?
2025-01-03 18:30 improve DEBUG1 logging of parallel workers for CREATE INDEX? Sami Imseih <[email protected]>
2025-01-08 15:21 ` Re: improve DEBUG1 logging of parallel workers for CREATE INDEX? Bertrand Drouvot <[email protected]>
2025-01-08 17:36 ` Re: improve DEBUG1 logging of parallel workers for CREATE INDEX? Sami Imseih <[email protected]>
2025-01-08 20:21 ` Re: improve DEBUG1 logging of parallel workers for CREATE INDEX? Guillaume Lelarge <[email protected]>
2025-01-08 20:35 ` Re: improve DEBUG1 logging of parallel workers for CREATE INDEX? Tom Lane <[email protected]>
2025-01-08 20:44 ` Re: improve DEBUG1 logging of parallel workers for CREATE INDEX? Sami Imseih <[email protected]>
2025-01-08 20:58 ` Re: improve DEBUG1 logging of parallel workers for CREATE INDEX? Tom Lane <[email protected]>
2025-01-09 03:24 ` Re: improve DEBUG1 logging of parallel workers for CREATE INDEX? Sami Imseih <[email protected]>
2025-01-11 10:29 ` Re: improve DEBUG1 logging of parallel workers for CREATE INDEX? Guillaume Lelarge <[email protected]>
2025-01-18 23:42 ` Re: improve DEBUG1 logging of parallel workers for CREATE INDEX? Sami Imseih <[email protected]>
2025-01-19 00:52 ` Re: improve DEBUG1 logging of parallel workers for CREATE INDEX? Michael Paquier <[email protected]>
2025-01-19 05:40 ` Re: improve DEBUG1 logging of parallel workers for CREATE INDEX? Sami Imseih <[email protected]>
2025-01-19 10:30 ` Re: improve DEBUG1 logging of parallel workers for CREATE INDEX? Guillaume Lelarge <[email protected]>
@ 2025-01-20 03:27 ` Sami Imseih <[email protected]>
2025-01-20 07:24 ` Re: improve DEBUG1 logging of parallel workers for CREATE INDEX? Guillaume Lelarge <[email protected]>
0 siblings, 1 reply; 70+ messages in thread
From: Sami Imseih @ 2025-01-20 03:27 UTC (permalink / raw)
To: Guillaume Lelarge <[email protected]>; +Cc: Michael Paquier <[email protected]>; Bertrand Drouvot <[email protected]>; Tom Lane <[email protected]>; pgsql-hackers
> > > Hmm. I am reading Tom's opinion that goes toward not going in this
> > > direction for more commands, with the point to extend EXPLAIN to show
> > > this kind of information:
> > > https://www.postgresql.org/message-id/[email protected]
> >
> > That sounds like the ability to do something like EXPLAIN CREATE INDEX ...
> > is that correct?
> >
> Yes, and I suppose this would be quite some work to do. Though I see how
> interesting it would be, and a nice project to hack on.
I have been contemplating this and do see how something like EXPLAIN
could be useful, as it would give users the ability to examine
what a command will do, i.e. EXPLAIN CREATE INDEX will tell you
how many parallel workers are planned and EXPLAIN (ANALYZE) CREATE
INDEX will actually execute the command and provide the actual parallel
workers launched. Tom also mentioned the example of a user wanting to know if
a table rewrite will occur. There are other cases where this could be useful.
FWIW, Oracle does support EXPLAIN PLAN FOR CREATE INDEX and
EXPLAIN PLAN FOR REBUILD (for index rebuilds) [1][2], but this functionality
is not officially documented [3]. Oracle also provides a documented function to
estimate the size of an index [4]. I do not have access to an Oracle instance to
verify the state of current versions of Oracle, but I do vaguely remember this
from when I worked on Oracle many years ago :)
Now, I also realized that v1 does not include logging for ALTER TABLE commands
that add an index, such as "ALTER TABLE foo ADD CONSTRAINT foo_uq UNIQUE (c1)"
or attaching a partition and ensuring the index is built. So this
needs to be added to
the current proposal.
So far these are 2 options being discussed:
1/
The current proposal of:
CREATE INDEX (VERBOSE) and ALTER TABLE (VERBOSE)
and output debugging at INFO level.
2/
EXPLAIN
This is a lot more work. Currently QueryDesc has all the info we
need to generate the execution plan. But, we will likely need a
new struct that can track the debugging info to relay it back
up to explain. And this has to be generic enough for all cases.
Also, implementing an EXPLAIN and EXPLAIN ANALYZE for such commands
may not even be feasible. Plans are different in that they are generated
up-front and we can choose to execute them or not. This is not the same
for the other types of commands being discussed.
I am not convinced it's worth the effort.
thoughts? other approaches?
[1] https://oracle-randolf.blogspot.com/2009/02/explain-plan-on-ddls.html
[2] https://stackoverflow.com/questions/827123/how-can-i-estimate-the-size-of-an-oracle-index
[3] https://docs.oracle.com/en/database/oracle/oracle-database/19/sqlrf/EXPLAIN-PLAN.html
[4] https://docs.oracle.com/en/database/oracle/oracle-database/21/arpls/DBMS_SPACE.html#GUID-B6EB7527-BC...
Regards,
Sami
^ permalink raw reply [nested|flat] 70+ messages in thread
* Re: improve DEBUG1 logging of parallel workers for CREATE INDEX?
2025-01-03 18:30 improve DEBUG1 logging of parallel workers for CREATE INDEX? Sami Imseih <[email protected]>
2025-01-08 15:21 ` Re: improve DEBUG1 logging of parallel workers for CREATE INDEX? Bertrand Drouvot <[email protected]>
2025-01-08 17:36 ` Re: improve DEBUG1 logging of parallel workers for CREATE INDEX? Sami Imseih <[email protected]>
2025-01-08 20:21 ` Re: improve DEBUG1 logging of parallel workers for CREATE INDEX? Guillaume Lelarge <[email protected]>
2025-01-08 20:35 ` Re: improve DEBUG1 logging of parallel workers for CREATE INDEX? Tom Lane <[email protected]>
2025-01-08 20:44 ` Re: improve DEBUG1 logging of parallel workers for CREATE INDEX? Sami Imseih <[email protected]>
2025-01-08 20:58 ` Re: improve DEBUG1 logging of parallel workers for CREATE INDEX? Tom Lane <[email protected]>
2025-01-09 03:24 ` Re: improve DEBUG1 logging of parallel workers for CREATE INDEX? Sami Imseih <[email protected]>
2025-01-11 10:29 ` Re: improve DEBUG1 logging of parallel workers for CREATE INDEX? Guillaume Lelarge <[email protected]>
2025-01-18 23:42 ` Re: improve DEBUG1 logging of parallel workers for CREATE INDEX? Sami Imseih <[email protected]>
2025-01-19 00:52 ` Re: improve DEBUG1 logging of parallel workers for CREATE INDEX? Michael Paquier <[email protected]>
2025-01-19 05:40 ` Re: improve DEBUG1 logging of parallel workers for CREATE INDEX? Sami Imseih <[email protected]>
2025-01-19 10:30 ` Re: improve DEBUG1 logging of parallel workers for CREATE INDEX? Guillaume Lelarge <[email protected]>
2025-01-20 03:27 ` Re: improve DEBUG1 logging of parallel workers for CREATE INDEX? Sami Imseih <[email protected]>
@ 2025-01-20 07:24 ` Guillaume Lelarge <[email protected]>
2025-01-22 00:33 ` Re: improve DEBUG1 logging of parallel workers for CREATE INDEX? Sami Imseih <[email protected]>
0 siblings, 1 reply; 70+ messages in thread
From: Guillaume Lelarge @ 2025-01-20 07:24 UTC (permalink / raw)
To: Sami Imseih <[email protected]>; +Cc: Michael Paquier <[email protected]>; Bertrand Drouvot <[email protected]>; Tom Lane <[email protected]>; pgsql-hackers
Le lun. 20 janv. 2025 à 04:28, Sami Imseih <[email protected]> a écrit :
> > > > Hmm. I am reading Tom's opinion that goes toward not going in this
> > > > direction for more commands, with the point to extend EXPLAIN to show
> > > > this kind of information:
> > > >
> https://www.postgresql.org/message-id/[email protected]
> > >
> > > That sounds like the ability to do something like EXPLAIN CREATE INDEX
> ...
> > > is that correct?
> > >
>
> > Yes, and I suppose this would be quite some work to do. Though I see how
> > interesting it would be, and a nice project to hack on.
>
> I have been contemplating this and do see how something like EXPLAIN
> could be useful, as it would give users the ability to examine
> what a command will do, i.e. EXPLAIN CREATE INDEX will tell you
> how many parallel workers are planned and EXPLAIN (ANALYZE) CREATE
> INDEX will actually execute the command and provide the actual parallel
> workers launched. Tom also mentioned the example of a user wanting to know
> if
> a table rewrite will occur. There are other cases where this could be
> useful.
>
> FWIW, Oracle does support EXPLAIN PLAN FOR CREATE INDEX and
> EXPLAIN PLAN FOR REBUILD (for index rebuilds) [1][2], but this
> functionality
> is not officially documented [3]. Oracle also provides a documented
> function to
> estimate the size of an index [4]. I do not have access to an Oracle
> instance to
> verify the state of current versions of Oracle, but I do vaguely remember
> this
> from when I worked on Oracle many years ago :)
>
> Now, I also realized that v1 does not include logging for ALTER TABLE
> commands
> that add an index, such as "ALTER TABLE foo ADD CONSTRAINT foo_uq UNIQUE
> (c1)"
> or attaching a partition and ensuring the index is built. So this
> needs to be added to
> the current proposal.
>
> So far these are 2 options being discussed:
>
> 1/
> The current proposal of:
>
> CREATE INDEX (VERBOSE) and ALTER TABLE (VERBOSE)
> and output debugging at INFO level.
>
>
I guess the major issue is that it won't scale fine.
> 2/
> EXPLAIN
>
> This is a lot more work. Currently QueryDesc has all the info we
> need to generate the execution plan. But, we will likely need a
> new struct that can track the debugging info to relay it back
> up to explain. And this has to be generic enough for all cases.
>
> Also, implementing an EXPLAIN and EXPLAIN ANALYZE for such commands
> may not even be feasible. Plans are different in that they are generated
> up-front and we can choose to execute them or not. This is not the same
> for the other types of commands being discussed.
>
> I am not convinced it's worth the effort.
>
>
> thoughts? other approaches?
>
>
You might be interested by this thread "Thinking about EXPLAIN ALTER TABLE":
https://www.postgresql.org/message-id/CAM-w4HNm1M5J-ow8UjTcqRe3JPxkVCrGe56tRpPUSePSdGcZ_w%40mail.gma...
I didn't had the time to read the whole thread, but it might be quite
interesting.
>
> [1] https://oracle-randolf.blogspot.com/2009/02/explain-plan-on-ddls.html
> [2]
> https://stackoverflow.com/questions/827123/how-can-i-estimate-the-size-of-an-oracle-index
> [3]
> https://docs.oracle.com/en/database/oracle/oracle-database/19/sqlrf/EXPLAIN-PLAN.html
> [4]
> https://docs.oracle.com/en/database/oracle/oracle-database/21/arpls/DBMS_SPACE.html#GUID-B6EB7527-BC...
>
> Regards,
>
> Sami
>
--
Guillaume.
^ permalink raw reply [nested|flat] 70+ messages in thread
* Re: improve DEBUG1 logging of parallel workers for CREATE INDEX?
2025-01-03 18:30 improve DEBUG1 logging of parallel workers for CREATE INDEX? Sami Imseih <[email protected]>
2025-01-08 15:21 ` Re: improve DEBUG1 logging of parallel workers for CREATE INDEX? Bertrand Drouvot <[email protected]>
2025-01-08 17:36 ` Re: improve DEBUG1 logging of parallel workers for CREATE INDEX? Sami Imseih <[email protected]>
2025-01-08 20:21 ` Re: improve DEBUG1 logging of parallel workers for CREATE INDEX? Guillaume Lelarge <[email protected]>
2025-01-08 20:35 ` Re: improve DEBUG1 logging of parallel workers for CREATE INDEX? Tom Lane <[email protected]>
2025-01-08 20:44 ` Re: improve DEBUG1 logging of parallel workers for CREATE INDEX? Sami Imseih <[email protected]>
2025-01-08 20:58 ` Re: improve DEBUG1 logging of parallel workers for CREATE INDEX? Tom Lane <[email protected]>
2025-01-09 03:24 ` Re: improve DEBUG1 logging of parallel workers for CREATE INDEX? Sami Imseih <[email protected]>
2025-01-11 10:29 ` Re: improve DEBUG1 logging of parallel workers for CREATE INDEX? Guillaume Lelarge <[email protected]>
2025-01-18 23:42 ` Re: improve DEBUG1 logging of parallel workers for CREATE INDEX? Sami Imseih <[email protected]>
2025-01-19 00:52 ` Re: improve DEBUG1 logging of parallel workers for CREATE INDEX? Michael Paquier <[email protected]>
2025-01-19 05:40 ` Re: improve DEBUG1 logging of parallel workers for CREATE INDEX? Sami Imseih <[email protected]>
2025-01-19 10:30 ` Re: improve DEBUG1 logging of parallel workers for CREATE INDEX? Guillaume Lelarge <[email protected]>
2025-01-20 03:27 ` Re: improve DEBUG1 logging of parallel workers for CREATE INDEX? Sami Imseih <[email protected]>
2025-01-20 07:24 ` Re: improve DEBUG1 logging of parallel workers for CREATE INDEX? Guillaume Lelarge <[email protected]>
@ 2025-01-22 00:33 ` Sami Imseih <[email protected]>
2025-02-03 20:29 ` Re: improve DEBUG1 logging of parallel workers for CREATE INDEX? Sami Imseih <[email protected]>
0 siblings, 1 reply; 70+ messages in thread
From: Sami Imseih @ 2025-01-22 00:33 UTC (permalink / raw)
To: Guillaume Lelarge <[email protected]>; +Cc: Michael Paquier <[email protected]>; Bertrand Drouvot <[email protected]>; Tom Lane <[email protected]>; pgsql-hackers
> You might be interested by this thread "Thinking about EXPLAIN ALTER TABLE":
> https://www.postgresql.org/message-id/CAM-w4HNm1M5J-ow8UjTcqRe3JPxkVCrGe56tRpPUSePSdGcZ_w%40mail.gma...
I reviewed this thread, and the primary issue with the EXPLAIN command lies
in the inability to predict all the steps the ALTER TABLE will take as some
are made in phase 2 or 3. It is unlikely that all significant
decisions can be made
in phase 1.
However, I don't think that EXPLAIN addresses the same problem as the proposed
VERBOSE option. Consider, for instance, a user who intends to perform a schema
change that includes a long sequence of ALTERs. These steps will depend on each
other. Tom raises this point of dependent ALTERs [2] but for a
different reason.
I think however, this is an important point. How will EXPLAIN help
here? It may not tell
you the truth, because it does not actually do the work and can't know
the future
state of the schema.
VERBOSE on the other hand will perform the steps, and a user
can test these changes in a test environment or a schema-only restore
and know exactly what to expect in production.
As this thread has expanded beyond its original scope,
I believe this broader discussion is valuable. Rather than merely
addressing the DEBUG1 issue, we can work toward a more complete
and beneficial solution.
Thoughts?
[0] https://www.postgresql.org/message-id/26597.1544460770%40sss.pgh.pa.us
Regards,
Sami
^ permalink raw reply [nested|flat] 70+ messages in thread
* Re: improve DEBUG1 logging of parallel workers for CREATE INDEX?
2025-01-03 18:30 improve DEBUG1 logging of parallel workers for CREATE INDEX? Sami Imseih <[email protected]>
2025-01-08 15:21 ` Re: improve DEBUG1 logging of parallel workers for CREATE INDEX? Bertrand Drouvot <[email protected]>
2025-01-08 17:36 ` Re: improve DEBUG1 logging of parallel workers for CREATE INDEX? Sami Imseih <[email protected]>
2025-01-08 20:21 ` Re: improve DEBUG1 logging of parallel workers for CREATE INDEX? Guillaume Lelarge <[email protected]>
2025-01-08 20:35 ` Re: improve DEBUG1 logging of parallel workers for CREATE INDEX? Tom Lane <[email protected]>
2025-01-08 20:44 ` Re: improve DEBUG1 logging of parallel workers for CREATE INDEX? Sami Imseih <[email protected]>
2025-01-08 20:58 ` Re: improve DEBUG1 logging of parallel workers for CREATE INDEX? Tom Lane <[email protected]>
2025-01-09 03:24 ` Re: improve DEBUG1 logging of parallel workers for CREATE INDEX? Sami Imseih <[email protected]>
2025-01-11 10:29 ` Re: improve DEBUG1 logging of parallel workers for CREATE INDEX? Guillaume Lelarge <[email protected]>
2025-01-18 23:42 ` Re: improve DEBUG1 logging of parallel workers for CREATE INDEX? Sami Imseih <[email protected]>
2025-01-19 00:52 ` Re: improve DEBUG1 logging of parallel workers for CREATE INDEX? Michael Paquier <[email protected]>
2025-01-19 05:40 ` Re: improve DEBUG1 logging of parallel workers for CREATE INDEX? Sami Imseih <[email protected]>
2025-01-19 10:30 ` Re: improve DEBUG1 logging of parallel workers for CREATE INDEX? Guillaume Lelarge <[email protected]>
2025-01-20 03:27 ` Re: improve DEBUG1 logging of parallel workers for CREATE INDEX? Sami Imseih <[email protected]>
2025-01-20 07:24 ` Re: improve DEBUG1 logging of parallel workers for CREATE INDEX? Guillaume Lelarge <[email protected]>
2025-01-22 00:33 ` Re: improve DEBUG1 logging of parallel workers for CREATE INDEX? Sami Imseih <[email protected]>
@ 2025-02-03 20:29 ` Sami Imseih <[email protected]>
0 siblings, 0 replies; 70+ messages in thread
From: Sami Imseih @ 2025-02-03 20:29 UTC (permalink / raw)
To: Guillaume Lelarge <[email protected]>; +Cc: Michael Paquier <[email protected]>; Bertrand Drouvot <[email protected]>; Tom Lane <[email protected]>; pgsql-hackers
At this point I am planning on withdrawing this patch
from the March commitfest. I don't think fixing the REINDEX
debug1 output makes a whole lot of sense. I still think more logging
for (CREATE|ALTER) (INDEX|TABLE) will be a good to have but there
needs to be more discussion about the best approach.
Regards,
Sami
^ permalink raw reply [nested|flat] 70+ messages in thread
* Re: improve DEBUG1 logging of parallel workers for CREATE INDEX?
2025-01-03 18:30 improve DEBUG1 logging of parallel workers for CREATE INDEX? Sami Imseih <[email protected]>
2025-01-08 15:21 ` Re: improve DEBUG1 logging of parallel workers for CREATE INDEX? Bertrand Drouvot <[email protected]>
2025-01-08 17:36 ` Re: improve DEBUG1 logging of parallel workers for CREATE INDEX? Sami Imseih <[email protected]>
2025-01-08 20:21 ` Re: improve DEBUG1 logging of parallel workers for CREATE INDEX? Guillaume Lelarge <[email protected]>
2025-01-08 20:35 ` Re: improve DEBUG1 logging of parallel workers for CREATE INDEX? Tom Lane <[email protected]>
@ 2025-01-08 20:46 ` Guillaume Lelarge <[email protected]>
1 sibling, 0 replies; 70+ messages in thread
From: Guillaume Lelarge @ 2025-01-08 20:46 UTC (permalink / raw)
To: Tom Lane <[email protected]>; +Cc: Sami Imseih <[email protected]>; Bertrand Drouvot <[email protected]>; pgsql-hackers
Le mer. 8 janv. 2025 à 21:35, Tom Lane <[email protected]> a écrit :
> Guillaume Lelarge <[email protected]> writes:
> > I tend to agree it might be better than Benoit's patch on the index
> > messages, though I'm afraid that DEBUG1 level won't work for many users.
> > DEBUGx are for developers, not users. A better log level for me would be
> > LOG.
>
> Please, no. That would result in spamming the postmaster log
> for perfectly ordinary actions, with no usable way to prevent it.
>
>
Not sure why I was thinking it would only go to the client. You're right,
it will also go to the logs, and it will be stored by default, which would
be bad. Sorry about this...
> I think DEBUG is perfectly appropriate for this sort of message.
>
>
Still think DEBUG level is pretty bad for a user experience. Anyway, I much
prefer a DEBUG message than no message at all :)
--
Guillaume.
^ permalink raw reply [nested|flat] 70+ messages in thread
end of thread, other threads:[~2025-02-03 20:29 UTC | newest]
Thread overview: 70+ messages (download: mbox mbox.gz follow: Atom feed)
-- links below jump to the message on this page --
2019-09-10 03:58 [PATCH v17 2/4] Move page-reader out of XLogReadRecord Kyotaro Horiguchi <[email protected]>
2019-09-10 03:58 [PATCH v9 2/4] Move page-reader out of XLogReadRecord Kyotaro Horiguchi <[email protected]>
2019-09-10 03:58 [PATCH v15 2/4] Move page-reader out of XLogReadRecord Kyotaro Horiguchi <[email protected]>
2019-09-10 03:58 [PATCH v10 2/4] Move page-reader out of XLogReadRecord Kyotaro Horiguchi <[email protected]>
2019-09-10 03:58 [PATCH v17 2/4] Move page-reader out of XLogReadRecord Kyotaro Horiguchi <[email protected]>
2019-09-10 03:58 [PATCH v15 2/4] Move page-reader out of XLogReadRecord Kyotaro Horiguchi <[email protected]>
2019-09-10 03:58 [PATCH v15 2/4] Move page-reader out of XLogReadRecord Kyotaro Horiguchi <[email protected]>
2019-09-10 03:58 [PATCH v17 2/4] Move page-reader out of XLogReadRecord Kyotaro Horiguchi <[email protected]>
2019-09-10 03:58 [PATCH v17 2/4] Move page-reader out of XLogReadRecord Kyotaro Horiguchi <[email protected]>
2019-09-10 03:58 [PATCH v15 2/4] Move page-reader out of XLogReadRecord Kyotaro Horiguchi <[email protected]>
2019-09-10 03:58 [PATCH v15 2/4] Move page-reader out of XLogReadRecord Kyotaro Horiguchi <[email protected]>
2019-09-10 03:58 [PATCH v15 2/4] Move page-reader out of XLogReadRecord Kyotaro Horiguchi <[email protected]>
2019-09-10 03:58 [PATCH v15 2/4] Move page-reader out of XLogReadRecord Kyotaro Horiguchi <[email protected]>
2019-09-10 03:58 [PATCH v15 2/4] Move page-reader out of XLogReadRecord Kyotaro Horiguchi <[email protected]>
2019-09-10 03:58 [PATCH v17 2/4] Move page-reader out of XLogReadRecord Kyotaro Horiguchi <[email protected]>
2019-09-10 03:58 [PATCH v17 2/4] Move page-reader out of XLogReadRecord Kyotaro Horiguchi <[email protected]>
2019-09-10 03:58 [PATCH v17 2/4] Move page-reader out of XLogReadRecord Kyotaro Horiguchi <[email protected]>
2019-09-10 03:58 [PATCH v17 2/4] Move page-reader out of XLogReadRecord Kyotaro Horiguchi <[email protected]>
2019-09-10 03:58 [PATCH v17 2/4] Move page-reader out of XLogReadRecord Kyotaro Horiguchi <[email protected]>
2019-09-10 03:58 [PATCH v17 2/4] Move page-reader out of XLogReadRecord Kyotaro Horiguchi <[email protected]>
2019-09-10 03:58 [PATCH v8 2/4] Move page-reader out of XLogReadRecord Kyotaro Horiguchi <[email protected]>
2019-09-10 03:58 [PATCH v17 2/4] Move page-reader out of XLogReadRecord Kyotaro Horiguchi <[email protected]>
2019-09-10 03:58 [PATCH v15 2/4] Move page-reader out of XLogReadRecord Kyotaro Horiguchi <[email protected]>
2019-09-10 03:58 [PATCH v13 2/4] Move page-reader out of XLogReadRecord Kyotaro Horiguchi <[email protected]>
2019-09-10 03:58 [PATCH v17 2/4] Move page-reader out of XLogReadRecord Kyotaro Horiguchi <[email protected]>
2019-09-10 03:58 [PATCH v17 2/4] Move page-reader out of XLogReadRecord Kyotaro Horiguchi <[email protected]>
2019-09-10 03:58 [PATCH v17 2/4] Move page-reader out of XLogReadRecord Kyotaro Horiguchi <[email protected]>
2019-09-10 03:58 [PATCH v17 2/4] Move page-reader out of XLogReadRecord Kyotaro Horiguchi <[email protected]>
2019-09-10 03:58 [PATCH v17 2/4] Move page-reader out of XLogReadRecord Kyotaro Horiguchi <[email protected]>
2019-09-10 03:58 [PATCH v15 2/4] Move page-reader out of XLogReadRecord Kyotaro Horiguchi <[email protected]>
2019-09-10 03:58 [PATCH v15 2/4] Move page-reader out of XLogReadRecord Kyotaro Horiguchi <[email protected]>
2019-09-10 03:58 [PATCH v17 2/4] Move page-reader out of XLogReadRecord Kyotaro Horiguchi <[email protected]>
2019-09-10 03:58 [PATCH v17 2/4] Move page-reader out of XLogReadRecord Kyotaro Horiguchi <[email protected]>
2019-09-10 03:58 [PATCH v15 2/4] Move page-reader out of XLogReadRecord Kyotaro Horiguchi <[email protected]>
2019-09-10 03:58 [PATCH v16 2/4] Move page-reader out of XLogReadRecord Kyotaro Horiguchi <[email protected]>
2019-09-10 03:58 [PATCH v17 2/4] Move page-reader out of XLogReadRecord Kyotaro Horiguchi <[email protected]>
2019-09-10 03:58 [PATCH v15 2/4] Move page-reader out of XLogReadRecord Kyotaro Horiguchi <[email protected]>
2019-09-10 03:58 [PATCH v17 2/4] Move page-reader out of XLogReadRecord Kyotaro Horiguchi <[email protected]>
2019-09-10 03:58 [PATCH v15 2/4] Move page-reader out of XLogReadRecord Kyotaro Horiguchi <[email protected]>
2019-09-10 03:58 [PATCH v15 2/4] Move page-reader out of XLogReadRecord Kyotaro Horiguchi <[email protected]>
2019-09-10 03:58 [PATCH v17 2/4] Move page-reader out of XLogReadRecord Kyotaro Horiguchi <[email protected]>
2019-09-10 03:58 [PATCH v15 2/4] Move page-reader out of XLogReadRecord Kyotaro Horiguchi <[email protected]>
2019-09-10 03:58 [PATCH v17 2/4] Move page-reader out of XLogReadRecord Kyotaro Horiguchi <[email protected]>
2019-09-10 03:58 [PATCH v15 2/4] Move page-reader out of XLogReadRecord Kyotaro Horiguchi <[email protected]>
2019-09-10 03:58 [PATCH v15 2/4] Move page-reader out of XLogReadRecord Kyotaro Horiguchi <[email protected]>
2019-09-10 03:58 [PATCH v17 2/4] Move page-reader out of XLogReadRecord Kyotaro Horiguchi <[email protected]>
2019-09-10 03:58 [PATCH v15 2/4] Move page-reader out of XLogReadRecord Kyotaro Horiguchi <[email protected]>
2019-09-10 03:58 [PATCH v17 2/4] Move page-reader out of XLogReadRecord Kyotaro Horiguchi <[email protected]>
2019-09-10 03:58 [PATCH v19 2/5] Move page-reader out of XLogReadRecord(). Kyotaro Horiguchi <[email protected]>
2019-09-10 03:58 [PATCH v15 2/4] Move page-reader out of XLogReadRecord Kyotaro Horiguchi <[email protected]>
2021-09-30 03:04 [PATCH v18 2/5] Move page-reader out of XLogReadRecord(). Kyotaro Horiguchi <[email protected]>
2025-01-03 18:30 improve DEBUG1 logging of parallel workers for CREATE INDEX? Sami Imseih <[email protected]>
2025-01-08 15:21 ` Re: improve DEBUG1 logging of parallel workers for CREATE INDEX? Bertrand Drouvot <[email protected]>
2025-01-08 17:36 ` Re: improve DEBUG1 logging of parallel workers for CREATE INDEX? Sami Imseih <[email protected]>
2025-01-08 20:21 ` Re: improve DEBUG1 logging of parallel workers for CREATE INDEX? Guillaume Lelarge <[email protected]>
2025-01-08 20:35 ` Re: improve DEBUG1 logging of parallel workers for CREATE INDEX? Tom Lane <[email protected]>
2025-01-08 20:44 ` Re: improve DEBUG1 logging of parallel workers for CREATE INDEX? Sami Imseih <[email protected]>
2025-01-08 20:48 ` Re: improve DEBUG1 logging of parallel workers for CREATE INDEX? Guillaume Lelarge <[email protected]>
2025-01-08 20:58 ` Re: improve DEBUG1 logging of parallel workers for CREATE INDEX? Tom Lane <[email protected]>
2025-01-09 03:24 ` Re: improve DEBUG1 logging of parallel workers for CREATE INDEX? Sami Imseih <[email protected]>
2025-01-11 10:29 ` Re: improve DEBUG1 logging of parallel workers for CREATE INDEX? Guillaume Lelarge <[email protected]>
2025-01-18 23:42 ` Re: improve DEBUG1 logging of parallel workers for CREATE INDEX? Sami Imseih <[email protected]>
2025-01-19 00:52 ` Re: improve DEBUG1 logging of parallel workers for CREATE INDEX? Michael Paquier <[email protected]>
2025-01-19 05:40 ` Re: improve DEBUG1 logging of parallel workers for CREATE INDEX? Sami Imseih <[email protected]>
2025-01-19 10:30 ` Re: improve DEBUG1 logging of parallel workers for CREATE INDEX? Guillaume Lelarge <[email protected]>
2025-01-20 03:27 ` Re: improve DEBUG1 logging of parallel workers for CREATE INDEX? Sami Imseih <[email protected]>
2025-01-20 07:24 ` Re: improve DEBUG1 logging of parallel workers for CREATE INDEX? Guillaume Lelarge <[email protected]>
2025-01-22 00:33 ` Re: improve DEBUG1 logging of parallel workers for CREATE INDEX? Sami Imseih <[email protected]>
2025-02-03 20:29 ` Re: improve DEBUG1 logging of parallel workers for CREATE INDEX? Sami Imseih <[email protected]>
2025-01-08 20:46 ` Re: improve DEBUG1 logging of parallel workers for CREATE INDEX? Guillaume Lelarge <[email protected]>
This inbox is served by agora; see mirroring instructions
for how to clone and mirror all data and code used for this inbox