agora inbox for [email protected]
help / color / mirror / Atom feed[PATCH 1/2] Add start timestamp to PGPROC
69+ messages / 6 participants
[nested] [flat]
* [PATCH 1/2] Add start timestamp to PGPROC
@ 2019-02-01 00:45 Kyotaro Horiguchi <[email protected]>
0 siblings, 0 replies; 69+ messages in thread
From: Kyotaro Horiguchi @ 2019-02-01 00:45 UTC (permalink / raw)
A ProcArray element may be allocated to another session while
investigating. Pid is not fully relialble for cheking that because of
the possibility of reusing. Add start timestamp to resolve the issue.
---
src/backend/storage/lmgr/proc.c | 2 ++
src/include/storage/proc.h | 1 +
2 files changed, 3 insertions(+)
diff --git a/src/backend/storage/lmgr/proc.c b/src/backend/storage/lmgr/proc.c
index 89c80fb687..a63309a432 100644
--- a/src/backend/storage/lmgr/proc.c
+++ b/src/backend/storage/lmgr/proc.c
@@ -373,6 +373,8 @@ InitProcess(void)
MyProc->fpLocalTransactionId = InvalidLocalTransactionId;
MyPgXact->xid = InvalidTransactionId;
MyPgXact->xmin = InvalidTransactionId;
+ /* set this before pid to get rid of wrong pair with pid */
+ MyProc->starttimestamp = MyStartTimestamp;
MyProc->pid = MyProcPid;
/* backendId, databaseId and roleId will be filled in later */
MyProc->backendId = InvalidBackendId;
diff --git a/src/include/storage/proc.h b/src/include/storage/proc.h
index d203acbb30..2cebdd2646 100644
--- a/src/include/storage/proc.h
+++ b/src/include/storage/proc.h
@@ -108,6 +108,7 @@ struct PGPROC
* else InvalidLocalTransactionId */
int pid; /* Backend's process ID; 0 if prepared xact */
int pgprocno;
+ uint64 starttimestamp; /* Backend's start timestamp */
/* These fields are zero while a backend is still starting up: */
BackendId backendId; /* This backend's backend ID (if assigned) */
--
2.16.3
----Next_Part(Fri_Feb_01_17_30_12_2019_719)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
filename="0002-Remote-GUC-via-shmem-ALTER-SESSION-NoXact-SET-versio.patch"
^ permalink raw reply [nested|flat] 69+ messages in thread
* [PATCH v17 1/4] Move callback-call from ReadPageInternal to XLogReadRecord.
@ 2019-09-05 11:21 Kyotaro Horiguchi <[email protected]>
0 siblings, 0 replies; 69+ messages in thread
From: Kyotaro Horiguchi @ 2019-09-05 11:21 UTC (permalink / raw)
The current WAL record reader reads page data using a call back
function. Although it is not so problematic alone, it would be a
problem if we are going to do add tasks like encryption which is
performed on page data before WAL reader reads them. To avoid that the
record reader facility has to have a new code path corresponds to
every new callback, this patch separates page reader from WAL record
reading facility by modifying the current WAL record reader to a state
machine.
As the first step of that change, this patch moves the page reader
function out of ReadPageInternal, then the remaining tasks of the
function are taken over by the new function XLogNeedData. As the
result XLogPageRead directly calls the page reader callback function
according to the feedback from XLogNeedData.
---
src/backend/access/transam/xlog.c | 16 +-
src/backend/access/transam/xlogreader.c | 281 ++++++++++++++----------
src/backend/access/transam/xlogutils.c | 12 +-
src/backend/replication/walsender.c | 10 +-
src/bin/pg_rewind/parsexlog.c | 21 +-
src/bin/pg_waldump/pg_waldump.c | 8 +-
src/include/access/xlogreader.h | 25 ++-
src/include/access/xlogutils.h | 2 +-
8 files changed, 222 insertions(+), 153 deletions(-)
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 377afb8732..606328a65a 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -917,7 +917,7 @@ static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath,
static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli,
XLogSource source, bool notfoundOk);
static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, XLogSource source);
-static int XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
+static bool XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
bool fetching_ckpt, XLogRecPtr tliRecPtr);
@@ -4350,7 +4350,6 @@ ReadRecord(XLogReaderState *xlogreader, int emode,
XLogRecord *record;
XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
- /* Pass through parameters to XLogPageRead */
private->fetching_ckpt = fetching_ckpt;
private->emode = emode;
private->randAccess = (xlogreader->ReadRecPtr == InvalidXLogRecPtr);
@@ -11987,7 +11986,7 @@ CancelBackup(void)
* XLogPageRead() to try fetching the record from another source, or to
* sleep and retry.
*/
-static int
+static bool
XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *readBuf)
{
@@ -12046,7 +12045,8 @@ retry:
readLen = 0;
readSource = XLOG_FROM_ANY;
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
}
@@ -12141,7 +12141,8 @@ retry:
goto next_record_is_invalid;
}
- return readLen;
+ xlogreader->readLen = readLen;
+ return true;
next_record_is_invalid:
lastSourceFailed = true;
@@ -12155,8 +12156,9 @@ next_record_is_invalid:
/* In standby-mode, keep trying */
if (StandbyMode)
goto retry;
- else
- return -1;
+
+ xlogreader->readLen = -1;
+ return false;
}
/*
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index 42738eb940..7d04995b7b 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -36,8 +36,8 @@
static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
pg_attribute_printf(2, 3);
static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
-static int ReadPageInternal(XLogReaderState *state, XLogRecPtr pageptr,
- int reqLen);
+static bool XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr,
+ int reqLen, bool header_inclusive);
static void XLogReaderInvalReadState(XLogReaderState *state);
static bool ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
XLogRecPtr PrevRecPtr, XLogRecord *record, bool randAccess);
@@ -276,7 +276,6 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
uint32 targetRecOff;
uint32 pageHeaderSize;
bool gotheader;
- int readOff;
/*
* randAccess indicates whether to verify the previous-record pointer of
@@ -326,14 +325,20 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
* byte to cover the whole record header, or at least the part of it that
* fits on the same page.
*/
- readOff = ReadPageInternal(state, targetPagePtr,
- Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ));
- if (readOff < 0)
+ while (XLogNeedData(state, targetPagePtr,
+ Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+ targetRecOff != 0))
+ {
+ if (!state->routine.page_read(state, state->readPagePtr, state->readLen,
+ RecPtr, state->readBuf))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
/*
- * ReadPageInternal always returns at least the page header, so we can
- * examine it now.
+ * We have at least the page header, so we can examine it now.
*/
pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
if (targetRecOff == 0)
@@ -359,8 +364,8 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
goto err;
}
- /* ReadPageInternal has verified the page header */
- Assert(pageHeaderSize <= readOff);
+ /* XLogNeedData has verified the page header */
+ Assert(pageHeaderSize <= state->readLen);
/*
* Read the record length.
@@ -432,18 +437,27 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
do
{
+ int rest_len = total_len - gotlen;
+
/* Calculate pointer to beginning of next page */
targetPagePtr += XLOG_BLCKSZ;
/* Wait for the next page to become available */
- readOff = ReadPageInternal(state, targetPagePtr,
- Min(total_len - gotlen + SizeOfXLogShortPHD,
- XLOG_BLCKSZ));
+ while (XLogNeedData(state, targetPagePtr,
+ Min(rest_len, XLOG_BLCKSZ),
+ false))
+ {
+ if (!state->routine.page_read(state, state->readPagePtr,
+ state->readLen,
+ state->ReadRecPtr,
+ state->readBuf))
+ break;
+ }
- if (readOff < 0)
+ if (!state->page_verified)
goto err;
- Assert(SizeOfXLogShortPHD <= readOff);
+ Assert(SizeOfXLogShortPHD <= state->readLen);
/* Check that the continuation on next page looks valid */
pageHeader = (XLogPageHeader) state->readBuf;
@@ -473,21 +487,14 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
/* Append the continuation from this page to the buffer */
pageHeaderSize = XLogPageHeaderSize(pageHeader);
- if (readOff < pageHeaderSize)
- readOff = ReadPageInternal(state, targetPagePtr,
- pageHeaderSize);
-
- Assert(pageHeaderSize <= readOff);
+ Assert(pageHeaderSize <= state->readLen);
contdata = (char *) state->readBuf + pageHeaderSize;
len = XLOG_BLCKSZ - pageHeaderSize;
if (pageHeader->xlp_rem_len < len)
len = pageHeader->xlp_rem_len;
- if (readOff < pageHeaderSize + len)
- readOff = ReadPageInternal(state, targetPagePtr,
- pageHeaderSize + len);
-
+ Assert (pageHeaderSize + len <= state->readLen);
memcpy(buffer, (char *) contdata, len);
buffer += len;
gotlen += len;
@@ -517,9 +524,16 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
else
{
/* Wait for the record data to become available */
- readOff = ReadPageInternal(state, targetPagePtr,
- Min(targetRecOff + total_len, XLOG_BLCKSZ));
- if (readOff < 0)
+ while (XLogNeedData(state, targetPagePtr,
+ Min(targetRecOff + total_len, XLOG_BLCKSZ), true))
+ {
+ if (!state->routine.page_read(state, state->readPagePtr,
+ state->readLen,
+ state->ReadRecPtr, state->readBuf))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
/* Record does not cross a page boundary */
@@ -562,109 +576,138 @@ err:
}
/*
- * Read a single xlog page including at least [pageptr, reqLen] of valid data
- * via the page_read() callback.
+ * Checks that an xlog page loaded in state->readBuf is including at least
+ * [pageptr, reqLen] and the page is valid. header_inclusive indicates that
+ * reqLen is calculated including page header length.
*
- * Returns -1 if the required page cannot be read for some reason; errormsg_buf
- * is set in that case (unless the error occurs in the page_read callback).
+ * Returns false if the buffer already contains the requested data, or found
+ * error. state->page_verified is set to true for the former and false for the
+ * latter.
*
- * We fetch the page from a reader-local cache if we know we have the required
- * data and if there hasn't been any error since caching the data.
+ * Otherwise returns true and requests data loaded onto state->readBuf by
+ * state->readPagePtr and state->readLen. The caller shall call this function
+ * again after filling the buffer at least with that portion of data and set
+ * state->readLen to the length of actually loaded data.
+ *
+ * If header_inclusive is false, corrects reqLen internally by adding the
+ * actual page header length and may request caller for new data.
*/
-static int
-ReadPageInternal(XLogReaderState *state, XLogRecPtr pageptr, int reqLen)
+static bool
+XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
+ bool header_inclusive)
{
- int readLen;
uint32 targetPageOff;
XLogSegNo targetSegNo;
- XLogPageHeader hdr;
+ uint32 addLen = 0;
- Assert((pageptr % XLOG_BLCKSZ) == 0);
+ /* Some data is loaded, but page header is not verified yet. */
+ if (!state->page_verified &&
+ !XLogRecPtrIsInvalid(state->readPagePtr) && state->readLen >= 0)
+ {
+ uint32 pageHeaderSize;
+ /* just loaded new data so needs to verify page header */
+
+ /* The caller must have loaded at least page header */
+ Assert (state->readLen >= SizeOfXLogShortPHD);
+
+ /*
+ * We have enough data to check the header length. Recheck the loaded
+ * length against the actual header length.
+ */
+ pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+
+ /* Request more data if we don't have the full header. */
+ if (state->readLen < pageHeaderSize)
+ {
+ state->readLen = pageHeaderSize;
+ return true;
+ }
+
+ /* Now that we know we have the full header, validate it. */
+ if (!XLogReaderValidatePageHeader(state, state->readPagePtr,
+ (char *) state->readBuf))
+ {
+ /* That's bad. Force reading the page again. */
+ XLogReaderInvalReadState(state);
+
+ return false;
+ }
+
+ state->page_verified = true;
+
+ XLByteToSeg(state->readPagePtr, state->seg.ws_segno,
+ state->segcxt.ws_segsize);
+ }
+
+ /*
+ * The loaded page may not be the one caller is supposing to read when we
+ * are verifying the first page of new segment. In that case, skip further
+ * verification and immediately load the target page.
+ */
+ if (state->page_verified && pageptr == state->readPagePtr)
+ {
+ /*
+ * calculate additional length for page header keeping the total
+ * length within the block size.
+ */
+ if (!header_inclusive)
+ {
+ uint32 pageHeaderSize =
+ XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+
+ addLen = pageHeaderSize;
+ if (reqLen + pageHeaderSize <= XLOG_BLCKSZ)
+ addLen = pageHeaderSize;
+ else
+ addLen = XLOG_BLCKSZ - reqLen;
+
+ Assert(addLen >= 0);
+ }
+
+ /* Return if we already have it. */
+ if (reqLen + addLen <= state->readLen)
+ return false;
+ }
+
+ /* Data is not in our buffer, request the caller for it. */
XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize);
targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize);
+ Assert((pageptr % XLOG_BLCKSZ) == 0);
- /* check whether we have all the requested data already */
- if (targetSegNo == state->seg.ws_segno &&
- targetPageOff == state->segoff && reqLen <= state->readLen)
- return state->readLen;
+ /*
+ * Every time we request to load new data of a page to the caller, even if
+ * we looked at a part of it before, we need to do verification on the next
+ * invocation as the caller might now be rereading data from a different
+ * source.
+ */
+ state->page_verified = false;
/*
- * Data is not in our buffer.
- *
- * Every time we actually read the segment, even if we looked at parts of
- * it before, we need to do verification as the page_read callback might
- * now be rereading data from a different source.
- *
* Whenever switching to a new WAL segment, we read the first page of the
* file and validate its header, even if that's not where the target
* record is. This is so that we can check the additional identification
* info that is present in the first page's "long" header.
+ * Don't do this if the caller requested the first page in the segment.
*/
if (targetSegNo != state->seg.ws_segno && targetPageOff != 0)
{
- XLogRecPtr targetSegmentPtr = pageptr - targetPageOff;
-
- readLen = state->routine.page_read(state, targetSegmentPtr, XLOG_BLCKSZ,
- state->currRecPtr,
- state->readBuf);
- if (readLen < 0)
- goto err;
-
- /* we can be sure to have enough WAL available, we scrolled back */
- Assert(readLen == XLOG_BLCKSZ);
-
- if (!XLogReaderValidatePageHeader(state, targetSegmentPtr,
- state->readBuf))
- goto err;
+ /*
+ * Then we'll see that the targetSegNo now matches the ws_segno, and
+ * will not come back here, but will request the actual target page.
+ */
+ state->readPagePtr = pageptr - targetPageOff;
+ state->readLen = XLOG_BLCKSZ;
+ return true;
}
/*
- * First, read the requested data length, but at least a short page header
- * so that we can validate it.
+ * Request the caller to load the page. We need at least a short page
+ * header so that we can validate it.
*/
- readLen = state->routine.page_read(state, pageptr, Max(reqLen, SizeOfXLogShortPHD),
- state->currRecPtr,
- state->readBuf);
- if (readLen < 0)
- goto err;
-
- Assert(readLen <= XLOG_BLCKSZ);
-
- /* Do we have enough data to check the header length? */
- if (readLen <= SizeOfXLogShortPHD)
- goto err;
-
- Assert(readLen >= reqLen);
-
- hdr = (XLogPageHeader) state->readBuf;
-
- /* still not enough */
- if (readLen < XLogPageHeaderSize(hdr))
- {
- readLen = state->routine.page_read(state, pageptr, XLogPageHeaderSize(hdr),
- state->currRecPtr,
- state->readBuf);
- if (readLen < 0)
- goto err;
- }
-
- /*
- * Now that we know we have the full header, validate it.
- */
- if (!XLogReaderValidatePageHeader(state, pageptr, (char *) hdr))
- goto err;
-
- /* update read state information */
- state->seg.ws_segno = targetSegNo;
- state->segoff = targetPageOff;
- state->readLen = readLen;
-
- return readLen;
-
-err:
- XLogReaderInvalReadState(state);
- return -1;
+ state->readPagePtr = pageptr;
+ state->readLen = Max(reqLen + addLen, SizeOfXLogShortPHD);
+ return true;
}
/*
@@ -673,9 +716,7 @@ err:
static void
XLogReaderInvalReadState(XLogReaderState *state)
{
- state->seg.ws_segno = 0;
- state->segoff = 0;
- state->readLen = 0;
+ state->readPagePtr = InvalidXLogRecPtr;
}
/*
@@ -953,7 +994,6 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
XLogRecPtr targetPagePtr;
int targetRecOff;
uint32 pageHeaderSize;
- int readLen;
/*
* Compute targetRecOff. It should typically be equal or greater than
@@ -961,7 +1001,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
* that, except when caller has explicitly specified the offset that
* falls somewhere there or when we are skipping multi-page
* continuation record. It doesn't matter though because
- * ReadPageInternal() is prepared to handle that and will read at
+ * XLogNeedData() is prepared to handle that and will read at
* least short page-header worth of data
*/
targetRecOff = tmpRecPtr % XLOG_BLCKSZ;
@@ -969,19 +1009,24 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
/* scroll back to page boundary */
targetPagePtr = tmpRecPtr - targetRecOff;
- /* Read the page containing the record */
- readLen = ReadPageInternal(state, targetPagePtr, targetRecOff);
- if (readLen < 0)
+ while(XLogNeedData(state, targetPagePtr, targetRecOff,
+ targetRecOff != 0))
+ {
+ if (!state->routine.page_read(state, state->readPagePtr,
+ state->readLen,
+ state->ReadRecPtr, state->readBuf))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
header = (XLogPageHeader) state->readBuf;
pageHeaderSize = XLogPageHeaderSize(header);
- /* make sure we have enough data for the page header */
- readLen = ReadPageInternal(state, targetPagePtr, pageHeaderSize);
- if (readLen < 0)
- goto err;
+ /* we should have read the page header */
+ Assert (state->readLen >= pageHeaderSize);
/* skip over potential continuation data */
if (header->xlp_info & XLP_FIRST_IS_CONTRECORD)
diff --git a/src/backend/access/transam/xlogutils.c b/src/backend/access/transam/xlogutils.c
index d17d660f46..46eda33f25 100644
--- a/src/backend/access/transam/xlogutils.c
+++ b/src/backend/access/transam/xlogutils.c
@@ -686,8 +686,8 @@ XLogTruncateRelation(RelFileNode rnode, ForkNumber forkNum,
void
XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wantLength)
{
- const XLogRecPtr lastReadPage = (state->seg.ws_segno *
- state->segcxt.ws_segsize + state->segoff);
+ const XLogRecPtr lastReadPage = state->seg.ws_segno *
+ state->segcxt.ws_segsize + state->readLen;
Assert(wantPage != InvalidXLogRecPtr && wantPage % XLOG_BLCKSZ == 0);
Assert(wantLength <= XLOG_BLCKSZ);
@@ -824,7 +824,7 @@ wal_segment_close(XLogReaderState *state)
* exists for normal backends, so we have to do a check/sleep/repeat style of
* loop for now.
*/
-int
+bool
read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *cur_page)
{
@@ -926,7 +926,8 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
else if (targetPagePtr + reqLen > read_upto)
{
/* not enough data there */
- return -1;
+ state->readLen = -1;
+ return false;
}
else
{
@@ -944,7 +945,8 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
WALReadRaiseError(&errinfo);
/* number of valid bytes in the buffer */
- return count;
+ state->readLen = count;
+ return true;
}
/*
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index 23baa4498a..c36b91ae5e 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -806,7 +806,7 @@ StartReplication(StartReplicationCmd *cmd)
* which has to do a plain sleep/busy loop, because the walsender's latch gets
* set every time WAL is flushed.
*/
-static int
+static bool
logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *cur_page)
{
@@ -826,7 +826,10 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
/* fail if not (implies we are going to shut down) */
if (flushptr < targetPagePtr + reqLen)
- return -1;
+ {
+ state->readLen = -1;
+ return false;
+ }
if (targetPagePtr + XLOG_BLCKSZ <= flushptr)
count = XLOG_BLCKSZ; /* more than one block available */
@@ -854,7 +857,8 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
XLByteToSeg(targetPagePtr, segno, state->segcxt.ws_segsize);
CheckXLogRemoved(segno, state->seg.ws_tli);
- return count;
+ state->readLen = count;
+ return true;
}
/*
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index 59ebac7d6a..cf119848b0 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -47,7 +47,7 @@ typedef struct XLogPageReadPrivate
int tliIndex;
} XLogPageReadPrivate;
-static int SimpleXLogPageRead(XLogReaderState *xlogreader,
+static bool SimpleXLogPageRead(XLogReaderState *xlogreader,
XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
@@ -246,7 +246,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
}
/* XLogReader callback function, to read a WAL page */
-static int
+static bool
SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf)
{
@@ -306,7 +306,8 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
if (private->restoreCommand == NULL)
{
pg_log_error("could not open file \"%s\": %m", xlogfpath);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
/*
@@ -319,7 +320,10 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
private->restoreCommand);
if (xlogreadfd < 0)
- return -1;
+ {
+ xlogreader->readLen = -1;
+ return false;
+ }
else
pg_log_debug("using file \"%s\" restored from archive",
xlogfpath);
@@ -335,7 +339,8 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
if (lseek(xlogreadfd, (off_t) targetPageOff, SEEK_SET) < 0)
{
pg_log_error("could not seek in file \"%s\": %m", xlogfpath);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
@@ -348,13 +353,15 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
pg_log_error("could not read file \"%s\": read %d of %zu",
xlogfpath, r, (Size) XLOG_BLCKSZ);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
Assert(targetSegNo == xlogreadsegno);
xlogreader->seg.ws_tli = targetHistory[private->tliIndex].tli;
- return XLOG_BLCKSZ;
+ xlogreader->readLen = XLOG_BLCKSZ;
+ return true;
}
/*
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index 610f65e471..0f0e0723a3 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -331,7 +331,7 @@ WALDumpCloseSegment(XLogReaderState *state)
}
/* pg_waldump's XLogReaderRoutine->page_read callback */
-static int
+static bool
WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetPtr, char *readBuff)
{
@@ -348,7 +348,8 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
else
{
private->endptr_reached = true;
- return -1;
+ state->readLen = -1;
+ return false;
}
}
@@ -373,7 +374,8 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
(Size) errinfo.wre_req);
}
- return count;
+ state->readLen = count;
+ return true;
}
/*
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index 21d200d3df..23d9e70a59 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -57,8 +57,8 @@ typedef struct WALSegmentContext
typedef struct XLogReaderState XLogReaderState;
-/* Function type definitions for various xlogreader interactions */
-typedef int (*XLogPageReadCB) (XLogReaderState *xlogreader,
+/* Function type definition for the read_page callback */
+typedef bool (*XLogPageReadCB) (XLogReaderState *xlogreader,
XLogRecPtr targetPagePtr,
int reqLen,
XLogRecPtr targetRecPtr,
@@ -175,6 +175,20 @@ struct XLogReaderState
XLogRecPtr ReadRecPtr; /* start of last record read */
XLogRecPtr EndRecPtr; /* end+1 of last record read */
+ /* ----------------------------------------
+ * Communication with page reader
+ * readBuf is XLOG_BLCKSZ bytes, valid up to at least readLen bytes.
+ * ----------------------------------------
+ */
+ /* variables to communicate with page reader */
+ XLogRecPtr readPagePtr; /* page pointer to read */
+ int32 readLen; /* bytes requested to reader, or actual bytes
+ * read by reader, which must be larger than
+ * the request, or -1 on error */
+ TimeLineID readPageTLI; /* TLI for data currently in readBuf */
+ char *readBuf; /* buffer to store data */
+ bool page_verified; /* is the page on the buffer verified? */
+
/* ----------------------------------------
* Decoded representation of current record
@@ -203,13 +217,6 @@ struct XLogReaderState
* ----------------------------------------
*/
- /*
- * Buffer for currently read page (XLOG_BLCKSZ bytes, valid up to at least
- * readLen bytes)
- */
- char *readBuf;
- uint32 readLen;
-
/* last read XLOG position for data currently in readBuf */
WALSegmentContext segcxt;
WALOpenSegment seg;
diff --git a/src/include/access/xlogutils.h b/src/include/access/xlogutils.h
index 9ac602b674..364a21c4ea 100644
--- a/src/include/access/xlogutils.h
+++ b/src/include/access/xlogutils.h
@@ -47,7 +47,7 @@ extern Buffer XLogReadBufferExtended(RelFileNode rnode, ForkNumber forknum,
extern Relation CreateFakeRelcacheEntry(RelFileNode rnode);
extern void FreeFakeRelcacheEntry(Relation fakerel);
-extern int read_local_xlog_page(XLogReaderState *state,
+extern bool read_local_xlog_page(XLogReaderState *state,
XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *cur_page);
extern void wal_segment_open(XLogReaderState *state,
--
2.27.0
----Next_Part(Thu_Mar__4_11_28_53_2021_014)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
filename="v17-0002-Move-page-reader-out-of-XLogReadRecord.patch"
^ permalink raw reply [nested|flat] 69+ messages in thread
* [PATCH v19 1/5] Move callback-call from ReadPageInternal to XLogReadRecord.
@ 2019-09-05 11:21 Kyotaro Horiguchi <[email protected]>
0 siblings, 0 replies; 69+ messages in thread
From: Kyotaro Horiguchi @ 2019-09-05 11:21 UTC (permalink / raw)
The current WAL record reader reads page data using a call back
function. Redesign the interface so that it asks the caller for more
data when required. This model works better for proposed projects that
encryption, prefetching and other new features that would require
extending the callback interface for each case.
As the first step of that change, this patch moves the page reader
function out of ReadPageInternal(), then the remaining tasks of the
function are taken over by the new function XLogNeedData().
Author: Kyotaro HORIGUCHI <[email protected]>
Author: Heikki Linnakangas <[email protected]>
Reviewed-by: Antonin Houska <[email protected]>
Reviewed-by: Alvaro Herrera <[email protected]>
Reviewed-by: Takashi Menjo <[email protected]>
Reviewed-by: Thomas Munro <[email protected]>
Discussion: https://postgr.es/m/20190418.210257.43726183.horiguchi.kyotaro%40lab.ntt.co.jp
---
src/backend/access/transam/xlog.c | 16 +-
src/backend/access/transam/xlogreader.c | 325 +++++++++++++++---------
src/backend/access/transam/xlogutils.c | 12 +-
src/backend/replication/walsender.c | 10 +-
src/bin/pg_rewind/parsexlog.c | 21 +-
src/bin/pg_waldump/pg_waldump.c | 8 +-
src/include/access/xlogreader.h | 32 ++-
src/include/access/xlogutils.h | 2 +-
8 files changed, 267 insertions(+), 159 deletions(-)
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index c1d4415a43..8085ca1117 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -920,7 +920,7 @@ static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath,
static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli,
XLogSource source, bool notfoundOk);
static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, XLogSource source);
-static int XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
+static bool XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
bool fetching_ckpt, XLogRecPtr tliRecPtr);
@@ -4375,7 +4375,6 @@ ReadRecord(XLogReaderState *xlogreader, int emode,
XLogRecord *record;
XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
- /* Pass through parameters to XLogPageRead */
private->fetching_ckpt = fetching_ckpt;
private->emode = emode;
private->randAccess = (xlogreader->ReadRecPtr == InvalidXLogRecPtr);
@@ -12107,7 +12106,7 @@ CancelBackup(void)
* XLogPageRead() to try fetching the record from another source, or to
* sleep and retry.
*/
-static int
+static bool
XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *readBuf)
{
@@ -12166,7 +12165,8 @@ retry:
readLen = 0;
readSource = XLOG_FROM_ANY;
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
}
@@ -12261,7 +12261,8 @@ retry:
goto next_record_is_invalid;
}
- return readLen;
+ xlogreader->readLen = readLen;
+ return true;
next_record_is_invalid:
lastSourceFailed = true;
@@ -12275,8 +12276,9 @@ next_record_is_invalid:
/* In standby-mode, keep trying */
if (StandbyMode)
goto retry;
- else
- return -1;
+
+ xlogreader->readLen = -1;
+ return false;
}
/*
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index 42738eb940..f2345ab09e 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -36,8 +36,8 @@
static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
pg_attribute_printf(2, 3);
static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
-static int ReadPageInternal(XLogReaderState *state, XLogRecPtr pageptr,
- int reqLen);
+static bool XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr,
+ int reqLen, bool header_inclusive);
static void XLogReaderInvalReadState(XLogReaderState *state);
static bool ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
XLogRecPtr PrevRecPtr, XLogRecord *record, bool randAccess);
@@ -261,8 +261,48 @@ XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
* If the reading fails for some other reason, NULL is also returned, and
* *errormsg is set to a string with details of the failure.
*
+ * Returns XLREAD_NEED_DATA if more data is needed to finish reading the
+ * current record. In that case, state->readPagePtr and state->readLen inform
+ * the desired position and minimum length of data needed. The caller shall
+ * read in the requested data and set state->readBuf to point to a buffer
+ * containing it. The caller must also set state->seg->ws_tli and
+ * state->readLen to indicate the timeline that it was read from, and the
+ * length of data that is now available (which must be >= given readLen),
+ * respectively.
+ *
+ * If invalid data is encountered, returns XLREAD_FAIL with *record being set to
+ * NULL. *errormsg is set to a string with details of the failure.
* The returned pointer (or *errormsg) points to an internal buffer that's
* valid until the next call to XLogReadRecord.
+ *
+ *
+ * This function runs a state machine consists of the following states.
+ *
+ * XLREAD_NEXT_RECORD :
+ * The initial state, if called with valid RecPtr, try to read a record at
+ * that position. If invalid RecPtr is given try to read a record just after
+ * the last one previously read.
+ * This state ens after setting ReadRecPtr. Then goes to XLREAD_TOT_LEN.
+ *
+ * XLREAD_TOT_LEN:
+ * Examining record header. Ends after reading record total
+ * length. recordRemainLen and recordGotLen are initialized.
+ *
+ * XLREAD_FIRST_FRAGMENT:
+ * Reading the first fragment. Ends with finishing reading a single
+ * record. Goes to XLREAD_NEXT_RECORD if that's all or
+ * XLREAD_CONTINUATION if we have continuation.
+
+ * XLREAD_CONTINUATION:
+ * Reading continuation of record. Ends with finishing the whole record then
+ * goes to XLREAD_NEXT_RECORD. During this state, recordRemainLen indicates
+ * how much is left and readRecordBuf holds the partially assert
+ * record.recordContRecPtr points to the beginning of the next page where to
+ * continue.
+ *
+ * If wrong data found in any state, the state machine stays at the current
+ * state. This behavior allows to continue reading a reacord switching among
+ * different souces, while streaming replication.
*/
XLogRecord *
XLogReadRecord(XLogReaderState *state, char **errormsg)
@@ -276,7 +316,6 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
uint32 targetRecOff;
uint32 pageHeaderSize;
bool gotheader;
- int readOff;
/*
* randAccess indicates whether to verify the previous-record pointer of
@@ -326,14 +365,20 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
* byte to cover the whole record header, or at least the part of it that
* fits on the same page.
*/
- readOff = ReadPageInternal(state, targetPagePtr,
- Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ));
- if (readOff < 0)
+ while (XLogNeedData(state, targetPagePtr,
+ Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+ targetRecOff != 0))
+ {
+ if (!state->routine.page_read(state, state->readPagePtr, state->readLen,
+ RecPtr, state->readBuf))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
/*
- * ReadPageInternal always returns at least the page header, so we can
- * examine it now.
+ * We have at least the page header, so we can examine it now.
*/
pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
if (targetRecOff == 0)
@@ -359,8 +404,8 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
goto err;
}
- /* ReadPageInternal has verified the page header */
- Assert(pageHeaderSize <= readOff);
+ /* XLogNeedData has verified the page header */
+ Assert(pageHeaderSize <= state->readLen);
/*
* Read the record length.
@@ -432,18 +477,27 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
do
{
+ int rest_len = total_len - gotlen;
+
/* Calculate pointer to beginning of next page */
targetPagePtr += XLOG_BLCKSZ;
/* Wait for the next page to become available */
- readOff = ReadPageInternal(state, targetPagePtr,
- Min(total_len - gotlen + SizeOfXLogShortPHD,
- XLOG_BLCKSZ));
+ while (XLogNeedData(state, targetPagePtr,
+ Min(rest_len, XLOG_BLCKSZ),
+ false))
+ {
+ if (!state->routine.page_read(state, state->readPagePtr,
+ state->readLen,
+ state->ReadRecPtr,
+ state->readBuf))
+ break;
+ }
- if (readOff < 0)
+ if (!state->page_verified)
goto err;
- Assert(SizeOfXLogShortPHD <= readOff);
+ Assert(SizeOfXLogShortPHD <= state->readLen);
/* Check that the continuation on next page looks valid */
pageHeader = (XLogPageHeader) state->readBuf;
@@ -473,21 +527,14 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
/* Append the continuation from this page to the buffer */
pageHeaderSize = XLogPageHeaderSize(pageHeader);
- if (readOff < pageHeaderSize)
- readOff = ReadPageInternal(state, targetPagePtr,
- pageHeaderSize);
-
- Assert(pageHeaderSize <= readOff);
+ Assert(pageHeaderSize <= state->readLen);
contdata = (char *) state->readBuf + pageHeaderSize;
len = XLOG_BLCKSZ - pageHeaderSize;
if (pageHeader->xlp_rem_len < len)
len = pageHeader->xlp_rem_len;
- if (readOff < pageHeaderSize + len)
- readOff = ReadPageInternal(state, targetPagePtr,
- pageHeaderSize + len);
-
+ Assert(pageHeaderSize + len <= state->readLen);
memcpy(buffer, (char *) contdata, len);
buffer += len;
gotlen += len;
@@ -517,9 +564,16 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
else
{
/* Wait for the record data to become available */
- readOff = ReadPageInternal(state, targetPagePtr,
- Min(targetRecOff + total_len, XLOG_BLCKSZ));
- if (readOff < 0)
+ while (XLogNeedData(state, targetPagePtr,
+ Min(targetRecOff + total_len, XLOG_BLCKSZ), true))
+ {
+ if (!state->routine.page_read(state, state->readPagePtr,
+ state->readLen,
+ state->ReadRecPtr, state->readBuf))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
/* Record does not cross a page boundary */
@@ -562,109 +616,138 @@ err:
}
/*
- * Read a single xlog page including at least [pageptr, reqLen] of valid data
- * via the page_read() callback.
+ * Checks that an xlog page loaded in state->readBuf is including at least
+ * [pageptr, reqLen] and the page is valid. header_inclusive indicates that
+ * reqLen is calculated including page header length.
*
- * Returns -1 if the required page cannot be read for some reason; errormsg_buf
- * is set in that case (unless the error occurs in the page_read callback).
+ * Returns false if the buffer already contains the requested data, or found
+ * error. state->page_verified is set to true for the former and false for the
+ * latter.
*
- * We fetch the page from a reader-local cache if we know we have the required
- * data and if there hasn't been any error since caching the data.
+ * Otherwise returns true and requests data loaded onto state->readBuf by
+ * state->readPagePtr and state->readLen. The caller shall call this function
+ * again after filling the buffer at least with that portion of data and set
+ * state->readLen to the length of actually loaded data.
+ *
+ * If header_inclusive is false, corrects reqLen internally by adding the
+ * actual page header length and may request caller for new data.
*/
-static int
-ReadPageInternal(XLogReaderState *state, XLogRecPtr pageptr, int reqLen)
+static bool
+XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
+ bool header_inclusive)
{
- int readLen;
uint32 targetPageOff;
XLogSegNo targetSegNo;
- XLogPageHeader hdr;
+ uint32 addLen = 0;
- Assert((pageptr % XLOG_BLCKSZ) == 0);
+ /* Some data is loaded, but page header is not verified yet. */
+ if (!state->page_verified &&
+ !XLogRecPtrIsInvalid(state->readPagePtr) && state->readLen >= 0)
+ {
+ uint32 pageHeaderSize;
+ /* just loaded new data so needs to verify page header */
+
+ /* The caller must have loaded at least page header */
+ Assert(state->readLen >= SizeOfXLogShortPHD);
+
+ /*
+ * We have enough data to check the header length. Recheck the loaded
+ * length against the actual header length.
+ */
+ pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+
+ /* Request more data if we don't have the full header. */
+ if (state->readLen < pageHeaderSize)
+ {
+ state->readLen = pageHeaderSize;
+ return true;
+ }
+
+ /* Now that we know we have the full header, validate it. */
+ if (!XLogReaderValidatePageHeader(state, state->readPagePtr,
+ (char *) state->readBuf))
+ {
+ /* That's bad. Force reading the page again. */
+ XLogReaderInvalReadState(state);
+
+ return false;
+ }
+
+ state->page_verified = true;
+
+ XLByteToSeg(state->readPagePtr, state->seg.ws_segno,
+ state->segcxt.ws_segsize);
+ }
+
+ /*
+ * The loaded page may not be the one caller is supposing to read when we
+ * are verifying the first page of new segment. In that case, skip further
+ * verification and immediately load the target page.
+ */
+ if (state->page_verified && pageptr == state->readPagePtr)
+ {
+ /*
+ * calculate additional length for page header keeping the total
+ * length within the block size.
+ */
+ if (!header_inclusive)
+ {
+ uint32 pageHeaderSize =
+ XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+
+ addLen = pageHeaderSize;
+ if (reqLen + pageHeaderSize <= XLOG_BLCKSZ)
+ addLen = pageHeaderSize;
+ else
+ addLen = XLOG_BLCKSZ - reqLen;
+
+ Assert(addLen >= 0);
+ }
+
+ /* Return if we already have it. */
+ if (reqLen + addLen <= state->readLen)
+ return false;
+ }
+
+ /* Data is not in our buffer, request the caller for it. */
XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize);
targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize);
+ Assert((pageptr % XLOG_BLCKSZ) == 0);
- /* check whether we have all the requested data already */
- if (targetSegNo == state->seg.ws_segno &&
- targetPageOff == state->segoff && reqLen <= state->readLen)
- return state->readLen;
+ /*
+ * Every time we request to load new data of a page to the caller, even if
+ * we looked at a part of it before, we need to do verification on the
+ * next invocation as the caller might now be rereading data from a
+ * different source.
+ */
+ state->page_verified = false;
/*
- * Data is not in our buffer.
- *
- * Every time we actually read the segment, even if we looked at parts of
- * it before, we need to do verification as the page_read callback might
- * now be rereading data from a different source.
- *
* Whenever switching to a new WAL segment, we read the first page of the
* file and validate its header, even if that's not where the target
* record is. This is so that we can check the additional identification
- * info that is present in the first page's "long" header.
+ * info that is present in the first page's "long" header. Don't do this
+ * if the caller requested the first page in the segment.
*/
if (targetSegNo != state->seg.ws_segno && targetPageOff != 0)
{
- XLogRecPtr targetSegmentPtr = pageptr - targetPageOff;
-
- readLen = state->routine.page_read(state, targetSegmentPtr, XLOG_BLCKSZ,
- state->currRecPtr,
- state->readBuf);
- if (readLen < 0)
- goto err;
-
- /* we can be sure to have enough WAL available, we scrolled back */
- Assert(readLen == XLOG_BLCKSZ);
-
- if (!XLogReaderValidatePageHeader(state, targetSegmentPtr,
- state->readBuf))
- goto err;
+ /*
+ * Then we'll see that the targetSegNo now matches the ws_segno, and
+ * will not come back here, but will request the actual target page.
+ */
+ state->readPagePtr = pageptr - targetPageOff;
+ state->readLen = XLOG_BLCKSZ;
+ return true;
}
/*
- * First, read the requested data length, but at least a short page header
- * so that we can validate it.
+ * Request the caller to load the page. We need at least a short page
+ * header so that we can validate it.
*/
- readLen = state->routine.page_read(state, pageptr, Max(reqLen, SizeOfXLogShortPHD),
- state->currRecPtr,
- state->readBuf);
- if (readLen < 0)
- goto err;
-
- Assert(readLen <= XLOG_BLCKSZ);
-
- /* Do we have enough data to check the header length? */
- if (readLen <= SizeOfXLogShortPHD)
- goto err;
-
- Assert(readLen >= reqLen);
-
- hdr = (XLogPageHeader) state->readBuf;
-
- /* still not enough */
- if (readLen < XLogPageHeaderSize(hdr))
- {
- readLen = state->routine.page_read(state, pageptr, XLogPageHeaderSize(hdr),
- state->currRecPtr,
- state->readBuf);
- if (readLen < 0)
- goto err;
- }
-
- /*
- * Now that we know we have the full header, validate it.
- */
- if (!XLogReaderValidatePageHeader(state, pageptr, (char *) hdr))
- goto err;
-
- /* update read state information */
- state->seg.ws_segno = targetSegNo;
- state->segoff = targetPageOff;
- state->readLen = readLen;
-
- return readLen;
-
-err:
- XLogReaderInvalReadState(state);
- return -1;
+ state->readPagePtr = pageptr;
+ state->readLen = Max(reqLen + addLen, SizeOfXLogShortPHD);
+ return true;
}
/*
@@ -673,9 +756,7 @@ err:
static void
XLogReaderInvalReadState(XLogReaderState *state)
{
- state->seg.ws_segno = 0;
- state->segoff = 0;
- state->readLen = 0;
+ state->readPagePtr = InvalidXLogRecPtr;
}
/*
@@ -953,7 +1034,6 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
XLogRecPtr targetPagePtr;
int targetRecOff;
uint32 pageHeaderSize;
- int readLen;
/*
* Compute targetRecOff. It should typically be equal or greater than
@@ -961,27 +1041,32 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
* that, except when caller has explicitly specified the offset that
* falls somewhere there or when we are skipping multi-page
* continuation record. It doesn't matter though because
- * ReadPageInternal() is prepared to handle that and will read at
- * least short page-header worth of data
+ * XLogNeedData() is prepared to handle that and will read at least
+ * short page-header worth of data
*/
targetRecOff = tmpRecPtr % XLOG_BLCKSZ;
/* scroll back to page boundary */
targetPagePtr = tmpRecPtr - targetRecOff;
- /* Read the page containing the record */
- readLen = ReadPageInternal(state, targetPagePtr, targetRecOff);
- if (readLen < 0)
+ while (XLogNeedData(state, targetPagePtr, targetRecOff,
+ targetRecOff != 0))
+ {
+ if (!state->routine.page_read(state, state->readPagePtr,
+ state->readLen,
+ state->ReadRecPtr, state->readBuf))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
header = (XLogPageHeader) state->readBuf;
pageHeaderSize = XLogPageHeaderSize(header);
- /* make sure we have enough data for the page header */
- readLen = ReadPageInternal(state, targetPagePtr, pageHeaderSize);
- if (readLen < 0)
- goto err;
+ /* we should have read the page header */
+ Assert(state->readLen >= pageHeaderSize);
/* skip over potential continuation data */
if (header->xlp_info & XLP_FIRST_IS_CONTRECORD)
diff --git a/src/backend/access/transam/xlogutils.c b/src/backend/access/transam/xlogutils.c
index d17d660f46..46eda33f25 100644
--- a/src/backend/access/transam/xlogutils.c
+++ b/src/backend/access/transam/xlogutils.c
@@ -686,8 +686,8 @@ XLogTruncateRelation(RelFileNode rnode, ForkNumber forkNum,
void
XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wantLength)
{
- const XLogRecPtr lastReadPage = (state->seg.ws_segno *
- state->segcxt.ws_segsize + state->segoff);
+ const XLogRecPtr lastReadPage = state->seg.ws_segno *
+ state->segcxt.ws_segsize + state->readLen;
Assert(wantPage != InvalidXLogRecPtr && wantPage % XLOG_BLCKSZ == 0);
Assert(wantLength <= XLOG_BLCKSZ);
@@ -824,7 +824,7 @@ wal_segment_close(XLogReaderState *state)
* exists for normal backends, so we have to do a check/sleep/repeat style of
* loop for now.
*/
-int
+bool
read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *cur_page)
{
@@ -926,7 +926,8 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
else if (targetPagePtr + reqLen > read_upto)
{
/* not enough data there */
- return -1;
+ state->readLen = -1;
+ return false;
}
else
{
@@ -944,7 +945,8 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
WALReadRaiseError(&errinfo);
/* number of valid bytes in the buffer */
- return count;
+ state->readLen = count;
+ return true;
}
/*
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index 4bf8a18e01..a4d6f30957 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -806,7 +806,7 @@ StartReplication(StartReplicationCmd *cmd)
* which has to do a plain sleep/busy loop, because the walsender's latch gets
* set every time WAL is flushed.
*/
-static int
+static bool
logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *cur_page)
{
@@ -826,7 +826,10 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
/* fail if not (implies we are going to shut down) */
if (flushptr < targetPagePtr + reqLen)
- return -1;
+ {
+ state->readLen = -1;
+ return false;
+ }
if (targetPagePtr + XLOG_BLCKSZ <= flushptr)
count = XLOG_BLCKSZ; /* more than one block available */
@@ -854,7 +857,8 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
XLByteToSeg(targetPagePtr, segno, state->segcxt.ws_segsize);
CheckXLogRemoved(segno, state->seg.ws_tli);
- return count;
+ state->readLen = count;
+ return true;
}
/*
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index 59ebac7d6a..cf119848b0 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -47,7 +47,7 @@ typedef struct XLogPageReadPrivate
int tliIndex;
} XLogPageReadPrivate;
-static int SimpleXLogPageRead(XLogReaderState *xlogreader,
+static bool SimpleXLogPageRead(XLogReaderState *xlogreader,
XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
@@ -246,7 +246,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
}
/* XLogReader callback function, to read a WAL page */
-static int
+static bool
SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf)
{
@@ -306,7 +306,8 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
if (private->restoreCommand == NULL)
{
pg_log_error("could not open file \"%s\": %m", xlogfpath);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
/*
@@ -319,7 +320,10 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
private->restoreCommand);
if (xlogreadfd < 0)
- return -1;
+ {
+ xlogreader->readLen = -1;
+ return false;
+ }
else
pg_log_debug("using file \"%s\" restored from archive",
xlogfpath);
@@ -335,7 +339,8 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
if (lseek(xlogreadfd, (off_t) targetPageOff, SEEK_SET) < 0)
{
pg_log_error("could not seek in file \"%s\": %m", xlogfpath);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
@@ -348,13 +353,15 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
pg_log_error("could not read file \"%s\": read %d of %zu",
xlogfpath, r, (Size) XLOG_BLCKSZ);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
Assert(targetSegNo == xlogreadsegno);
xlogreader->seg.ws_tli = targetHistory[private->tliIndex].tli;
- return XLOG_BLCKSZ;
+ xlogreader->readLen = XLOG_BLCKSZ;
+ return true;
}
/*
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index f8b8afe4a7..75ece5c658 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -331,7 +331,7 @@ WALDumpCloseSegment(XLogReaderState *state)
}
/* pg_waldump's XLogReaderRoutine->page_read callback */
-static int
+static bool
WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetPtr, char *readBuff)
{
@@ -348,7 +348,8 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
else
{
private->endptr_reached = true;
- return -1;
+ state->readLen = -1;
+ return false;
}
}
@@ -373,7 +374,8 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
(Size) errinfo.wre_req);
}
- return count;
+ state->readLen = count;
+ return true;
}
/*
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index 21d200d3df..5d9e0d3292 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -57,12 +57,12 @@ typedef struct WALSegmentContext
typedef struct XLogReaderState XLogReaderState;
-/* Function type definitions for various xlogreader interactions */
-typedef int (*XLogPageReadCB) (XLogReaderState *xlogreader,
- XLogRecPtr targetPagePtr,
- int reqLen,
- XLogRecPtr targetRecPtr,
- char *readBuf);
+/* Function type definition for the read_page callback */
+typedef bool (*XLogPageReadCB) (XLogReaderState *xlogreader,
+ XLogRecPtr targetPagePtr,
+ int reqLen,
+ XLogRecPtr targetRecPtr,
+ char *readBuf);
typedef void (*WALSegmentOpenCB) (XLogReaderState *xlogreader,
XLogSegNo nextSegNo,
TimeLineID *tli_p);
@@ -175,6 +175,19 @@ struct XLogReaderState
XLogRecPtr ReadRecPtr; /* start of last record read */
XLogRecPtr EndRecPtr; /* end+1 of last record read */
+ /* ----------------------------------------
+ * Communication with page reader
+ * readBuf is XLOG_BLCKSZ bytes, valid up to at least readLen bytes.
+ * ----------------------------------------
+ */
+ /* variables to communicate with page reader */
+ XLogRecPtr readPagePtr; /* page pointer to read */
+ int32 readLen; /* bytes requested to reader, or actual bytes
+ * read by reader, which must be larger than
+ * the request, or -1 on error */
+ char *readBuf; /* buffer to store data */
+ bool page_verified; /* is the page on the buffer verified? */
+
/* ----------------------------------------
* Decoded representation of current record
@@ -203,13 +216,6 @@ struct XLogReaderState
* ----------------------------------------
*/
- /*
- * Buffer for currently read page (XLOG_BLCKSZ bytes, valid up to at least
- * readLen bytes)
- */
- char *readBuf;
- uint32 readLen;
-
/* last read XLOG position for data currently in readBuf */
WALSegmentContext segcxt;
WALOpenSegment seg;
diff --git a/src/include/access/xlogutils.h b/src/include/access/xlogutils.h
index 9ac602b674..364a21c4ea 100644
--- a/src/include/access/xlogutils.h
+++ b/src/include/access/xlogutils.h
@@ -47,7 +47,7 @@ extern Buffer XLogReadBufferExtended(RelFileNode rnode, ForkNumber forknum,
extern Relation CreateFakeRelcacheEntry(RelFileNode rnode);
extern void FreeFakeRelcacheEntry(Relation fakerel);
-extern int read_local_xlog_page(XLogReaderState *state,
+extern bool read_local_xlog_page(XLogReaderState *state,
XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *cur_page);
extern void wal_segment_open(XLogReaderState *state,
--
2.27.0
----Next_Part(Wed_Apr__7_17_50_25_2021_942)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
filename="v19-0002-Move-page-reader-out-of-XLogReadRecord.patch"
^ permalink raw reply [nested|flat] 69+ messages in thread
* [PATCH v17 1/4] Move callback-call from ReadPageInternal to XLogReadRecord.
@ 2019-09-05 11:21 Kyotaro Horiguchi <[email protected]>
0 siblings, 0 replies; 69+ messages in thread
From: Kyotaro Horiguchi @ 2019-09-05 11:21 UTC (permalink / raw)
The current WAL record reader reads page data using a call back
function. Although it is not so problematic alone, it would be a
problem if we are going to do add tasks like encryption which is
performed on page data before WAL reader reads them. To avoid that the
record reader facility has to have a new code path corresponds to
every new callback, this patch separates page reader from WAL record
reading facility by modifying the current WAL record reader to a state
machine.
As the first step of that change, this patch moves the page reader
function out of ReadPageInternal, then the remaining tasks of the
function are taken over by the new function XLogNeedData. As the
result XLogPageRead directly calls the page reader callback function
according to the feedback from XLogNeedData.
---
src/backend/access/transam/xlog.c | 16 +-
src/backend/access/transam/xlogreader.c | 281 ++++++++++++++----------
src/backend/access/transam/xlogutils.c | 12 +-
src/backend/replication/walsender.c | 10 +-
src/bin/pg_rewind/parsexlog.c | 21 +-
src/bin/pg_waldump/pg_waldump.c | 8 +-
src/include/access/xlogreader.h | 25 ++-
src/include/access/xlogutils.h | 2 +-
8 files changed, 222 insertions(+), 153 deletions(-)
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 377afb8732..606328a65a 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -917,7 +917,7 @@ static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath,
static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli,
XLogSource source, bool notfoundOk);
static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, XLogSource source);
-static int XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
+static bool XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
bool fetching_ckpt, XLogRecPtr tliRecPtr);
@@ -4350,7 +4350,6 @@ ReadRecord(XLogReaderState *xlogreader, int emode,
XLogRecord *record;
XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
- /* Pass through parameters to XLogPageRead */
private->fetching_ckpt = fetching_ckpt;
private->emode = emode;
private->randAccess = (xlogreader->ReadRecPtr == InvalidXLogRecPtr);
@@ -11987,7 +11986,7 @@ CancelBackup(void)
* XLogPageRead() to try fetching the record from another source, or to
* sleep and retry.
*/
-static int
+static bool
XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *readBuf)
{
@@ -12046,7 +12045,8 @@ retry:
readLen = 0;
readSource = XLOG_FROM_ANY;
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
}
@@ -12141,7 +12141,8 @@ retry:
goto next_record_is_invalid;
}
- return readLen;
+ xlogreader->readLen = readLen;
+ return true;
next_record_is_invalid:
lastSourceFailed = true;
@@ -12155,8 +12156,9 @@ next_record_is_invalid:
/* In standby-mode, keep trying */
if (StandbyMode)
goto retry;
- else
- return -1;
+
+ xlogreader->readLen = -1;
+ return false;
}
/*
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index 42738eb940..7d04995b7b 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -36,8 +36,8 @@
static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
pg_attribute_printf(2, 3);
static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
-static int ReadPageInternal(XLogReaderState *state, XLogRecPtr pageptr,
- int reqLen);
+static bool XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr,
+ int reqLen, bool header_inclusive);
static void XLogReaderInvalReadState(XLogReaderState *state);
static bool ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
XLogRecPtr PrevRecPtr, XLogRecord *record, bool randAccess);
@@ -276,7 +276,6 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
uint32 targetRecOff;
uint32 pageHeaderSize;
bool gotheader;
- int readOff;
/*
* randAccess indicates whether to verify the previous-record pointer of
@@ -326,14 +325,20 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
* byte to cover the whole record header, or at least the part of it that
* fits on the same page.
*/
- readOff = ReadPageInternal(state, targetPagePtr,
- Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ));
- if (readOff < 0)
+ while (XLogNeedData(state, targetPagePtr,
+ Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+ targetRecOff != 0))
+ {
+ if (!state->routine.page_read(state, state->readPagePtr, state->readLen,
+ RecPtr, state->readBuf))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
/*
- * ReadPageInternal always returns at least the page header, so we can
- * examine it now.
+ * We have at least the page header, so we can examine it now.
*/
pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
if (targetRecOff == 0)
@@ -359,8 +364,8 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
goto err;
}
- /* ReadPageInternal has verified the page header */
- Assert(pageHeaderSize <= readOff);
+ /* XLogNeedData has verified the page header */
+ Assert(pageHeaderSize <= state->readLen);
/*
* Read the record length.
@@ -432,18 +437,27 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
do
{
+ int rest_len = total_len - gotlen;
+
/* Calculate pointer to beginning of next page */
targetPagePtr += XLOG_BLCKSZ;
/* Wait for the next page to become available */
- readOff = ReadPageInternal(state, targetPagePtr,
- Min(total_len - gotlen + SizeOfXLogShortPHD,
- XLOG_BLCKSZ));
+ while (XLogNeedData(state, targetPagePtr,
+ Min(rest_len, XLOG_BLCKSZ),
+ false))
+ {
+ if (!state->routine.page_read(state, state->readPagePtr,
+ state->readLen,
+ state->ReadRecPtr,
+ state->readBuf))
+ break;
+ }
- if (readOff < 0)
+ if (!state->page_verified)
goto err;
- Assert(SizeOfXLogShortPHD <= readOff);
+ Assert(SizeOfXLogShortPHD <= state->readLen);
/* Check that the continuation on next page looks valid */
pageHeader = (XLogPageHeader) state->readBuf;
@@ -473,21 +487,14 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
/* Append the continuation from this page to the buffer */
pageHeaderSize = XLogPageHeaderSize(pageHeader);
- if (readOff < pageHeaderSize)
- readOff = ReadPageInternal(state, targetPagePtr,
- pageHeaderSize);
-
- Assert(pageHeaderSize <= readOff);
+ Assert(pageHeaderSize <= state->readLen);
contdata = (char *) state->readBuf + pageHeaderSize;
len = XLOG_BLCKSZ - pageHeaderSize;
if (pageHeader->xlp_rem_len < len)
len = pageHeader->xlp_rem_len;
- if (readOff < pageHeaderSize + len)
- readOff = ReadPageInternal(state, targetPagePtr,
- pageHeaderSize + len);
-
+ Assert (pageHeaderSize + len <= state->readLen);
memcpy(buffer, (char *) contdata, len);
buffer += len;
gotlen += len;
@@ -517,9 +524,16 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
else
{
/* Wait for the record data to become available */
- readOff = ReadPageInternal(state, targetPagePtr,
- Min(targetRecOff + total_len, XLOG_BLCKSZ));
- if (readOff < 0)
+ while (XLogNeedData(state, targetPagePtr,
+ Min(targetRecOff + total_len, XLOG_BLCKSZ), true))
+ {
+ if (!state->routine.page_read(state, state->readPagePtr,
+ state->readLen,
+ state->ReadRecPtr, state->readBuf))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
/* Record does not cross a page boundary */
@@ -562,109 +576,138 @@ err:
}
/*
- * Read a single xlog page including at least [pageptr, reqLen] of valid data
- * via the page_read() callback.
+ * Checks that an xlog page loaded in state->readBuf is including at least
+ * [pageptr, reqLen] and the page is valid. header_inclusive indicates that
+ * reqLen is calculated including page header length.
*
- * Returns -1 if the required page cannot be read for some reason; errormsg_buf
- * is set in that case (unless the error occurs in the page_read callback).
+ * Returns false if the buffer already contains the requested data, or found
+ * error. state->page_verified is set to true for the former and false for the
+ * latter.
*
- * We fetch the page from a reader-local cache if we know we have the required
- * data and if there hasn't been any error since caching the data.
+ * Otherwise returns true and requests data loaded onto state->readBuf by
+ * state->readPagePtr and state->readLen. The caller shall call this function
+ * again after filling the buffer at least with that portion of data and set
+ * state->readLen to the length of actually loaded data.
+ *
+ * If header_inclusive is false, corrects reqLen internally by adding the
+ * actual page header length and may request caller for new data.
*/
-static int
-ReadPageInternal(XLogReaderState *state, XLogRecPtr pageptr, int reqLen)
+static bool
+XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
+ bool header_inclusive)
{
- int readLen;
uint32 targetPageOff;
XLogSegNo targetSegNo;
- XLogPageHeader hdr;
+ uint32 addLen = 0;
- Assert((pageptr % XLOG_BLCKSZ) == 0);
+ /* Some data is loaded, but page header is not verified yet. */
+ if (!state->page_verified &&
+ !XLogRecPtrIsInvalid(state->readPagePtr) && state->readLen >= 0)
+ {
+ uint32 pageHeaderSize;
+ /* just loaded new data so needs to verify page header */
+
+ /* The caller must have loaded at least page header */
+ Assert (state->readLen >= SizeOfXLogShortPHD);
+
+ /*
+ * We have enough data to check the header length. Recheck the loaded
+ * length against the actual header length.
+ */
+ pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+
+ /* Request more data if we don't have the full header. */
+ if (state->readLen < pageHeaderSize)
+ {
+ state->readLen = pageHeaderSize;
+ return true;
+ }
+
+ /* Now that we know we have the full header, validate it. */
+ if (!XLogReaderValidatePageHeader(state, state->readPagePtr,
+ (char *) state->readBuf))
+ {
+ /* That's bad. Force reading the page again. */
+ XLogReaderInvalReadState(state);
+
+ return false;
+ }
+
+ state->page_verified = true;
+
+ XLByteToSeg(state->readPagePtr, state->seg.ws_segno,
+ state->segcxt.ws_segsize);
+ }
+
+ /*
+ * The loaded page may not be the one caller is supposing to read when we
+ * are verifying the first page of new segment. In that case, skip further
+ * verification and immediately load the target page.
+ */
+ if (state->page_verified && pageptr == state->readPagePtr)
+ {
+ /*
+ * calculate additional length for page header keeping the total
+ * length within the block size.
+ */
+ if (!header_inclusive)
+ {
+ uint32 pageHeaderSize =
+ XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+
+ addLen = pageHeaderSize;
+ if (reqLen + pageHeaderSize <= XLOG_BLCKSZ)
+ addLen = pageHeaderSize;
+ else
+ addLen = XLOG_BLCKSZ - reqLen;
+
+ Assert(addLen >= 0);
+ }
+
+ /* Return if we already have it. */
+ if (reqLen + addLen <= state->readLen)
+ return false;
+ }
+
+ /* Data is not in our buffer, request the caller for it. */
XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize);
targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize);
+ Assert((pageptr % XLOG_BLCKSZ) == 0);
- /* check whether we have all the requested data already */
- if (targetSegNo == state->seg.ws_segno &&
- targetPageOff == state->segoff && reqLen <= state->readLen)
- return state->readLen;
+ /*
+ * Every time we request to load new data of a page to the caller, even if
+ * we looked at a part of it before, we need to do verification on the next
+ * invocation as the caller might now be rereading data from a different
+ * source.
+ */
+ state->page_verified = false;
/*
- * Data is not in our buffer.
- *
- * Every time we actually read the segment, even if we looked at parts of
- * it before, we need to do verification as the page_read callback might
- * now be rereading data from a different source.
- *
* Whenever switching to a new WAL segment, we read the first page of the
* file and validate its header, even if that's not where the target
* record is. This is so that we can check the additional identification
* info that is present in the first page's "long" header.
+ * Don't do this if the caller requested the first page in the segment.
*/
if (targetSegNo != state->seg.ws_segno && targetPageOff != 0)
{
- XLogRecPtr targetSegmentPtr = pageptr - targetPageOff;
-
- readLen = state->routine.page_read(state, targetSegmentPtr, XLOG_BLCKSZ,
- state->currRecPtr,
- state->readBuf);
- if (readLen < 0)
- goto err;
-
- /* we can be sure to have enough WAL available, we scrolled back */
- Assert(readLen == XLOG_BLCKSZ);
-
- if (!XLogReaderValidatePageHeader(state, targetSegmentPtr,
- state->readBuf))
- goto err;
+ /*
+ * Then we'll see that the targetSegNo now matches the ws_segno, and
+ * will not come back here, but will request the actual target page.
+ */
+ state->readPagePtr = pageptr - targetPageOff;
+ state->readLen = XLOG_BLCKSZ;
+ return true;
}
/*
- * First, read the requested data length, but at least a short page header
- * so that we can validate it.
+ * Request the caller to load the page. We need at least a short page
+ * header so that we can validate it.
*/
- readLen = state->routine.page_read(state, pageptr, Max(reqLen, SizeOfXLogShortPHD),
- state->currRecPtr,
- state->readBuf);
- if (readLen < 0)
- goto err;
-
- Assert(readLen <= XLOG_BLCKSZ);
-
- /* Do we have enough data to check the header length? */
- if (readLen <= SizeOfXLogShortPHD)
- goto err;
-
- Assert(readLen >= reqLen);
-
- hdr = (XLogPageHeader) state->readBuf;
-
- /* still not enough */
- if (readLen < XLogPageHeaderSize(hdr))
- {
- readLen = state->routine.page_read(state, pageptr, XLogPageHeaderSize(hdr),
- state->currRecPtr,
- state->readBuf);
- if (readLen < 0)
- goto err;
- }
-
- /*
- * Now that we know we have the full header, validate it.
- */
- if (!XLogReaderValidatePageHeader(state, pageptr, (char *) hdr))
- goto err;
-
- /* update read state information */
- state->seg.ws_segno = targetSegNo;
- state->segoff = targetPageOff;
- state->readLen = readLen;
-
- return readLen;
-
-err:
- XLogReaderInvalReadState(state);
- return -1;
+ state->readPagePtr = pageptr;
+ state->readLen = Max(reqLen + addLen, SizeOfXLogShortPHD);
+ return true;
}
/*
@@ -673,9 +716,7 @@ err:
static void
XLogReaderInvalReadState(XLogReaderState *state)
{
- state->seg.ws_segno = 0;
- state->segoff = 0;
- state->readLen = 0;
+ state->readPagePtr = InvalidXLogRecPtr;
}
/*
@@ -953,7 +994,6 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
XLogRecPtr targetPagePtr;
int targetRecOff;
uint32 pageHeaderSize;
- int readLen;
/*
* Compute targetRecOff. It should typically be equal or greater than
@@ -961,7 +1001,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
* that, except when caller has explicitly specified the offset that
* falls somewhere there or when we are skipping multi-page
* continuation record. It doesn't matter though because
- * ReadPageInternal() is prepared to handle that and will read at
+ * XLogNeedData() is prepared to handle that and will read at
* least short page-header worth of data
*/
targetRecOff = tmpRecPtr % XLOG_BLCKSZ;
@@ -969,19 +1009,24 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
/* scroll back to page boundary */
targetPagePtr = tmpRecPtr - targetRecOff;
- /* Read the page containing the record */
- readLen = ReadPageInternal(state, targetPagePtr, targetRecOff);
- if (readLen < 0)
+ while(XLogNeedData(state, targetPagePtr, targetRecOff,
+ targetRecOff != 0))
+ {
+ if (!state->routine.page_read(state, state->readPagePtr,
+ state->readLen,
+ state->ReadRecPtr, state->readBuf))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
header = (XLogPageHeader) state->readBuf;
pageHeaderSize = XLogPageHeaderSize(header);
- /* make sure we have enough data for the page header */
- readLen = ReadPageInternal(state, targetPagePtr, pageHeaderSize);
- if (readLen < 0)
- goto err;
+ /* we should have read the page header */
+ Assert (state->readLen >= pageHeaderSize);
/* skip over potential continuation data */
if (header->xlp_info & XLP_FIRST_IS_CONTRECORD)
diff --git a/src/backend/access/transam/xlogutils.c b/src/backend/access/transam/xlogutils.c
index d17d660f46..46eda33f25 100644
--- a/src/backend/access/transam/xlogutils.c
+++ b/src/backend/access/transam/xlogutils.c
@@ -686,8 +686,8 @@ XLogTruncateRelation(RelFileNode rnode, ForkNumber forkNum,
void
XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wantLength)
{
- const XLogRecPtr lastReadPage = (state->seg.ws_segno *
- state->segcxt.ws_segsize + state->segoff);
+ const XLogRecPtr lastReadPage = state->seg.ws_segno *
+ state->segcxt.ws_segsize + state->readLen;
Assert(wantPage != InvalidXLogRecPtr && wantPage % XLOG_BLCKSZ == 0);
Assert(wantLength <= XLOG_BLCKSZ);
@@ -824,7 +824,7 @@ wal_segment_close(XLogReaderState *state)
* exists for normal backends, so we have to do a check/sleep/repeat style of
* loop for now.
*/
-int
+bool
read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *cur_page)
{
@@ -926,7 +926,8 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
else if (targetPagePtr + reqLen > read_upto)
{
/* not enough data there */
- return -1;
+ state->readLen = -1;
+ return false;
}
else
{
@@ -944,7 +945,8 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
WALReadRaiseError(&errinfo);
/* number of valid bytes in the buffer */
- return count;
+ state->readLen = count;
+ return true;
}
/*
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index 23baa4498a..c36b91ae5e 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -806,7 +806,7 @@ StartReplication(StartReplicationCmd *cmd)
* which has to do a plain sleep/busy loop, because the walsender's latch gets
* set every time WAL is flushed.
*/
-static int
+static bool
logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *cur_page)
{
@@ -826,7 +826,10 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
/* fail if not (implies we are going to shut down) */
if (flushptr < targetPagePtr + reqLen)
- return -1;
+ {
+ state->readLen = -1;
+ return false;
+ }
if (targetPagePtr + XLOG_BLCKSZ <= flushptr)
count = XLOG_BLCKSZ; /* more than one block available */
@@ -854,7 +857,8 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
XLByteToSeg(targetPagePtr, segno, state->segcxt.ws_segsize);
CheckXLogRemoved(segno, state->seg.ws_tli);
- return count;
+ state->readLen = count;
+ return true;
}
/*
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index 59ebac7d6a..cf119848b0 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -47,7 +47,7 @@ typedef struct XLogPageReadPrivate
int tliIndex;
} XLogPageReadPrivate;
-static int SimpleXLogPageRead(XLogReaderState *xlogreader,
+static bool SimpleXLogPageRead(XLogReaderState *xlogreader,
XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
@@ -246,7 +246,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
}
/* XLogReader callback function, to read a WAL page */
-static int
+static bool
SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf)
{
@@ -306,7 +306,8 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
if (private->restoreCommand == NULL)
{
pg_log_error("could not open file \"%s\": %m", xlogfpath);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
/*
@@ -319,7 +320,10 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
private->restoreCommand);
if (xlogreadfd < 0)
- return -1;
+ {
+ xlogreader->readLen = -1;
+ return false;
+ }
else
pg_log_debug("using file \"%s\" restored from archive",
xlogfpath);
@@ -335,7 +339,8 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
if (lseek(xlogreadfd, (off_t) targetPageOff, SEEK_SET) < 0)
{
pg_log_error("could not seek in file \"%s\": %m", xlogfpath);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
@@ -348,13 +353,15 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
pg_log_error("could not read file \"%s\": read %d of %zu",
xlogfpath, r, (Size) XLOG_BLCKSZ);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
Assert(targetSegNo == xlogreadsegno);
xlogreader->seg.ws_tli = targetHistory[private->tliIndex].tli;
- return XLOG_BLCKSZ;
+ xlogreader->readLen = XLOG_BLCKSZ;
+ return true;
}
/*
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index 610f65e471..0f0e0723a3 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -331,7 +331,7 @@ WALDumpCloseSegment(XLogReaderState *state)
}
/* pg_waldump's XLogReaderRoutine->page_read callback */
-static int
+static bool
WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetPtr, char *readBuff)
{
@@ -348,7 +348,8 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
else
{
private->endptr_reached = true;
- return -1;
+ state->readLen = -1;
+ return false;
}
}
@@ -373,7 +374,8 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
(Size) errinfo.wre_req);
}
- return count;
+ state->readLen = count;
+ return true;
}
/*
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index 21d200d3df..23d9e70a59 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -57,8 +57,8 @@ typedef struct WALSegmentContext
typedef struct XLogReaderState XLogReaderState;
-/* Function type definitions for various xlogreader interactions */
-typedef int (*XLogPageReadCB) (XLogReaderState *xlogreader,
+/* Function type definition for the read_page callback */
+typedef bool (*XLogPageReadCB) (XLogReaderState *xlogreader,
XLogRecPtr targetPagePtr,
int reqLen,
XLogRecPtr targetRecPtr,
@@ -175,6 +175,20 @@ struct XLogReaderState
XLogRecPtr ReadRecPtr; /* start of last record read */
XLogRecPtr EndRecPtr; /* end+1 of last record read */
+ /* ----------------------------------------
+ * Communication with page reader
+ * readBuf is XLOG_BLCKSZ bytes, valid up to at least readLen bytes.
+ * ----------------------------------------
+ */
+ /* variables to communicate with page reader */
+ XLogRecPtr readPagePtr; /* page pointer to read */
+ int32 readLen; /* bytes requested to reader, or actual bytes
+ * read by reader, which must be larger than
+ * the request, or -1 on error */
+ TimeLineID readPageTLI; /* TLI for data currently in readBuf */
+ char *readBuf; /* buffer to store data */
+ bool page_verified; /* is the page on the buffer verified? */
+
/* ----------------------------------------
* Decoded representation of current record
@@ -203,13 +217,6 @@ struct XLogReaderState
* ----------------------------------------
*/
- /*
- * Buffer for currently read page (XLOG_BLCKSZ bytes, valid up to at least
- * readLen bytes)
- */
- char *readBuf;
- uint32 readLen;
-
/* last read XLOG position for data currently in readBuf */
WALSegmentContext segcxt;
WALOpenSegment seg;
diff --git a/src/include/access/xlogutils.h b/src/include/access/xlogutils.h
index 9ac602b674..364a21c4ea 100644
--- a/src/include/access/xlogutils.h
+++ b/src/include/access/xlogutils.h
@@ -47,7 +47,7 @@ extern Buffer XLogReadBufferExtended(RelFileNode rnode, ForkNumber forknum,
extern Relation CreateFakeRelcacheEntry(RelFileNode rnode);
extern void FreeFakeRelcacheEntry(Relation fakerel);
-extern int read_local_xlog_page(XLogReaderState *state,
+extern bool read_local_xlog_page(XLogReaderState *state,
XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *cur_page);
extern void wal_segment_open(XLogReaderState *state,
--
2.27.0
----Next_Part(Thu_Mar__4_11_28_53_2021_014)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
filename="v17-0002-Move-page-reader-out-of-XLogReadRecord.patch"
^ permalink raw reply [nested|flat] 69+ messages in thread
* [PATCH v16 1/4] Move callback-call from ReadPageInternal to XLogReadRecord.
@ 2019-09-05 11:21 Kyotaro Horiguchi <[email protected]>
0 siblings, 0 replies; 69+ messages in thread
From: Kyotaro Horiguchi @ 2019-09-05 11:21 UTC (permalink / raw)
The current WAL record reader reads page data using a call back
function. Although it is not so problematic alone, it would be a
problem if we are going to do add tasks like encryption which is
performed on page data before WAL reader reads them. To avoid that the
record reader facility has to have a new code path corresponds to
every new callback, this patch separates page reader from WAL record
reading facility by modifying the current WAL record reader to a state
machine.
As the first step of that change, this patch moves the page reader
function out of ReadPageInternal, then the remaining tasks of the
function are taken over by the new function XLogNeedData. As the
result XLogPageRead directly calls the page reader callback function
according to the feedback from XLogNeedData.
---
src/backend/access/transam/xlog.c | 16 +-
src/backend/access/transam/xlogreader.c | 281 ++++++++++++++----------
src/backend/access/transam/xlogutils.c | 12 +-
src/backend/replication/walsender.c | 10 +-
src/bin/pg_rewind/parsexlog.c | 21 +-
src/bin/pg_waldump/pg_waldump.c | 8 +-
src/include/access/xlogreader.h | 25 ++-
src/include/access/xlogutils.h | 2 +-
8 files changed, 222 insertions(+), 153 deletions(-)
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 09c01ed4ae..a91e86b290 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -909,7 +909,7 @@ static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath,
static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli,
XLogSource source, bool notfoundOk);
static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, XLogSource source);
-static int XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
+static bool XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
bool fetching_ckpt, XLogRecPtr tliRecPtr);
@@ -4329,7 +4329,6 @@ ReadRecord(XLogReaderState *xlogreader, int emode,
XLogRecord *record;
XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
- /* Pass through parameters to XLogPageRead */
private->fetching_ckpt = fetching_ckpt;
private->emode = emode;
private->randAccess = (xlogreader->ReadRecPtr == InvalidXLogRecPtr);
@@ -11866,7 +11865,7 @@ CancelBackup(void)
* XLogPageRead() to try fetching the record from another source, or to
* sleep and retry.
*/
-static int
+static bool
XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *readBuf)
{
@@ -11925,7 +11924,8 @@ retry:
readLen = 0;
readSource = XLOG_FROM_ANY;
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
}
@@ -12020,7 +12020,8 @@ retry:
goto next_record_is_invalid;
}
- return readLen;
+ xlogreader->readLen = readLen;
+ return true;
next_record_is_invalid:
lastSourceFailed = true;
@@ -12034,8 +12035,9 @@ next_record_is_invalid:
/* In standby-mode, keep trying */
if (StandbyMode)
goto retry;
- else
- return -1;
+
+ xlogreader->readLen = -1;
+ return false;
}
/*
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index a63ad8cfd0..f403261626 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -36,8 +36,8 @@
static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
pg_attribute_printf(2, 3);
static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
-static int ReadPageInternal(XLogReaderState *state, XLogRecPtr pageptr,
- int reqLen);
+static bool XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr,
+ int reqLen, bool header_inclusive);
static void XLogReaderInvalReadState(XLogReaderState *state);
static bool ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
XLogRecPtr PrevRecPtr, XLogRecord *record, bool randAccess);
@@ -276,7 +276,6 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
uint32 targetRecOff;
uint32 pageHeaderSize;
bool gotheader;
- int readOff;
/*
* randAccess indicates whether to verify the previous-record pointer of
@@ -326,14 +325,20 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
* byte to cover the whole record header, or at least the part of it that
* fits on the same page.
*/
- readOff = ReadPageInternal(state, targetPagePtr,
- Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ));
- if (readOff < 0)
+ while (XLogNeedData(state, targetPagePtr,
+ Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+ targetRecOff != 0))
+ {
+ if (!state->routine.page_read(state, state->readPagePtr, state->readLen,
+ RecPtr, state->readBuf))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
/*
- * ReadPageInternal always returns at least the page header, so we can
- * examine it now.
+ * We have at least the page header, so we can examine it now.
*/
pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
if (targetRecOff == 0)
@@ -359,8 +364,8 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
goto err;
}
- /* ReadPageInternal has verified the page header */
- Assert(pageHeaderSize <= readOff);
+ /* XLogNeedData has verified the page header */
+ Assert(pageHeaderSize <= state->readLen);
/*
* Read the record length.
@@ -433,18 +438,27 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
do
{
+ int rest_len = total_len - gotlen;
+
/* Calculate pointer to beginning of next page */
targetPagePtr += XLOG_BLCKSZ;
/* Wait for the next page to become available */
- readOff = ReadPageInternal(state, targetPagePtr,
- Min(total_len - gotlen + SizeOfXLogShortPHD,
- XLOG_BLCKSZ));
+ while (XLogNeedData(state, targetPagePtr,
+ Min(rest_len, XLOG_BLCKSZ),
+ false))
+ {
+ if (!state->routine.page_read(state, state->readPagePtr,
+ state->readLen,
+ state->ReadRecPtr,
+ state->readBuf))
+ break;
+ }
- if (readOff < 0)
+ if (!state->page_verified)
goto err;
- Assert(SizeOfXLogShortPHD <= readOff);
+ Assert(SizeOfXLogShortPHD <= state->readLen);
/* Check that the continuation on next page looks valid */
pageHeader = (XLogPageHeader) state->readBuf;
@@ -474,21 +488,14 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
/* Append the continuation from this page to the buffer */
pageHeaderSize = XLogPageHeaderSize(pageHeader);
- if (readOff < pageHeaderSize)
- readOff = ReadPageInternal(state, targetPagePtr,
- pageHeaderSize);
-
- Assert(pageHeaderSize <= readOff);
+ Assert(pageHeaderSize <= state->readLen);
contdata = (char *) state->readBuf + pageHeaderSize;
len = XLOG_BLCKSZ - pageHeaderSize;
if (pageHeader->xlp_rem_len < len)
len = pageHeader->xlp_rem_len;
- if (readOff < pageHeaderSize + len)
- readOff = ReadPageInternal(state, targetPagePtr,
- pageHeaderSize + len);
-
+ Assert (pageHeaderSize + len <= state->readLen);
memcpy(buffer, (char *) contdata, len);
buffer += len;
gotlen += len;
@@ -518,9 +525,16 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
else
{
/* Wait for the record data to become available */
- readOff = ReadPageInternal(state, targetPagePtr,
- Min(targetRecOff + total_len, XLOG_BLCKSZ));
- if (readOff < 0)
+ while (XLogNeedData(state, targetPagePtr,
+ Min(targetRecOff + total_len, XLOG_BLCKSZ), true))
+ {
+ if (!state->routine.page_read(state, state->readPagePtr,
+ state->readLen,
+ state->ReadRecPtr, state->readBuf))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
/* Record does not cross a page boundary */
@@ -563,109 +577,138 @@ err:
}
/*
- * Read a single xlog page including at least [pageptr, reqLen] of valid data
- * via the page_read() callback.
+ * Checks that an xlog page loaded in state->readBuf is including at least
+ * [pageptr, reqLen] and the page is valid. header_inclusive indicates that
+ * reqLen is calculated including page header length.
*
- * Returns -1 if the required page cannot be read for some reason; errormsg_buf
- * is set in that case (unless the error occurs in the page_read callback).
+ * Returns false if the buffer already contains the requested data, or found
+ * error. state->page_verified is set to true for the former and false for the
+ * latter.
*
- * We fetch the page from a reader-local cache if we know we have the required
- * data and if there hasn't been any error since caching the data.
+ * Otherwise returns true and requests data loaded onto state->readBuf by
+ * state->readPagePtr and state->readLen. The caller shall call this function
+ * again after filling the buffer at least with that portion of data and set
+ * state->readLen to the length of actually loaded data.
+ *
+ * If header_inclusive is false, corrects reqLen internally by adding the
+ * actual page header length and may request caller for new data.
*/
-static int
-ReadPageInternal(XLogReaderState *state, XLogRecPtr pageptr, int reqLen)
+static bool
+XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
+ bool header_inclusive)
{
- int readLen;
uint32 targetPageOff;
XLogSegNo targetSegNo;
- XLogPageHeader hdr;
+ uint32 addLen = 0;
- Assert((pageptr % XLOG_BLCKSZ) == 0);
+ /* Some data is loaded, but page header is not verified yet. */
+ if (!state->page_verified &&
+ !XLogRecPtrIsInvalid(state->readPagePtr) && state->readLen >= 0)
+ {
+ uint32 pageHeaderSize;
+ /* just loaded new data so needs to verify page header */
+
+ /* The caller must have loaded at least page header */
+ Assert (state->readLen >= SizeOfXLogShortPHD);
+
+ /*
+ * We have enough data to check the header length. Recheck the loaded
+ * length against the actual header length.
+ */
+ pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+
+ /* Request more data if we don't have the full header. */
+ if (state->readLen < pageHeaderSize)
+ {
+ state->readLen = pageHeaderSize;
+ return true;
+ }
+
+ /* Now that we know we have the full header, validate it. */
+ if (!XLogReaderValidatePageHeader(state, state->readPagePtr,
+ (char *) state->readBuf))
+ {
+ /* That's bad. Force reading the page again. */
+ XLogReaderInvalReadState(state);
+
+ return false;
+ }
+
+ state->page_verified = true;
+
+ XLByteToSeg(state->readPagePtr, state->seg.ws_segno,
+ state->segcxt.ws_segsize);
+ }
+
+ /*
+ * The loaded page may not be the one caller is supposing to read when we
+ * are verifying the first page of new segment. In that case, skip further
+ * verification and immediately load the target page.
+ */
+ if (state->page_verified && pageptr == state->readPagePtr)
+ {
+ /*
+ * calculate additional length for page header keeping the total
+ * length within the block size.
+ */
+ if (!header_inclusive)
+ {
+ uint32 pageHeaderSize =
+ XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+
+ addLen = pageHeaderSize;
+ if (reqLen + pageHeaderSize <= XLOG_BLCKSZ)
+ addLen = pageHeaderSize;
+ else
+ addLen = XLOG_BLCKSZ - reqLen;
+
+ Assert(addLen >= 0);
+ }
+
+ /* Return if we already have it. */
+ if (reqLen + addLen <= state->readLen)
+ return false;
+ }
+
+ /* Data is not in our buffer, request the caller for it. */
XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize);
targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize);
+ Assert((pageptr % XLOG_BLCKSZ) == 0);
- /* check whether we have all the requested data already */
- if (targetSegNo == state->seg.ws_segno &&
- targetPageOff == state->segoff && reqLen <= state->readLen)
- return state->readLen;
+ /*
+ * Every time we request to load new data of a page to the caller, even if
+ * we looked at a part of it before, we need to do verification on the next
+ * invocation as the caller might now be rereading data from a different
+ * source.
+ */
+ state->page_verified = false;
/*
- * Data is not in our buffer.
- *
- * Every time we actually read the segment, even if we looked at parts of
- * it before, we need to do verification as the page_read callback might
- * now be rereading data from a different source.
- *
* Whenever switching to a new WAL segment, we read the first page of the
* file and validate its header, even if that's not where the target
* record is. This is so that we can check the additional identification
* info that is present in the first page's "long" header.
+ * Don't do this if the caller requested the first page in the segment.
*/
if (targetSegNo != state->seg.ws_segno && targetPageOff != 0)
{
- XLogRecPtr targetSegmentPtr = pageptr - targetPageOff;
-
- readLen = state->routine.page_read(state, targetSegmentPtr, XLOG_BLCKSZ,
- state->currRecPtr,
- state->readBuf);
- if (readLen < 0)
- goto err;
-
- /* we can be sure to have enough WAL available, we scrolled back */
- Assert(readLen == XLOG_BLCKSZ);
-
- if (!XLogReaderValidatePageHeader(state, targetSegmentPtr,
- state->readBuf))
- goto err;
+ /*
+ * Then we'll see that the targetSegNo now matches the ws_segno, and
+ * will not come back here, but will request the actual target page.
+ */
+ state->readPagePtr = pageptr - targetPageOff;
+ state->readLen = XLOG_BLCKSZ;
+ return true;
}
/*
- * First, read the requested data length, but at least a short page header
- * so that we can validate it.
+ * Request the caller to load the page. We need at least a short page
+ * header so that we can validate it.
*/
- readLen = state->routine.page_read(state, pageptr, Max(reqLen, SizeOfXLogShortPHD),
- state->currRecPtr,
- state->readBuf);
- if (readLen < 0)
- goto err;
-
- Assert(readLen <= XLOG_BLCKSZ);
-
- /* Do we have enough data to check the header length? */
- if (readLen <= SizeOfXLogShortPHD)
- goto err;
-
- Assert(readLen >= reqLen);
-
- hdr = (XLogPageHeader) state->readBuf;
-
- /* still not enough */
- if (readLen < XLogPageHeaderSize(hdr))
- {
- readLen = state->routine.page_read(state, pageptr, XLogPageHeaderSize(hdr),
- state->currRecPtr,
- state->readBuf);
- if (readLen < 0)
- goto err;
- }
-
- /*
- * Now that we know we have the full header, validate it.
- */
- if (!XLogReaderValidatePageHeader(state, pageptr, (char *) hdr))
- goto err;
-
- /* update read state information */
- state->seg.ws_segno = targetSegNo;
- state->segoff = targetPageOff;
- state->readLen = readLen;
-
- return readLen;
-
-err:
- XLogReaderInvalReadState(state);
- return -1;
+ state->readPagePtr = pageptr;
+ state->readLen = Max(reqLen + addLen, SizeOfXLogShortPHD);
+ return true;
}
/*
@@ -674,9 +717,7 @@ err:
static void
XLogReaderInvalReadState(XLogReaderState *state)
{
- state->seg.ws_segno = 0;
- state->segoff = 0;
- state->readLen = 0;
+ state->readPagePtr = InvalidXLogRecPtr;
}
/*
@@ -957,7 +998,6 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
XLogRecPtr targetPagePtr;
int targetRecOff;
uint32 pageHeaderSize;
- int readLen;
/*
* Compute targetRecOff. It should typically be equal or greater than
@@ -965,7 +1005,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
* that, except when caller has explicitly specified the offset that
* falls somewhere there or when we are skipping multi-page
* continuation record. It doesn't matter though because
- * ReadPageInternal() is prepared to handle that and will read at
+ * XLogNeedData() is prepared to handle that and will read at
* least short page-header worth of data
*/
targetRecOff = tmpRecPtr % XLOG_BLCKSZ;
@@ -973,19 +1013,24 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
/* scroll back to page boundary */
targetPagePtr = tmpRecPtr - targetRecOff;
- /* Read the page containing the record */
- readLen = ReadPageInternal(state, targetPagePtr, targetRecOff);
- if (readLen < 0)
+ while(XLogNeedData(state, targetPagePtr, targetRecOff,
+ targetRecOff != 0))
+ {
+ if (!state->routine.page_read(state, state->readPagePtr,
+ state->readLen,
+ state->ReadRecPtr, state->readBuf))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
header = (XLogPageHeader) state->readBuf;
pageHeaderSize = XLogPageHeaderSize(header);
- /* make sure we have enough data for the page header */
- readLen = ReadPageInternal(state, targetPagePtr, pageHeaderSize);
- if (readLen < 0)
- goto err;
+ /* we should have read the page header */
+ Assert (state->readLen >= pageHeaderSize);
/* skip over potential continuation data */
if (header->xlp_info & XLP_FIRST_IS_CONTRECORD)
diff --git a/src/backend/access/transam/xlogutils.c b/src/backend/access/transam/xlogutils.c
index 7e915bcadf..4c6d072720 100644
--- a/src/backend/access/transam/xlogutils.c
+++ b/src/backend/access/transam/xlogutils.c
@@ -688,8 +688,8 @@ XLogTruncateRelation(RelFileNode rnode, ForkNumber forkNum,
void
XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wantLength)
{
- const XLogRecPtr lastReadPage = (state->seg.ws_segno *
- state->segcxt.ws_segsize + state->segoff);
+ const XLogRecPtr lastReadPage = state->seg.ws_segno *
+ state->segcxt.ws_segsize + state->readLen;
Assert(wantPage != InvalidXLogRecPtr && wantPage % XLOG_BLCKSZ == 0);
Assert(wantLength <= XLOG_BLCKSZ);
@@ -827,7 +827,7 @@ wal_segment_close(XLogReaderState *state)
* exists for normal backends, so we have to do a check/sleep/repeat style of
* loop for now.
*/
-int
+bool
read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *cur_page)
{
@@ -929,7 +929,8 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
else if (targetPagePtr + reqLen > read_upto)
{
/* not enough data there */
- return -1;
+ state->readLen = -1;
+ return false;
}
else
{
@@ -947,7 +948,8 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
WALReadRaiseError(&errinfo);
/* number of valid bytes in the buffer */
- return count;
+ state->readLen = count;
+ return true;
}
/*
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index 3f756b470a..76f94fa61f 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -810,7 +810,7 @@ StartReplication(StartReplicationCmd *cmd)
* which has to do a plain sleep/busy loop, because the walsender's latch gets
* set every time WAL is flushed.
*/
-static int
+static bool
logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *cur_page)
{
@@ -830,7 +830,10 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
/* fail if not (implies we are going to shut down) */
if (flushptr < targetPagePtr + reqLen)
- return -1;
+ {
+ state->readLen = -1;
+ return false;
+ }
if (targetPagePtr + XLOG_BLCKSZ <= flushptr)
count = XLOG_BLCKSZ; /* more than one block available */
@@ -858,7 +861,8 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
XLByteToSeg(targetPagePtr, segno, state->segcxt.ws_segsize);
CheckXLogRemoved(segno, state->seg.ws_tli);
- return count;
+ state->readLen = count;
+ return true;
}
/*
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index 2229c86f9a..4be2eede92 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -47,7 +47,7 @@ typedef struct XLogPageReadPrivate
int tliIndex;
} XLogPageReadPrivate;
-static int SimpleXLogPageRead(XLogReaderState *xlogreader,
+static bool SimpleXLogPageRead(XLogReaderState *xlogreader,
XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
@@ -237,7 +237,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
}
/* XLogReader callback function, to read a WAL page */
-static int
+static bool
SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf)
{
@@ -297,7 +297,8 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
if (private->restoreCommand == NULL)
{
pg_log_error("could not open file \"%s\": %m", xlogfpath);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
/*
@@ -310,7 +311,10 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
private->restoreCommand);
if (xlogreadfd < 0)
- return -1;
+ {
+ xlogreader->readLen = -1;
+ return false;
+ }
else
pg_log_debug("using file \"%s\" restored from archive",
xlogfpath);
@@ -326,7 +330,8 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
if (lseek(xlogreadfd, (off_t) targetPageOff, SEEK_SET) < 0)
{
pg_log_error("could not seek in file \"%s\": %m", xlogfpath);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
@@ -339,13 +344,15 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
pg_log_error("could not read file \"%s\": read %d of %zu",
xlogfpath, r, (Size) XLOG_BLCKSZ);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
Assert(targetSegNo == xlogreadsegno);
xlogreader->seg.ws_tli = targetHistory[private->tliIndex].tli;
- return XLOG_BLCKSZ;
+ xlogreader->readLen = XLOG_BLCKSZ;
+ return true;
}
/*
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index 31e99c2a6d..da929bf4cd 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -331,7 +331,7 @@ WALDumpCloseSegment(XLogReaderState *state)
}
/* pg_waldump's XLogReaderRoutine->page_read callback */
-static int
+static bool
WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetPtr, char *readBuff)
{
@@ -348,7 +348,8 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
else
{
private->endptr_reached = true;
- return -1;
+ state->readLen = -1;
+ return false;
}
}
@@ -373,7 +374,8 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
(Size) errinfo.wre_req);
}
- return count;
+ state->readLen = count;
+ return true;
}
/*
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index b976882229..f8f6a6264c 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -57,8 +57,8 @@ typedef struct WALSegmentContext
typedef struct XLogReaderState XLogReaderState;
-/* Function type definitions for various xlogreader interactions */
-typedef int (*XLogPageReadCB) (XLogReaderState *xlogreader,
+/* Function type definition for the read_page callback */
+typedef bool (*XLogPageReadCB) (XLogReaderState *xlogreader,
XLogRecPtr targetPagePtr,
int reqLen,
XLogRecPtr targetRecPtr,
@@ -175,6 +175,20 @@ struct XLogReaderState
XLogRecPtr ReadRecPtr; /* start of last record read */
XLogRecPtr EndRecPtr; /* end+1 of last record read */
+ /* ----------------------------------------
+ * Communication with page reader
+ * readBuf is XLOG_BLCKSZ bytes, valid up to at least readLen bytes.
+ * ----------------------------------------
+ */
+ /* variables to communicate with page reader */
+ XLogRecPtr readPagePtr; /* page pointer to read */
+ int32 readLen; /* bytes requested to reader, or actual bytes
+ * read by reader, which must be larger than
+ * the request, or -1 on error */
+ TimeLineID readPageTLI; /* TLI for data currently in readBuf */
+ char *readBuf; /* buffer to store data */
+ bool page_verified; /* is the page on the buffer verified? */
+
/* ----------------------------------------
* Decoded representation of current record
@@ -203,13 +217,6 @@ struct XLogReaderState
* ----------------------------------------
*/
- /*
- * Buffer for currently read page (XLOG_BLCKSZ bytes, valid up to at least
- * readLen bytes)
- */
- char *readBuf;
- uint32 readLen;
-
/* last read XLOG position for data currently in readBuf */
WALSegmentContext segcxt;
WALOpenSegment seg;
diff --git a/src/include/access/xlogutils.h b/src/include/access/xlogutils.h
index e59b6cf3a9..c287edf206 100644
--- a/src/include/access/xlogutils.h
+++ b/src/include/access/xlogutils.h
@@ -47,7 +47,7 @@ extern Buffer XLogReadBufferExtended(RelFileNode rnode, ForkNumber forknum,
extern Relation CreateFakeRelcacheEntry(RelFileNode rnode);
extern void FreeFakeRelcacheEntry(Relation fakerel);
-extern int read_local_xlog_page(XLogReaderState *state,
+extern bool read_local_xlog_page(XLogReaderState *state,
XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *cur_page);
extern void wal_segment_open(XLogReaderState *state,
--
2.18.4
----Next_Part(Tue_Sep__8_16_35_16_2020_701)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
filename="v16-0002-Move-page-reader-out-of-XLogReadRecord.patch"
^ permalink raw reply [nested|flat] 69+ messages in thread
* [PATCH v17 1/4] Move callback-call from ReadPageInternal to XLogReadRecord.
@ 2019-09-05 11:21 Kyotaro Horiguchi <[email protected]>
0 siblings, 0 replies; 69+ messages in thread
From: Kyotaro Horiguchi @ 2019-09-05 11:21 UTC (permalink / raw)
The current WAL record reader reads page data using a call back
function. Although it is not so problematic alone, it would be a
problem if we are going to do add tasks like encryption which is
performed on page data before WAL reader reads them. To avoid that the
record reader facility has to have a new code path corresponds to
every new callback, this patch separates page reader from WAL record
reading facility by modifying the current WAL record reader to a state
machine.
As the first step of that change, this patch moves the page reader
function out of ReadPageInternal, then the remaining tasks of the
function are taken over by the new function XLogNeedData. As the
result XLogPageRead directly calls the page reader callback function
according to the feedback from XLogNeedData.
---
src/backend/access/transam/xlog.c | 16 +-
src/backend/access/transam/xlogreader.c | 281 ++++++++++++++----------
src/backend/access/transam/xlogutils.c | 12 +-
src/backend/replication/walsender.c | 10 +-
src/bin/pg_rewind/parsexlog.c | 21 +-
src/bin/pg_waldump/pg_waldump.c | 8 +-
src/include/access/xlogreader.h | 25 ++-
src/include/access/xlogutils.h | 2 +-
8 files changed, 222 insertions(+), 153 deletions(-)
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 377afb8732..606328a65a 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -917,7 +917,7 @@ static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath,
static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli,
XLogSource source, bool notfoundOk);
static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, XLogSource source);
-static int XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
+static bool XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
bool fetching_ckpt, XLogRecPtr tliRecPtr);
@@ -4350,7 +4350,6 @@ ReadRecord(XLogReaderState *xlogreader, int emode,
XLogRecord *record;
XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
- /* Pass through parameters to XLogPageRead */
private->fetching_ckpt = fetching_ckpt;
private->emode = emode;
private->randAccess = (xlogreader->ReadRecPtr == InvalidXLogRecPtr);
@@ -11987,7 +11986,7 @@ CancelBackup(void)
* XLogPageRead() to try fetching the record from another source, or to
* sleep and retry.
*/
-static int
+static bool
XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *readBuf)
{
@@ -12046,7 +12045,8 @@ retry:
readLen = 0;
readSource = XLOG_FROM_ANY;
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
}
@@ -12141,7 +12141,8 @@ retry:
goto next_record_is_invalid;
}
- return readLen;
+ xlogreader->readLen = readLen;
+ return true;
next_record_is_invalid:
lastSourceFailed = true;
@@ -12155,8 +12156,9 @@ next_record_is_invalid:
/* In standby-mode, keep trying */
if (StandbyMode)
goto retry;
- else
- return -1;
+
+ xlogreader->readLen = -1;
+ return false;
}
/*
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index 42738eb940..7d04995b7b 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -36,8 +36,8 @@
static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
pg_attribute_printf(2, 3);
static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
-static int ReadPageInternal(XLogReaderState *state, XLogRecPtr pageptr,
- int reqLen);
+static bool XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr,
+ int reqLen, bool header_inclusive);
static void XLogReaderInvalReadState(XLogReaderState *state);
static bool ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
XLogRecPtr PrevRecPtr, XLogRecord *record, bool randAccess);
@@ -276,7 +276,6 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
uint32 targetRecOff;
uint32 pageHeaderSize;
bool gotheader;
- int readOff;
/*
* randAccess indicates whether to verify the previous-record pointer of
@@ -326,14 +325,20 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
* byte to cover the whole record header, or at least the part of it that
* fits on the same page.
*/
- readOff = ReadPageInternal(state, targetPagePtr,
- Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ));
- if (readOff < 0)
+ while (XLogNeedData(state, targetPagePtr,
+ Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+ targetRecOff != 0))
+ {
+ if (!state->routine.page_read(state, state->readPagePtr, state->readLen,
+ RecPtr, state->readBuf))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
/*
- * ReadPageInternal always returns at least the page header, so we can
- * examine it now.
+ * We have at least the page header, so we can examine it now.
*/
pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
if (targetRecOff == 0)
@@ -359,8 +364,8 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
goto err;
}
- /* ReadPageInternal has verified the page header */
- Assert(pageHeaderSize <= readOff);
+ /* XLogNeedData has verified the page header */
+ Assert(pageHeaderSize <= state->readLen);
/*
* Read the record length.
@@ -432,18 +437,27 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
do
{
+ int rest_len = total_len - gotlen;
+
/* Calculate pointer to beginning of next page */
targetPagePtr += XLOG_BLCKSZ;
/* Wait for the next page to become available */
- readOff = ReadPageInternal(state, targetPagePtr,
- Min(total_len - gotlen + SizeOfXLogShortPHD,
- XLOG_BLCKSZ));
+ while (XLogNeedData(state, targetPagePtr,
+ Min(rest_len, XLOG_BLCKSZ),
+ false))
+ {
+ if (!state->routine.page_read(state, state->readPagePtr,
+ state->readLen,
+ state->ReadRecPtr,
+ state->readBuf))
+ break;
+ }
- if (readOff < 0)
+ if (!state->page_verified)
goto err;
- Assert(SizeOfXLogShortPHD <= readOff);
+ Assert(SizeOfXLogShortPHD <= state->readLen);
/* Check that the continuation on next page looks valid */
pageHeader = (XLogPageHeader) state->readBuf;
@@ -473,21 +487,14 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
/* Append the continuation from this page to the buffer */
pageHeaderSize = XLogPageHeaderSize(pageHeader);
- if (readOff < pageHeaderSize)
- readOff = ReadPageInternal(state, targetPagePtr,
- pageHeaderSize);
-
- Assert(pageHeaderSize <= readOff);
+ Assert(pageHeaderSize <= state->readLen);
contdata = (char *) state->readBuf + pageHeaderSize;
len = XLOG_BLCKSZ - pageHeaderSize;
if (pageHeader->xlp_rem_len < len)
len = pageHeader->xlp_rem_len;
- if (readOff < pageHeaderSize + len)
- readOff = ReadPageInternal(state, targetPagePtr,
- pageHeaderSize + len);
-
+ Assert (pageHeaderSize + len <= state->readLen);
memcpy(buffer, (char *) contdata, len);
buffer += len;
gotlen += len;
@@ -517,9 +524,16 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
else
{
/* Wait for the record data to become available */
- readOff = ReadPageInternal(state, targetPagePtr,
- Min(targetRecOff + total_len, XLOG_BLCKSZ));
- if (readOff < 0)
+ while (XLogNeedData(state, targetPagePtr,
+ Min(targetRecOff + total_len, XLOG_BLCKSZ), true))
+ {
+ if (!state->routine.page_read(state, state->readPagePtr,
+ state->readLen,
+ state->ReadRecPtr, state->readBuf))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
/* Record does not cross a page boundary */
@@ -562,109 +576,138 @@ err:
}
/*
- * Read a single xlog page including at least [pageptr, reqLen] of valid data
- * via the page_read() callback.
+ * Checks that an xlog page loaded in state->readBuf is including at least
+ * [pageptr, reqLen] and the page is valid. header_inclusive indicates that
+ * reqLen is calculated including page header length.
*
- * Returns -1 if the required page cannot be read for some reason; errormsg_buf
- * is set in that case (unless the error occurs in the page_read callback).
+ * Returns false if the buffer already contains the requested data, or found
+ * error. state->page_verified is set to true for the former and false for the
+ * latter.
*
- * We fetch the page from a reader-local cache if we know we have the required
- * data and if there hasn't been any error since caching the data.
+ * Otherwise returns true and requests data loaded onto state->readBuf by
+ * state->readPagePtr and state->readLen. The caller shall call this function
+ * again after filling the buffer at least with that portion of data and set
+ * state->readLen to the length of actually loaded data.
+ *
+ * If header_inclusive is false, corrects reqLen internally by adding the
+ * actual page header length and may request caller for new data.
*/
-static int
-ReadPageInternal(XLogReaderState *state, XLogRecPtr pageptr, int reqLen)
+static bool
+XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
+ bool header_inclusive)
{
- int readLen;
uint32 targetPageOff;
XLogSegNo targetSegNo;
- XLogPageHeader hdr;
+ uint32 addLen = 0;
- Assert((pageptr % XLOG_BLCKSZ) == 0);
+ /* Some data is loaded, but page header is not verified yet. */
+ if (!state->page_verified &&
+ !XLogRecPtrIsInvalid(state->readPagePtr) && state->readLen >= 0)
+ {
+ uint32 pageHeaderSize;
+ /* just loaded new data so needs to verify page header */
+
+ /* The caller must have loaded at least page header */
+ Assert (state->readLen >= SizeOfXLogShortPHD);
+
+ /*
+ * We have enough data to check the header length. Recheck the loaded
+ * length against the actual header length.
+ */
+ pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+
+ /* Request more data if we don't have the full header. */
+ if (state->readLen < pageHeaderSize)
+ {
+ state->readLen = pageHeaderSize;
+ return true;
+ }
+
+ /* Now that we know we have the full header, validate it. */
+ if (!XLogReaderValidatePageHeader(state, state->readPagePtr,
+ (char *) state->readBuf))
+ {
+ /* That's bad. Force reading the page again. */
+ XLogReaderInvalReadState(state);
+
+ return false;
+ }
+
+ state->page_verified = true;
+
+ XLByteToSeg(state->readPagePtr, state->seg.ws_segno,
+ state->segcxt.ws_segsize);
+ }
+
+ /*
+ * The loaded page may not be the one caller is supposing to read when we
+ * are verifying the first page of new segment. In that case, skip further
+ * verification and immediately load the target page.
+ */
+ if (state->page_verified && pageptr == state->readPagePtr)
+ {
+ /*
+ * calculate additional length for page header keeping the total
+ * length within the block size.
+ */
+ if (!header_inclusive)
+ {
+ uint32 pageHeaderSize =
+ XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+
+ addLen = pageHeaderSize;
+ if (reqLen + pageHeaderSize <= XLOG_BLCKSZ)
+ addLen = pageHeaderSize;
+ else
+ addLen = XLOG_BLCKSZ - reqLen;
+
+ Assert(addLen >= 0);
+ }
+
+ /* Return if we already have it. */
+ if (reqLen + addLen <= state->readLen)
+ return false;
+ }
+
+ /* Data is not in our buffer, request the caller for it. */
XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize);
targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize);
+ Assert((pageptr % XLOG_BLCKSZ) == 0);
- /* check whether we have all the requested data already */
- if (targetSegNo == state->seg.ws_segno &&
- targetPageOff == state->segoff && reqLen <= state->readLen)
- return state->readLen;
+ /*
+ * Every time we request to load new data of a page to the caller, even if
+ * we looked at a part of it before, we need to do verification on the next
+ * invocation as the caller might now be rereading data from a different
+ * source.
+ */
+ state->page_verified = false;
/*
- * Data is not in our buffer.
- *
- * Every time we actually read the segment, even if we looked at parts of
- * it before, we need to do verification as the page_read callback might
- * now be rereading data from a different source.
- *
* Whenever switching to a new WAL segment, we read the first page of the
* file and validate its header, even if that's not where the target
* record is. This is so that we can check the additional identification
* info that is present in the first page's "long" header.
+ * Don't do this if the caller requested the first page in the segment.
*/
if (targetSegNo != state->seg.ws_segno && targetPageOff != 0)
{
- XLogRecPtr targetSegmentPtr = pageptr - targetPageOff;
-
- readLen = state->routine.page_read(state, targetSegmentPtr, XLOG_BLCKSZ,
- state->currRecPtr,
- state->readBuf);
- if (readLen < 0)
- goto err;
-
- /* we can be sure to have enough WAL available, we scrolled back */
- Assert(readLen == XLOG_BLCKSZ);
-
- if (!XLogReaderValidatePageHeader(state, targetSegmentPtr,
- state->readBuf))
- goto err;
+ /*
+ * Then we'll see that the targetSegNo now matches the ws_segno, and
+ * will not come back here, but will request the actual target page.
+ */
+ state->readPagePtr = pageptr - targetPageOff;
+ state->readLen = XLOG_BLCKSZ;
+ return true;
}
/*
- * First, read the requested data length, but at least a short page header
- * so that we can validate it.
+ * Request the caller to load the page. We need at least a short page
+ * header so that we can validate it.
*/
- readLen = state->routine.page_read(state, pageptr, Max(reqLen, SizeOfXLogShortPHD),
- state->currRecPtr,
- state->readBuf);
- if (readLen < 0)
- goto err;
-
- Assert(readLen <= XLOG_BLCKSZ);
-
- /* Do we have enough data to check the header length? */
- if (readLen <= SizeOfXLogShortPHD)
- goto err;
-
- Assert(readLen >= reqLen);
-
- hdr = (XLogPageHeader) state->readBuf;
-
- /* still not enough */
- if (readLen < XLogPageHeaderSize(hdr))
- {
- readLen = state->routine.page_read(state, pageptr, XLogPageHeaderSize(hdr),
- state->currRecPtr,
- state->readBuf);
- if (readLen < 0)
- goto err;
- }
-
- /*
- * Now that we know we have the full header, validate it.
- */
- if (!XLogReaderValidatePageHeader(state, pageptr, (char *) hdr))
- goto err;
-
- /* update read state information */
- state->seg.ws_segno = targetSegNo;
- state->segoff = targetPageOff;
- state->readLen = readLen;
-
- return readLen;
-
-err:
- XLogReaderInvalReadState(state);
- return -1;
+ state->readPagePtr = pageptr;
+ state->readLen = Max(reqLen + addLen, SizeOfXLogShortPHD);
+ return true;
}
/*
@@ -673,9 +716,7 @@ err:
static void
XLogReaderInvalReadState(XLogReaderState *state)
{
- state->seg.ws_segno = 0;
- state->segoff = 0;
- state->readLen = 0;
+ state->readPagePtr = InvalidXLogRecPtr;
}
/*
@@ -953,7 +994,6 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
XLogRecPtr targetPagePtr;
int targetRecOff;
uint32 pageHeaderSize;
- int readLen;
/*
* Compute targetRecOff. It should typically be equal or greater than
@@ -961,7 +1001,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
* that, except when caller has explicitly specified the offset that
* falls somewhere there or when we are skipping multi-page
* continuation record. It doesn't matter though because
- * ReadPageInternal() is prepared to handle that and will read at
+ * XLogNeedData() is prepared to handle that and will read at
* least short page-header worth of data
*/
targetRecOff = tmpRecPtr % XLOG_BLCKSZ;
@@ -969,19 +1009,24 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
/* scroll back to page boundary */
targetPagePtr = tmpRecPtr - targetRecOff;
- /* Read the page containing the record */
- readLen = ReadPageInternal(state, targetPagePtr, targetRecOff);
- if (readLen < 0)
+ while(XLogNeedData(state, targetPagePtr, targetRecOff,
+ targetRecOff != 0))
+ {
+ if (!state->routine.page_read(state, state->readPagePtr,
+ state->readLen,
+ state->ReadRecPtr, state->readBuf))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
header = (XLogPageHeader) state->readBuf;
pageHeaderSize = XLogPageHeaderSize(header);
- /* make sure we have enough data for the page header */
- readLen = ReadPageInternal(state, targetPagePtr, pageHeaderSize);
- if (readLen < 0)
- goto err;
+ /* we should have read the page header */
+ Assert (state->readLen >= pageHeaderSize);
/* skip over potential continuation data */
if (header->xlp_info & XLP_FIRST_IS_CONTRECORD)
diff --git a/src/backend/access/transam/xlogutils.c b/src/backend/access/transam/xlogutils.c
index d17d660f46..46eda33f25 100644
--- a/src/backend/access/transam/xlogutils.c
+++ b/src/backend/access/transam/xlogutils.c
@@ -686,8 +686,8 @@ XLogTruncateRelation(RelFileNode rnode, ForkNumber forkNum,
void
XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wantLength)
{
- const XLogRecPtr lastReadPage = (state->seg.ws_segno *
- state->segcxt.ws_segsize + state->segoff);
+ const XLogRecPtr lastReadPage = state->seg.ws_segno *
+ state->segcxt.ws_segsize + state->readLen;
Assert(wantPage != InvalidXLogRecPtr && wantPage % XLOG_BLCKSZ == 0);
Assert(wantLength <= XLOG_BLCKSZ);
@@ -824,7 +824,7 @@ wal_segment_close(XLogReaderState *state)
* exists for normal backends, so we have to do a check/sleep/repeat style of
* loop for now.
*/
-int
+bool
read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *cur_page)
{
@@ -926,7 +926,8 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
else if (targetPagePtr + reqLen > read_upto)
{
/* not enough data there */
- return -1;
+ state->readLen = -1;
+ return false;
}
else
{
@@ -944,7 +945,8 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
WALReadRaiseError(&errinfo);
/* number of valid bytes in the buffer */
- return count;
+ state->readLen = count;
+ return true;
}
/*
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index 23baa4498a..c36b91ae5e 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -806,7 +806,7 @@ StartReplication(StartReplicationCmd *cmd)
* which has to do a plain sleep/busy loop, because the walsender's latch gets
* set every time WAL is flushed.
*/
-static int
+static bool
logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *cur_page)
{
@@ -826,7 +826,10 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
/* fail if not (implies we are going to shut down) */
if (flushptr < targetPagePtr + reqLen)
- return -1;
+ {
+ state->readLen = -1;
+ return false;
+ }
if (targetPagePtr + XLOG_BLCKSZ <= flushptr)
count = XLOG_BLCKSZ; /* more than one block available */
@@ -854,7 +857,8 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
XLByteToSeg(targetPagePtr, segno, state->segcxt.ws_segsize);
CheckXLogRemoved(segno, state->seg.ws_tli);
- return count;
+ state->readLen = count;
+ return true;
}
/*
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index 59ebac7d6a..cf119848b0 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -47,7 +47,7 @@ typedef struct XLogPageReadPrivate
int tliIndex;
} XLogPageReadPrivate;
-static int SimpleXLogPageRead(XLogReaderState *xlogreader,
+static bool SimpleXLogPageRead(XLogReaderState *xlogreader,
XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
@@ -246,7 +246,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
}
/* XLogReader callback function, to read a WAL page */
-static int
+static bool
SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf)
{
@@ -306,7 +306,8 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
if (private->restoreCommand == NULL)
{
pg_log_error("could not open file \"%s\": %m", xlogfpath);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
/*
@@ -319,7 +320,10 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
private->restoreCommand);
if (xlogreadfd < 0)
- return -1;
+ {
+ xlogreader->readLen = -1;
+ return false;
+ }
else
pg_log_debug("using file \"%s\" restored from archive",
xlogfpath);
@@ -335,7 +339,8 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
if (lseek(xlogreadfd, (off_t) targetPageOff, SEEK_SET) < 0)
{
pg_log_error("could not seek in file \"%s\": %m", xlogfpath);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
@@ -348,13 +353,15 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
pg_log_error("could not read file \"%s\": read %d of %zu",
xlogfpath, r, (Size) XLOG_BLCKSZ);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
Assert(targetSegNo == xlogreadsegno);
xlogreader->seg.ws_tli = targetHistory[private->tliIndex].tli;
- return XLOG_BLCKSZ;
+ xlogreader->readLen = XLOG_BLCKSZ;
+ return true;
}
/*
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index 610f65e471..0f0e0723a3 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -331,7 +331,7 @@ WALDumpCloseSegment(XLogReaderState *state)
}
/* pg_waldump's XLogReaderRoutine->page_read callback */
-static int
+static bool
WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetPtr, char *readBuff)
{
@@ -348,7 +348,8 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
else
{
private->endptr_reached = true;
- return -1;
+ state->readLen = -1;
+ return false;
}
}
@@ -373,7 +374,8 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
(Size) errinfo.wre_req);
}
- return count;
+ state->readLen = count;
+ return true;
}
/*
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index 21d200d3df..23d9e70a59 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -57,8 +57,8 @@ typedef struct WALSegmentContext
typedef struct XLogReaderState XLogReaderState;
-/* Function type definitions for various xlogreader interactions */
-typedef int (*XLogPageReadCB) (XLogReaderState *xlogreader,
+/* Function type definition for the read_page callback */
+typedef bool (*XLogPageReadCB) (XLogReaderState *xlogreader,
XLogRecPtr targetPagePtr,
int reqLen,
XLogRecPtr targetRecPtr,
@@ -175,6 +175,20 @@ struct XLogReaderState
XLogRecPtr ReadRecPtr; /* start of last record read */
XLogRecPtr EndRecPtr; /* end+1 of last record read */
+ /* ----------------------------------------
+ * Communication with page reader
+ * readBuf is XLOG_BLCKSZ bytes, valid up to at least readLen bytes.
+ * ----------------------------------------
+ */
+ /* variables to communicate with page reader */
+ XLogRecPtr readPagePtr; /* page pointer to read */
+ int32 readLen; /* bytes requested to reader, or actual bytes
+ * read by reader, which must be larger than
+ * the request, or -1 on error */
+ TimeLineID readPageTLI; /* TLI for data currently in readBuf */
+ char *readBuf; /* buffer to store data */
+ bool page_verified; /* is the page on the buffer verified? */
+
/* ----------------------------------------
* Decoded representation of current record
@@ -203,13 +217,6 @@ struct XLogReaderState
* ----------------------------------------
*/
- /*
- * Buffer for currently read page (XLOG_BLCKSZ bytes, valid up to at least
- * readLen bytes)
- */
- char *readBuf;
- uint32 readLen;
-
/* last read XLOG position for data currently in readBuf */
WALSegmentContext segcxt;
WALOpenSegment seg;
diff --git a/src/include/access/xlogutils.h b/src/include/access/xlogutils.h
index 9ac602b674..364a21c4ea 100644
--- a/src/include/access/xlogutils.h
+++ b/src/include/access/xlogutils.h
@@ -47,7 +47,7 @@ extern Buffer XLogReadBufferExtended(RelFileNode rnode, ForkNumber forknum,
extern Relation CreateFakeRelcacheEntry(RelFileNode rnode);
extern void FreeFakeRelcacheEntry(Relation fakerel);
-extern int read_local_xlog_page(XLogReaderState *state,
+extern bool read_local_xlog_page(XLogReaderState *state,
XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *cur_page);
extern void wal_segment_open(XLogReaderState *state,
--
2.27.0
----Next_Part(Thu_Mar__4_11_28_53_2021_014)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
filename="v17-0002-Move-page-reader-out-of-XLogReadRecord.patch"
^ permalink raw reply [nested|flat] 69+ messages in thread
* [PATCH v15 1/4] Move callback-call from ReadPageInternal to XLogReadRecord.
@ 2019-09-05 11:21 Kyotaro Horiguchi <[email protected]>
0 siblings, 0 replies; 69+ messages in thread
From: Kyotaro Horiguchi @ 2019-09-05 11:21 UTC (permalink / raw)
The current WAL record reader reads page data using a call back
function. Although it is not so problematic alone, it would be a
problem if we are going to do add tasks like encryption which is
performed on page data before WAL reader reads them. To avoid that the
record reader facility has to have a new code path corresponds to
every new callback, this patch separates page reader from WAL record
reading facility by modifying the current WAL record reader to a state
machine.
As the first step of that change, this patch moves the page reader
function out of ReadPageInternal, then the remaining tasks of the
function are taken over by the new function XLogNeedData. As the
result XLogPageRead directly calls the page reader callback function
according to the feedback from XLogNeedData.
---
src/backend/access/transam/xlog.c | 16 +-
src/backend/access/transam/xlogreader.c | 281 ++++++++++++++----------
src/backend/access/transam/xlogutils.c | 12 +-
src/backend/replication/walsender.c | 10 +-
src/bin/pg_rewind/parsexlog.c | 21 +-
src/bin/pg_waldump/pg_waldump.c | 8 +-
src/include/access/xlogreader.h | 25 ++-
src/include/access/xlogutils.h | 2 +-
8 files changed, 222 insertions(+), 153 deletions(-)
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index fd93bcfaeb..e570e56a24 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -912,7 +912,7 @@ static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath,
static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli,
XLogSource source, bool notfoundOk);
static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, XLogSource source);
-static int XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
+static bool XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
bool fetching_ckpt, XLogRecPtr tliRecPtr);
@@ -4332,7 +4332,6 @@ ReadRecord(XLogReaderState *xlogreader, int emode,
XLogRecord *record;
XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
- /* Pass through parameters to XLogPageRead */
private->fetching_ckpt = fetching_ckpt;
private->emode = emode;
private->randAccess = (xlogreader->ReadRecPtr == InvalidXLogRecPtr);
@@ -11860,7 +11859,7 @@ CancelBackup(void)
* XLogPageRead() to try fetching the record from another source, or to
* sleep and retry.
*/
-static int
+static bool
XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *readBuf)
{
@@ -11919,7 +11918,8 @@ retry:
readLen = 0;
readSource = XLOG_FROM_ANY;
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
}
@@ -12014,7 +12014,8 @@ retry:
goto next_record_is_invalid;
}
- return readLen;
+ xlogreader->readLen = readLen;
+ return true;
next_record_is_invalid:
lastSourceFailed = true;
@@ -12028,8 +12029,9 @@ next_record_is_invalid:
/* In standby-mode, keep trying */
if (StandbyMode)
goto retry;
- else
- return -1;
+
+ xlogreader->readLen = -1;
+ return false;
}
/*
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index cb76be4f46..3d599325ee 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -36,8 +36,8 @@
static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
pg_attribute_printf(2, 3);
static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
-static int ReadPageInternal(XLogReaderState *state, XLogRecPtr pageptr,
- int reqLen);
+static bool XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr,
+ int reqLen, bool header_inclusive);
static void XLogReaderInvalReadState(XLogReaderState *state);
static bool ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
XLogRecPtr PrevRecPtr, XLogRecord *record, bool randAccess);
@@ -276,7 +276,6 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
uint32 targetRecOff;
uint32 pageHeaderSize;
bool gotheader;
- int readOff;
/*
* randAccess indicates whether to verify the previous-record pointer of
@@ -326,14 +325,20 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
* byte to cover the whole record header, or at least the part of it that
* fits on the same page.
*/
- readOff = ReadPageInternal(state, targetPagePtr,
- Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ));
- if (readOff < 0)
+ while (XLogNeedData(state, targetPagePtr,
+ Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+ targetRecOff != 0))
+ {
+ if (!state->routine.page_read(state, state->readPagePtr, state->readLen,
+ RecPtr, state->readBuf))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
/*
- * ReadPageInternal always returns at least the page header, so we can
- * examine it now.
+ * We have at least the page header, so we can examine it now.
*/
pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
if (targetRecOff == 0)
@@ -359,8 +364,8 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
goto err;
}
- /* ReadPageInternal has verified the page header */
- Assert(pageHeaderSize <= readOff);
+ /* XLogNeedData has verified the page header */
+ Assert(pageHeaderSize <= state->readLen);
/*
* Read the record length.
@@ -433,18 +438,27 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
do
{
+ int rest_len = total_len - gotlen;
+
/* Calculate pointer to beginning of next page */
targetPagePtr += XLOG_BLCKSZ;
/* Wait for the next page to become available */
- readOff = ReadPageInternal(state, targetPagePtr,
- Min(total_len - gotlen + SizeOfXLogShortPHD,
- XLOG_BLCKSZ));
+ while (XLogNeedData(state, targetPagePtr,
+ Min(rest_len, XLOG_BLCKSZ),
+ false))
+ {
+ if (!state->routine.page_read(state, state->readPagePtr,
+ state->readLen,
+ state->ReadRecPtr,
+ state->readBuf))
+ break;
+ }
- if (readOff < 0)
+ if (!state->page_verified)
goto err;
- Assert(SizeOfXLogShortPHD <= readOff);
+ Assert(SizeOfXLogShortPHD <= state->readLen);
/* Check that the continuation on next page looks valid */
pageHeader = (XLogPageHeader) state->readBuf;
@@ -473,21 +487,14 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
/* Append the continuation from this page to the buffer */
pageHeaderSize = XLogPageHeaderSize(pageHeader);
- if (readOff < pageHeaderSize)
- readOff = ReadPageInternal(state, targetPagePtr,
- pageHeaderSize);
-
- Assert(pageHeaderSize <= readOff);
+ Assert(pageHeaderSize <= state->readLen);
contdata = (char *) state->readBuf + pageHeaderSize;
len = XLOG_BLCKSZ - pageHeaderSize;
if (pageHeader->xlp_rem_len < len)
len = pageHeader->xlp_rem_len;
- if (readOff < pageHeaderSize + len)
- readOff = ReadPageInternal(state, targetPagePtr,
- pageHeaderSize + len);
-
+ Assert (pageHeaderSize + len <= state->readLen);
memcpy(buffer, (char *) contdata, len);
buffer += len;
gotlen += len;
@@ -517,9 +524,16 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
else
{
/* Wait for the record data to become available */
- readOff = ReadPageInternal(state, targetPagePtr,
- Min(targetRecOff + total_len, XLOG_BLCKSZ));
- if (readOff < 0)
+ while (XLogNeedData(state, targetPagePtr,
+ Min(targetRecOff + total_len, XLOG_BLCKSZ), true))
+ {
+ if (!state->routine.page_read(state, state->readPagePtr,
+ state->readLen,
+ state->ReadRecPtr, state->readBuf))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
/* Record does not cross a page boundary */
@@ -562,109 +576,138 @@ err:
}
/*
- * Read a single xlog page including at least [pageptr, reqLen] of valid data
- * via the page_read() callback.
+ * Checks that an xlog page loaded in state->readBuf is including at least
+ * [pageptr, reqLen] and the page is valid. header_inclusive indicates that
+ * reqLen is calculated including page header length.
*
- * Returns -1 if the required page cannot be read for some reason; errormsg_buf
- * is set in that case (unless the error occurs in the page_read callback).
+ * Returns false if the buffer already contains the requested data, or found
+ * error. state->page_verified is set to true for the former and false for the
+ * latter.
*
- * We fetch the page from a reader-local cache if we know we have the required
- * data and if there hasn't been any error since caching the data.
+ * Otherwise returns true and requests data loaded onto state->readBuf by
+ * state->readPagePtr and state->readLen. The caller shall call this function
+ * again after filling the buffer at least with that portion of data and set
+ * state->readLen to the length of actually loaded data.
+ *
+ * If header_inclusive is false, corrects reqLen internally by adding the
+ * actual page header length and may request caller for new data.
*/
-static int
-ReadPageInternal(XLogReaderState *state, XLogRecPtr pageptr, int reqLen)
+static bool
+XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
+ bool header_inclusive)
{
- int readLen;
uint32 targetPageOff;
XLogSegNo targetSegNo;
- XLogPageHeader hdr;
+ uint32 addLen = 0;
- Assert((pageptr % XLOG_BLCKSZ) == 0);
+ /* Some data is loaded, but page header is not verified yet. */
+ if (!state->page_verified &&
+ !XLogRecPtrIsInvalid(state->readPagePtr) && state->readLen >= 0)
+ {
+ uint32 pageHeaderSize;
+ /* just loaded new data so needs to verify page header */
+
+ /* The caller must have loaded at least page header */
+ Assert (state->readLen >= SizeOfXLogShortPHD);
+
+ /*
+ * We have enough data to check the header length. Recheck the loaded
+ * length against the actual header length.
+ */
+ pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+
+ /* Request more data if we don't have the full header. */
+ if (state->readLen < pageHeaderSize)
+ {
+ state->readLen = pageHeaderSize;
+ return true;
+ }
+
+ /* Now that we know we have the full header, validate it. */
+ if (!XLogReaderValidatePageHeader(state, state->readPagePtr,
+ (char *) state->readBuf))
+ {
+ /* That's bad. Force reading the page again. */
+ XLogReaderInvalReadState(state);
+
+ return false;
+ }
+
+ state->page_verified = true;
+
+ XLByteToSeg(state->readPagePtr, state->seg.ws_segno,
+ state->segcxt.ws_segsize);
+ }
+
+ /*
+ * The loaded page may not be the one caller is supposing to read when we
+ * are verifying the first page of new segment. In that case, skip further
+ * verification and immediately load the target page.
+ */
+ if (state->page_verified && pageptr == state->readPagePtr)
+ {
+ /*
+ * calculate additional length for page header keeping the total
+ * length within the block size.
+ */
+ if (!header_inclusive)
+ {
+ uint32 pageHeaderSize =
+ XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+
+ addLen = pageHeaderSize;
+ if (reqLen + pageHeaderSize <= XLOG_BLCKSZ)
+ addLen = pageHeaderSize;
+ else
+ addLen = XLOG_BLCKSZ - reqLen;
+
+ Assert(addLen >= 0);
+ }
+
+ /* Return if we already have it. */
+ if (reqLen + addLen <= state->readLen)
+ return false;
+ }
+
+ /* Data is not in our buffer, request the caller for it. */
XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize);
targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize);
+ Assert((pageptr % XLOG_BLCKSZ) == 0);
- /* check whether we have all the requested data already */
- if (targetSegNo == state->seg.ws_segno &&
- targetPageOff == state->segoff && reqLen <= state->readLen)
- return state->readLen;
+ /*
+ * Every time we request to load new data of a page to the caller, even if
+ * we looked at a part of it before, we need to do verification on the next
+ * invocation as the caller might now be rereading data from a different
+ * source.
+ */
+ state->page_verified = false;
/*
- * Data is not in our buffer.
- *
- * Every time we actually read the segment, even if we looked at parts of
- * it before, we need to do verification as the page_read callback might
- * now be rereading data from a different source.
- *
* Whenever switching to a new WAL segment, we read the first page of the
* file and validate its header, even if that's not where the target
* record is. This is so that we can check the additional identification
* info that is present in the first page's "long" header.
+ * Don't do this if the caller requested the first page in the segment.
*/
if (targetSegNo != state->seg.ws_segno && targetPageOff != 0)
{
- XLogRecPtr targetSegmentPtr = pageptr - targetPageOff;
-
- readLen = state->routine.page_read(state, targetSegmentPtr, XLOG_BLCKSZ,
- state->currRecPtr,
- state->readBuf);
- if (readLen < 0)
- goto err;
-
- /* we can be sure to have enough WAL available, we scrolled back */
- Assert(readLen == XLOG_BLCKSZ);
-
- if (!XLogReaderValidatePageHeader(state, targetSegmentPtr,
- state->readBuf))
- goto err;
+ /*
+ * Then we'll see that the targetSegNo now matches the ws_segno, and
+ * will not come back here, but will request the actual target page.
+ */
+ state->readPagePtr = pageptr - targetPageOff;
+ state->readLen = XLOG_BLCKSZ;
+ return true;
}
/*
- * First, read the requested data length, but at least a short page header
- * so that we can validate it.
+ * Request the caller to load the page. We need at least a short page
+ * header so that we can validate it.
*/
- readLen = state->routine.page_read(state, pageptr, Max(reqLen, SizeOfXLogShortPHD),
- state->currRecPtr,
- state->readBuf);
- if (readLen < 0)
- goto err;
-
- Assert(readLen <= XLOG_BLCKSZ);
-
- /* Do we have enough data to check the header length? */
- if (readLen <= SizeOfXLogShortPHD)
- goto err;
-
- Assert(readLen >= reqLen);
-
- hdr = (XLogPageHeader) state->readBuf;
-
- /* still not enough */
- if (readLen < XLogPageHeaderSize(hdr))
- {
- readLen = state->routine.page_read(state, pageptr, XLogPageHeaderSize(hdr),
- state->currRecPtr,
- state->readBuf);
- if (readLen < 0)
- goto err;
- }
-
- /*
- * Now that we know we have the full header, validate it.
- */
- if (!XLogReaderValidatePageHeader(state, pageptr, (char *) hdr))
- goto err;
-
- /* update read state information */
- state->seg.ws_segno = targetSegNo;
- state->segoff = targetPageOff;
- state->readLen = readLen;
-
- return readLen;
-
-err:
- XLogReaderInvalReadState(state);
- return -1;
+ state->readPagePtr = pageptr;
+ state->readLen = Max(reqLen + addLen, SizeOfXLogShortPHD);
+ return true;
}
/*
@@ -673,9 +716,7 @@ err:
static void
XLogReaderInvalReadState(XLogReaderState *state)
{
- state->seg.ws_segno = 0;
- state->segoff = 0;
- state->readLen = 0;
+ state->readPagePtr = InvalidXLogRecPtr;
}
/*
@@ -956,7 +997,6 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
XLogRecPtr targetPagePtr;
int targetRecOff;
uint32 pageHeaderSize;
- int readLen;
/*
* Compute targetRecOff. It should typically be equal or greater than
@@ -964,7 +1004,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
* that, except when caller has explicitly specified the offset that
* falls somewhere there or when we are skipping multi-page
* continuation record. It doesn't matter though because
- * ReadPageInternal() is prepared to handle that and will read at
+ * XLogNeedData() is prepared to handle that and will read at
* least short page-header worth of data
*/
targetRecOff = tmpRecPtr % XLOG_BLCKSZ;
@@ -972,19 +1012,24 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
/* scroll back to page boundary */
targetPagePtr = tmpRecPtr - targetRecOff;
- /* Read the page containing the record */
- readLen = ReadPageInternal(state, targetPagePtr, targetRecOff);
- if (readLen < 0)
+ while(XLogNeedData(state, targetPagePtr, targetRecOff,
+ targetRecOff != 0))
+ {
+ if (!state->routine.page_read(state, state->readPagePtr,
+ state->readLen,
+ state->ReadRecPtr, state->readBuf))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
header = (XLogPageHeader) state->readBuf;
pageHeaderSize = XLogPageHeaderSize(header);
- /* make sure we have enough data for the page header */
- readLen = ReadPageInternal(state, targetPagePtr, pageHeaderSize);
- if (readLen < 0)
- goto err;
+ /* we should have read the page header */
+ Assert (state->readLen >= pageHeaderSize);
/* skip over potential continuation data */
if (header->xlp_info & XLP_FIRST_IS_CONTRECORD)
diff --git a/src/backend/access/transam/xlogutils.c b/src/backend/access/transam/xlogutils.c
index 322b0e8ff5..1b10b6df20 100644
--- a/src/backend/access/transam/xlogutils.c
+++ b/src/backend/access/transam/xlogutils.c
@@ -689,8 +689,8 @@ XLogTruncateRelation(RelFileNode rnode, ForkNumber forkNum,
void
XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wantLength)
{
- const XLogRecPtr lastReadPage = (state->seg.ws_segno *
- state->segcxt.ws_segsize + state->segoff);
+ const XLogRecPtr lastReadPage = state->seg.ws_segno *
+ state->segcxt.ws_segsize + state->readLen;
Assert(wantPage != InvalidXLogRecPtr && wantPage % XLOG_BLCKSZ == 0);
Assert(wantLength <= XLOG_BLCKSZ);
@@ -828,7 +828,7 @@ wal_segment_close(XLogReaderState *state)
* exists for normal backends, so we have to do a check/sleep/repeat style of
* loop for now.
*/
-int
+bool
read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *cur_page)
{
@@ -930,7 +930,8 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
else if (targetPagePtr + reqLen > read_upto)
{
/* not enough data there */
- return -1;
+ state->readLen = -1;
+ return false;
}
else
{
@@ -948,7 +949,8 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
WALReadRaiseError(&errinfo);
/* number of valid bytes in the buffer */
- return count;
+ state->readLen = count;
+ return true;
}
/*
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index e2477c47e0..bf2711bddd 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -811,7 +811,7 @@ StartReplication(StartReplicationCmd *cmd)
* which has to do a plain sleep/busy loop, because the walsender's latch gets
* set every time WAL is flushed.
*/
-static int
+static bool
logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *cur_page)
{
@@ -831,7 +831,10 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
/* fail if not (implies we are going to shut down) */
if (flushptr < targetPagePtr + reqLen)
- return -1;
+ {
+ state->readLen = -1;
+ return false;
+ }
if (targetPagePtr + XLOG_BLCKSZ <= flushptr)
count = XLOG_BLCKSZ; /* more than one block available */
@@ -859,7 +862,8 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
XLByteToSeg(targetPagePtr, segno, state->segcxt.ws_segsize);
CheckXLogRemoved(segno, state->seg.ws_tli);
- return count;
+ state->readLen = count;
+ return true;
}
/*
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index bc6f976994..0b7e73ae79 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -46,7 +46,7 @@ typedef struct XLogPageReadPrivate
int tliIndex;
} XLogPageReadPrivate;
-static int SimpleXLogPageRead(XLogReaderState *xlogreader,
+static bool SimpleXLogPageRead(XLogReaderState *xlogreader,
XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
@@ -236,7 +236,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
}
/* XLogReader callback function, to read a WAL page */
-static int
+static bool
SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf)
{
@@ -296,7 +296,8 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
if (private->restoreCommand == NULL)
{
pg_log_error("could not open file \"%s\": %m", xlogfpath);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
/*
@@ -309,7 +310,10 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
private->restoreCommand);
if (xlogreadfd < 0)
- return -1;
+ {
+ xlogreader->readLen = -1;
+ return false;
+ }
else
pg_log_debug("using file \"%s\" restored from archive",
xlogfpath);
@@ -325,7 +329,8 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
if (lseek(xlogreadfd, (off_t) targetPageOff, SEEK_SET) < 0)
{
pg_log_error("could not seek in file \"%s\": %m", xlogfpath);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
@@ -338,13 +343,15 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
pg_log_error("could not read file \"%s\": read %d of %zu",
xlogfpath, r, (Size) XLOG_BLCKSZ);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
Assert(targetSegNo == xlogreadsegno);
xlogreader->seg.ws_tli = targetHistory[private->tliIndex].tli;
- return XLOG_BLCKSZ;
+ xlogreader->readLen = XLOG_BLCKSZ;
+ return true;
}
/*
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index d1a0678935..ab0c28d9b7 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -331,7 +331,7 @@ WALDumpCloseSegment(XLogReaderState *state)
}
/* pg_waldump's XLogReaderRoutine->page_read callback */
-static int
+static bool
WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetPtr, char *readBuff)
{
@@ -348,7 +348,8 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
else
{
private->endptr_reached = true;
- return -1;
+ state->readLen = -1;
+ return false;
}
}
@@ -373,7 +374,8 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
(Size) errinfo.wre_req);
}
- return count;
+ state->readLen = count;
+ return true;
}
/*
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index b0f2a6ed43..6c8848e14f 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -57,8 +57,8 @@ typedef struct WALSegmentContext
typedef struct XLogReaderState XLogReaderState;
-/* Function type definitions for various xlogreader interactions */
-typedef int (*XLogPageReadCB) (XLogReaderState *xlogreader,
+/* Function type definition for the read_page callback */
+typedef bool (*XLogPageReadCB) (XLogReaderState *xlogreader,
XLogRecPtr targetPagePtr,
int reqLen,
XLogRecPtr targetRecPtr,
@@ -175,6 +175,20 @@ struct XLogReaderState
XLogRecPtr ReadRecPtr; /* start of last record read */
XLogRecPtr EndRecPtr; /* end+1 of last record read */
+ /* ----------------------------------------
+ * Communication with page reader
+ * readBuf is XLOG_BLCKSZ bytes, valid up to at least readLen bytes.
+ * ----------------------------------------
+ */
+ /* variables to communicate with page reader */
+ XLogRecPtr readPagePtr; /* page pointer to read */
+ int32 readLen; /* bytes requested to reader, or actual bytes
+ * read by reader, which must be larger than
+ * the request, or -1 on error */
+ TimeLineID readPageTLI; /* TLI for data currently in readBuf */
+ char *readBuf; /* buffer to store data */
+ bool page_verified; /* is the page on the buffer verified? */
+
/* ----------------------------------------
* Decoded representation of current record
@@ -201,13 +215,6 @@ struct XLogReaderState
* ----------------------------------------
*/
- /*
- * Buffer for currently read page (XLOG_BLCKSZ bytes, valid up to at least
- * readLen bytes)
- */
- char *readBuf;
- uint32 readLen;
-
/* last read XLOG position for data currently in readBuf */
WALSegmentContext segcxt;
WALOpenSegment seg;
diff --git a/src/include/access/xlogutils.h b/src/include/access/xlogutils.h
index e59b6cf3a9..c287edf206 100644
--- a/src/include/access/xlogutils.h
+++ b/src/include/access/xlogutils.h
@@ -47,7 +47,7 @@ extern Buffer XLogReadBufferExtended(RelFileNode rnode, ForkNumber forknum,
extern Relation CreateFakeRelcacheEntry(RelFileNode rnode);
extern void FreeFakeRelcacheEntry(Relation fakerel);
-extern int read_local_xlog_page(XLogReaderState *state,
+extern bool read_local_xlog_page(XLogReaderState *state,
XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *cur_page);
extern void wal_segment_open(XLogReaderState *state,
--
2.18.4
----Next_Part(Thu_Jul__2_13_53_30_2020_072)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
filename="v15-0002-Move-page-reader-out-of-XLogReadRecord.patch"
^ permalink raw reply [nested|flat] 69+ messages in thread
* [PATCH v17 1/4] Move callback-call from ReadPageInternal to XLogReadRecord.
@ 2019-09-05 11:21 Kyotaro Horiguchi <[email protected]>
0 siblings, 0 replies; 69+ messages in thread
From: Kyotaro Horiguchi @ 2019-09-05 11:21 UTC (permalink / raw)
The current WAL record reader reads page data using a call back
function. Although it is not so problematic alone, it would be a
problem if we are going to do add tasks like encryption which is
performed on page data before WAL reader reads them. To avoid that the
record reader facility has to have a new code path corresponds to
every new callback, this patch separates page reader from WAL record
reading facility by modifying the current WAL record reader to a state
machine.
As the first step of that change, this patch moves the page reader
function out of ReadPageInternal, then the remaining tasks of the
function are taken over by the new function XLogNeedData. As the
result XLogPageRead directly calls the page reader callback function
according to the feedback from XLogNeedData.
---
src/backend/access/transam/xlog.c | 16 +-
src/backend/access/transam/xlogreader.c | 281 ++++++++++++++----------
src/backend/access/transam/xlogutils.c | 12 +-
src/backend/replication/walsender.c | 10 +-
src/bin/pg_rewind/parsexlog.c | 21 +-
src/bin/pg_waldump/pg_waldump.c | 8 +-
src/include/access/xlogreader.h | 25 ++-
src/include/access/xlogutils.h | 2 +-
8 files changed, 222 insertions(+), 153 deletions(-)
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 377afb8732..606328a65a 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -917,7 +917,7 @@ static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath,
static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli,
XLogSource source, bool notfoundOk);
static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, XLogSource source);
-static int XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
+static bool XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
bool fetching_ckpt, XLogRecPtr tliRecPtr);
@@ -4350,7 +4350,6 @@ ReadRecord(XLogReaderState *xlogreader, int emode,
XLogRecord *record;
XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
- /* Pass through parameters to XLogPageRead */
private->fetching_ckpt = fetching_ckpt;
private->emode = emode;
private->randAccess = (xlogreader->ReadRecPtr == InvalidXLogRecPtr);
@@ -11987,7 +11986,7 @@ CancelBackup(void)
* XLogPageRead() to try fetching the record from another source, or to
* sleep and retry.
*/
-static int
+static bool
XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *readBuf)
{
@@ -12046,7 +12045,8 @@ retry:
readLen = 0;
readSource = XLOG_FROM_ANY;
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
}
@@ -12141,7 +12141,8 @@ retry:
goto next_record_is_invalid;
}
- return readLen;
+ xlogreader->readLen = readLen;
+ return true;
next_record_is_invalid:
lastSourceFailed = true;
@@ -12155,8 +12156,9 @@ next_record_is_invalid:
/* In standby-mode, keep trying */
if (StandbyMode)
goto retry;
- else
- return -1;
+
+ xlogreader->readLen = -1;
+ return false;
}
/*
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index 42738eb940..7d04995b7b 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -36,8 +36,8 @@
static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
pg_attribute_printf(2, 3);
static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
-static int ReadPageInternal(XLogReaderState *state, XLogRecPtr pageptr,
- int reqLen);
+static bool XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr,
+ int reqLen, bool header_inclusive);
static void XLogReaderInvalReadState(XLogReaderState *state);
static bool ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
XLogRecPtr PrevRecPtr, XLogRecord *record, bool randAccess);
@@ -276,7 +276,6 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
uint32 targetRecOff;
uint32 pageHeaderSize;
bool gotheader;
- int readOff;
/*
* randAccess indicates whether to verify the previous-record pointer of
@@ -326,14 +325,20 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
* byte to cover the whole record header, or at least the part of it that
* fits on the same page.
*/
- readOff = ReadPageInternal(state, targetPagePtr,
- Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ));
- if (readOff < 0)
+ while (XLogNeedData(state, targetPagePtr,
+ Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+ targetRecOff != 0))
+ {
+ if (!state->routine.page_read(state, state->readPagePtr, state->readLen,
+ RecPtr, state->readBuf))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
/*
- * ReadPageInternal always returns at least the page header, so we can
- * examine it now.
+ * We have at least the page header, so we can examine it now.
*/
pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
if (targetRecOff == 0)
@@ -359,8 +364,8 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
goto err;
}
- /* ReadPageInternal has verified the page header */
- Assert(pageHeaderSize <= readOff);
+ /* XLogNeedData has verified the page header */
+ Assert(pageHeaderSize <= state->readLen);
/*
* Read the record length.
@@ -432,18 +437,27 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
do
{
+ int rest_len = total_len - gotlen;
+
/* Calculate pointer to beginning of next page */
targetPagePtr += XLOG_BLCKSZ;
/* Wait for the next page to become available */
- readOff = ReadPageInternal(state, targetPagePtr,
- Min(total_len - gotlen + SizeOfXLogShortPHD,
- XLOG_BLCKSZ));
+ while (XLogNeedData(state, targetPagePtr,
+ Min(rest_len, XLOG_BLCKSZ),
+ false))
+ {
+ if (!state->routine.page_read(state, state->readPagePtr,
+ state->readLen,
+ state->ReadRecPtr,
+ state->readBuf))
+ break;
+ }
- if (readOff < 0)
+ if (!state->page_verified)
goto err;
- Assert(SizeOfXLogShortPHD <= readOff);
+ Assert(SizeOfXLogShortPHD <= state->readLen);
/* Check that the continuation on next page looks valid */
pageHeader = (XLogPageHeader) state->readBuf;
@@ -473,21 +487,14 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
/* Append the continuation from this page to the buffer */
pageHeaderSize = XLogPageHeaderSize(pageHeader);
- if (readOff < pageHeaderSize)
- readOff = ReadPageInternal(state, targetPagePtr,
- pageHeaderSize);
-
- Assert(pageHeaderSize <= readOff);
+ Assert(pageHeaderSize <= state->readLen);
contdata = (char *) state->readBuf + pageHeaderSize;
len = XLOG_BLCKSZ - pageHeaderSize;
if (pageHeader->xlp_rem_len < len)
len = pageHeader->xlp_rem_len;
- if (readOff < pageHeaderSize + len)
- readOff = ReadPageInternal(state, targetPagePtr,
- pageHeaderSize + len);
-
+ Assert (pageHeaderSize + len <= state->readLen);
memcpy(buffer, (char *) contdata, len);
buffer += len;
gotlen += len;
@@ -517,9 +524,16 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
else
{
/* Wait for the record data to become available */
- readOff = ReadPageInternal(state, targetPagePtr,
- Min(targetRecOff + total_len, XLOG_BLCKSZ));
- if (readOff < 0)
+ while (XLogNeedData(state, targetPagePtr,
+ Min(targetRecOff + total_len, XLOG_BLCKSZ), true))
+ {
+ if (!state->routine.page_read(state, state->readPagePtr,
+ state->readLen,
+ state->ReadRecPtr, state->readBuf))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
/* Record does not cross a page boundary */
@@ -562,109 +576,138 @@ err:
}
/*
- * Read a single xlog page including at least [pageptr, reqLen] of valid data
- * via the page_read() callback.
+ * Checks that an xlog page loaded in state->readBuf is including at least
+ * [pageptr, reqLen] and the page is valid. header_inclusive indicates that
+ * reqLen is calculated including page header length.
*
- * Returns -1 if the required page cannot be read for some reason; errormsg_buf
- * is set in that case (unless the error occurs in the page_read callback).
+ * Returns false if the buffer already contains the requested data, or found
+ * error. state->page_verified is set to true for the former and false for the
+ * latter.
*
- * We fetch the page from a reader-local cache if we know we have the required
- * data and if there hasn't been any error since caching the data.
+ * Otherwise returns true and requests data loaded onto state->readBuf by
+ * state->readPagePtr and state->readLen. The caller shall call this function
+ * again after filling the buffer at least with that portion of data and set
+ * state->readLen to the length of actually loaded data.
+ *
+ * If header_inclusive is false, corrects reqLen internally by adding the
+ * actual page header length and may request caller for new data.
*/
-static int
-ReadPageInternal(XLogReaderState *state, XLogRecPtr pageptr, int reqLen)
+static bool
+XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
+ bool header_inclusive)
{
- int readLen;
uint32 targetPageOff;
XLogSegNo targetSegNo;
- XLogPageHeader hdr;
+ uint32 addLen = 0;
- Assert((pageptr % XLOG_BLCKSZ) == 0);
+ /* Some data is loaded, but page header is not verified yet. */
+ if (!state->page_verified &&
+ !XLogRecPtrIsInvalid(state->readPagePtr) && state->readLen >= 0)
+ {
+ uint32 pageHeaderSize;
+ /* just loaded new data so needs to verify page header */
+
+ /* The caller must have loaded at least page header */
+ Assert (state->readLen >= SizeOfXLogShortPHD);
+
+ /*
+ * We have enough data to check the header length. Recheck the loaded
+ * length against the actual header length.
+ */
+ pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+
+ /* Request more data if we don't have the full header. */
+ if (state->readLen < pageHeaderSize)
+ {
+ state->readLen = pageHeaderSize;
+ return true;
+ }
+
+ /* Now that we know we have the full header, validate it. */
+ if (!XLogReaderValidatePageHeader(state, state->readPagePtr,
+ (char *) state->readBuf))
+ {
+ /* That's bad. Force reading the page again. */
+ XLogReaderInvalReadState(state);
+
+ return false;
+ }
+
+ state->page_verified = true;
+
+ XLByteToSeg(state->readPagePtr, state->seg.ws_segno,
+ state->segcxt.ws_segsize);
+ }
+
+ /*
+ * The loaded page may not be the one caller is supposing to read when we
+ * are verifying the first page of new segment. In that case, skip further
+ * verification and immediately load the target page.
+ */
+ if (state->page_verified && pageptr == state->readPagePtr)
+ {
+ /*
+ * calculate additional length for page header keeping the total
+ * length within the block size.
+ */
+ if (!header_inclusive)
+ {
+ uint32 pageHeaderSize =
+ XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+
+ addLen = pageHeaderSize;
+ if (reqLen + pageHeaderSize <= XLOG_BLCKSZ)
+ addLen = pageHeaderSize;
+ else
+ addLen = XLOG_BLCKSZ - reqLen;
+
+ Assert(addLen >= 0);
+ }
+
+ /* Return if we already have it. */
+ if (reqLen + addLen <= state->readLen)
+ return false;
+ }
+
+ /* Data is not in our buffer, request the caller for it. */
XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize);
targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize);
+ Assert((pageptr % XLOG_BLCKSZ) == 0);
- /* check whether we have all the requested data already */
- if (targetSegNo == state->seg.ws_segno &&
- targetPageOff == state->segoff && reqLen <= state->readLen)
- return state->readLen;
+ /*
+ * Every time we request to load new data of a page to the caller, even if
+ * we looked at a part of it before, we need to do verification on the next
+ * invocation as the caller might now be rereading data from a different
+ * source.
+ */
+ state->page_verified = false;
/*
- * Data is not in our buffer.
- *
- * Every time we actually read the segment, even if we looked at parts of
- * it before, we need to do verification as the page_read callback might
- * now be rereading data from a different source.
- *
* Whenever switching to a new WAL segment, we read the first page of the
* file and validate its header, even if that's not where the target
* record is. This is so that we can check the additional identification
* info that is present in the first page's "long" header.
+ * Don't do this if the caller requested the first page in the segment.
*/
if (targetSegNo != state->seg.ws_segno && targetPageOff != 0)
{
- XLogRecPtr targetSegmentPtr = pageptr - targetPageOff;
-
- readLen = state->routine.page_read(state, targetSegmentPtr, XLOG_BLCKSZ,
- state->currRecPtr,
- state->readBuf);
- if (readLen < 0)
- goto err;
-
- /* we can be sure to have enough WAL available, we scrolled back */
- Assert(readLen == XLOG_BLCKSZ);
-
- if (!XLogReaderValidatePageHeader(state, targetSegmentPtr,
- state->readBuf))
- goto err;
+ /*
+ * Then we'll see that the targetSegNo now matches the ws_segno, and
+ * will not come back here, but will request the actual target page.
+ */
+ state->readPagePtr = pageptr - targetPageOff;
+ state->readLen = XLOG_BLCKSZ;
+ return true;
}
/*
- * First, read the requested data length, but at least a short page header
- * so that we can validate it.
+ * Request the caller to load the page. We need at least a short page
+ * header so that we can validate it.
*/
- readLen = state->routine.page_read(state, pageptr, Max(reqLen, SizeOfXLogShortPHD),
- state->currRecPtr,
- state->readBuf);
- if (readLen < 0)
- goto err;
-
- Assert(readLen <= XLOG_BLCKSZ);
-
- /* Do we have enough data to check the header length? */
- if (readLen <= SizeOfXLogShortPHD)
- goto err;
-
- Assert(readLen >= reqLen);
-
- hdr = (XLogPageHeader) state->readBuf;
-
- /* still not enough */
- if (readLen < XLogPageHeaderSize(hdr))
- {
- readLen = state->routine.page_read(state, pageptr, XLogPageHeaderSize(hdr),
- state->currRecPtr,
- state->readBuf);
- if (readLen < 0)
- goto err;
- }
-
- /*
- * Now that we know we have the full header, validate it.
- */
- if (!XLogReaderValidatePageHeader(state, pageptr, (char *) hdr))
- goto err;
-
- /* update read state information */
- state->seg.ws_segno = targetSegNo;
- state->segoff = targetPageOff;
- state->readLen = readLen;
-
- return readLen;
-
-err:
- XLogReaderInvalReadState(state);
- return -1;
+ state->readPagePtr = pageptr;
+ state->readLen = Max(reqLen + addLen, SizeOfXLogShortPHD);
+ return true;
}
/*
@@ -673,9 +716,7 @@ err:
static void
XLogReaderInvalReadState(XLogReaderState *state)
{
- state->seg.ws_segno = 0;
- state->segoff = 0;
- state->readLen = 0;
+ state->readPagePtr = InvalidXLogRecPtr;
}
/*
@@ -953,7 +994,6 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
XLogRecPtr targetPagePtr;
int targetRecOff;
uint32 pageHeaderSize;
- int readLen;
/*
* Compute targetRecOff. It should typically be equal or greater than
@@ -961,7 +1001,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
* that, except when caller has explicitly specified the offset that
* falls somewhere there or when we are skipping multi-page
* continuation record. It doesn't matter though because
- * ReadPageInternal() is prepared to handle that and will read at
+ * XLogNeedData() is prepared to handle that and will read at
* least short page-header worth of data
*/
targetRecOff = tmpRecPtr % XLOG_BLCKSZ;
@@ -969,19 +1009,24 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
/* scroll back to page boundary */
targetPagePtr = tmpRecPtr - targetRecOff;
- /* Read the page containing the record */
- readLen = ReadPageInternal(state, targetPagePtr, targetRecOff);
- if (readLen < 0)
+ while(XLogNeedData(state, targetPagePtr, targetRecOff,
+ targetRecOff != 0))
+ {
+ if (!state->routine.page_read(state, state->readPagePtr,
+ state->readLen,
+ state->ReadRecPtr, state->readBuf))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
header = (XLogPageHeader) state->readBuf;
pageHeaderSize = XLogPageHeaderSize(header);
- /* make sure we have enough data for the page header */
- readLen = ReadPageInternal(state, targetPagePtr, pageHeaderSize);
- if (readLen < 0)
- goto err;
+ /* we should have read the page header */
+ Assert (state->readLen >= pageHeaderSize);
/* skip over potential continuation data */
if (header->xlp_info & XLP_FIRST_IS_CONTRECORD)
diff --git a/src/backend/access/transam/xlogutils.c b/src/backend/access/transam/xlogutils.c
index d17d660f46..46eda33f25 100644
--- a/src/backend/access/transam/xlogutils.c
+++ b/src/backend/access/transam/xlogutils.c
@@ -686,8 +686,8 @@ XLogTruncateRelation(RelFileNode rnode, ForkNumber forkNum,
void
XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wantLength)
{
- const XLogRecPtr lastReadPage = (state->seg.ws_segno *
- state->segcxt.ws_segsize + state->segoff);
+ const XLogRecPtr lastReadPage = state->seg.ws_segno *
+ state->segcxt.ws_segsize + state->readLen;
Assert(wantPage != InvalidXLogRecPtr && wantPage % XLOG_BLCKSZ == 0);
Assert(wantLength <= XLOG_BLCKSZ);
@@ -824,7 +824,7 @@ wal_segment_close(XLogReaderState *state)
* exists for normal backends, so we have to do a check/sleep/repeat style of
* loop for now.
*/
-int
+bool
read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *cur_page)
{
@@ -926,7 +926,8 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
else if (targetPagePtr + reqLen > read_upto)
{
/* not enough data there */
- return -1;
+ state->readLen = -1;
+ return false;
}
else
{
@@ -944,7 +945,8 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
WALReadRaiseError(&errinfo);
/* number of valid bytes in the buffer */
- return count;
+ state->readLen = count;
+ return true;
}
/*
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index 23baa4498a..c36b91ae5e 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -806,7 +806,7 @@ StartReplication(StartReplicationCmd *cmd)
* which has to do a plain sleep/busy loop, because the walsender's latch gets
* set every time WAL is flushed.
*/
-static int
+static bool
logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *cur_page)
{
@@ -826,7 +826,10 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
/* fail if not (implies we are going to shut down) */
if (flushptr < targetPagePtr + reqLen)
- return -1;
+ {
+ state->readLen = -1;
+ return false;
+ }
if (targetPagePtr + XLOG_BLCKSZ <= flushptr)
count = XLOG_BLCKSZ; /* more than one block available */
@@ -854,7 +857,8 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
XLByteToSeg(targetPagePtr, segno, state->segcxt.ws_segsize);
CheckXLogRemoved(segno, state->seg.ws_tli);
- return count;
+ state->readLen = count;
+ return true;
}
/*
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index 59ebac7d6a..cf119848b0 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -47,7 +47,7 @@ typedef struct XLogPageReadPrivate
int tliIndex;
} XLogPageReadPrivate;
-static int SimpleXLogPageRead(XLogReaderState *xlogreader,
+static bool SimpleXLogPageRead(XLogReaderState *xlogreader,
XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
@@ -246,7 +246,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
}
/* XLogReader callback function, to read a WAL page */
-static int
+static bool
SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf)
{
@@ -306,7 +306,8 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
if (private->restoreCommand == NULL)
{
pg_log_error("could not open file \"%s\": %m", xlogfpath);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
/*
@@ -319,7 +320,10 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
private->restoreCommand);
if (xlogreadfd < 0)
- return -1;
+ {
+ xlogreader->readLen = -1;
+ return false;
+ }
else
pg_log_debug("using file \"%s\" restored from archive",
xlogfpath);
@@ -335,7 +339,8 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
if (lseek(xlogreadfd, (off_t) targetPageOff, SEEK_SET) < 0)
{
pg_log_error("could not seek in file \"%s\": %m", xlogfpath);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
@@ -348,13 +353,15 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
pg_log_error("could not read file \"%s\": read %d of %zu",
xlogfpath, r, (Size) XLOG_BLCKSZ);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
Assert(targetSegNo == xlogreadsegno);
xlogreader->seg.ws_tli = targetHistory[private->tliIndex].tli;
- return XLOG_BLCKSZ;
+ xlogreader->readLen = XLOG_BLCKSZ;
+ return true;
}
/*
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index 610f65e471..0f0e0723a3 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -331,7 +331,7 @@ WALDumpCloseSegment(XLogReaderState *state)
}
/* pg_waldump's XLogReaderRoutine->page_read callback */
-static int
+static bool
WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetPtr, char *readBuff)
{
@@ -348,7 +348,8 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
else
{
private->endptr_reached = true;
- return -1;
+ state->readLen = -1;
+ return false;
}
}
@@ -373,7 +374,8 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
(Size) errinfo.wre_req);
}
- return count;
+ state->readLen = count;
+ return true;
}
/*
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index 21d200d3df..23d9e70a59 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -57,8 +57,8 @@ typedef struct WALSegmentContext
typedef struct XLogReaderState XLogReaderState;
-/* Function type definitions for various xlogreader interactions */
-typedef int (*XLogPageReadCB) (XLogReaderState *xlogreader,
+/* Function type definition for the read_page callback */
+typedef bool (*XLogPageReadCB) (XLogReaderState *xlogreader,
XLogRecPtr targetPagePtr,
int reqLen,
XLogRecPtr targetRecPtr,
@@ -175,6 +175,20 @@ struct XLogReaderState
XLogRecPtr ReadRecPtr; /* start of last record read */
XLogRecPtr EndRecPtr; /* end+1 of last record read */
+ /* ----------------------------------------
+ * Communication with page reader
+ * readBuf is XLOG_BLCKSZ bytes, valid up to at least readLen bytes.
+ * ----------------------------------------
+ */
+ /* variables to communicate with page reader */
+ XLogRecPtr readPagePtr; /* page pointer to read */
+ int32 readLen; /* bytes requested to reader, or actual bytes
+ * read by reader, which must be larger than
+ * the request, or -1 on error */
+ TimeLineID readPageTLI; /* TLI for data currently in readBuf */
+ char *readBuf; /* buffer to store data */
+ bool page_verified; /* is the page on the buffer verified? */
+
/* ----------------------------------------
* Decoded representation of current record
@@ -203,13 +217,6 @@ struct XLogReaderState
* ----------------------------------------
*/
- /*
- * Buffer for currently read page (XLOG_BLCKSZ bytes, valid up to at least
- * readLen bytes)
- */
- char *readBuf;
- uint32 readLen;
-
/* last read XLOG position for data currently in readBuf */
WALSegmentContext segcxt;
WALOpenSegment seg;
diff --git a/src/include/access/xlogutils.h b/src/include/access/xlogutils.h
index 9ac602b674..364a21c4ea 100644
--- a/src/include/access/xlogutils.h
+++ b/src/include/access/xlogutils.h
@@ -47,7 +47,7 @@ extern Buffer XLogReadBufferExtended(RelFileNode rnode, ForkNumber forknum,
extern Relation CreateFakeRelcacheEntry(RelFileNode rnode);
extern void FreeFakeRelcacheEntry(Relation fakerel);
-extern int read_local_xlog_page(XLogReaderState *state,
+extern bool read_local_xlog_page(XLogReaderState *state,
XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *cur_page);
extern void wal_segment_open(XLogReaderState *state,
--
2.27.0
----Next_Part(Thu_Mar__4_11_28_53_2021_014)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
filename="v17-0002-Move-page-reader-out-of-XLogReadRecord.patch"
^ permalink raw reply [nested|flat] 69+ messages in thread
* [PATCH v15 1/4] Move callback-call from ReadPageInternal to XLogReadRecord.
@ 2019-09-05 11:21 Kyotaro Horiguchi <[email protected]>
0 siblings, 0 replies; 69+ messages in thread
From: Kyotaro Horiguchi @ 2019-09-05 11:21 UTC (permalink / raw)
The current WAL record reader reads page data using a call back
function. Although it is not so problematic alone, it would be a
problem if we are going to do add tasks like encryption which is
performed on page data before WAL reader reads them. To avoid that the
record reader facility has to have a new code path corresponds to
every new callback, this patch separates page reader from WAL record
reading facility by modifying the current WAL record reader to a state
machine.
As the first step of that change, this patch moves the page reader
function out of ReadPageInternal, then the remaining tasks of the
function are taken over by the new function XLogNeedData. As the
result XLogPageRead directly calls the page reader callback function
according to the feedback from XLogNeedData.
---
src/backend/access/transam/xlog.c | 16 +-
src/backend/access/transam/xlogreader.c | 281 ++++++++++++++----------
src/backend/access/transam/xlogutils.c | 12 +-
src/backend/replication/walsender.c | 10 +-
src/bin/pg_rewind/parsexlog.c | 21 +-
src/bin/pg_waldump/pg_waldump.c | 8 +-
src/include/access/xlogreader.h | 25 ++-
src/include/access/xlogutils.h | 2 +-
8 files changed, 222 insertions(+), 153 deletions(-)
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index fd93bcfaeb..e570e56a24 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -912,7 +912,7 @@ static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath,
static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli,
XLogSource source, bool notfoundOk);
static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, XLogSource source);
-static int XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
+static bool XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
bool fetching_ckpt, XLogRecPtr tliRecPtr);
@@ -4332,7 +4332,6 @@ ReadRecord(XLogReaderState *xlogreader, int emode,
XLogRecord *record;
XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
- /* Pass through parameters to XLogPageRead */
private->fetching_ckpt = fetching_ckpt;
private->emode = emode;
private->randAccess = (xlogreader->ReadRecPtr == InvalidXLogRecPtr);
@@ -11860,7 +11859,7 @@ CancelBackup(void)
* XLogPageRead() to try fetching the record from another source, or to
* sleep and retry.
*/
-static int
+static bool
XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *readBuf)
{
@@ -11919,7 +11918,8 @@ retry:
readLen = 0;
readSource = XLOG_FROM_ANY;
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
}
@@ -12014,7 +12014,8 @@ retry:
goto next_record_is_invalid;
}
- return readLen;
+ xlogreader->readLen = readLen;
+ return true;
next_record_is_invalid:
lastSourceFailed = true;
@@ -12028,8 +12029,9 @@ next_record_is_invalid:
/* In standby-mode, keep trying */
if (StandbyMode)
goto retry;
- else
- return -1;
+
+ xlogreader->readLen = -1;
+ return false;
}
/*
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index cb76be4f46..3d599325ee 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -36,8 +36,8 @@
static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
pg_attribute_printf(2, 3);
static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
-static int ReadPageInternal(XLogReaderState *state, XLogRecPtr pageptr,
- int reqLen);
+static bool XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr,
+ int reqLen, bool header_inclusive);
static void XLogReaderInvalReadState(XLogReaderState *state);
static bool ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
XLogRecPtr PrevRecPtr, XLogRecord *record, bool randAccess);
@@ -276,7 +276,6 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
uint32 targetRecOff;
uint32 pageHeaderSize;
bool gotheader;
- int readOff;
/*
* randAccess indicates whether to verify the previous-record pointer of
@@ -326,14 +325,20 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
* byte to cover the whole record header, or at least the part of it that
* fits on the same page.
*/
- readOff = ReadPageInternal(state, targetPagePtr,
- Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ));
- if (readOff < 0)
+ while (XLogNeedData(state, targetPagePtr,
+ Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+ targetRecOff != 0))
+ {
+ if (!state->routine.page_read(state, state->readPagePtr, state->readLen,
+ RecPtr, state->readBuf))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
/*
- * ReadPageInternal always returns at least the page header, so we can
- * examine it now.
+ * We have at least the page header, so we can examine it now.
*/
pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
if (targetRecOff == 0)
@@ -359,8 +364,8 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
goto err;
}
- /* ReadPageInternal has verified the page header */
- Assert(pageHeaderSize <= readOff);
+ /* XLogNeedData has verified the page header */
+ Assert(pageHeaderSize <= state->readLen);
/*
* Read the record length.
@@ -433,18 +438,27 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
do
{
+ int rest_len = total_len - gotlen;
+
/* Calculate pointer to beginning of next page */
targetPagePtr += XLOG_BLCKSZ;
/* Wait for the next page to become available */
- readOff = ReadPageInternal(state, targetPagePtr,
- Min(total_len - gotlen + SizeOfXLogShortPHD,
- XLOG_BLCKSZ));
+ while (XLogNeedData(state, targetPagePtr,
+ Min(rest_len, XLOG_BLCKSZ),
+ false))
+ {
+ if (!state->routine.page_read(state, state->readPagePtr,
+ state->readLen,
+ state->ReadRecPtr,
+ state->readBuf))
+ break;
+ }
- if (readOff < 0)
+ if (!state->page_verified)
goto err;
- Assert(SizeOfXLogShortPHD <= readOff);
+ Assert(SizeOfXLogShortPHD <= state->readLen);
/* Check that the continuation on next page looks valid */
pageHeader = (XLogPageHeader) state->readBuf;
@@ -473,21 +487,14 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
/* Append the continuation from this page to the buffer */
pageHeaderSize = XLogPageHeaderSize(pageHeader);
- if (readOff < pageHeaderSize)
- readOff = ReadPageInternal(state, targetPagePtr,
- pageHeaderSize);
-
- Assert(pageHeaderSize <= readOff);
+ Assert(pageHeaderSize <= state->readLen);
contdata = (char *) state->readBuf + pageHeaderSize;
len = XLOG_BLCKSZ - pageHeaderSize;
if (pageHeader->xlp_rem_len < len)
len = pageHeader->xlp_rem_len;
- if (readOff < pageHeaderSize + len)
- readOff = ReadPageInternal(state, targetPagePtr,
- pageHeaderSize + len);
-
+ Assert (pageHeaderSize + len <= state->readLen);
memcpy(buffer, (char *) contdata, len);
buffer += len;
gotlen += len;
@@ -517,9 +524,16 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
else
{
/* Wait for the record data to become available */
- readOff = ReadPageInternal(state, targetPagePtr,
- Min(targetRecOff + total_len, XLOG_BLCKSZ));
- if (readOff < 0)
+ while (XLogNeedData(state, targetPagePtr,
+ Min(targetRecOff + total_len, XLOG_BLCKSZ), true))
+ {
+ if (!state->routine.page_read(state, state->readPagePtr,
+ state->readLen,
+ state->ReadRecPtr, state->readBuf))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
/* Record does not cross a page boundary */
@@ -562,109 +576,138 @@ err:
}
/*
- * Read a single xlog page including at least [pageptr, reqLen] of valid data
- * via the page_read() callback.
+ * Checks that an xlog page loaded in state->readBuf is including at least
+ * [pageptr, reqLen] and the page is valid. header_inclusive indicates that
+ * reqLen is calculated including page header length.
*
- * Returns -1 if the required page cannot be read for some reason; errormsg_buf
- * is set in that case (unless the error occurs in the page_read callback).
+ * Returns false if the buffer already contains the requested data, or found
+ * error. state->page_verified is set to true for the former and false for the
+ * latter.
*
- * We fetch the page from a reader-local cache if we know we have the required
- * data and if there hasn't been any error since caching the data.
+ * Otherwise returns true and requests data loaded onto state->readBuf by
+ * state->readPagePtr and state->readLen. The caller shall call this function
+ * again after filling the buffer at least with that portion of data and set
+ * state->readLen to the length of actually loaded data.
+ *
+ * If header_inclusive is false, corrects reqLen internally by adding the
+ * actual page header length and may request caller for new data.
*/
-static int
-ReadPageInternal(XLogReaderState *state, XLogRecPtr pageptr, int reqLen)
+static bool
+XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
+ bool header_inclusive)
{
- int readLen;
uint32 targetPageOff;
XLogSegNo targetSegNo;
- XLogPageHeader hdr;
+ uint32 addLen = 0;
- Assert((pageptr % XLOG_BLCKSZ) == 0);
+ /* Some data is loaded, but page header is not verified yet. */
+ if (!state->page_verified &&
+ !XLogRecPtrIsInvalid(state->readPagePtr) && state->readLen >= 0)
+ {
+ uint32 pageHeaderSize;
+ /* just loaded new data so needs to verify page header */
+
+ /* The caller must have loaded at least page header */
+ Assert (state->readLen >= SizeOfXLogShortPHD);
+
+ /*
+ * We have enough data to check the header length. Recheck the loaded
+ * length against the actual header length.
+ */
+ pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+
+ /* Request more data if we don't have the full header. */
+ if (state->readLen < pageHeaderSize)
+ {
+ state->readLen = pageHeaderSize;
+ return true;
+ }
+
+ /* Now that we know we have the full header, validate it. */
+ if (!XLogReaderValidatePageHeader(state, state->readPagePtr,
+ (char *) state->readBuf))
+ {
+ /* That's bad. Force reading the page again. */
+ XLogReaderInvalReadState(state);
+
+ return false;
+ }
+
+ state->page_verified = true;
+
+ XLByteToSeg(state->readPagePtr, state->seg.ws_segno,
+ state->segcxt.ws_segsize);
+ }
+
+ /*
+ * The loaded page may not be the one caller is supposing to read when we
+ * are verifying the first page of new segment. In that case, skip further
+ * verification and immediately load the target page.
+ */
+ if (state->page_verified && pageptr == state->readPagePtr)
+ {
+ /*
+ * calculate additional length for page header keeping the total
+ * length within the block size.
+ */
+ if (!header_inclusive)
+ {
+ uint32 pageHeaderSize =
+ XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+
+ addLen = pageHeaderSize;
+ if (reqLen + pageHeaderSize <= XLOG_BLCKSZ)
+ addLen = pageHeaderSize;
+ else
+ addLen = XLOG_BLCKSZ - reqLen;
+
+ Assert(addLen >= 0);
+ }
+
+ /* Return if we already have it. */
+ if (reqLen + addLen <= state->readLen)
+ return false;
+ }
+
+ /* Data is not in our buffer, request the caller for it. */
XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize);
targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize);
+ Assert((pageptr % XLOG_BLCKSZ) == 0);
- /* check whether we have all the requested data already */
- if (targetSegNo == state->seg.ws_segno &&
- targetPageOff == state->segoff && reqLen <= state->readLen)
- return state->readLen;
+ /*
+ * Every time we request to load new data of a page to the caller, even if
+ * we looked at a part of it before, we need to do verification on the next
+ * invocation as the caller might now be rereading data from a different
+ * source.
+ */
+ state->page_verified = false;
/*
- * Data is not in our buffer.
- *
- * Every time we actually read the segment, even if we looked at parts of
- * it before, we need to do verification as the page_read callback might
- * now be rereading data from a different source.
- *
* Whenever switching to a new WAL segment, we read the first page of the
* file and validate its header, even if that's not where the target
* record is. This is so that we can check the additional identification
* info that is present in the first page's "long" header.
+ * Don't do this if the caller requested the first page in the segment.
*/
if (targetSegNo != state->seg.ws_segno && targetPageOff != 0)
{
- XLogRecPtr targetSegmentPtr = pageptr - targetPageOff;
-
- readLen = state->routine.page_read(state, targetSegmentPtr, XLOG_BLCKSZ,
- state->currRecPtr,
- state->readBuf);
- if (readLen < 0)
- goto err;
-
- /* we can be sure to have enough WAL available, we scrolled back */
- Assert(readLen == XLOG_BLCKSZ);
-
- if (!XLogReaderValidatePageHeader(state, targetSegmentPtr,
- state->readBuf))
- goto err;
+ /*
+ * Then we'll see that the targetSegNo now matches the ws_segno, and
+ * will not come back here, but will request the actual target page.
+ */
+ state->readPagePtr = pageptr - targetPageOff;
+ state->readLen = XLOG_BLCKSZ;
+ return true;
}
/*
- * First, read the requested data length, but at least a short page header
- * so that we can validate it.
+ * Request the caller to load the page. We need at least a short page
+ * header so that we can validate it.
*/
- readLen = state->routine.page_read(state, pageptr, Max(reqLen, SizeOfXLogShortPHD),
- state->currRecPtr,
- state->readBuf);
- if (readLen < 0)
- goto err;
-
- Assert(readLen <= XLOG_BLCKSZ);
-
- /* Do we have enough data to check the header length? */
- if (readLen <= SizeOfXLogShortPHD)
- goto err;
-
- Assert(readLen >= reqLen);
-
- hdr = (XLogPageHeader) state->readBuf;
-
- /* still not enough */
- if (readLen < XLogPageHeaderSize(hdr))
- {
- readLen = state->routine.page_read(state, pageptr, XLogPageHeaderSize(hdr),
- state->currRecPtr,
- state->readBuf);
- if (readLen < 0)
- goto err;
- }
-
- /*
- * Now that we know we have the full header, validate it.
- */
- if (!XLogReaderValidatePageHeader(state, pageptr, (char *) hdr))
- goto err;
-
- /* update read state information */
- state->seg.ws_segno = targetSegNo;
- state->segoff = targetPageOff;
- state->readLen = readLen;
-
- return readLen;
-
-err:
- XLogReaderInvalReadState(state);
- return -1;
+ state->readPagePtr = pageptr;
+ state->readLen = Max(reqLen + addLen, SizeOfXLogShortPHD);
+ return true;
}
/*
@@ -673,9 +716,7 @@ err:
static void
XLogReaderInvalReadState(XLogReaderState *state)
{
- state->seg.ws_segno = 0;
- state->segoff = 0;
- state->readLen = 0;
+ state->readPagePtr = InvalidXLogRecPtr;
}
/*
@@ -956,7 +997,6 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
XLogRecPtr targetPagePtr;
int targetRecOff;
uint32 pageHeaderSize;
- int readLen;
/*
* Compute targetRecOff. It should typically be equal or greater than
@@ -964,7 +1004,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
* that, except when caller has explicitly specified the offset that
* falls somewhere there or when we are skipping multi-page
* continuation record. It doesn't matter though because
- * ReadPageInternal() is prepared to handle that and will read at
+ * XLogNeedData() is prepared to handle that and will read at
* least short page-header worth of data
*/
targetRecOff = tmpRecPtr % XLOG_BLCKSZ;
@@ -972,19 +1012,24 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
/* scroll back to page boundary */
targetPagePtr = tmpRecPtr - targetRecOff;
- /* Read the page containing the record */
- readLen = ReadPageInternal(state, targetPagePtr, targetRecOff);
- if (readLen < 0)
+ while(XLogNeedData(state, targetPagePtr, targetRecOff,
+ targetRecOff != 0))
+ {
+ if (!state->routine.page_read(state, state->readPagePtr,
+ state->readLen,
+ state->ReadRecPtr, state->readBuf))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
header = (XLogPageHeader) state->readBuf;
pageHeaderSize = XLogPageHeaderSize(header);
- /* make sure we have enough data for the page header */
- readLen = ReadPageInternal(state, targetPagePtr, pageHeaderSize);
- if (readLen < 0)
- goto err;
+ /* we should have read the page header */
+ Assert (state->readLen >= pageHeaderSize);
/* skip over potential continuation data */
if (header->xlp_info & XLP_FIRST_IS_CONTRECORD)
diff --git a/src/backend/access/transam/xlogutils.c b/src/backend/access/transam/xlogutils.c
index 322b0e8ff5..1b10b6df20 100644
--- a/src/backend/access/transam/xlogutils.c
+++ b/src/backend/access/transam/xlogutils.c
@@ -689,8 +689,8 @@ XLogTruncateRelation(RelFileNode rnode, ForkNumber forkNum,
void
XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wantLength)
{
- const XLogRecPtr lastReadPage = (state->seg.ws_segno *
- state->segcxt.ws_segsize + state->segoff);
+ const XLogRecPtr lastReadPage = state->seg.ws_segno *
+ state->segcxt.ws_segsize + state->readLen;
Assert(wantPage != InvalidXLogRecPtr && wantPage % XLOG_BLCKSZ == 0);
Assert(wantLength <= XLOG_BLCKSZ);
@@ -828,7 +828,7 @@ wal_segment_close(XLogReaderState *state)
* exists for normal backends, so we have to do a check/sleep/repeat style of
* loop for now.
*/
-int
+bool
read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *cur_page)
{
@@ -930,7 +930,8 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
else if (targetPagePtr + reqLen > read_upto)
{
/* not enough data there */
- return -1;
+ state->readLen = -1;
+ return false;
}
else
{
@@ -948,7 +949,8 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
WALReadRaiseError(&errinfo);
/* number of valid bytes in the buffer */
- return count;
+ state->readLen = count;
+ return true;
}
/*
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index e2477c47e0..bf2711bddd 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -811,7 +811,7 @@ StartReplication(StartReplicationCmd *cmd)
* which has to do a plain sleep/busy loop, because the walsender's latch gets
* set every time WAL is flushed.
*/
-static int
+static bool
logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *cur_page)
{
@@ -831,7 +831,10 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
/* fail if not (implies we are going to shut down) */
if (flushptr < targetPagePtr + reqLen)
- return -1;
+ {
+ state->readLen = -1;
+ return false;
+ }
if (targetPagePtr + XLOG_BLCKSZ <= flushptr)
count = XLOG_BLCKSZ; /* more than one block available */
@@ -859,7 +862,8 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
XLByteToSeg(targetPagePtr, segno, state->segcxt.ws_segsize);
CheckXLogRemoved(segno, state->seg.ws_tli);
- return count;
+ state->readLen = count;
+ return true;
}
/*
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index bc6f976994..0b7e73ae79 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -46,7 +46,7 @@ typedef struct XLogPageReadPrivate
int tliIndex;
} XLogPageReadPrivate;
-static int SimpleXLogPageRead(XLogReaderState *xlogreader,
+static bool SimpleXLogPageRead(XLogReaderState *xlogreader,
XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
@@ -236,7 +236,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
}
/* XLogReader callback function, to read a WAL page */
-static int
+static bool
SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf)
{
@@ -296,7 +296,8 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
if (private->restoreCommand == NULL)
{
pg_log_error("could not open file \"%s\": %m", xlogfpath);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
/*
@@ -309,7 +310,10 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
private->restoreCommand);
if (xlogreadfd < 0)
- return -1;
+ {
+ xlogreader->readLen = -1;
+ return false;
+ }
else
pg_log_debug("using file \"%s\" restored from archive",
xlogfpath);
@@ -325,7 +329,8 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
if (lseek(xlogreadfd, (off_t) targetPageOff, SEEK_SET) < 0)
{
pg_log_error("could not seek in file \"%s\": %m", xlogfpath);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
@@ -338,13 +343,15 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
pg_log_error("could not read file \"%s\": read %d of %zu",
xlogfpath, r, (Size) XLOG_BLCKSZ);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
Assert(targetSegNo == xlogreadsegno);
xlogreader->seg.ws_tli = targetHistory[private->tliIndex].tli;
- return XLOG_BLCKSZ;
+ xlogreader->readLen = XLOG_BLCKSZ;
+ return true;
}
/*
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index d1a0678935..ab0c28d9b7 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -331,7 +331,7 @@ WALDumpCloseSegment(XLogReaderState *state)
}
/* pg_waldump's XLogReaderRoutine->page_read callback */
-static int
+static bool
WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetPtr, char *readBuff)
{
@@ -348,7 +348,8 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
else
{
private->endptr_reached = true;
- return -1;
+ state->readLen = -1;
+ return false;
}
}
@@ -373,7 +374,8 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
(Size) errinfo.wre_req);
}
- return count;
+ state->readLen = count;
+ return true;
}
/*
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index b0f2a6ed43..6c8848e14f 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -57,8 +57,8 @@ typedef struct WALSegmentContext
typedef struct XLogReaderState XLogReaderState;
-/* Function type definitions for various xlogreader interactions */
-typedef int (*XLogPageReadCB) (XLogReaderState *xlogreader,
+/* Function type definition for the read_page callback */
+typedef bool (*XLogPageReadCB) (XLogReaderState *xlogreader,
XLogRecPtr targetPagePtr,
int reqLen,
XLogRecPtr targetRecPtr,
@@ -175,6 +175,20 @@ struct XLogReaderState
XLogRecPtr ReadRecPtr; /* start of last record read */
XLogRecPtr EndRecPtr; /* end+1 of last record read */
+ /* ----------------------------------------
+ * Communication with page reader
+ * readBuf is XLOG_BLCKSZ bytes, valid up to at least readLen bytes.
+ * ----------------------------------------
+ */
+ /* variables to communicate with page reader */
+ XLogRecPtr readPagePtr; /* page pointer to read */
+ int32 readLen; /* bytes requested to reader, or actual bytes
+ * read by reader, which must be larger than
+ * the request, or -1 on error */
+ TimeLineID readPageTLI; /* TLI for data currently in readBuf */
+ char *readBuf; /* buffer to store data */
+ bool page_verified; /* is the page on the buffer verified? */
+
/* ----------------------------------------
* Decoded representation of current record
@@ -201,13 +215,6 @@ struct XLogReaderState
* ----------------------------------------
*/
- /*
- * Buffer for currently read page (XLOG_BLCKSZ bytes, valid up to at least
- * readLen bytes)
- */
- char *readBuf;
- uint32 readLen;
-
/* last read XLOG position for data currently in readBuf */
WALSegmentContext segcxt;
WALOpenSegment seg;
diff --git a/src/include/access/xlogutils.h b/src/include/access/xlogutils.h
index e59b6cf3a9..c287edf206 100644
--- a/src/include/access/xlogutils.h
+++ b/src/include/access/xlogutils.h
@@ -47,7 +47,7 @@ extern Buffer XLogReadBufferExtended(RelFileNode rnode, ForkNumber forknum,
extern Relation CreateFakeRelcacheEntry(RelFileNode rnode);
extern void FreeFakeRelcacheEntry(Relation fakerel);
-extern int read_local_xlog_page(XLogReaderState *state,
+extern bool read_local_xlog_page(XLogReaderState *state,
XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *cur_page);
extern void wal_segment_open(XLogReaderState *state,
--
2.18.4
----Next_Part(Thu_Jul__2_13_53_30_2020_072)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
filename="v15-0002-Move-page-reader-out-of-XLogReadRecord.patch"
^ permalink raw reply [nested|flat] 69+ messages in thread
* [PATCH v17 1/4] Move callback-call from ReadPageInternal to XLogReadRecord.
@ 2019-09-05 11:21 Kyotaro Horiguchi <[email protected]>
0 siblings, 0 replies; 69+ messages in thread
From: Kyotaro Horiguchi @ 2019-09-05 11:21 UTC (permalink / raw)
The current WAL record reader reads page data using a call back
function. Although it is not so problematic alone, it would be a
problem if we are going to do add tasks like encryption which is
performed on page data before WAL reader reads them. To avoid that the
record reader facility has to have a new code path corresponds to
every new callback, this patch separates page reader from WAL record
reading facility by modifying the current WAL record reader to a state
machine.
As the first step of that change, this patch moves the page reader
function out of ReadPageInternal, then the remaining tasks of the
function are taken over by the new function XLogNeedData. As the
result XLogPageRead directly calls the page reader callback function
according to the feedback from XLogNeedData.
---
src/backend/access/transam/xlog.c | 16 +-
src/backend/access/transam/xlogreader.c | 281 ++++++++++++++----------
src/backend/access/transam/xlogutils.c | 12 +-
src/backend/replication/walsender.c | 10 +-
src/bin/pg_rewind/parsexlog.c | 21 +-
src/bin/pg_waldump/pg_waldump.c | 8 +-
src/include/access/xlogreader.h | 25 ++-
src/include/access/xlogutils.h | 2 +-
8 files changed, 222 insertions(+), 153 deletions(-)
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 377afb8732..606328a65a 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -917,7 +917,7 @@ static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath,
static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli,
XLogSource source, bool notfoundOk);
static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, XLogSource source);
-static int XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
+static bool XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
bool fetching_ckpt, XLogRecPtr tliRecPtr);
@@ -4350,7 +4350,6 @@ ReadRecord(XLogReaderState *xlogreader, int emode,
XLogRecord *record;
XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
- /* Pass through parameters to XLogPageRead */
private->fetching_ckpt = fetching_ckpt;
private->emode = emode;
private->randAccess = (xlogreader->ReadRecPtr == InvalidXLogRecPtr);
@@ -11987,7 +11986,7 @@ CancelBackup(void)
* XLogPageRead() to try fetching the record from another source, or to
* sleep and retry.
*/
-static int
+static bool
XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *readBuf)
{
@@ -12046,7 +12045,8 @@ retry:
readLen = 0;
readSource = XLOG_FROM_ANY;
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
}
@@ -12141,7 +12141,8 @@ retry:
goto next_record_is_invalid;
}
- return readLen;
+ xlogreader->readLen = readLen;
+ return true;
next_record_is_invalid:
lastSourceFailed = true;
@@ -12155,8 +12156,9 @@ next_record_is_invalid:
/* In standby-mode, keep trying */
if (StandbyMode)
goto retry;
- else
- return -1;
+
+ xlogreader->readLen = -1;
+ return false;
}
/*
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index 42738eb940..7d04995b7b 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -36,8 +36,8 @@
static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
pg_attribute_printf(2, 3);
static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
-static int ReadPageInternal(XLogReaderState *state, XLogRecPtr pageptr,
- int reqLen);
+static bool XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr,
+ int reqLen, bool header_inclusive);
static void XLogReaderInvalReadState(XLogReaderState *state);
static bool ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
XLogRecPtr PrevRecPtr, XLogRecord *record, bool randAccess);
@@ -276,7 +276,6 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
uint32 targetRecOff;
uint32 pageHeaderSize;
bool gotheader;
- int readOff;
/*
* randAccess indicates whether to verify the previous-record pointer of
@@ -326,14 +325,20 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
* byte to cover the whole record header, or at least the part of it that
* fits on the same page.
*/
- readOff = ReadPageInternal(state, targetPagePtr,
- Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ));
- if (readOff < 0)
+ while (XLogNeedData(state, targetPagePtr,
+ Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+ targetRecOff != 0))
+ {
+ if (!state->routine.page_read(state, state->readPagePtr, state->readLen,
+ RecPtr, state->readBuf))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
/*
- * ReadPageInternal always returns at least the page header, so we can
- * examine it now.
+ * We have at least the page header, so we can examine it now.
*/
pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
if (targetRecOff == 0)
@@ -359,8 +364,8 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
goto err;
}
- /* ReadPageInternal has verified the page header */
- Assert(pageHeaderSize <= readOff);
+ /* XLogNeedData has verified the page header */
+ Assert(pageHeaderSize <= state->readLen);
/*
* Read the record length.
@@ -432,18 +437,27 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
do
{
+ int rest_len = total_len - gotlen;
+
/* Calculate pointer to beginning of next page */
targetPagePtr += XLOG_BLCKSZ;
/* Wait for the next page to become available */
- readOff = ReadPageInternal(state, targetPagePtr,
- Min(total_len - gotlen + SizeOfXLogShortPHD,
- XLOG_BLCKSZ));
+ while (XLogNeedData(state, targetPagePtr,
+ Min(rest_len, XLOG_BLCKSZ),
+ false))
+ {
+ if (!state->routine.page_read(state, state->readPagePtr,
+ state->readLen,
+ state->ReadRecPtr,
+ state->readBuf))
+ break;
+ }
- if (readOff < 0)
+ if (!state->page_verified)
goto err;
- Assert(SizeOfXLogShortPHD <= readOff);
+ Assert(SizeOfXLogShortPHD <= state->readLen);
/* Check that the continuation on next page looks valid */
pageHeader = (XLogPageHeader) state->readBuf;
@@ -473,21 +487,14 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
/* Append the continuation from this page to the buffer */
pageHeaderSize = XLogPageHeaderSize(pageHeader);
- if (readOff < pageHeaderSize)
- readOff = ReadPageInternal(state, targetPagePtr,
- pageHeaderSize);
-
- Assert(pageHeaderSize <= readOff);
+ Assert(pageHeaderSize <= state->readLen);
contdata = (char *) state->readBuf + pageHeaderSize;
len = XLOG_BLCKSZ - pageHeaderSize;
if (pageHeader->xlp_rem_len < len)
len = pageHeader->xlp_rem_len;
- if (readOff < pageHeaderSize + len)
- readOff = ReadPageInternal(state, targetPagePtr,
- pageHeaderSize + len);
-
+ Assert (pageHeaderSize + len <= state->readLen);
memcpy(buffer, (char *) contdata, len);
buffer += len;
gotlen += len;
@@ -517,9 +524,16 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
else
{
/* Wait for the record data to become available */
- readOff = ReadPageInternal(state, targetPagePtr,
- Min(targetRecOff + total_len, XLOG_BLCKSZ));
- if (readOff < 0)
+ while (XLogNeedData(state, targetPagePtr,
+ Min(targetRecOff + total_len, XLOG_BLCKSZ), true))
+ {
+ if (!state->routine.page_read(state, state->readPagePtr,
+ state->readLen,
+ state->ReadRecPtr, state->readBuf))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
/* Record does not cross a page boundary */
@@ -562,109 +576,138 @@ err:
}
/*
- * Read a single xlog page including at least [pageptr, reqLen] of valid data
- * via the page_read() callback.
+ * Checks that an xlog page loaded in state->readBuf is including at least
+ * [pageptr, reqLen] and the page is valid. header_inclusive indicates that
+ * reqLen is calculated including page header length.
*
- * Returns -1 if the required page cannot be read for some reason; errormsg_buf
- * is set in that case (unless the error occurs in the page_read callback).
+ * Returns false if the buffer already contains the requested data, or found
+ * error. state->page_verified is set to true for the former and false for the
+ * latter.
*
- * We fetch the page from a reader-local cache if we know we have the required
- * data and if there hasn't been any error since caching the data.
+ * Otherwise returns true and requests data loaded onto state->readBuf by
+ * state->readPagePtr and state->readLen. The caller shall call this function
+ * again after filling the buffer at least with that portion of data and set
+ * state->readLen to the length of actually loaded data.
+ *
+ * If header_inclusive is false, corrects reqLen internally by adding the
+ * actual page header length and may request caller for new data.
*/
-static int
-ReadPageInternal(XLogReaderState *state, XLogRecPtr pageptr, int reqLen)
+static bool
+XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
+ bool header_inclusive)
{
- int readLen;
uint32 targetPageOff;
XLogSegNo targetSegNo;
- XLogPageHeader hdr;
+ uint32 addLen = 0;
- Assert((pageptr % XLOG_BLCKSZ) == 0);
+ /* Some data is loaded, but page header is not verified yet. */
+ if (!state->page_verified &&
+ !XLogRecPtrIsInvalid(state->readPagePtr) && state->readLen >= 0)
+ {
+ uint32 pageHeaderSize;
+ /* just loaded new data so needs to verify page header */
+
+ /* The caller must have loaded at least page header */
+ Assert (state->readLen >= SizeOfXLogShortPHD);
+
+ /*
+ * We have enough data to check the header length. Recheck the loaded
+ * length against the actual header length.
+ */
+ pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+
+ /* Request more data if we don't have the full header. */
+ if (state->readLen < pageHeaderSize)
+ {
+ state->readLen = pageHeaderSize;
+ return true;
+ }
+
+ /* Now that we know we have the full header, validate it. */
+ if (!XLogReaderValidatePageHeader(state, state->readPagePtr,
+ (char *) state->readBuf))
+ {
+ /* That's bad. Force reading the page again. */
+ XLogReaderInvalReadState(state);
+
+ return false;
+ }
+
+ state->page_verified = true;
+
+ XLByteToSeg(state->readPagePtr, state->seg.ws_segno,
+ state->segcxt.ws_segsize);
+ }
+
+ /*
+ * The loaded page may not be the one caller is supposing to read when we
+ * are verifying the first page of new segment. In that case, skip further
+ * verification and immediately load the target page.
+ */
+ if (state->page_verified && pageptr == state->readPagePtr)
+ {
+ /*
+ * calculate additional length for page header keeping the total
+ * length within the block size.
+ */
+ if (!header_inclusive)
+ {
+ uint32 pageHeaderSize =
+ XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+
+ addLen = pageHeaderSize;
+ if (reqLen + pageHeaderSize <= XLOG_BLCKSZ)
+ addLen = pageHeaderSize;
+ else
+ addLen = XLOG_BLCKSZ - reqLen;
+
+ Assert(addLen >= 0);
+ }
+
+ /* Return if we already have it. */
+ if (reqLen + addLen <= state->readLen)
+ return false;
+ }
+
+ /* Data is not in our buffer, request the caller for it. */
XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize);
targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize);
+ Assert((pageptr % XLOG_BLCKSZ) == 0);
- /* check whether we have all the requested data already */
- if (targetSegNo == state->seg.ws_segno &&
- targetPageOff == state->segoff && reqLen <= state->readLen)
- return state->readLen;
+ /*
+ * Every time we request to load new data of a page to the caller, even if
+ * we looked at a part of it before, we need to do verification on the next
+ * invocation as the caller might now be rereading data from a different
+ * source.
+ */
+ state->page_verified = false;
/*
- * Data is not in our buffer.
- *
- * Every time we actually read the segment, even if we looked at parts of
- * it before, we need to do verification as the page_read callback might
- * now be rereading data from a different source.
- *
* Whenever switching to a new WAL segment, we read the first page of the
* file and validate its header, even if that's not where the target
* record is. This is so that we can check the additional identification
* info that is present in the first page's "long" header.
+ * Don't do this if the caller requested the first page in the segment.
*/
if (targetSegNo != state->seg.ws_segno && targetPageOff != 0)
{
- XLogRecPtr targetSegmentPtr = pageptr - targetPageOff;
-
- readLen = state->routine.page_read(state, targetSegmentPtr, XLOG_BLCKSZ,
- state->currRecPtr,
- state->readBuf);
- if (readLen < 0)
- goto err;
-
- /* we can be sure to have enough WAL available, we scrolled back */
- Assert(readLen == XLOG_BLCKSZ);
-
- if (!XLogReaderValidatePageHeader(state, targetSegmentPtr,
- state->readBuf))
- goto err;
+ /*
+ * Then we'll see that the targetSegNo now matches the ws_segno, and
+ * will not come back here, but will request the actual target page.
+ */
+ state->readPagePtr = pageptr - targetPageOff;
+ state->readLen = XLOG_BLCKSZ;
+ return true;
}
/*
- * First, read the requested data length, but at least a short page header
- * so that we can validate it.
+ * Request the caller to load the page. We need at least a short page
+ * header so that we can validate it.
*/
- readLen = state->routine.page_read(state, pageptr, Max(reqLen, SizeOfXLogShortPHD),
- state->currRecPtr,
- state->readBuf);
- if (readLen < 0)
- goto err;
-
- Assert(readLen <= XLOG_BLCKSZ);
-
- /* Do we have enough data to check the header length? */
- if (readLen <= SizeOfXLogShortPHD)
- goto err;
-
- Assert(readLen >= reqLen);
-
- hdr = (XLogPageHeader) state->readBuf;
-
- /* still not enough */
- if (readLen < XLogPageHeaderSize(hdr))
- {
- readLen = state->routine.page_read(state, pageptr, XLogPageHeaderSize(hdr),
- state->currRecPtr,
- state->readBuf);
- if (readLen < 0)
- goto err;
- }
-
- /*
- * Now that we know we have the full header, validate it.
- */
- if (!XLogReaderValidatePageHeader(state, pageptr, (char *) hdr))
- goto err;
-
- /* update read state information */
- state->seg.ws_segno = targetSegNo;
- state->segoff = targetPageOff;
- state->readLen = readLen;
-
- return readLen;
-
-err:
- XLogReaderInvalReadState(state);
- return -1;
+ state->readPagePtr = pageptr;
+ state->readLen = Max(reqLen + addLen, SizeOfXLogShortPHD);
+ return true;
}
/*
@@ -673,9 +716,7 @@ err:
static void
XLogReaderInvalReadState(XLogReaderState *state)
{
- state->seg.ws_segno = 0;
- state->segoff = 0;
- state->readLen = 0;
+ state->readPagePtr = InvalidXLogRecPtr;
}
/*
@@ -953,7 +994,6 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
XLogRecPtr targetPagePtr;
int targetRecOff;
uint32 pageHeaderSize;
- int readLen;
/*
* Compute targetRecOff. It should typically be equal or greater than
@@ -961,7 +1001,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
* that, except when caller has explicitly specified the offset that
* falls somewhere there or when we are skipping multi-page
* continuation record. It doesn't matter though because
- * ReadPageInternal() is prepared to handle that and will read at
+ * XLogNeedData() is prepared to handle that and will read at
* least short page-header worth of data
*/
targetRecOff = tmpRecPtr % XLOG_BLCKSZ;
@@ -969,19 +1009,24 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
/* scroll back to page boundary */
targetPagePtr = tmpRecPtr - targetRecOff;
- /* Read the page containing the record */
- readLen = ReadPageInternal(state, targetPagePtr, targetRecOff);
- if (readLen < 0)
+ while(XLogNeedData(state, targetPagePtr, targetRecOff,
+ targetRecOff != 0))
+ {
+ if (!state->routine.page_read(state, state->readPagePtr,
+ state->readLen,
+ state->ReadRecPtr, state->readBuf))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
header = (XLogPageHeader) state->readBuf;
pageHeaderSize = XLogPageHeaderSize(header);
- /* make sure we have enough data for the page header */
- readLen = ReadPageInternal(state, targetPagePtr, pageHeaderSize);
- if (readLen < 0)
- goto err;
+ /* we should have read the page header */
+ Assert (state->readLen >= pageHeaderSize);
/* skip over potential continuation data */
if (header->xlp_info & XLP_FIRST_IS_CONTRECORD)
diff --git a/src/backend/access/transam/xlogutils.c b/src/backend/access/transam/xlogutils.c
index d17d660f46..46eda33f25 100644
--- a/src/backend/access/transam/xlogutils.c
+++ b/src/backend/access/transam/xlogutils.c
@@ -686,8 +686,8 @@ XLogTruncateRelation(RelFileNode rnode, ForkNumber forkNum,
void
XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wantLength)
{
- const XLogRecPtr lastReadPage = (state->seg.ws_segno *
- state->segcxt.ws_segsize + state->segoff);
+ const XLogRecPtr lastReadPage = state->seg.ws_segno *
+ state->segcxt.ws_segsize + state->readLen;
Assert(wantPage != InvalidXLogRecPtr && wantPage % XLOG_BLCKSZ == 0);
Assert(wantLength <= XLOG_BLCKSZ);
@@ -824,7 +824,7 @@ wal_segment_close(XLogReaderState *state)
* exists for normal backends, so we have to do a check/sleep/repeat style of
* loop for now.
*/
-int
+bool
read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *cur_page)
{
@@ -926,7 +926,8 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
else if (targetPagePtr + reqLen > read_upto)
{
/* not enough data there */
- return -1;
+ state->readLen = -1;
+ return false;
}
else
{
@@ -944,7 +945,8 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
WALReadRaiseError(&errinfo);
/* number of valid bytes in the buffer */
- return count;
+ state->readLen = count;
+ return true;
}
/*
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index 23baa4498a..c36b91ae5e 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -806,7 +806,7 @@ StartReplication(StartReplicationCmd *cmd)
* which has to do a plain sleep/busy loop, because the walsender's latch gets
* set every time WAL is flushed.
*/
-static int
+static bool
logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *cur_page)
{
@@ -826,7 +826,10 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
/* fail if not (implies we are going to shut down) */
if (flushptr < targetPagePtr + reqLen)
- return -1;
+ {
+ state->readLen = -1;
+ return false;
+ }
if (targetPagePtr + XLOG_BLCKSZ <= flushptr)
count = XLOG_BLCKSZ; /* more than one block available */
@@ -854,7 +857,8 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
XLByteToSeg(targetPagePtr, segno, state->segcxt.ws_segsize);
CheckXLogRemoved(segno, state->seg.ws_tli);
- return count;
+ state->readLen = count;
+ return true;
}
/*
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index 59ebac7d6a..cf119848b0 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -47,7 +47,7 @@ typedef struct XLogPageReadPrivate
int tliIndex;
} XLogPageReadPrivate;
-static int SimpleXLogPageRead(XLogReaderState *xlogreader,
+static bool SimpleXLogPageRead(XLogReaderState *xlogreader,
XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
@@ -246,7 +246,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
}
/* XLogReader callback function, to read a WAL page */
-static int
+static bool
SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf)
{
@@ -306,7 +306,8 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
if (private->restoreCommand == NULL)
{
pg_log_error("could not open file \"%s\": %m", xlogfpath);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
/*
@@ -319,7 +320,10 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
private->restoreCommand);
if (xlogreadfd < 0)
- return -1;
+ {
+ xlogreader->readLen = -1;
+ return false;
+ }
else
pg_log_debug("using file \"%s\" restored from archive",
xlogfpath);
@@ -335,7 +339,8 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
if (lseek(xlogreadfd, (off_t) targetPageOff, SEEK_SET) < 0)
{
pg_log_error("could not seek in file \"%s\": %m", xlogfpath);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
@@ -348,13 +353,15 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
pg_log_error("could not read file \"%s\": read %d of %zu",
xlogfpath, r, (Size) XLOG_BLCKSZ);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
Assert(targetSegNo == xlogreadsegno);
xlogreader->seg.ws_tli = targetHistory[private->tliIndex].tli;
- return XLOG_BLCKSZ;
+ xlogreader->readLen = XLOG_BLCKSZ;
+ return true;
}
/*
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index 610f65e471..0f0e0723a3 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -331,7 +331,7 @@ WALDumpCloseSegment(XLogReaderState *state)
}
/* pg_waldump's XLogReaderRoutine->page_read callback */
-static int
+static bool
WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetPtr, char *readBuff)
{
@@ -348,7 +348,8 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
else
{
private->endptr_reached = true;
- return -1;
+ state->readLen = -1;
+ return false;
}
}
@@ -373,7 +374,8 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
(Size) errinfo.wre_req);
}
- return count;
+ state->readLen = count;
+ return true;
}
/*
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index 21d200d3df..23d9e70a59 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -57,8 +57,8 @@ typedef struct WALSegmentContext
typedef struct XLogReaderState XLogReaderState;
-/* Function type definitions for various xlogreader interactions */
-typedef int (*XLogPageReadCB) (XLogReaderState *xlogreader,
+/* Function type definition for the read_page callback */
+typedef bool (*XLogPageReadCB) (XLogReaderState *xlogreader,
XLogRecPtr targetPagePtr,
int reqLen,
XLogRecPtr targetRecPtr,
@@ -175,6 +175,20 @@ struct XLogReaderState
XLogRecPtr ReadRecPtr; /* start of last record read */
XLogRecPtr EndRecPtr; /* end+1 of last record read */
+ /* ----------------------------------------
+ * Communication with page reader
+ * readBuf is XLOG_BLCKSZ bytes, valid up to at least readLen bytes.
+ * ----------------------------------------
+ */
+ /* variables to communicate with page reader */
+ XLogRecPtr readPagePtr; /* page pointer to read */
+ int32 readLen; /* bytes requested to reader, or actual bytes
+ * read by reader, which must be larger than
+ * the request, or -1 on error */
+ TimeLineID readPageTLI; /* TLI for data currently in readBuf */
+ char *readBuf; /* buffer to store data */
+ bool page_verified; /* is the page on the buffer verified? */
+
/* ----------------------------------------
* Decoded representation of current record
@@ -203,13 +217,6 @@ struct XLogReaderState
* ----------------------------------------
*/
- /*
- * Buffer for currently read page (XLOG_BLCKSZ bytes, valid up to at least
- * readLen bytes)
- */
- char *readBuf;
- uint32 readLen;
-
/* last read XLOG position for data currently in readBuf */
WALSegmentContext segcxt;
WALOpenSegment seg;
diff --git a/src/include/access/xlogutils.h b/src/include/access/xlogutils.h
index 9ac602b674..364a21c4ea 100644
--- a/src/include/access/xlogutils.h
+++ b/src/include/access/xlogutils.h
@@ -47,7 +47,7 @@ extern Buffer XLogReadBufferExtended(RelFileNode rnode, ForkNumber forknum,
extern Relation CreateFakeRelcacheEntry(RelFileNode rnode);
extern void FreeFakeRelcacheEntry(Relation fakerel);
-extern int read_local_xlog_page(XLogReaderState *state,
+extern bool read_local_xlog_page(XLogReaderState *state,
XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *cur_page);
extern void wal_segment_open(XLogReaderState *state,
--
2.27.0
----Next_Part(Thu_Mar__4_11_28_53_2021_014)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
filename="v17-0002-Move-page-reader-out-of-XLogReadRecord.patch"
^ permalink raw reply [nested|flat] 69+ messages in thread
* [PATCH v15 1/4] Move callback-call from ReadPageInternal to XLogReadRecord.
@ 2019-09-05 11:21 Kyotaro Horiguchi <[email protected]>
0 siblings, 0 replies; 69+ messages in thread
From: Kyotaro Horiguchi @ 2019-09-05 11:21 UTC (permalink / raw)
The current WAL record reader reads page data using a call back
function. Although it is not so problematic alone, it would be a
problem if we are going to do add tasks like encryption which is
performed on page data before WAL reader reads them. To avoid that the
record reader facility has to have a new code path corresponds to
every new callback, this patch separates page reader from WAL record
reading facility by modifying the current WAL record reader to a state
machine.
As the first step of that change, this patch moves the page reader
function out of ReadPageInternal, then the remaining tasks of the
function are taken over by the new function XLogNeedData. As the
result XLogPageRead directly calls the page reader callback function
according to the feedback from XLogNeedData.
---
src/backend/access/transam/xlog.c | 16 +-
src/backend/access/transam/xlogreader.c | 281 ++++++++++++++----------
src/backend/access/transam/xlogutils.c | 12 +-
src/backend/replication/walsender.c | 10 +-
src/bin/pg_rewind/parsexlog.c | 21 +-
src/bin/pg_waldump/pg_waldump.c | 8 +-
src/include/access/xlogreader.h | 25 ++-
src/include/access/xlogutils.h | 2 +-
8 files changed, 222 insertions(+), 153 deletions(-)
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index fd93bcfaeb..e570e56a24 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -912,7 +912,7 @@ static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath,
static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli,
XLogSource source, bool notfoundOk);
static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, XLogSource source);
-static int XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
+static bool XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
bool fetching_ckpt, XLogRecPtr tliRecPtr);
@@ -4332,7 +4332,6 @@ ReadRecord(XLogReaderState *xlogreader, int emode,
XLogRecord *record;
XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
- /* Pass through parameters to XLogPageRead */
private->fetching_ckpt = fetching_ckpt;
private->emode = emode;
private->randAccess = (xlogreader->ReadRecPtr == InvalidXLogRecPtr);
@@ -11860,7 +11859,7 @@ CancelBackup(void)
* XLogPageRead() to try fetching the record from another source, or to
* sleep and retry.
*/
-static int
+static bool
XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *readBuf)
{
@@ -11919,7 +11918,8 @@ retry:
readLen = 0;
readSource = XLOG_FROM_ANY;
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
}
@@ -12014,7 +12014,8 @@ retry:
goto next_record_is_invalid;
}
- return readLen;
+ xlogreader->readLen = readLen;
+ return true;
next_record_is_invalid:
lastSourceFailed = true;
@@ -12028,8 +12029,9 @@ next_record_is_invalid:
/* In standby-mode, keep trying */
if (StandbyMode)
goto retry;
- else
- return -1;
+
+ xlogreader->readLen = -1;
+ return false;
}
/*
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index cb76be4f46..3d599325ee 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -36,8 +36,8 @@
static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
pg_attribute_printf(2, 3);
static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
-static int ReadPageInternal(XLogReaderState *state, XLogRecPtr pageptr,
- int reqLen);
+static bool XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr,
+ int reqLen, bool header_inclusive);
static void XLogReaderInvalReadState(XLogReaderState *state);
static bool ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
XLogRecPtr PrevRecPtr, XLogRecord *record, bool randAccess);
@@ -276,7 +276,6 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
uint32 targetRecOff;
uint32 pageHeaderSize;
bool gotheader;
- int readOff;
/*
* randAccess indicates whether to verify the previous-record pointer of
@@ -326,14 +325,20 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
* byte to cover the whole record header, or at least the part of it that
* fits on the same page.
*/
- readOff = ReadPageInternal(state, targetPagePtr,
- Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ));
- if (readOff < 0)
+ while (XLogNeedData(state, targetPagePtr,
+ Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+ targetRecOff != 0))
+ {
+ if (!state->routine.page_read(state, state->readPagePtr, state->readLen,
+ RecPtr, state->readBuf))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
/*
- * ReadPageInternal always returns at least the page header, so we can
- * examine it now.
+ * We have at least the page header, so we can examine it now.
*/
pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
if (targetRecOff == 0)
@@ -359,8 +364,8 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
goto err;
}
- /* ReadPageInternal has verified the page header */
- Assert(pageHeaderSize <= readOff);
+ /* XLogNeedData has verified the page header */
+ Assert(pageHeaderSize <= state->readLen);
/*
* Read the record length.
@@ -433,18 +438,27 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
do
{
+ int rest_len = total_len - gotlen;
+
/* Calculate pointer to beginning of next page */
targetPagePtr += XLOG_BLCKSZ;
/* Wait for the next page to become available */
- readOff = ReadPageInternal(state, targetPagePtr,
- Min(total_len - gotlen + SizeOfXLogShortPHD,
- XLOG_BLCKSZ));
+ while (XLogNeedData(state, targetPagePtr,
+ Min(rest_len, XLOG_BLCKSZ),
+ false))
+ {
+ if (!state->routine.page_read(state, state->readPagePtr,
+ state->readLen,
+ state->ReadRecPtr,
+ state->readBuf))
+ break;
+ }
- if (readOff < 0)
+ if (!state->page_verified)
goto err;
- Assert(SizeOfXLogShortPHD <= readOff);
+ Assert(SizeOfXLogShortPHD <= state->readLen);
/* Check that the continuation on next page looks valid */
pageHeader = (XLogPageHeader) state->readBuf;
@@ -473,21 +487,14 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
/* Append the continuation from this page to the buffer */
pageHeaderSize = XLogPageHeaderSize(pageHeader);
- if (readOff < pageHeaderSize)
- readOff = ReadPageInternal(state, targetPagePtr,
- pageHeaderSize);
-
- Assert(pageHeaderSize <= readOff);
+ Assert(pageHeaderSize <= state->readLen);
contdata = (char *) state->readBuf + pageHeaderSize;
len = XLOG_BLCKSZ - pageHeaderSize;
if (pageHeader->xlp_rem_len < len)
len = pageHeader->xlp_rem_len;
- if (readOff < pageHeaderSize + len)
- readOff = ReadPageInternal(state, targetPagePtr,
- pageHeaderSize + len);
-
+ Assert (pageHeaderSize + len <= state->readLen);
memcpy(buffer, (char *) contdata, len);
buffer += len;
gotlen += len;
@@ -517,9 +524,16 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
else
{
/* Wait for the record data to become available */
- readOff = ReadPageInternal(state, targetPagePtr,
- Min(targetRecOff + total_len, XLOG_BLCKSZ));
- if (readOff < 0)
+ while (XLogNeedData(state, targetPagePtr,
+ Min(targetRecOff + total_len, XLOG_BLCKSZ), true))
+ {
+ if (!state->routine.page_read(state, state->readPagePtr,
+ state->readLen,
+ state->ReadRecPtr, state->readBuf))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
/* Record does not cross a page boundary */
@@ -562,109 +576,138 @@ err:
}
/*
- * Read a single xlog page including at least [pageptr, reqLen] of valid data
- * via the page_read() callback.
+ * Checks that an xlog page loaded in state->readBuf is including at least
+ * [pageptr, reqLen] and the page is valid. header_inclusive indicates that
+ * reqLen is calculated including page header length.
*
- * Returns -1 if the required page cannot be read for some reason; errormsg_buf
- * is set in that case (unless the error occurs in the page_read callback).
+ * Returns false if the buffer already contains the requested data, or found
+ * error. state->page_verified is set to true for the former and false for the
+ * latter.
*
- * We fetch the page from a reader-local cache if we know we have the required
- * data and if there hasn't been any error since caching the data.
+ * Otherwise returns true and requests data loaded onto state->readBuf by
+ * state->readPagePtr and state->readLen. The caller shall call this function
+ * again after filling the buffer at least with that portion of data and set
+ * state->readLen to the length of actually loaded data.
+ *
+ * If header_inclusive is false, corrects reqLen internally by adding the
+ * actual page header length and may request caller for new data.
*/
-static int
-ReadPageInternal(XLogReaderState *state, XLogRecPtr pageptr, int reqLen)
+static bool
+XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
+ bool header_inclusive)
{
- int readLen;
uint32 targetPageOff;
XLogSegNo targetSegNo;
- XLogPageHeader hdr;
+ uint32 addLen = 0;
- Assert((pageptr % XLOG_BLCKSZ) == 0);
+ /* Some data is loaded, but page header is not verified yet. */
+ if (!state->page_verified &&
+ !XLogRecPtrIsInvalid(state->readPagePtr) && state->readLen >= 0)
+ {
+ uint32 pageHeaderSize;
+ /* just loaded new data so needs to verify page header */
+
+ /* The caller must have loaded at least page header */
+ Assert (state->readLen >= SizeOfXLogShortPHD);
+
+ /*
+ * We have enough data to check the header length. Recheck the loaded
+ * length against the actual header length.
+ */
+ pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+
+ /* Request more data if we don't have the full header. */
+ if (state->readLen < pageHeaderSize)
+ {
+ state->readLen = pageHeaderSize;
+ return true;
+ }
+
+ /* Now that we know we have the full header, validate it. */
+ if (!XLogReaderValidatePageHeader(state, state->readPagePtr,
+ (char *) state->readBuf))
+ {
+ /* That's bad. Force reading the page again. */
+ XLogReaderInvalReadState(state);
+
+ return false;
+ }
+
+ state->page_verified = true;
+
+ XLByteToSeg(state->readPagePtr, state->seg.ws_segno,
+ state->segcxt.ws_segsize);
+ }
+
+ /*
+ * The loaded page may not be the one caller is supposing to read when we
+ * are verifying the first page of new segment. In that case, skip further
+ * verification and immediately load the target page.
+ */
+ if (state->page_verified && pageptr == state->readPagePtr)
+ {
+ /*
+ * calculate additional length for page header keeping the total
+ * length within the block size.
+ */
+ if (!header_inclusive)
+ {
+ uint32 pageHeaderSize =
+ XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+
+ addLen = pageHeaderSize;
+ if (reqLen + pageHeaderSize <= XLOG_BLCKSZ)
+ addLen = pageHeaderSize;
+ else
+ addLen = XLOG_BLCKSZ - reqLen;
+
+ Assert(addLen >= 0);
+ }
+
+ /* Return if we already have it. */
+ if (reqLen + addLen <= state->readLen)
+ return false;
+ }
+
+ /* Data is not in our buffer, request the caller for it. */
XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize);
targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize);
+ Assert((pageptr % XLOG_BLCKSZ) == 0);
- /* check whether we have all the requested data already */
- if (targetSegNo == state->seg.ws_segno &&
- targetPageOff == state->segoff && reqLen <= state->readLen)
- return state->readLen;
+ /*
+ * Every time we request to load new data of a page to the caller, even if
+ * we looked at a part of it before, we need to do verification on the next
+ * invocation as the caller might now be rereading data from a different
+ * source.
+ */
+ state->page_verified = false;
/*
- * Data is not in our buffer.
- *
- * Every time we actually read the segment, even if we looked at parts of
- * it before, we need to do verification as the page_read callback might
- * now be rereading data from a different source.
- *
* Whenever switching to a new WAL segment, we read the first page of the
* file and validate its header, even if that's not where the target
* record is. This is so that we can check the additional identification
* info that is present in the first page's "long" header.
+ * Don't do this if the caller requested the first page in the segment.
*/
if (targetSegNo != state->seg.ws_segno && targetPageOff != 0)
{
- XLogRecPtr targetSegmentPtr = pageptr - targetPageOff;
-
- readLen = state->routine.page_read(state, targetSegmentPtr, XLOG_BLCKSZ,
- state->currRecPtr,
- state->readBuf);
- if (readLen < 0)
- goto err;
-
- /* we can be sure to have enough WAL available, we scrolled back */
- Assert(readLen == XLOG_BLCKSZ);
-
- if (!XLogReaderValidatePageHeader(state, targetSegmentPtr,
- state->readBuf))
- goto err;
+ /*
+ * Then we'll see that the targetSegNo now matches the ws_segno, and
+ * will not come back here, but will request the actual target page.
+ */
+ state->readPagePtr = pageptr - targetPageOff;
+ state->readLen = XLOG_BLCKSZ;
+ return true;
}
/*
- * First, read the requested data length, but at least a short page header
- * so that we can validate it.
+ * Request the caller to load the page. We need at least a short page
+ * header so that we can validate it.
*/
- readLen = state->routine.page_read(state, pageptr, Max(reqLen, SizeOfXLogShortPHD),
- state->currRecPtr,
- state->readBuf);
- if (readLen < 0)
- goto err;
-
- Assert(readLen <= XLOG_BLCKSZ);
-
- /* Do we have enough data to check the header length? */
- if (readLen <= SizeOfXLogShortPHD)
- goto err;
-
- Assert(readLen >= reqLen);
-
- hdr = (XLogPageHeader) state->readBuf;
-
- /* still not enough */
- if (readLen < XLogPageHeaderSize(hdr))
- {
- readLen = state->routine.page_read(state, pageptr, XLogPageHeaderSize(hdr),
- state->currRecPtr,
- state->readBuf);
- if (readLen < 0)
- goto err;
- }
-
- /*
- * Now that we know we have the full header, validate it.
- */
- if (!XLogReaderValidatePageHeader(state, pageptr, (char *) hdr))
- goto err;
-
- /* update read state information */
- state->seg.ws_segno = targetSegNo;
- state->segoff = targetPageOff;
- state->readLen = readLen;
-
- return readLen;
-
-err:
- XLogReaderInvalReadState(state);
- return -1;
+ state->readPagePtr = pageptr;
+ state->readLen = Max(reqLen + addLen, SizeOfXLogShortPHD);
+ return true;
}
/*
@@ -673,9 +716,7 @@ err:
static void
XLogReaderInvalReadState(XLogReaderState *state)
{
- state->seg.ws_segno = 0;
- state->segoff = 0;
- state->readLen = 0;
+ state->readPagePtr = InvalidXLogRecPtr;
}
/*
@@ -956,7 +997,6 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
XLogRecPtr targetPagePtr;
int targetRecOff;
uint32 pageHeaderSize;
- int readLen;
/*
* Compute targetRecOff. It should typically be equal or greater than
@@ -964,7 +1004,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
* that, except when caller has explicitly specified the offset that
* falls somewhere there or when we are skipping multi-page
* continuation record. It doesn't matter though because
- * ReadPageInternal() is prepared to handle that and will read at
+ * XLogNeedData() is prepared to handle that and will read at
* least short page-header worth of data
*/
targetRecOff = tmpRecPtr % XLOG_BLCKSZ;
@@ -972,19 +1012,24 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
/* scroll back to page boundary */
targetPagePtr = tmpRecPtr - targetRecOff;
- /* Read the page containing the record */
- readLen = ReadPageInternal(state, targetPagePtr, targetRecOff);
- if (readLen < 0)
+ while(XLogNeedData(state, targetPagePtr, targetRecOff,
+ targetRecOff != 0))
+ {
+ if (!state->routine.page_read(state, state->readPagePtr,
+ state->readLen,
+ state->ReadRecPtr, state->readBuf))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
header = (XLogPageHeader) state->readBuf;
pageHeaderSize = XLogPageHeaderSize(header);
- /* make sure we have enough data for the page header */
- readLen = ReadPageInternal(state, targetPagePtr, pageHeaderSize);
- if (readLen < 0)
- goto err;
+ /* we should have read the page header */
+ Assert (state->readLen >= pageHeaderSize);
/* skip over potential continuation data */
if (header->xlp_info & XLP_FIRST_IS_CONTRECORD)
diff --git a/src/backend/access/transam/xlogutils.c b/src/backend/access/transam/xlogutils.c
index 322b0e8ff5..1b10b6df20 100644
--- a/src/backend/access/transam/xlogutils.c
+++ b/src/backend/access/transam/xlogutils.c
@@ -689,8 +689,8 @@ XLogTruncateRelation(RelFileNode rnode, ForkNumber forkNum,
void
XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wantLength)
{
- const XLogRecPtr lastReadPage = (state->seg.ws_segno *
- state->segcxt.ws_segsize + state->segoff);
+ const XLogRecPtr lastReadPage = state->seg.ws_segno *
+ state->segcxt.ws_segsize + state->readLen;
Assert(wantPage != InvalidXLogRecPtr && wantPage % XLOG_BLCKSZ == 0);
Assert(wantLength <= XLOG_BLCKSZ);
@@ -828,7 +828,7 @@ wal_segment_close(XLogReaderState *state)
* exists for normal backends, so we have to do a check/sleep/repeat style of
* loop for now.
*/
-int
+bool
read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *cur_page)
{
@@ -930,7 +930,8 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
else if (targetPagePtr + reqLen > read_upto)
{
/* not enough data there */
- return -1;
+ state->readLen = -1;
+ return false;
}
else
{
@@ -948,7 +949,8 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
WALReadRaiseError(&errinfo);
/* number of valid bytes in the buffer */
- return count;
+ state->readLen = count;
+ return true;
}
/*
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index e2477c47e0..bf2711bddd 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -811,7 +811,7 @@ StartReplication(StartReplicationCmd *cmd)
* which has to do a plain sleep/busy loop, because the walsender's latch gets
* set every time WAL is flushed.
*/
-static int
+static bool
logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *cur_page)
{
@@ -831,7 +831,10 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
/* fail if not (implies we are going to shut down) */
if (flushptr < targetPagePtr + reqLen)
- return -1;
+ {
+ state->readLen = -1;
+ return false;
+ }
if (targetPagePtr + XLOG_BLCKSZ <= flushptr)
count = XLOG_BLCKSZ; /* more than one block available */
@@ -859,7 +862,8 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
XLByteToSeg(targetPagePtr, segno, state->segcxt.ws_segsize);
CheckXLogRemoved(segno, state->seg.ws_tli);
- return count;
+ state->readLen = count;
+ return true;
}
/*
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index bc6f976994..0b7e73ae79 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -46,7 +46,7 @@ typedef struct XLogPageReadPrivate
int tliIndex;
} XLogPageReadPrivate;
-static int SimpleXLogPageRead(XLogReaderState *xlogreader,
+static bool SimpleXLogPageRead(XLogReaderState *xlogreader,
XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
@@ -236,7 +236,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
}
/* XLogReader callback function, to read a WAL page */
-static int
+static bool
SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf)
{
@@ -296,7 +296,8 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
if (private->restoreCommand == NULL)
{
pg_log_error("could not open file \"%s\": %m", xlogfpath);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
/*
@@ -309,7 +310,10 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
private->restoreCommand);
if (xlogreadfd < 0)
- return -1;
+ {
+ xlogreader->readLen = -1;
+ return false;
+ }
else
pg_log_debug("using file \"%s\" restored from archive",
xlogfpath);
@@ -325,7 +329,8 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
if (lseek(xlogreadfd, (off_t) targetPageOff, SEEK_SET) < 0)
{
pg_log_error("could not seek in file \"%s\": %m", xlogfpath);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
@@ -338,13 +343,15 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
pg_log_error("could not read file \"%s\": read %d of %zu",
xlogfpath, r, (Size) XLOG_BLCKSZ);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
Assert(targetSegNo == xlogreadsegno);
xlogreader->seg.ws_tli = targetHistory[private->tliIndex].tli;
- return XLOG_BLCKSZ;
+ xlogreader->readLen = XLOG_BLCKSZ;
+ return true;
}
/*
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index d1a0678935..ab0c28d9b7 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -331,7 +331,7 @@ WALDumpCloseSegment(XLogReaderState *state)
}
/* pg_waldump's XLogReaderRoutine->page_read callback */
-static int
+static bool
WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetPtr, char *readBuff)
{
@@ -348,7 +348,8 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
else
{
private->endptr_reached = true;
- return -1;
+ state->readLen = -1;
+ return false;
}
}
@@ -373,7 +374,8 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
(Size) errinfo.wre_req);
}
- return count;
+ state->readLen = count;
+ return true;
}
/*
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index b0f2a6ed43..6c8848e14f 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -57,8 +57,8 @@ typedef struct WALSegmentContext
typedef struct XLogReaderState XLogReaderState;
-/* Function type definitions for various xlogreader interactions */
-typedef int (*XLogPageReadCB) (XLogReaderState *xlogreader,
+/* Function type definition for the read_page callback */
+typedef bool (*XLogPageReadCB) (XLogReaderState *xlogreader,
XLogRecPtr targetPagePtr,
int reqLen,
XLogRecPtr targetRecPtr,
@@ -175,6 +175,20 @@ struct XLogReaderState
XLogRecPtr ReadRecPtr; /* start of last record read */
XLogRecPtr EndRecPtr; /* end+1 of last record read */
+ /* ----------------------------------------
+ * Communication with page reader
+ * readBuf is XLOG_BLCKSZ bytes, valid up to at least readLen bytes.
+ * ----------------------------------------
+ */
+ /* variables to communicate with page reader */
+ XLogRecPtr readPagePtr; /* page pointer to read */
+ int32 readLen; /* bytes requested to reader, or actual bytes
+ * read by reader, which must be larger than
+ * the request, or -1 on error */
+ TimeLineID readPageTLI; /* TLI for data currently in readBuf */
+ char *readBuf; /* buffer to store data */
+ bool page_verified; /* is the page on the buffer verified? */
+
/* ----------------------------------------
* Decoded representation of current record
@@ -201,13 +215,6 @@ struct XLogReaderState
* ----------------------------------------
*/
- /*
- * Buffer for currently read page (XLOG_BLCKSZ bytes, valid up to at least
- * readLen bytes)
- */
- char *readBuf;
- uint32 readLen;
-
/* last read XLOG position for data currently in readBuf */
WALSegmentContext segcxt;
WALOpenSegment seg;
diff --git a/src/include/access/xlogutils.h b/src/include/access/xlogutils.h
index e59b6cf3a9..c287edf206 100644
--- a/src/include/access/xlogutils.h
+++ b/src/include/access/xlogutils.h
@@ -47,7 +47,7 @@ extern Buffer XLogReadBufferExtended(RelFileNode rnode, ForkNumber forknum,
extern Relation CreateFakeRelcacheEntry(RelFileNode rnode);
extern void FreeFakeRelcacheEntry(Relation fakerel);
-extern int read_local_xlog_page(XLogReaderState *state,
+extern bool read_local_xlog_page(XLogReaderState *state,
XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *cur_page);
extern void wal_segment_open(XLogReaderState *state,
--
2.18.4
----Next_Part(Thu_Jul__2_13_53_30_2020_072)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
filename="v15-0002-Move-page-reader-out-of-XLogReadRecord.patch"
^ permalink raw reply [nested|flat] 69+ messages in thread
* [PATCH v17 1/4] Move callback-call from ReadPageInternal to XLogReadRecord.
@ 2019-09-05 11:21 Kyotaro Horiguchi <[email protected]>
0 siblings, 0 replies; 69+ messages in thread
From: Kyotaro Horiguchi @ 2019-09-05 11:21 UTC (permalink / raw)
The current WAL record reader reads page data using a call back
function. Although it is not so problematic alone, it would be a
problem if we are going to do add tasks like encryption which is
performed on page data before WAL reader reads them. To avoid that the
record reader facility has to have a new code path corresponds to
every new callback, this patch separates page reader from WAL record
reading facility by modifying the current WAL record reader to a state
machine.
As the first step of that change, this patch moves the page reader
function out of ReadPageInternal, then the remaining tasks of the
function are taken over by the new function XLogNeedData. As the
result XLogPageRead directly calls the page reader callback function
according to the feedback from XLogNeedData.
---
src/backend/access/transam/xlog.c | 16 +-
src/backend/access/transam/xlogreader.c | 281 ++++++++++++++----------
src/backend/access/transam/xlogutils.c | 12 +-
src/backend/replication/walsender.c | 10 +-
src/bin/pg_rewind/parsexlog.c | 21 +-
src/bin/pg_waldump/pg_waldump.c | 8 +-
src/include/access/xlogreader.h | 25 ++-
src/include/access/xlogutils.h | 2 +-
8 files changed, 222 insertions(+), 153 deletions(-)
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 377afb8732..606328a65a 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -917,7 +917,7 @@ static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath,
static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli,
XLogSource source, bool notfoundOk);
static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, XLogSource source);
-static int XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
+static bool XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
bool fetching_ckpt, XLogRecPtr tliRecPtr);
@@ -4350,7 +4350,6 @@ ReadRecord(XLogReaderState *xlogreader, int emode,
XLogRecord *record;
XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
- /* Pass through parameters to XLogPageRead */
private->fetching_ckpt = fetching_ckpt;
private->emode = emode;
private->randAccess = (xlogreader->ReadRecPtr == InvalidXLogRecPtr);
@@ -11987,7 +11986,7 @@ CancelBackup(void)
* XLogPageRead() to try fetching the record from another source, or to
* sleep and retry.
*/
-static int
+static bool
XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *readBuf)
{
@@ -12046,7 +12045,8 @@ retry:
readLen = 0;
readSource = XLOG_FROM_ANY;
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
}
@@ -12141,7 +12141,8 @@ retry:
goto next_record_is_invalid;
}
- return readLen;
+ xlogreader->readLen = readLen;
+ return true;
next_record_is_invalid:
lastSourceFailed = true;
@@ -12155,8 +12156,9 @@ next_record_is_invalid:
/* In standby-mode, keep trying */
if (StandbyMode)
goto retry;
- else
- return -1;
+
+ xlogreader->readLen = -1;
+ return false;
}
/*
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index 42738eb940..7d04995b7b 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -36,8 +36,8 @@
static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
pg_attribute_printf(2, 3);
static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
-static int ReadPageInternal(XLogReaderState *state, XLogRecPtr pageptr,
- int reqLen);
+static bool XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr,
+ int reqLen, bool header_inclusive);
static void XLogReaderInvalReadState(XLogReaderState *state);
static bool ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
XLogRecPtr PrevRecPtr, XLogRecord *record, bool randAccess);
@@ -276,7 +276,6 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
uint32 targetRecOff;
uint32 pageHeaderSize;
bool gotheader;
- int readOff;
/*
* randAccess indicates whether to verify the previous-record pointer of
@@ -326,14 +325,20 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
* byte to cover the whole record header, or at least the part of it that
* fits on the same page.
*/
- readOff = ReadPageInternal(state, targetPagePtr,
- Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ));
- if (readOff < 0)
+ while (XLogNeedData(state, targetPagePtr,
+ Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+ targetRecOff != 0))
+ {
+ if (!state->routine.page_read(state, state->readPagePtr, state->readLen,
+ RecPtr, state->readBuf))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
/*
- * ReadPageInternal always returns at least the page header, so we can
- * examine it now.
+ * We have at least the page header, so we can examine it now.
*/
pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
if (targetRecOff == 0)
@@ -359,8 +364,8 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
goto err;
}
- /* ReadPageInternal has verified the page header */
- Assert(pageHeaderSize <= readOff);
+ /* XLogNeedData has verified the page header */
+ Assert(pageHeaderSize <= state->readLen);
/*
* Read the record length.
@@ -432,18 +437,27 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
do
{
+ int rest_len = total_len - gotlen;
+
/* Calculate pointer to beginning of next page */
targetPagePtr += XLOG_BLCKSZ;
/* Wait for the next page to become available */
- readOff = ReadPageInternal(state, targetPagePtr,
- Min(total_len - gotlen + SizeOfXLogShortPHD,
- XLOG_BLCKSZ));
+ while (XLogNeedData(state, targetPagePtr,
+ Min(rest_len, XLOG_BLCKSZ),
+ false))
+ {
+ if (!state->routine.page_read(state, state->readPagePtr,
+ state->readLen,
+ state->ReadRecPtr,
+ state->readBuf))
+ break;
+ }
- if (readOff < 0)
+ if (!state->page_verified)
goto err;
- Assert(SizeOfXLogShortPHD <= readOff);
+ Assert(SizeOfXLogShortPHD <= state->readLen);
/* Check that the continuation on next page looks valid */
pageHeader = (XLogPageHeader) state->readBuf;
@@ -473,21 +487,14 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
/* Append the continuation from this page to the buffer */
pageHeaderSize = XLogPageHeaderSize(pageHeader);
- if (readOff < pageHeaderSize)
- readOff = ReadPageInternal(state, targetPagePtr,
- pageHeaderSize);
-
- Assert(pageHeaderSize <= readOff);
+ Assert(pageHeaderSize <= state->readLen);
contdata = (char *) state->readBuf + pageHeaderSize;
len = XLOG_BLCKSZ - pageHeaderSize;
if (pageHeader->xlp_rem_len < len)
len = pageHeader->xlp_rem_len;
- if (readOff < pageHeaderSize + len)
- readOff = ReadPageInternal(state, targetPagePtr,
- pageHeaderSize + len);
-
+ Assert (pageHeaderSize + len <= state->readLen);
memcpy(buffer, (char *) contdata, len);
buffer += len;
gotlen += len;
@@ -517,9 +524,16 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
else
{
/* Wait for the record data to become available */
- readOff = ReadPageInternal(state, targetPagePtr,
- Min(targetRecOff + total_len, XLOG_BLCKSZ));
- if (readOff < 0)
+ while (XLogNeedData(state, targetPagePtr,
+ Min(targetRecOff + total_len, XLOG_BLCKSZ), true))
+ {
+ if (!state->routine.page_read(state, state->readPagePtr,
+ state->readLen,
+ state->ReadRecPtr, state->readBuf))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
/* Record does not cross a page boundary */
@@ -562,109 +576,138 @@ err:
}
/*
- * Read a single xlog page including at least [pageptr, reqLen] of valid data
- * via the page_read() callback.
+ * Checks that an xlog page loaded in state->readBuf is including at least
+ * [pageptr, reqLen] and the page is valid. header_inclusive indicates that
+ * reqLen is calculated including page header length.
*
- * Returns -1 if the required page cannot be read for some reason; errormsg_buf
- * is set in that case (unless the error occurs in the page_read callback).
+ * Returns false if the buffer already contains the requested data, or found
+ * error. state->page_verified is set to true for the former and false for the
+ * latter.
*
- * We fetch the page from a reader-local cache if we know we have the required
- * data and if there hasn't been any error since caching the data.
+ * Otherwise returns true and requests data loaded onto state->readBuf by
+ * state->readPagePtr and state->readLen. The caller shall call this function
+ * again after filling the buffer at least with that portion of data and set
+ * state->readLen to the length of actually loaded data.
+ *
+ * If header_inclusive is false, corrects reqLen internally by adding the
+ * actual page header length and may request caller for new data.
*/
-static int
-ReadPageInternal(XLogReaderState *state, XLogRecPtr pageptr, int reqLen)
+static bool
+XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
+ bool header_inclusive)
{
- int readLen;
uint32 targetPageOff;
XLogSegNo targetSegNo;
- XLogPageHeader hdr;
+ uint32 addLen = 0;
- Assert((pageptr % XLOG_BLCKSZ) == 0);
+ /* Some data is loaded, but page header is not verified yet. */
+ if (!state->page_verified &&
+ !XLogRecPtrIsInvalid(state->readPagePtr) && state->readLen >= 0)
+ {
+ uint32 pageHeaderSize;
+ /* just loaded new data so needs to verify page header */
+
+ /* The caller must have loaded at least page header */
+ Assert (state->readLen >= SizeOfXLogShortPHD);
+
+ /*
+ * We have enough data to check the header length. Recheck the loaded
+ * length against the actual header length.
+ */
+ pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+
+ /* Request more data if we don't have the full header. */
+ if (state->readLen < pageHeaderSize)
+ {
+ state->readLen = pageHeaderSize;
+ return true;
+ }
+
+ /* Now that we know we have the full header, validate it. */
+ if (!XLogReaderValidatePageHeader(state, state->readPagePtr,
+ (char *) state->readBuf))
+ {
+ /* That's bad. Force reading the page again. */
+ XLogReaderInvalReadState(state);
+
+ return false;
+ }
+
+ state->page_verified = true;
+
+ XLByteToSeg(state->readPagePtr, state->seg.ws_segno,
+ state->segcxt.ws_segsize);
+ }
+
+ /*
+ * The loaded page may not be the one caller is supposing to read when we
+ * are verifying the first page of new segment. In that case, skip further
+ * verification and immediately load the target page.
+ */
+ if (state->page_verified && pageptr == state->readPagePtr)
+ {
+ /*
+ * calculate additional length for page header keeping the total
+ * length within the block size.
+ */
+ if (!header_inclusive)
+ {
+ uint32 pageHeaderSize =
+ XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+
+ addLen = pageHeaderSize;
+ if (reqLen + pageHeaderSize <= XLOG_BLCKSZ)
+ addLen = pageHeaderSize;
+ else
+ addLen = XLOG_BLCKSZ - reqLen;
+
+ Assert(addLen >= 0);
+ }
+
+ /* Return if we already have it. */
+ if (reqLen + addLen <= state->readLen)
+ return false;
+ }
+
+ /* Data is not in our buffer, request the caller for it. */
XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize);
targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize);
+ Assert((pageptr % XLOG_BLCKSZ) == 0);
- /* check whether we have all the requested data already */
- if (targetSegNo == state->seg.ws_segno &&
- targetPageOff == state->segoff && reqLen <= state->readLen)
- return state->readLen;
+ /*
+ * Every time we request to load new data of a page to the caller, even if
+ * we looked at a part of it before, we need to do verification on the next
+ * invocation as the caller might now be rereading data from a different
+ * source.
+ */
+ state->page_verified = false;
/*
- * Data is not in our buffer.
- *
- * Every time we actually read the segment, even if we looked at parts of
- * it before, we need to do verification as the page_read callback might
- * now be rereading data from a different source.
- *
* Whenever switching to a new WAL segment, we read the first page of the
* file and validate its header, even if that's not where the target
* record is. This is so that we can check the additional identification
* info that is present in the first page's "long" header.
+ * Don't do this if the caller requested the first page in the segment.
*/
if (targetSegNo != state->seg.ws_segno && targetPageOff != 0)
{
- XLogRecPtr targetSegmentPtr = pageptr - targetPageOff;
-
- readLen = state->routine.page_read(state, targetSegmentPtr, XLOG_BLCKSZ,
- state->currRecPtr,
- state->readBuf);
- if (readLen < 0)
- goto err;
-
- /* we can be sure to have enough WAL available, we scrolled back */
- Assert(readLen == XLOG_BLCKSZ);
-
- if (!XLogReaderValidatePageHeader(state, targetSegmentPtr,
- state->readBuf))
- goto err;
+ /*
+ * Then we'll see that the targetSegNo now matches the ws_segno, and
+ * will not come back here, but will request the actual target page.
+ */
+ state->readPagePtr = pageptr - targetPageOff;
+ state->readLen = XLOG_BLCKSZ;
+ return true;
}
/*
- * First, read the requested data length, but at least a short page header
- * so that we can validate it.
+ * Request the caller to load the page. We need at least a short page
+ * header so that we can validate it.
*/
- readLen = state->routine.page_read(state, pageptr, Max(reqLen, SizeOfXLogShortPHD),
- state->currRecPtr,
- state->readBuf);
- if (readLen < 0)
- goto err;
-
- Assert(readLen <= XLOG_BLCKSZ);
-
- /* Do we have enough data to check the header length? */
- if (readLen <= SizeOfXLogShortPHD)
- goto err;
-
- Assert(readLen >= reqLen);
-
- hdr = (XLogPageHeader) state->readBuf;
-
- /* still not enough */
- if (readLen < XLogPageHeaderSize(hdr))
- {
- readLen = state->routine.page_read(state, pageptr, XLogPageHeaderSize(hdr),
- state->currRecPtr,
- state->readBuf);
- if (readLen < 0)
- goto err;
- }
-
- /*
- * Now that we know we have the full header, validate it.
- */
- if (!XLogReaderValidatePageHeader(state, pageptr, (char *) hdr))
- goto err;
-
- /* update read state information */
- state->seg.ws_segno = targetSegNo;
- state->segoff = targetPageOff;
- state->readLen = readLen;
-
- return readLen;
-
-err:
- XLogReaderInvalReadState(state);
- return -1;
+ state->readPagePtr = pageptr;
+ state->readLen = Max(reqLen + addLen, SizeOfXLogShortPHD);
+ return true;
}
/*
@@ -673,9 +716,7 @@ err:
static void
XLogReaderInvalReadState(XLogReaderState *state)
{
- state->seg.ws_segno = 0;
- state->segoff = 0;
- state->readLen = 0;
+ state->readPagePtr = InvalidXLogRecPtr;
}
/*
@@ -953,7 +994,6 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
XLogRecPtr targetPagePtr;
int targetRecOff;
uint32 pageHeaderSize;
- int readLen;
/*
* Compute targetRecOff. It should typically be equal or greater than
@@ -961,7 +1001,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
* that, except when caller has explicitly specified the offset that
* falls somewhere there or when we are skipping multi-page
* continuation record. It doesn't matter though because
- * ReadPageInternal() is prepared to handle that and will read at
+ * XLogNeedData() is prepared to handle that and will read at
* least short page-header worth of data
*/
targetRecOff = tmpRecPtr % XLOG_BLCKSZ;
@@ -969,19 +1009,24 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
/* scroll back to page boundary */
targetPagePtr = tmpRecPtr - targetRecOff;
- /* Read the page containing the record */
- readLen = ReadPageInternal(state, targetPagePtr, targetRecOff);
- if (readLen < 0)
+ while(XLogNeedData(state, targetPagePtr, targetRecOff,
+ targetRecOff != 0))
+ {
+ if (!state->routine.page_read(state, state->readPagePtr,
+ state->readLen,
+ state->ReadRecPtr, state->readBuf))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
header = (XLogPageHeader) state->readBuf;
pageHeaderSize = XLogPageHeaderSize(header);
- /* make sure we have enough data for the page header */
- readLen = ReadPageInternal(state, targetPagePtr, pageHeaderSize);
- if (readLen < 0)
- goto err;
+ /* we should have read the page header */
+ Assert (state->readLen >= pageHeaderSize);
/* skip over potential continuation data */
if (header->xlp_info & XLP_FIRST_IS_CONTRECORD)
diff --git a/src/backend/access/transam/xlogutils.c b/src/backend/access/transam/xlogutils.c
index d17d660f46..46eda33f25 100644
--- a/src/backend/access/transam/xlogutils.c
+++ b/src/backend/access/transam/xlogutils.c
@@ -686,8 +686,8 @@ XLogTruncateRelation(RelFileNode rnode, ForkNumber forkNum,
void
XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wantLength)
{
- const XLogRecPtr lastReadPage = (state->seg.ws_segno *
- state->segcxt.ws_segsize + state->segoff);
+ const XLogRecPtr lastReadPage = state->seg.ws_segno *
+ state->segcxt.ws_segsize + state->readLen;
Assert(wantPage != InvalidXLogRecPtr && wantPage % XLOG_BLCKSZ == 0);
Assert(wantLength <= XLOG_BLCKSZ);
@@ -824,7 +824,7 @@ wal_segment_close(XLogReaderState *state)
* exists for normal backends, so we have to do a check/sleep/repeat style of
* loop for now.
*/
-int
+bool
read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *cur_page)
{
@@ -926,7 +926,8 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
else if (targetPagePtr + reqLen > read_upto)
{
/* not enough data there */
- return -1;
+ state->readLen = -1;
+ return false;
}
else
{
@@ -944,7 +945,8 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
WALReadRaiseError(&errinfo);
/* number of valid bytes in the buffer */
- return count;
+ state->readLen = count;
+ return true;
}
/*
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index 23baa4498a..c36b91ae5e 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -806,7 +806,7 @@ StartReplication(StartReplicationCmd *cmd)
* which has to do a plain sleep/busy loop, because the walsender's latch gets
* set every time WAL is flushed.
*/
-static int
+static bool
logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *cur_page)
{
@@ -826,7 +826,10 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
/* fail if not (implies we are going to shut down) */
if (flushptr < targetPagePtr + reqLen)
- return -1;
+ {
+ state->readLen = -1;
+ return false;
+ }
if (targetPagePtr + XLOG_BLCKSZ <= flushptr)
count = XLOG_BLCKSZ; /* more than one block available */
@@ -854,7 +857,8 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
XLByteToSeg(targetPagePtr, segno, state->segcxt.ws_segsize);
CheckXLogRemoved(segno, state->seg.ws_tli);
- return count;
+ state->readLen = count;
+ return true;
}
/*
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index 59ebac7d6a..cf119848b0 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -47,7 +47,7 @@ typedef struct XLogPageReadPrivate
int tliIndex;
} XLogPageReadPrivate;
-static int SimpleXLogPageRead(XLogReaderState *xlogreader,
+static bool SimpleXLogPageRead(XLogReaderState *xlogreader,
XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
@@ -246,7 +246,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
}
/* XLogReader callback function, to read a WAL page */
-static int
+static bool
SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf)
{
@@ -306,7 +306,8 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
if (private->restoreCommand == NULL)
{
pg_log_error("could not open file \"%s\": %m", xlogfpath);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
/*
@@ -319,7 +320,10 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
private->restoreCommand);
if (xlogreadfd < 0)
- return -1;
+ {
+ xlogreader->readLen = -1;
+ return false;
+ }
else
pg_log_debug("using file \"%s\" restored from archive",
xlogfpath);
@@ -335,7 +339,8 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
if (lseek(xlogreadfd, (off_t) targetPageOff, SEEK_SET) < 0)
{
pg_log_error("could not seek in file \"%s\": %m", xlogfpath);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
@@ -348,13 +353,15 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
pg_log_error("could not read file \"%s\": read %d of %zu",
xlogfpath, r, (Size) XLOG_BLCKSZ);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
Assert(targetSegNo == xlogreadsegno);
xlogreader->seg.ws_tli = targetHistory[private->tliIndex].tli;
- return XLOG_BLCKSZ;
+ xlogreader->readLen = XLOG_BLCKSZ;
+ return true;
}
/*
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index 610f65e471..0f0e0723a3 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -331,7 +331,7 @@ WALDumpCloseSegment(XLogReaderState *state)
}
/* pg_waldump's XLogReaderRoutine->page_read callback */
-static int
+static bool
WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetPtr, char *readBuff)
{
@@ -348,7 +348,8 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
else
{
private->endptr_reached = true;
- return -1;
+ state->readLen = -1;
+ return false;
}
}
@@ -373,7 +374,8 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
(Size) errinfo.wre_req);
}
- return count;
+ state->readLen = count;
+ return true;
}
/*
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index 21d200d3df..23d9e70a59 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -57,8 +57,8 @@ typedef struct WALSegmentContext
typedef struct XLogReaderState XLogReaderState;
-/* Function type definitions for various xlogreader interactions */
-typedef int (*XLogPageReadCB) (XLogReaderState *xlogreader,
+/* Function type definition for the read_page callback */
+typedef bool (*XLogPageReadCB) (XLogReaderState *xlogreader,
XLogRecPtr targetPagePtr,
int reqLen,
XLogRecPtr targetRecPtr,
@@ -175,6 +175,20 @@ struct XLogReaderState
XLogRecPtr ReadRecPtr; /* start of last record read */
XLogRecPtr EndRecPtr; /* end+1 of last record read */
+ /* ----------------------------------------
+ * Communication with page reader
+ * readBuf is XLOG_BLCKSZ bytes, valid up to at least readLen bytes.
+ * ----------------------------------------
+ */
+ /* variables to communicate with page reader */
+ XLogRecPtr readPagePtr; /* page pointer to read */
+ int32 readLen; /* bytes requested to reader, or actual bytes
+ * read by reader, which must be larger than
+ * the request, or -1 on error */
+ TimeLineID readPageTLI; /* TLI for data currently in readBuf */
+ char *readBuf; /* buffer to store data */
+ bool page_verified; /* is the page on the buffer verified? */
+
/* ----------------------------------------
* Decoded representation of current record
@@ -203,13 +217,6 @@ struct XLogReaderState
* ----------------------------------------
*/
- /*
- * Buffer for currently read page (XLOG_BLCKSZ bytes, valid up to at least
- * readLen bytes)
- */
- char *readBuf;
- uint32 readLen;
-
/* last read XLOG position for data currently in readBuf */
WALSegmentContext segcxt;
WALOpenSegment seg;
diff --git a/src/include/access/xlogutils.h b/src/include/access/xlogutils.h
index 9ac602b674..364a21c4ea 100644
--- a/src/include/access/xlogutils.h
+++ b/src/include/access/xlogutils.h
@@ -47,7 +47,7 @@ extern Buffer XLogReadBufferExtended(RelFileNode rnode, ForkNumber forknum,
extern Relation CreateFakeRelcacheEntry(RelFileNode rnode);
extern void FreeFakeRelcacheEntry(Relation fakerel);
-extern int read_local_xlog_page(XLogReaderState *state,
+extern bool read_local_xlog_page(XLogReaderState *state,
XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *cur_page);
extern void wal_segment_open(XLogReaderState *state,
--
2.27.0
----Next_Part(Thu_Mar__4_11_28_53_2021_014)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
filename="v17-0002-Move-page-reader-out-of-XLogReadRecord.patch"
^ permalink raw reply [nested|flat] 69+ messages in thread
* [PATCH v15 1/4] Move callback-call from ReadPageInternal to XLogReadRecord.
@ 2019-09-05 11:21 Kyotaro Horiguchi <[email protected]>
0 siblings, 0 replies; 69+ messages in thread
From: Kyotaro Horiguchi @ 2019-09-05 11:21 UTC (permalink / raw)
The current WAL record reader reads page data using a call back
function. Although it is not so problematic alone, it would be a
problem if we are going to do add tasks like encryption which is
performed on page data before WAL reader reads them. To avoid that the
record reader facility has to have a new code path corresponds to
every new callback, this patch separates page reader from WAL record
reading facility by modifying the current WAL record reader to a state
machine.
As the first step of that change, this patch moves the page reader
function out of ReadPageInternal, then the remaining tasks of the
function are taken over by the new function XLogNeedData. As the
result XLogPageRead directly calls the page reader callback function
according to the feedback from XLogNeedData.
---
src/backend/access/transam/xlog.c | 16 +-
src/backend/access/transam/xlogreader.c | 281 ++++++++++++++----------
src/backend/access/transam/xlogutils.c | 12 +-
src/backend/replication/walsender.c | 10 +-
src/bin/pg_rewind/parsexlog.c | 21 +-
src/bin/pg_waldump/pg_waldump.c | 8 +-
src/include/access/xlogreader.h | 25 ++-
src/include/access/xlogutils.h | 2 +-
8 files changed, 222 insertions(+), 153 deletions(-)
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index fd93bcfaeb..e570e56a24 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -912,7 +912,7 @@ static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath,
static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli,
XLogSource source, bool notfoundOk);
static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, XLogSource source);
-static int XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
+static bool XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
bool fetching_ckpt, XLogRecPtr tliRecPtr);
@@ -4332,7 +4332,6 @@ ReadRecord(XLogReaderState *xlogreader, int emode,
XLogRecord *record;
XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
- /* Pass through parameters to XLogPageRead */
private->fetching_ckpt = fetching_ckpt;
private->emode = emode;
private->randAccess = (xlogreader->ReadRecPtr == InvalidXLogRecPtr);
@@ -11860,7 +11859,7 @@ CancelBackup(void)
* XLogPageRead() to try fetching the record from another source, or to
* sleep and retry.
*/
-static int
+static bool
XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *readBuf)
{
@@ -11919,7 +11918,8 @@ retry:
readLen = 0;
readSource = XLOG_FROM_ANY;
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
}
@@ -12014,7 +12014,8 @@ retry:
goto next_record_is_invalid;
}
- return readLen;
+ xlogreader->readLen = readLen;
+ return true;
next_record_is_invalid:
lastSourceFailed = true;
@@ -12028,8 +12029,9 @@ next_record_is_invalid:
/* In standby-mode, keep trying */
if (StandbyMode)
goto retry;
- else
- return -1;
+
+ xlogreader->readLen = -1;
+ return false;
}
/*
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index cb76be4f46..3d599325ee 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -36,8 +36,8 @@
static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
pg_attribute_printf(2, 3);
static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
-static int ReadPageInternal(XLogReaderState *state, XLogRecPtr pageptr,
- int reqLen);
+static bool XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr,
+ int reqLen, bool header_inclusive);
static void XLogReaderInvalReadState(XLogReaderState *state);
static bool ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
XLogRecPtr PrevRecPtr, XLogRecord *record, bool randAccess);
@@ -276,7 +276,6 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
uint32 targetRecOff;
uint32 pageHeaderSize;
bool gotheader;
- int readOff;
/*
* randAccess indicates whether to verify the previous-record pointer of
@@ -326,14 +325,20 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
* byte to cover the whole record header, or at least the part of it that
* fits on the same page.
*/
- readOff = ReadPageInternal(state, targetPagePtr,
- Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ));
- if (readOff < 0)
+ while (XLogNeedData(state, targetPagePtr,
+ Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+ targetRecOff != 0))
+ {
+ if (!state->routine.page_read(state, state->readPagePtr, state->readLen,
+ RecPtr, state->readBuf))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
/*
- * ReadPageInternal always returns at least the page header, so we can
- * examine it now.
+ * We have at least the page header, so we can examine it now.
*/
pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
if (targetRecOff == 0)
@@ -359,8 +364,8 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
goto err;
}
- /* ReadPageInternal has verified the page header */
- Assert(pageHeaderSize <= readOff);
+ /* XLogNeedData has verified the page header */
+ Assert(pageHeaderSize <= state->readLen);
/*
* Read the record length.
@@ -433,18 +438,27 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
do
{
+ int rest_len = total_len - gotlen;
+
/* Calculate pointer to beginning of next page */
targetPagePtr += XLOG_BLCKSZ;
/* Wait for the next page to become available */
- readOff = ReadPageInternal(state, targetPagePtr,
- Min(total_len - gotlen + SizeOfXLogShortPHD,
- XLOG_BLCKSZ));
+ while (XLogNeedData(state, targetPagePtr,
+ Min(rest_len, XLOG_BLCKSZ),
+ false))
+ {
+ if (!state->routine.page_read(state, state->readPagePtr,
+ state->readLen,
+ state->ReadRecPtr,
+ state->readBuf))
+ break;
+ }
- if (readOff < 0)
+ if (!state->page_verified)
goto err;
- Assert(SizeOfXLogShortPHD <= readOff);
+ Assert(SizeOfXLogShortPHD <= state->readLen);
/* Check that the continuation on next page looks valid */
pageHeader = (XLogPageHeader) state->readBuf;
@@ -473,21 +487,14 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
/* Append the continuation from this page to the buffer */
pageHeaderSize = XLogPageHeaderSize(pageHeader);
- if (readOff < pageHeaderSize)
- readOff = ReadPageInternal(state, targetPagePtr,
- pageHeaderSize);
-
- Assert(pageHeaderSize <= readOff);
+ Assert(pageHeaderSize <= state->readLen);
contdata = (char *) state->readBuf + pageHeaderSize;
len = XLOG_BLCKSZ - pageHeaderSize;
if (pageHeader->xlp_rem_len < len)
len = pageHeader->xlp_rem_len;
- if (readOff < pageHeaderSize + len)
- readOff = ReadPageInternal(state, targetPagePtr,
- pageHeaderSize + len);
-
+ Assert (pageHeaderSize + len <= state->readLen);
memcpy(buffer, (char *) contdata, len);
buffer += len;
gotlen += len;
@@ -517,9 +524,16 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
else
{
/* Wait for the record data to become available */
- readOff = ReadPageInternal(state, targetPagePtr,
- Min(targetRecOff + total_len, XLOG_BLCKSZ));
- if (readOff < 0)
+ while (XLogNeedData(state, targetPagePtr,
+ Min(targetRecOff + total_len, XLOG_BLCKSZ), true))
+ {
+ if (!state->routine.page_read(state, state->readPagePtr,
+ state->readLen,
+ state->ReadRecPtr, state->readBuf))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
/* Record does not cross a page boundary */
@@ -562,109 +576,138 @@ err:
}
/*
- * Read a single xlog page including at least [pageptr, reqLen] of valid data
- * via the page_read() callback.
+ * Checks that an xlog page loaded in state->readBuf is including at least
+ * [pageptr, reqLen] and the page is valid. header_inclusive indicates that
+ * reqLen is calculated including page header length.
*
- * Returns -1 if the required page cannot be read for some reason; errormsg_buf
- * is set in that case (unless the error occurs in the page_read callback).
+ * Returns false if the buffer already contains the requested data, or found
+ * error. state->page_verified is set to true for the former and false for the
+ * latter.
*
- * We fetch the page from a reader-local cache if we know we have the required
- * data and if there hasn't been any error since caching the data.
+ * Otherwise returns true and requests data loaded onto state->readBuf by
+ * state->readPagePtr and state->readLen. The caller shall call this function
+ * again after filling the buffer at least with that portion of data and set
+ * state->readLen to the length of actually loaded data.
+ *
+ * If header_inclusive is false, corrects reqLen internally by adding the
+ * actual page header length and may request caller for new data.
*/
-static int
-ReadPageInternal(XLogReaderState *state, XLogRecPtr pageptr, int reqLen)
+static bool
+XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
+ bool header_inclusive)
{
- int readLen;
uint32 targetPageOff;
XLogSegNo targetSegNo;
- XLogPageHeader hdr;
+ uint32 addLen = 0;
- Assert((pageptr % XLOG_BLCKSZ) == 0);
+ /* Some data is loaded, but page header is not verified yet. */
+ if (!state->page_verified &&
+ !XLogRecPtrIsInvalid(state->readPagePtr) && state->readLen >= 0)
+ {
+ uint32 pageHeaderSize;
+ /* just loaded new data so needs to verify page header */
+
+ /* The caller must have loaded at least page header */
+ Assert (state->readLen >= SizeOfXLogShortPHD);
+
+ /*
+ * We have enough data to check the header length. Recheck the loaded
+ * length against the actual header length.
+ */
+ pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+
+ /* Request more data if we don't have the full header. */
+ if (state->readLen < pageHeaderSize)
+ {
+ state->readLen = pageHeaderSize;
+ return true;
+ }
+
+ /* Now that we know we have the full header, validate it. */
+ if (!XLogReaderValidatePageHeader(state, state->readPagePtr,
+ (char *) state->readBuf))
+ {
+ /* That's bad. Force reading the page again. */
+ XLogReaderInvalReadState(state);
+
+ return false;
+ }
+
+ state->page_verified = true;
+
+ XLByteToSeg(state->readPagePtr, state->seg.ws_segno,
+ state->segcxt.ws_segsize);
+ }
+
+ /*
+ * The loaded page may not be the one caller is supposing to read when we
+ * are verifying the first page of new segment. In that case, skip further
+ * verification and immediately load the target page.
+ */
+ if (state->page_verified && pageptr == state->readPagePtr)
+ {
+ /*
+ * calculate additional length for page header keeping the total
+ * length within the block size.
+ */
+ if (!header_inclusive)
+ {
+ uint32 pageHeaderSize =
+ XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+
+ addLen = pageHeaderSize;
+ if (reqLen + pageHeaderSize <= XLOG_BLCKSZ)
+ addLen = pageHeaderSize;
+ else
+ addLen = XLOG_BLCKSZ - reqLen;
+
+ Assert(addLen >= 0);
+ }
+
+ /* Return if we already have it. */
+ if (reqLen + addLen <= state->readLen)
+ return false;
+ }
+
+ /* Data is not in our buffer, request the caller for it. */
XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize);
targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize);
+ Assert((pageptr % XLOG_BLCKSZ) == 0);
- /* check whether we have all the requested data already */
- if (targetSegNo == state->seg.ws_segno &&
- targetPageOff == state->segoff && reqLen <= state->readLen)
- return state->readLen;
+ /*
+ * Every time we request to load new data of a page to the caller, even if
+ * we looked at a part of it before, we need to do verification on the next
+ * invocation as the caller might now be rereading data from a different
+ * source.
+ */
+ state->page_verified = false;
/*
- * Data is not in our buffer.
- *
- * Every time we actually read the segment, even if we looked at parts of
- * it before, we need to do verification as the page_read callback might
- * now be rereading data from a different source.
- *
* Whenever switching to a new WAL segment, we read the first page of the
* file and validate its header, even if that's not where the target
* record is. This is so that we can check the additional identification
* info that is present in the first page's "long" header.
+ * Don't do this if the caller requested the first page in the segment.
*/
if (targetSegNo != state->seg.ws_segno && targetPageOff != 0)
{
- XLogRecPtr targetSegmentPtr = pageptr - targetPageOff;
-
- readLen = state->routine.page_read(state, targetSegmentPtr, XLOG_BLCKSZ,
- state->currRecPtr,
- state->readBuf);
- if (readLen < 0)
- goto err;
-
- /* we can be sure to have enough WAL available, we scrolled back */
- Assert(readLen == XLOG_BLCKSZ);
-
- if (!XLogReaderValidatePageHeader(state, targetSegmentPtr,
- state->readBuf))
- goto err;
+ /*
+ * Then we'll see that the targetSegNo now matches the ws_segno, and
+ * will not come back here, but will request the actual target page.
+ */
+ state->readPagePtr = pageptr - targetPageOff;
+ state->readLen = XLOG_BLCKSZ;
+ return true;
}
/*
- * First, read the requested data length, but at least a short page header
- * so that we can validate it.
+ * Request the caller to load the page. We need at least a short page
+ * header so that we can validate it.
*/
- readLen = state->routine.page_read(state, pageptr, Max(reqLen, SizeOfXLogShortPHD),
- state->currRecPtr,
- state->readBuf);
- if (readLen < 0)
- goto err;
-
- Assert(readLen <= XLOG_BLCKSZ);
-
- /* Do we have enough data to check the header length? */
- if (readLen <= SizeOfXLogShortPHD)
- goto err;
-
- Assert(readLen >= reqLen);
-
- hdr = (XLogPageHeader) state->readBuf;
-
- /* still not enough */
- if (readLen < XLogPageHeaderSize(hdr))
- {
- readLen = state->routine.page_read(state, pageptr, XLogPageHeaderSize(hdr),
- state->currRecPtr,
- state->readBuf);
- if (readLen < 0)
- goto err;
- }
-
- /*
- * Now that we know we have the full header, validate it.
- */
- if (!XLogReaderValidatePageHeader(state, pageptr, (char *) hdr))
- goto err;
-
- /* update read state information */
- state->seg.ws_segno = targetSegNo;
- state->segoff = targetPageOff;
- state->readLen = readLen;
-
- return readLen;
-
-err:
- XLogReaderInvalReadState(state);
- return -1;
+ state->readPagePtr = pageptr;
+ state->readLen = Max(reqLen + addLen, SizeOfXLogShortPHD);
+ return true;
}
/*
@@ -673,9 +716,7 @@ err:
static void
XLogReaderInvalReadState(XLogReaderState *state)
{
- state->seg.ws_segno = 0;
- state->segoff = 0;
- state->readLen = 0;
+ state->readPagePtr = InvalidXLogRecPtr;
}
/*
@@ -956,7 +997,6 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
XLogRecPtr targetPagePtr;
int targetRecOff;
uint32 pageHeaderSize;
- int readLen;
/*
* Compute targetRecOff. It should typically be equal or greater than
@@ -964,7 +1004,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
* that, except when caller has explicitly specified the offset that
* falls somewhere there or when we are skipping multi-page
* continuation record. It doesn't matter though because
- * ReadPageInternal() is prepared to handle that and will read at
+ * XLogNeedData() is prepared to handle that and will read at
* least short page-header worth of data
*/
targetRecOff = tmpRecPtr % XLOG_BLCKSZ;
@@ -972,19 +1012,24 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
/* scroll back to page boundary */
targetPagePtr = tmpRecPtr - targetRecOff;
- /* Read the page containing the record */
- readLen = ReadPageInternal(state, targetPagePtr, targetRecOff);
- if (readLen < 0)
+ while(XLogNeedData(state, targetPagePtr, targetRecOff,
+ targetRecOff != 0))
+ {
+ if (!state->routine.page_read(state, state->readPagePtr,
+ state->readLen,
+ state->ReadRecPtr, state->readBuf))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
header = (XLogPageHeader) state->readBuf;
pageHeaderSize = XLogPageHeaderSize(header);
- /* make sure we have enough data for the page header */
- readLen = ReadPageInternal(state, targetPagePtr, pageHeaderSize);
- if (readLen < 0)
- goto err;
+ /* we should have read the page header */
+ Assert (state->readLen >= pageHeaderSize);
/* skip over potential continuation data */
if (header->xlp_info & XLP_FIRST_IS_CONTRECORD)
diff --git a/src/backend/access/transam/xlogutils.c b/src/backend/access/transam/xlogutils.c
index 322b0e8ff5..1b10b6df20 100644
--- a/src/backend/access/transam/xlogutils.c
+++ b/src/backend/access/transam/xlogutils.c
@@ -689,8 +689,8 @@ XLogTruncateRelation(RelFileNode rnode, ForkNumber forkNum,
void
XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wantLength)
{
- const XLogRecPtr lastReadPage = (state->seg.ws_segno *
- state->segcxt.ws_segsize + state->segoff);
+ const XLogRecPtr lastReadPage = state->seg.ws_segno *
+ state->segcxt.ws_segsize + state->readLen;
Assert(wantPage != InvalidXLogRecPtr && wantPage % XLOG_BLCKSZ == 0);
Assert(wantLength <= XLOG_BLCKSZ);
@@ -828,7 +828,7 @@ wal_segment_close(XLogReaderState *state)
* exists for normal backends, so we have to do a check/sleep/repeat style of
* loop for now.
*/
-int
+bool
read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *cur_page)
{
@@ -930,7 +930,8 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
else if (targetPagePtr + reqLen > read_upto)
{
/* not enough data there */
- return -1;
+ state->readLen = -1;
+ return false;
}
else
{
@@ -948,7 +949,8 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
WALReadRaiseError(&errinfo);
/* number of valid bytes in the buffer */
- return count;
+ state->readLen = count;
+ return true;
}
/*
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index e2477c47e0..bf2711bddd 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -811,7 +811,7 @@ StartReplication(StartReplicationCmd *cmd)
* which has to do a plain sleep/busy loop, because the walsender's latch gets
* set every time WAL is flushed.
*/
-static int
+static bool
logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *cur_page)
{
@@ -831,7 +831,10 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
/* fail if not (implies we are going to shut down) */
if (flushptr < targetPagePtr + reqLen)
- return -1;
+ {
+ state->readLen = -1;
+ return false;
+ }
if (targetPagePtr + XLOG_BLCKSZ <= flushptr)
count = XLOG_BLCKSZ; /* more than one block available */
@@ -859,7 +862,8 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
XLByteToSeg(targetPagePtr, segno, state->segcxt.ws_segsize);
CheckXLogRemoved(segno, state->seg.ws_tli);
- return count;
+ state->readLen = count;
+ return true;
}
/*
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index bc6f976994..0b7e73ae79 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -46,7 +46,7 @@ typedef struct XLogPageReadPrivate
int tliIndex;
} XLogPageReadPrivate;
-static int SimpleXLogPageRead(XLogReaderState *xlogreader,
+static bool SimpleXLogPageRead(XLogReaderState *xlogreader,
XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
@@ -236,7 +236,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
}
/* XLogReader callback function, to read a WAL page */
-static int
+static bool
SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf)
{
@@ -296,7 +296,8 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
if (private->restoreCommand == NULL)
{
pg_log_error("could not open file \"%s\": %m", xlogfpath);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
/*
@@ -309,7 +310,10 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
private->restoreCommand);
if (xlogreadfd < 0)
- return -1;
+ {
+ xlogreader->readLen = -1;
+ return false;
+ }
else
pg_log_debug("using file \"%s\" restored from archive",
xlogfpath);
@@ -325,7 +329,8 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
if (lseek(xlogreadfd, (off_t) targetPageOff, SEEK_SET) < 0)
{
pg_log_error("could not seek in file \"%s\": %m", xlogfpath);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
@@ -338,13 +343,15 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
pg_log_error("could not read file \"%s\": read %d of %zu",
xlogfpath, r, (Size) XLOG_BLCKSZ);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
Assert(targetSegNo == xlogreadsegno);
xlogreader->seg.ws_tli = targetHistory[private->tliIndex].tli;
- return XLOG_BLCKSZ;
+ xlogreader->readLen = XLOG_BLCKSZ;
+ return true;
}
/*
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index d1a0678935..ab0c28d9b7 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -331,7 +331,7 @@ WALDumpCloseSegment(XLogReaderState *state)
}
/* pg_waldump's XLogReaderRoutine->page_read callback */
-static int
+static bool
WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetPtr, char *readBuff)
{
@@ -348,7 +348,8 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
else
{
private->endptr_reached = true;
- return -1;
+ state->readLen = -1;
+ return false;
}
}
@@ -373,7 +374,8 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
(Size) errinfo.wre_req);
}
- return count;
+ state->readLen = count;
+ return true;
}
/*
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index b0f2a6ed43..6c8848e14f 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -57,8 +57,8 @@ typedef struct WALSegmentContext
typedef struct XLogReaderState XLogReaderState;
-/* Function type definitions for various xlogreader interactions */
-typedef int (*XLogPageReadCB) (XLogReaderState *xlogreader,
+/* Function type definition for the read_page callback */
+typedef bool (*XLogPageReadCB) (XLogReaderState *xlogreader,
XLogRecPtr targetPagePtr,
int reqLen,
XLogRecPtr targetRecPtr,
@@ -175,6 +175,20 @@ struct XLogReaderState
XLogRecPtr ReadRecPtr; /* start of last record read */
XLogRecPtr EndRecPtr; /* end+1 of last record read */
+ /* ----------------------------------------
+ * Communication with page reader
+ * readBuf is XLOG_BLCKSZ bytes, valid up to at least readLen bytes.
+ * ----------------------------------------
+ */
+ /* variables to communicate with page reader */
+ XLogRecPtr readPagePtr; /* page pointer to read */
+ int32 readLen; /* bytes requested to reader, or actual bytes
+ * read by reader, which must be larger than
+ * the request, or -1 on error */
+ TimeLineID readPageTLI; /* TLI for data currently in readBuf */
+ char *readBuf; /* buffer to store data */
+ bool page_verified; /* is the page on the buffer verified? */
+
/* ----------------------------------------
* Decoded representation of current record
@@ -201,13 +215,6 @@ struct XLogReaderState
* ----------------------------------------
*/
- /*
- * Buffer for currently read page (XLOG_BLCKSZ bytes, valid up to at least
- * readLen bytes)
- */
- char *readBuf;
- uint32 readLen;
-
/* last read XLOG position for data currently in readBuf */
WALSegmentContext segcxt;
WALOpenSegment seg;
diff --git a/src/include/access/xlogutils.h b/src/include/access/xlogutils.h
index e59b6cf3a9..c287edf206 100644
--- a/src/include/access/xlogutils.h
+++ b/src/include/access/xlogutils.h
@@ -47,7 +47,7 @@ extern Buffer XLogReadBufferExtended(RelFileNode rnode, ForkNumber forknum,
extern Relation CreateFakeRelcacheEntry(RelFileNode rnode);
extern void FreeFakeRelcacheEntry(Relation fakerel);
-extern int read_local_xlog_page(XLogReaderState *state,
+extern bool read_local_xlog_page(XLogReaderState *state,
XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *cur_page);
extern void wal_segment_open(XLogReaderState *state,
--
2.18.4
----Next_Part(Thu_Jul__2_13_53_30_2020_072)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
filename="v15-0002-Move-page-reader-out-of-XLogReadRecord.patch"
^ permalink raw reply [nested|flat] 69+ messages in thread
* [PATCH v17 1/4] Move callback-call from ReadPageInternal to XLogReadRecord.
@ 2019-09-05 11:21 Kyotaro Horiguchi <[email protected]>
0 siblings, 0 replies; 69+ messages in thread
From: Kyotaro Horiguchi @ 2019-09-05 11:21 UTC (permalink / raw)
The current WAL record reader reads page data using a call back
function. Although it is not so problematic alone, it would be a
problem if we are going to do add tasks like encryption which is
performed on page data before WAL reader reads them. To avoid that the
record reader facility has to have a new code path corresponds to
every new callback, this patch separates page reader from WAL record
reading facility by modifying the current WAL record reader to a state
machine.
As the first step of that change, this patch moves the page reader
function out of ReadPageInternal, then the remaining tasks of the
function are taken over by the new function XLogNeedData. As the
result XLogPageRead directly calls the page reader callback function
according to the feedback from XLogNeedData.
---
src/backend/access/transam/xlog.c | 16 +-
src/backend/access/transam/xlogreader.c | 281 ++++++++++++++----------
src/backend/access/transam/xlogutils.c | 12 +-
src/backend/replication/walsender.c | 10 +-
src/bin/pg_rewind/parsexlog.c | 21 +-
src/bin/pg_waldump/pg_waldump.c | 8 +-
src/include/access/xlogreader.h | 25 ++-
src/include/access/xlogutils.h | 2 +-
8 files changed, 222 insertions(+), 153 deletions(-)
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 377afb8732..606328a65a 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -917,7 +917,7 @@ static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath,
static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli,
XLogSource source, bool notfoundOk);
static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, XLogSource source);
-static int XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
+static bool XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
bool fetching_ckpt, XLogRecPtr tliRecPtr);
@@ -4350,7 +4350,6 @@ ReadRecord(XLogReaderState *xlogreader, int emode,
XLogRecord *record;
XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
- /* Pass through parameters to XLogPageRead */
private->fetching_ckpt = fetching_ckpt;
private->emode = emode;
private->randAccess = (xlogreader->ReadRecPtr == InvalidXLogRecPtr);
@@ -11987,7 +11986,7 @@ CancelBackup(void)
* XLogPageRead() to try fetching the record from another source, or to
* sleep and retry.
*/
-static int
+static bool
XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *readBuf)
{
@@ -12046,7 +12045,8 @@ retry:
readLen = 0;
readSource = XLOG_FROM_ANY;
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
}
@@ -12141,7 +12141,8 @@ retry:
goto next_record_is_invalid;
}
- return readLen;
+ xlogreader->readLen = readLen;
+ return true;
next_record_is_invalid:
lastSourceFailed = true;
@@ -12155,8 +12156,9 @@ next_record_is_invalid:
/* In standby-mode, keep trying */
if (StandbyMode)
goto retry;
- else
- return -1;
+
+ xlogreader->readLen = -1;
+ return false;
}
/*
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index 42738eb940..7d04995b7b 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -36,8 +36,8 @@
static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
pg_attribute_printf(2, 3);
static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
-static int ReadPageInternal(XLogReaderState *state, XLogRecPtr pageptr,
- int reqLen);
+static bool XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr,
+ int reqLen, bool header_inclusive);
static void XLogReaderInvalReadState(XLogReaderState *state);
static bool ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
XLogRecPtr PrevRecPtr, XLogRecord *record, bool randAccess);
@@ -276,7 +276,6 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
uint32 targetRecOff;
uint32 pageHeaderSize;
bool gotheader;
- int readOff;
/*
* randAccess indicates whether to verify the previous-record pointer of
@@ -326,14 +325,20 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
* byte to cover the whole record header, or at least the part of it that
* fits on the same page.
*/
- readOff = ReadPageInternal(state, targetPagePtr,
- Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ));
- if (readOff < 0)
+ while (XLogNeedData(state, targetPagePtr,
+ Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+ targetRecOff != 0))
+ {
+ if (!state->routine.page_read(state, state->readPagePtr, state->readLen,
+ RecPtr, state->readBuf))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
/*
- * ReadPageInternal always returns at least the page header, so we can
- * examine it now.
+ * We have at least the page header, so we can examine it now.
*/
pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
if (targetRecOff == 0)
@@ -359,8 +364,8 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
goto err;
}
- /* ReadPageInternal has verified the page header */
- Assert(pageHeaderSize <= readOff);
+ /* XLogNeedData has verified the page header */
+ Assert(pageHeaderSize <= state->readLen);
/*
* Read the record length.
@@ -432,18 +437,27 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
do
{
+ int rest_len = total_len - gotlen;
+
/* Calculate pointer to beginning of next page */
targetPagePtr += XLOG_BLCKSZ;
/* Wait for the next page to become available */
- readOff = ReadPageInternal(state, targetPagePtr,
- Min(total_len - gotlen + SizeOfXLogShortPHD,
- XLOG_BLCKSZ));
+ while (XLogNeedData(state, targetPagePtr,
+ Min(rest_len, XLOG_BLCKSZ),
+ false))
+ {
+ if (!state->routine.page_read(state, state->readPagePtr,
+ state->readLen,
+ state->ReadRecPtr,
+ state->readBuf))
+ break;
+ }
- if (readOff < 0)
+ if (!state->page_verified)
goto err;
- Assert(SizeOfXLogShortPHD <= readOff);
+ Assert(SizeOfXLogShortPHD <= state->readLen);
/* Check that the continuation on next page looks valid */
pageHeader = (XLogPageHeader) state->readBuf;
@@ -473,21 +487,14 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
/* Append the continuation from this page to the buffer */
pageHeaderSize = XLogPageHeaderSize(pageHeader);
- if (readOff < pageHeaderSize)
- readOff = ReadPageInternal(state, targetPagePtr,
- pageHeaderSize);
-
- Assert(pageHeaderSize <= readOff);
+ Assert(pageHeaderSize <= state->readLen);
contdata = (char *) state->readBuf + pageHeaderSize;
len = XLOG_BLCKSZ - pageHeaderSize;
if (pageHeader->xlp_rem_len < len)
len = pageHeader->xlp_rem_len;
- if (readOff < pageHeaderSize + len)
- readOff = ReadPageInternal(state, targetPagePtr,
- pageHeaderSize + len);
-
+ Assert (pageHeaderSize + len <= state->readLen);
memcpy(buffer, (char *) contdata, len);
buffer += len;
gotlen += len;
@@ -517,9 +524,16 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
else
{
/* Wait for the record data to become available */
- readOff = ReadPageInternal(state, targetPagePtr,
- Min(targetRecOff + total_len, XLOG_BLCKSZ));
- if (readOff < 0)
+ while (XLogNeedData(state, targetPagePtr,
+ Min(targetRecOff + total_len, XLOG_BLCKSZ), true))
+ {
+ if (!state->routine.page_read(state, state->readPagePtr,
+ state->readLen,
+ state->ReadRecPtr, state->readBuf))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
/* Record does not cross a page boundary */
@@ -562,109 +576,138 @@ err:
}
/*
- * Read a single xlog page including at least [pageptr, reqLen] of valid data
- * via the page_read() callback.
+ * Checks that an xlog page loaded in state->readBuf is including at least
+ * [pageptr, reqLen] and the page is valid. header_inclusive indicates that
+ * reqLen is calculated including page header length.
*
- * Returns -1 if the required page cannot be read for some reason; errormsg_buf
- * is set in that case (unless the error occurs in the page_read callback).
+ * Returns false if the buffer already contains the requested data, or found
+ * error. state->page_verified is set to true for the former and false for the
+ * latter.
*
- * We fetch the page from a reader-local cache if we know we have the required
- * data and if there hasn't been any error since caching the data.
+ * Otherwise returns true and requests data loaded onto state->readBuf by
+ * state->readPagePtr and state->readLen. The caller shall call this function
+ * again after filling the buffer at least with that portion of data and set
+ * state->readLen to the length of actually loaded data.
+ *
+ * If header_inclusive is false, corrects reqLen internally by adding the
+ * actual page header length and may request caller for new data.
*/
-static int
-ReadPageInternal(XLogReaderState *state, XLogRecPtr pageptr, int reqLen)
+static bool
+XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
+ bool header_inclusive)
{
- int readLen;
uint32 targetPageOff;
XLogSegNo targetSegNo;
- XLogPageHeader hdr;
+ uint32 addLen = 0;
- Assert((pageptr % XLOG_BLCKSZ) == 0);
+ /* Some data is loaded, but page header is not verified yet. */
+ if (!state->page_verified &&
+ !XLogRecPtrIsInvalid(state->readPagePtr) && state->readLen >= 0)
+ {
+ uint32 pageHeaderSize;
+ /* just loaded new data so needs to verify page header */
+
+ /* The caller must have loaded at least page header */
+ Assert (state->readLen >= SizeOfXLogShortPHD);
+
+ /*
+ * We have enough data to check the header length. Recheck the loaded
+ * length against the actual header length.
+ */
+ pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+
+ /* Request more data if we don't have the full header. */
+ if (state->readLen < pageHeaderSize)
+ {
+ state->readLen = pageHeaderSize;
+ return true;
+ }
+
+ /* Now that we know we have the full header, validate it. */
+ if (!XLogReaderValidatePageHeader(state, state->readPagePtr,
+ (char *) state->readBuf))
+ {
+ /* That's bad. Force reading the page again. */
+ XLogReaderInvalReadState(state);
+
+ return false;
+ }
+
+ state->page_verified = true;
+
+ XLByteToSeg(state->readPagePtr, state->seg.ws_segno,
+ state->segcxt.ws_segsize);
+ }
+
+ /*
+ * The loaded page may not be the one caller is supposing to read when we
+ * are verifying the first page of new segment. In that case, skip further
+ * verification and immediately load the target page.
+ */
+ if (state->page_verified && pageptr == state->readPagePtr)
+ {
+ /*
+ * calculate additional length for page header keeping the total
+ * length within the block size.
+ */
+ if (!header_inclusive)
+ {
+ uint32 pageHeaderSize =
+ XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+
+ addLen = pageHeaderSize;
+ if (reqLen + pageHeaderSize <= XLOG_BLCKSZ)
+ addLen = pageHeaderSize;
+ else
+ addLen = XLOG_BLCKSZ - reqLen;
+
+ Assert(addLen >= 0);
+ }
+
+ /* Return if we already have it. */
+ if (reqLen + addLen <= state->readLen)
+ return false;
+ }
+
+ /* Data is not in our buffer, request the caller for it. */
XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize);
targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize);
+ Assert((pageptr % XLOG_BLCKSZ) == 0);
- /* check whether we have all the requested data already */
- if (targetSegNo == state->seg.ws_segno &&
- targetPageOff == state->segoff && reqLen <= state->readLen)
- return state->readLen;
+ /*
+ * Every time we request to load new data of a page to the caller, even if
+ * we looked at a part of it before, we need to do verification on the next
+ * invocation as the caller might now be rereading data from a different
+ * source.
+ */
+ state->page_verified = false;
/*
- * Data is not in our buffer.
- *
- * Every time we actually read the segment, even if we looked at parts of
- * it before, we need to do verification as the page_read callback might
- * now be rereading data from a different source.
- *
* Whenever switching to a new WAL segment, we read the first page of the
* file and validate its header, even if that's not where the target
* record is. This is so that we can check the additional identification
* info that is present in the first page's "long" header.
+ * Don't do this if the caller requested the first page in the segment.
*/
if (targetSegNo != state->seg.ws_segno && targetPageOff != 0)
{
- XLogRecPtr targetSegmentPtr = pageptr - targetPageOff;
-
- readLen = state->routine.page_read(state, targetSegmentPtr, XLOG_BLCKSZ,
- state->currRecPtr,
- state->readBuf);
- if (readLen < 0)
- goto err;
-
- /* we can be sure to have enough WAL available, we scrolled back */
- Assert(readLen == XLOG_BLCKSZ);
-
- if (!XLogReaderValidatePageHeader(state, targetSegmentPtr,
- state->readBuf))
- goto err;
+ /*
+ * Then we'll see that the targetSegNo now matches the ws_segno, and
+ * will not come back here, but will request the actual target page.
+ */
+ state->readPagePtr = pageptr - targetPageOff;
+ state->readLen = XLOG_BLCKSZ;
+ return true;
}
/*
- * First, read the requested data length, but at least a short page header
- * so that we can validate it.
+ * Request the caller to load the page. We need at least a short page
+ * header so that we can validate it.
*/
- readLen = state->routine.page_read(state, pageptr, Max(reqLen, SizeOfXLogShortPHD),
- state->currRecPtr,
- state->readBuf);
- if (readLen < 0)
- goto err;
-
- Assert(readLen <= XLOG_BLCKSZ);
-
- /* Do we have enough data to check the header length? */
- if (readLen <= SizeOfXLogShortPHD)
- goto err;
-
- Assert(readLen >= reqLen);
-
- hdr = (XLogPageHeader) state->readBuf;
-
- /* still not enough */
- if (readLen < XLogPageHeaderSize(hdr))
- {
- readLen = state->routine.page_read(state, pageptr, XLogPageHeaderSize(hdr),
- state->currRecPtr,
- state->readBuf);
- if (readLen < 0)
- goto err;
- }
-
- /*
- * Now that we know we have the full header, validate it.
- */
- if (!XLogReaderValidatePageHeader(state, pageptr, (char *) hdr))
- goto err;
-
- /* update read state information */
- state->seg.ws_segno = targetSegNo;
- state->segoff = targetPageOff;
- state->readLen = readLen;
-
- return readLen;
-
-err:
- XLogReaderInvalReadState(state);
- return -1;
+ state->readPagePtr = pageptr;
+ state->readLen = Max(reqLen + addLen, SizeOfXLogShortPHD);
+ return true;
}
/*
@@ -673,9 +716,7 @@ err:
static void
XLogReaderInvalReadState(XLogReaderState *state)
{
- state->seg.ws_segno = 0;
- state->segoff = 0;
- state->readLen = 0;
+ state->readPagePtr = InvalidXLogRecPtr;
}
/*
@@ -953,7 +994,6 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
XLogRecPtr targetPagePtr;
int targetRecOff;
uint32 pageHeaderSize;
- int readLen;
/*
* Compute targetRecOff. It should typically be equal or greater than
@@ -961,7 +1001,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
* that, except when caller has explicitly specified the offset that
* falls somewhere there or when we are skipping multi-page
* continuation record. It doesn't matter though because
- * ReadPageInternal() is prepared to handle that and will read at
+ * XLogNeedData() is prepared to handle that and will read at
* least short page-header worth of data
*/
targetRecOff = tmpRecPtr % XLOG_BLCKSZ;
@@ -969,19 +1009,24 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
/* scroll back to page boundary */
targetPagePtr = tmpRecPtr - targetRecOff;
- /* Read the page containing the record */
- readLen = ReadPageInternal(state, targetPagePtr, targetRecOff);
- if (readLen < 0)
+ while(XLogNeedData(state, targetPagePtr, targetRecOff,
+ targetRecOff != 0))
+ {
+ if (!state->routine.page_read(state, state->readPagePtr,
+ state->readLen,
+ state->ReadRecPtr, state->readBuf))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
header = (XLogPageHeader) state->readBuf;
pageHeaderSize = XLogPageHeaderSize(header);
- /* make sure we have enough data for the page header */
- readLen = ReadPageInternal(state, targetPagePtr, pageHeaderSize);
- if (readLen < 0)
- goto err;
+ /* we should have read the page header */
+ Assert (state->readLen >= pageHeaderSize);
/* skip over potential continuation data */
if (header->xlp_info & XLP_FIRST_IS_CONTRECORD)
diff --git a/src/backend/access/transam/xlogutils.c b/src/backend/access/transam/xlogutils.c
index d17d660f46..46eda33f25 100644
--- a/src/backend/access/transam/xlogutils.c
+++ b/src/backend/access/transam/xlogutils.c
@@ -686,8 +686,8 @@ XLogTruncateRelation(RelFileNode rnode, ForkNumber forkNum,
void
XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wantLength)
{
- const XLogRecPtr lastReadPage = (state->seg.ws_segno *
- state->segcxt.ws_segsize + state->segoff);
+ const XLogRecPtr lastReadPage = state->seg.ws_segno *
+ state->segcxt.ws_segsize + state->readLen;
Assert(wantPage != InvalidXLogRecPtr && wantPage % XLOG_BLCKSZ == 0);
Assert(wantLength <= XLOG_BLCKSZ);
@@ -824,7 +824,7 @@ wal_segment_close(XLogReaderState *state)
* exists for normal backends, so we have to do a check/sleep/repeat style of
* loop for now.
*/
-int
+bool
read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *cur_page)
{
@@ -926,7 +926,8 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
else if (targetPagePtr + reqLen > read_upto)
{
/* not enough data there */
- return -1;
+ state->readLen = -1;
+ return false;
}
else
{
@@ -944,7 +945,8 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
WALReadRaiseError(&errinfo);
/* number of valid bytes in the buffer */
- return count;
+ state->readLen = count;
+ return true;
}
/*
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index 23baa4498a..c36b91ae5e 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -806,7 +806,7 @@ StartReplication(StartReplicationCmd *cmd)
* which has to do a plain sleep/busy loop, because the walsender's latch gets
* set every time WAL is flushed.
*/
-static int
+static bool
logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *cur_page)
{
@@ -826,7 +826,10 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
/* fail if not (implies we are going to shut down) */
if (flushptr < targetPagePtr + reqLen)
- return -1;
+ {
+ state->readLen = -1;
+ return false;
+ }
if (targetPagePtr + XLOG_BLCKSZ <= flushptr)
count = XLOG_BLCKSZ; /* more than one block available */
@@ -854,7 +857,8 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
XLByteToSeg(targetPagePtr, segno, state->segcxt.ws_segsize);
CheckXLogRemoved(segno, state->seg.ws_tli);
- return count;
+ state->readLen = count;
+ return true;
}
/*
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index 59ebac7d6a..cf119848b0 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -47,7 +47,7 @@ typedef struct XLogPageReadPrivate
int tliIndex;
} XLogPageReadPrivate;
-static int SimpleXLogPageRead(XLogReaderState *xlogreader,
+static bool SimpleXLogPageRead(XLogReaderState *xlogreader,
XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
@@ -246,7 +246,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
}
/* XLogReader callback function, to read a WAL page */
-static int
+static bool
SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf)
{
@@ -306,7 +306,8 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
if (private->restoreCommand == NULL)
{
pg_log_error("could not open file \"%s\": %m", xlogfpath);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
/*
@@ -319,7 +320,10 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
private->restoreCommand);
if (xlogreadfd < 0)
- return -1;
+ {
+ xlogreader->readLen = -1;
+ return false;
+ }
else
pg_log_debug("using file \"%s\" restored from archive",
xlogfpath);
@@ -335,7 +339,8 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
if (lseek(xlogreadfd, (off_t) targetPageOff, SEEK_SET) < 0)
{
pg_log_error("could not seek in file \"%s\": %m", xlogfpath);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
@@ -348,13 +353,15 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
pg_log_error("could not read file \"%s\": read %d of %zu",
xlogfpath, r, (Size) XLOG_BLCKSZ);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
Assert(targetSegNo == xlogreadsegno);
xlogreader->seg.ws_tli = targetHistory[private->tliIndex].tli;
- return XLOG_BLCKSZ;
+ xlogreader->readLen = XLOG_BLCKSZ;
+ return true;
}
/*
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index 610f65e471..0f0e0723a3 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -331,7 +331,7 @@ WALDumpCloseSegment(XLogReaderState *state)
}
/* pg_waldump's XLogReaderRoutine->page_read callback */
-static int
+static bool
WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetPtr, char *readBuff)
{
@@ -348,7 +348,8 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
else
{
private->endptr_reached = true;
- return -1;
+ state->readLen = -1;
+ return false;
}
}
@@ -373,7 +374,8 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
(Size) errinfo.wre_req);
}
- return count;
+ state->readLen = count;
+ return true;
}
/*
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index 21d200d3df..23d9e70a59 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -57,8 +57,8 @@ typedef struct WALSegmentContext
typedef struct XLogReaderState XLogReaderState;
-/* Function type definitions for various xlogreader interactions */
-typedef int (*XLogPageReadCB) (XLogReaderState *xlogreader,
+/* Function type definition for the read_page callback */
+typedef bool (*XLogPageReadCB) (XLogReaderState *xlogreader,
XLogRecPtr targetPagePtr,
int reqLen,
XLogRecPtr targetRecPtr,
@@ -175,6 +175,20 @@ struct XLogReaderState
XLogRecPtr ReadRecPtr; /* start of last record read */
XLogRecPtr EndRecPtr; /* end+1 of last record read */
+ /* ----------------------------------------
+ * Communication with page reader
+ * readBuf is XLOG_BLCKSZ bytes, valid up to at least readLen bytes.
+ * ----------------------------------------
+ */
+ /* variables to communicate with page reader */
+ XLogRecPtr readPagePtr; /* page pointer to read */
+ int32 readLen; /* bytes requested to reader, or actual bytes
+ * read by reader, which must be larger than
+ * the request, or -1 on error */
+ TimeLineID readPageTLI; /* TLI for data currently in readBuf */
+ char *readBuf; /* buffer to store data */
+ bool page_verified; /* is the page on the buffer verified? */
+
/* ----------------------------------------
* Decoded representation of current record
@@ -203,13 +217,6 @@ struct XLogReaderState
* ----------------------------------------
*/
- /*
- * Buffer for currently read page (XLOG_BLCKSZ bytes, valid up to at least
- * readLen bytes)
- */
- char *readBuf;
- uint32 readLen;
-
/* last read XLOG position for data currently in readBuf */
WALSegmentContext segcxt;
WALOpenSegment seg;
diff --git a/src/include/access/xlogutils.h b/src/include/access/xlogutils.h
index 9ac602b674..364a21c4ea 100644
--- a/src/include/access/xlogutils.h
+++ b/src/include/access/xlogutils.h
@@ -47,7 +47,7 @@ extern Buffer XLogReadBufferExtended(RelFileNode rnode, ForkNumber forknum,
extern Relation CreateFakeRelcacheEntry(RelFileNode rnode);
extern void FreeFakeRelcacheEntry(Relation fakerel);
-extern int read_local_xlog_page(XLogReaderState *state,
+extern bool read_local_xlog_page(XLogReaderState *state,
XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *cur_page);
extern void wal_segment_open(XLogReaderState *state,
--
2.27.0
----Next_Part(Thu_Mar__4_11_28_53_2021_014)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
filename="v17-0002-Move-page-reader-out-of-XLogReadRecord.patch"
^ permalink raw reply [nested|flat] 69+ messages in thread
* [PATCH v15 1/4] Move callback-call from ReadPageInternal to XLogReadRecord.
@ 2019-09-05 11:21 Kyotaro Horiguchi <[email protected]>
0 siblings, 0 replies; 69+ messages in thread
From: Kyotaro Horiguchi @ 2019-09-05 11:21 UTC (permalink / raw)
The current WAL record reader reads page data using a call back
function. Although it is not so problematic alone, it would be a
problem if we are going to do add tasks like encryption which is
performed on page data before WAL reader reads them. To avoid that the
record reader facility has to have a new code path corresponds to
every new callback, this patch separates page reader from WAL record
reading facility by modifying the current WAL record reader to a state
machine.
As the first step of that change, this patch moves the page reader
function out of ReadPageInternal, then the remaining tasks of the
function are taken over by the new function XLogNeedData. As the
result XLogPageRead directly calls the page reader callback function
according to the feedback from XLogNeedData.
---
src/backend/access/transam/xlog.c | 16 +-
src/backend/access/transam/xlogreader.c | 281 ++++++++++++++----------
src/backend/access/transam/xlogutils.c | 12 +-
src/backend/replication/walsender.c | 10 +-
src/bin/pg_rewind/parsexlog.c | 21 +-
src/bin/pg_waldump/pg_waldump.c | 8 +-
src/include/access/xlogreader.h | 25 ++-
src/include/access/xlogutils.h | 2 +-
8 files changed, 222 insertions(+), 153 deletions(-)
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index fd93bcfaeb..e570e56a24 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -912,7 +912,7 @@ static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath,
static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli,
XLogSource source, bool notfoundOk);
static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, XLogSource source);
-static int XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
+static bool XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
bool fetching_ckpt, XLogRecPtr tliRecPtr);
@@ -4332,7 +4332,6 @@ ReadRecord(XLogReaderState *xlogreader, int emode,
XLogRecord *record;
XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
- /* Pass through parameters to XLogPageRead */
private->fetching_ckpt = fetching_ckpt;
private->emode = emode;
private->randAccess = (xlogreader->ReadRecPtr == InvalidXLogRecPtr);
@@ -11860,7 +11859,7 @@ CancelBackup(void)
* XLogPageRead() to try fetching the record from another source, or to
* sleep and retry.
*/
-static int
+static bool
XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *readBuf)
{
@@ -11919,7 +11918,8 @@ retry:
readLen = 0;
readSource = XLOG_FROM_ANY;
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
}
@@ -12014,7 +12014,8 @@ retry:
goto next_record_is_invalid;
}
- return readLen;
+ xlogreader->readLen = readLen;
+ return true;
next_record_is_invalid:
lastSourceFailed = true;
@@ -12028,8 +12029,9 @@ next_record_is_invalid:
/* In standby-mode, keep trying */
if (StandbyMode)
goto retry;
- else
- return -1;
+
+ xlogreader->readLen = -1;
+ return false;
}
/*
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index cb76be4f46..3d599325ee 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -36,8 +36,8 @@
static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
pg_attribute_printf(2, 3);
static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
-static int ReadPageInternal(XLogReaderState *state, XLogRecPtr pageptr,
- int reqLen);
+static bool XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr,
+ int reqLen, bool header_inclusive);
static void XLogReaderInvalReadState(XLogReaderState *state);
static bool ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
XLogRecPtr PrevRecPtr, XLogRecord *record, bool randAccess);
@@ -276,7 +276,6 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
uint32 targetRecOff;
uint32 pageHeaderSize;
bool gotheader;
- int readOff;
/*
* randAccess indicates whether to verify the previous-record pointer of
@@ -326,14 +325,20 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
* byte to cover the whole record header, or at least the part of it that
* fits on the same page.
*/
- readOff = ReadPageInternal(state, targetPagePtr,
- Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ));
- if (readOff < 0)
+ while (XLogNeedData(state, targetPagePtr,
+ Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+ targetRecOff != 0))
+ {
+ if (!state->routine.page_read(state, state->readPagePtr, state->readLen,
+ RecPtr, state->readBuf))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
/*
- * ReadPageInternal always returns at least the page header, so we can
- * examine it now.
+ * We have at least the page header, so we can examine it now.
*/
pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
if (targetRecOff == 0)
@@ -359,8 +364,8 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
goto err;
}
- /* ReadPageInternal has verified the page header */
- Assert(pageHeaderSize <= readOff);
+ /* XLogNeedData has verified the page header */
+ Assert(pageHeaderSize <= state->readLen);
/*
* Read the record length.
@@ -433,18 +438,27 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
do
{
+ int rest_len = total_len - gotlen;
+
/* Calculate pointer to beginning of next page */
targetPagePtr += XLOG_BLCKSZ;
/* Wait for the next page to become available */
- readOff = ReadPageInternal(state, targetPagePtr,
- Min(total_len - gotlen + SizeOfXLogShortPHD,
- XLOG_BLCKSZ));
+ while (XLogNeedData(state, targetPagePtr,
+ Min(rest_len, XLOG_BLCKSZ),
+ false))
+ {
+ if (!state->routine.page_read(state, state->readPagePtr,
+ state->readLen,
+ state->ReadRecPtr,
+ state->readBuf))
+ break;
+ }
- if (readOff < 0)
+ if (!state->page_verified)
goto err;
- Assert(SizeOfXLogShortPHD <= readOff);
+ Assert(SizeOfXLogShortPHD <= state->readLen);
/* Check that the continuation on next page looks valid */
pageHeader = (XLogPageHeader) state->readBuf;
@@ -473,21 +487,14 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
/* Append the continuation from this page to the buffer */
pageHeaderSize = XLogPageHeaderSize(pageHeader);
- if (readOff < pageHeaderSize)
- readOff = ReadPageInternal(state, targetPagePtr,
- pageHeaderSize);
-
- Assert(pageHeaderSize <= readOff);
+ Assert(pageHeaderSize <= state->readLen);
contdata = (char *) state->readBuf + pageHeaderSize;
len = XLOG_BLCKSZ - pageHeaderSize;
if (pageHeader->xlp_rem_len < len)
len = pageHeader->xlp_rem_len;
- if (readOff < pageHeaderSize + len)
- readOff = ReadPageInternal(state, targetPagePtr,
- pageHeaderSize + len);
-
+ Assert (pageHeaderSize + len <= state->readLen);
memcpy(buffer, (char *) contdata, len);
buffer += len;
gotlen += len;
@@ -517,9 +524,16 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
else
{
/* Wait for the record data to become available */
- readOff = ReadPageInternal(state, targetPagePtr,
- Min(targetRecOff + total_len, XLOG_BLCKSZ));
- if (readOff < 0)
+ while (XLogNeedData(state, targetPagePtr,
+ Min(targetRecOff + total_len, XLOG_BLCKSZ), true))
+ {
+ if (!state->routine.page_read(state, state->readPagePtr,
+ state->readLen,
+ state->ReadRecPtr, state->readBuf))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
/* Record does not cross a page boundary */
@@ -562,109 +576,138 @@ err:
}
/*
- * Read a single xlog page including at least [pageptr, reqLen] of valid data
- * via the page_read() callback.
+ * Checks that an xlog page loaded in state->readBuf is including at least
+ * [pageptr, reqLen] and the page is valid. header_inclusive indicates that
+ * reqLen is calculated including page header length.
*
- * Returns -1 if the required page cannot be read for some reason; errormsg_buf
- * is set in that case (unless the error occurs in the page_read callback).
+ * Returns false if the buffer already contains the requested data, or found
+ * error. state->page_verified is set to true for the former and false for the
+ * latter.
*
- * We fetch the page from a reader-local cache if we know we have the required
- * data and if there hasn't been any error since caching the data.
+ * Otherwise returns true and requests data loaded onto state->readBuf by
+ * state->readPagePtr and state->readLen. The caller shall call this function
+ * again after filling the buffer at least with that portion of data and set
+ * state->readLen to the length of actually loaded data.
+ *
+ * If header_inclusive is false, corrects reqLen internally by adding the
+ * actual page header length and may request caller for new data.
*/
-static int
-ReadPageInternal(XLogReaderState *state, XLogRecPtr pageptr, int reqLen)
+static bool
+XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
+ bool header_inclusive)
{
- int readLen;
uint32 targetPageOff;
XLogSegNo targetSegNo;
- XLogPageHeader hdr;
+ uint32 addLen = 0;
- Assert((pageptr % XLOG_BLCKSZ) == 0);
+ /* Some data is loaded, but page header is not verified yet. */
+ if (!state->page_verified &&
+ !XLogRecPtrIsInvalid(state->readPagePtr) && state->readLen >= 0)
+ {
+ uint32 pageHeaderSize;
+ /* just loaded new data so needs to verify page header */
+
+ /* The caller must have loaded at least page header */
+ Assert (state->readLen >= SizeOfXLogShortPHD);
+
+ /*
+ * We have enough data to check the header length. Recheck the loaded
+ * length against the actual header length.
+ */
+ pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+
+ /* Request more data if we don't have the full header. */
+ if (state->readLen < pageHeaderSize)
+ {
+ state->readLen = pageHeaderSize;
+ return true;
+ }
+
+ /* Now that we know we have the full header, validate it. */
+ if (!XLogReaderValidatePageHeader(state, state->readPagePtr,
+ (char *) state->readBuf))
+ {
+ /* That's bad. Force reading the page again. */
+ XLogReaderInvalReadState(state);
+
+ return false;
+ }
+
+ state->page_verified = true;
+
+ XLByteToSeg(state->readPagePtr, state->seg.ws_segno,
+ state->segcxt.ws_segsize);
+ }
+
+ /*
+ * The loaded page may not be the one caller is supposing to read when we
+ * are verifying the first page of new segment. In that case, skip further
+ * verification and immediately load the target page.
+ */
+ if (state->page_verified && pageptr == state->readPagePtr)
+ {
+ /*
+ * calculate additional length for page header keeping the total
+ * length within the block size.
+ */
+ if (!header_inclusive)
+ {
+ uint32 pageHeaderSize =
+ XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+
+ addLen = pageHeaderSize;
+ if (reqLen + pageHeaderSize <= XLOG_BLCKSZ)
+ addLen = pageHeaderSize;
+ else
+ addLen = XLOG_BLCKSZ - reqLen;
+
+ Assert(addLen >= 0);
+ }
+
+ /* Return if we already have it. */
+ if (reqLen + addLen <= state->readLen)
+ return false;
+ }
+
+ /* Data is not in our buffer, request the caller for it. */
XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize);
targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize);
+ Assert((pageptr % XLOG_BLCKSZ) == 0);
- /* check whether we have all the requested data already */
- if (targetSegNo == state->seg.ws_segno &&
- targetPageOff == state->segoff && reqLen <= state->readLen)
- return state->readLen;
+ /*
+ * Every time we request to load new data of a page to the caller, even if
+ * we looked at a part of it before, we need to do verification on the next
+ * invocation as the caller might now be rereading data from a different
+ * source.
+ */
+ state->page_verified = false;
/*
- * Data is not in our buffer.
- *
- * Every time we actually read the segment, even if we looked at parts of
- * it before, we need to do verification as the page_read callback might
- * now be rereading data from a different source.
- *
* Whenever switching to a new WAL segment, we read the first page of the
* file and validate its header, even if that's not where the target
* record is. This is so that we can check the additional identification
* info that is present in the first page's "long" header.
+ * Don't do this if the caller requested the first page in the segment.
*/
if (targetSegNo != state->seg.ws_segno && targetPageOff != 0)
{
- XLogRecPtr targetSegmentPtr = pageptr - targetPageOff;
-
- readLen = state->routine.page_read(state, targetSegmentPtr, XLOG_BLCKSZ,
- state->currRecPtr,
- state->readBuf);
- if (readLen < 0)
- goto err;
-
- /* we can be sure to have enough WAL available, we scrolled back */
- Assert(readLen == XLOG_BLCKSZ);
-
- if (!XLogReaderValidatePageHeader(state, targetSegmentPtr,
- state->readBuf))
- goto err;
+ /*
+ * Then we'll see that the targetSegNo now matches the ws_segno, and
+ * will not come back here, but will request the actual target page.
+ */
+ state->readPagePtr = pageptr - targetPageOff;
+ state->readLen = XLOG_BLCKSZ;
+ return true;
}
/*
- * First, read the requested data length, but at least a short page header
- * so that we can validate it.
+ * Request the caller to load the page. We need at least a short page
+ * header so that we can validate it.
*/
- readLen = state->routine.page_read(state, pageptr, Max(reqLen, SizeOfXLogShortPHD),
- state->currRecPtr,
- state->readBuf);
- if (readLen < 0)
- goto err;
-
- Assert(readLen <= XLOG_BLCKSZ);
-
- /* Do we have enough data to check the header length? */
- if (readLen <= SizeOfXLogShortPHD)
- goto err;
-
- Assert(readLen >= reqLen);
-
- hdr = (XLogPageHeader) state->readBuf;
-
- /* still not enough */
- if (readLen < XLogPageHeaderSize(hdr))
- {
- readLen = state->routine.page_read(state, pageptr, XLogPageHeaderSize(hdr),
- state->currRecPtr,
- state->readBuf);
- if (readLen < 0)
- goto err;
- }
-
- /*
- * Now that we know we have the full header, validate it.
- */
- if (!XLogReaderValidatePageHeader(state, pageptr, (char *) hdr))
- goto err;
-
- /* update read state information */
- state->seg.ws_segno = targetSegNo;
- state->segoff = targetPageOff;
- state->readLen = readLen;
-
- return readLen;
-
-err:
- XLogReaderInvalReadState(state);
- return -1;
+ state->readPagePtr = pageptr;
+ state->readLen = Max(reqLen + addLen, SizeOfXLogShortPHD);
+ return true;
}
/*
@@ -673,9 +716,7 @@ err:
static void
XLogReaderInvalReadState(XLogReaderState *state)
{
- state->seg.ws_segno = 0;
- state->segoff = 0;
- state->readLen = 0;
+ state->readPagePtr = InvalidXLogRecPtr;
}
/*
@@ -956,7 +997,6 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
XLogRecPtr targetPagePtr;
int targetRecOff;
uint32 pageHeaderSize;
- int readLen;
/*
* Compute targetRecOff. It should typically be equal or greater than
@@ -964,7 +1004,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
* that, except when caller has explicitly specified the offset that
* falls somewhere there or when we are skipping multi-page
* continuation record. It doesn't matter though because
- * ReadPageInternal() is prepared to handle that and will read at
+ * XLogNeedData() is prepared to handle that and will read at
* least short page-header worth of data
*/
targetRecOff = tmpRecPtr % XLOG_BLCKSZ;
@@ -972,19 +1012,24 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
/* scroll back to page boundary */
targetPagePtr = tmpRecPtr - targetRecOff;
- /* Read the page containing the record */
- readLen = ReadPageInternal(state, targetPagePtr, targetRecOff);
- if (readLen < 0)
+ while(XLogNeedData(state, targetPagePtr, targetRecOff,
+ targetRecOff != 0))
+ {
+ if (!state->routine.page_read(state, state->readPagePtr,
+ state->readLen,
+ state->ReadRecPtr, state->readBuf))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
header = (XLogPageHeader) state->readBuf;
pageHeaderSize = XLogPageHeaderSize(header);
- /* make sure we have enough data for the page header */
- readLen = ReadPageInternal(state, targetPagePtr, pageHeaderSize);
- if (readLen < 0)
- goto err;
+ /* we should have read the page header */
+ Assert (state->readLen >= pageHeaderSize);
/* skip over potential continuation data */
if (header->xlp_info & XLP_FIRST_IS_CONTRECORD)
diff --git a/src/backend/access/transam/xlogutils.c b/src/backend/access/transam/xlogutils.c
index 322b0e8ff5..1b10b6df20 100644
--- a/src/backend/access/transam/xlogutils.c
+++ b/src/backend/access/transam/xlogutils.c
@@ -689,8 +689,8 @@ XLogTruncateRelation(RelFileNode rnode, ForkNumber forkNum,
void
XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wantLength)
{
- const XLogRecPtr lastReadPage = (state->seg.ws_segno *
- state->segcxt.ws_segsize + state->segoff);
+ const XLogRecPtr lastReadPage = state->seg.ws_segno *
+ state->segcxt.ws_segsize + state->readLen;
Assert(wantPage != InvalidXLogRecPtr && wantPage % XLOG_BLCKSZ == 0);
Assert(wantLength <= XLOG_BLCKSZ);
@@ -828,7 +828,7 @@ wal_segment_close(XLogReaderState *state)
* exists for normal backends, so we have to do a check/sleep/repeat style of
* loop for now.
*/
-int
+bool
read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *cur_page)
{
@@ -930,7 +930,8 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
else if (targetPagePtr + reqLen > read_upto)
{
/* not enough data there */
- return -1;
+ state->readLen = -1;
+ return false;
}
else
{
@@ -948,7 +949,8 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
WALReadRaiseError(&errinfo);
/* number of valid bytes in the buffer */
- return count;
+ state->readLen = count;
+ return true;
}
/*
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index e2477c47e0..bf2711bddd 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -811,7 +811,7 @@ StartReplication(StartReplicationCmd *cmd)
* which has to do a plain sleep/busy loop, because the walsender's latch gets
* set every time WAL is flushed.
*/
-static int
+static bool
logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *cur_page)
{
@@ -831,7 +831,10 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
/* fail if not (implies we are going to shut down) */
if (flushptr < targetPagePtr + reqLen)
- return -1;
+ {
+ state->readLen = -1;
+ return false;
+ }
if (targetPagePtr + XLOG_BLCKSZ <= flushptr)
count = XLOG_BLCKSZ; /* more than one block available */
@@ -859,7 +862,8 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
XLByteToSeg(targetPagePtr, segno, state->segcxt.ws_segsize);
CheckXLogRemoved(segno, state->seg.ws_tli);
- return count;
+ state->readLen = count;
+ return true;
}
/*
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index bc6f976994..0b7e73ae79 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -46,7 +46,7 @@ typedef struct XLogPageReadPrivate
int tliIndex;
} XLogPageReadPrivate;
-static int SimpleXLogPageRead(XLogReaderState *xlogreader,
+static bool SimpleXLogPageRead(XLogReaderState *xlogreader,
XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
@@ -236,7 +236,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
}
/* XLogReader callback function, to read a WAL page */
-static int
+static bool
SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf)
{
@@ -296,7 +296,8 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
if (private->restoreCommand == NULL)
{
pg_log_error("could not open file \"%s\": %m", xlogfpath);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
/*
@@ -309,7 +310,10 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
private->restoreCommand);
if (xlogreadfd < 0)
- return -1;
+ {
+ xlogreader->readLen = -1;
+ return false;
+ }
else
pg_log_debug("using file \"%s\" restored from archive",
xlogfpath);
@@ -325,7 +329,8 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
if (lseek(xlogreadfd, (off_t) targetPageOff, SEEK_SET) < 0)
{
pg_log_error("could not seek in file \"%s\": %m", xlogfpath);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
@@ -338,13 +343,15 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
pg_log_error("could not read file \"%s\": read %d of %zu",
xlogfpath, r, (Size) XLOG_BLCKSZ);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
Assert(targetSegNo == xlogreadsegno);
xlogreader->seg.ws_tli = targetHistory[private->tliIndex].tli;
- return XLOG_BLCKSZ;
+ xlogreader->readLen = XLOG_BLCKSZ;
+ return true;
}
/*
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index d1a0678935..ab0c28d9b7 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -331,7 +331,7 @@ WALDumpCloseSegment(XLogReaderState *state)
}
/* pg_waldump's XLogReaderRoutine->page_read callback */
-static int
+static bool
WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetPtr, char *readBuff)
{
@@ -348,7 +348,8 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
else
{
private->endptr_reached = true;
- return -1;
+ state->readLen = -1;
+ return false;
}
}
@@ -373,7 +374,8 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
(Size) errinfo.wre_req);
}
- return count;
+ state->readLen = count;
+ return true;
}
/*
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index b0f2a6ed43..6c8848e14f 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -57,8 +57,8 @@ typedef struct WALSegmentContext
typedef struct XLogReaderState XLogReaderState;
-/* Function type definitions for various xlogreader interactions */
-typedef int (*XLogPageReadCB) (XLogReaderState *xlogreader,
+/* Function type definition for the read_page callback */
+typedef bool (*XLogPageReadCB) (XLogReaderState *xlogreader,
XLogRecPtr targetPagePtr,
int reqLen,
XLogRecPtr targetRecPtr,
@@ -175,6 +175,20 @@ struct XLogReaderState
XLogRecPtr ReadRecPtr; /* start of last record read */
XLogRecPtr EndRecPtr; /* end+1 of last record read */
+ /* ----------------------------------------
+ * Communication with page reader
+ * readBuf is XLOG_BLCKSZ bytes, valid up to at least readLen bytes.
+ * ----------------------------------------
+ */
+ /* variables to communicate with page reader */
+ XLogRecPtr readPagePtr; /* page pointer to read */
+ int32 readLen; /* bytes requested to reader, or actual bytes
+ * read by reader, which must be larger than
+ * the request, or -1 on error */
+ TimeLineID readPageTLI; /* TLI for data currently in readBuf */
+ char *readBuf; /* buffer to store data */
+ bool page_verified; /* is the page on the buffer verified? */
+
/* ----------------------------------------
* Decoded representation of current record
@@ -201,13 +215,6 @@ struct XLogReaderState
* ----------------------------------------
*/
- /*
- * Buffer for currently read page (XLOG_BLCKSZ bytes, valid up to at least
- * readLen bytes)
- */
- char *readBuf;
- uint32 readLen;
-
/* last read XLOG position for data currently in readBuf */
WALSegmentContext segcxt;
WALOpenSegment seg;
diff --git a/src/include/access/xlogutils.h b/src/include/access/xlogutils.h
index e59b6cf3a9..c287edf206 100644
--- a/src/include/access/xlogutils.h
+++ b/src/include/access/xlogutils.h
@@ -47,7 +47,7 @@ extern Buffer XLogReadBufferExtended(RelFileNode rnode, ForkNumber forknum,
extern Relation CreateFakeRelcacheEntry(RelFileNode rnode);
extern void FreeFakeRelcacheEntry(Relation fakerel);
-extern int read_local_xlog_page(XLogReaderState *state,
+extern bool read_local_xlog_page(XLogReaderState *state,
XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *cur_page);
extern void wal_segment_open(XLogReaderState *state,
--
2.18.4
----Next_Part(Thu_Jul__2_13_53_30_2020_072)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
filename="v15-0002-Move-page-reader-out-of-XLogReadRecord.patch"
^ permalink raw reply [nested|flat] 69+ messages in thread
* [PATCH v17 1/4] Move callback-call from ReadPageInternal to XLogReadRecord.
@ 2019-09-05 11:21 Kyotaro Horiguchi <[email protected]>
0 siblings, 0 replies; 69+ messages in thread
From: Kyotaro Horiguchi @ 2019-09-05 11:21 UTC (permalink / raw)
The current WAL record reader reads page data using a call back
function. Although it is not so problematic alone, it would be a
problem if we are going to do add tasks like encryption which is
performed on page data before WAL reader reads them. To avoid that the
record reader facility has to have a new code path corresponds to
every new callback, this patch separates page reader from WAL record
reading facility by modifying the current WAL record reader to a state
machine.
As the first step of that change, this patch moves the page reader
function out of ReadPageInternal, then the remaining tasks of the
function are taken over by the new function XLogNeedData. As the
result XLogPageRead directly calls the page reader callback function
according to the feedback from XLogNeedData.
---
src/backend/access/transam/xlog.c | 16 +-
src/backend/access/transam/xlogreader.c | 281 ++++++++++++++----------
src/backend/access/transam/xlogutils.c | 12 +-
src/backend/replication/walsender.c | 10 +-
src/bin/pg_rewind/parsexlog.c | 21 +-
src/bin/pg_waldump/pg_waldump.c | 8 +-
src/include/access/xlogreader.h | 25 ++-
src/include/access/xlogutils.h | 2 +-
8 files changed, 222 insertions(+), 153 deletions(-)
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 377afb8732..606328a65a 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -917,7 +917,7 @@ static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath,
static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli,
XLogSource source, bool notfoundOk);
static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, XLogSource source);
-static int XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
+static bool XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
bool fetching_ckpt, XLogRecPtr tliRecPtr);
@@ -4350,7 +4350,6 @@ ReadRecord(XLogReaderState *xlogreader, int emode,
XLogRecord *record;
XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
- /* Pass through parameters to XLogPageRead */
private->fetching_ckpt = fetching_ckpt;
private->emode = emode;
private->randAccess = (xlogreader->ReadRecPtr == InvalidXLogRecPtr);
@@ -11987,7 +11986,7 @@ CancelBackup(void)
* XLogPageRead() to try fetching the record from another source, or to
* sleep and retry.
*/
-static int
+static bool
XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *readBuf)
{
@@ -12046,7 +12045,8 @@ retry:
readLen = 0;
readSource = XLOG_FROM_ANY;
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
}
@@ -12141,7 +12141,8 @@ retry:
goto next_record_is_invalid;
}
- return readLen;
+ xlogreader->readLen = readLen;
+ return true;
next_record_is_invalid:
lastSourceFailed = true;
@@ -12155,8 +12156,9 @@ next_record_is_invalid:
/* In standby-mode, keep trying */
if (StandbyMode)
goto retry;
- else
- return -1;
+
+ xlogreader->readLen = -1;
+ return false;
}
/*
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index 42738eb940..7d04995b7b 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -36,8 +36,8 @@
static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
pg_attribute_printf(2, 3);
static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
-static int ReadPageInternal(XLogReaderState *state, XLogRecPtr pageptr,
- int reqLen);
+static bool XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr,
+ int reqLen, bool header_inclusive);
static void XLogReaderInvalReadState(XLogReaderState *state);
static bool ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
XLogRecPtr PrevRecPtr, XLogRecord *record, bool randAccess);
@@ -276,7 +276,6 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
uint32 targetRecOff;
uint32 pageHeaderSize;
bool gotheader;
- int readOff;
/*
* randAccess indicates whether to verify the previous-record pointer of
@@ -326,14 +325,20 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
* byte to cover the whole record header, or at least the part of it that
* fits on the same page.
*/
- readOff = ReadPageInternal(state, targetPagePtr,
- Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ));
- if (readOff < 0)
+ while (XLogNeedData(state, targetPagePtr,
+ Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+ targetRecOff != 0))
+ {
+ if (!state->routine.page_read(state, state->readPagePtr, state->readLen,
+ RecPtr, state->readBuf))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
/*
- * ReadPageInternal always returns at least the page header, so we can
- * examine it now.
+ * We have at least the page header, so we can examine it now.
*/
pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
if (targetRecOff == 0)
@@ -359,8 +364,8 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
goto err;
}
- /* ReadPageInternal has verified the page header */
- Assert(pageHeaderSize <= readOff);
+ /* XLogNeedData has verified the page header */
+ Assert(pageHeaderSize <= state->readLen);
/*
* Read the record length.
@@ -432,18 +437,27 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
do
{
+ int rest_len = total_len - gotlen;
+
/* Calculate pointer to beginning of next page */
targetPagePtr += XLOG_BLCKSZ;
/* Wait for the next page to become available */
- readOff = ReadPageInternal(state, targetPagePtr,
- Min(total_len - gotlen + SizeOfXLogShortPHD,
- XLOG_BLCKSZ));
+ while (XLogNeedData(state, targetPagePtr,
+ Min(rest_len, XLOG_BLCKSZ),
+ false))
+ {
+ if (!state->routine.page_read(state, state->readPagePtr,
+ state->readLen,
+ state->ReadRecPtr,
+ state->readBuf))
+ break;
+ }
- if (readOff < 0)
+ if (!state->page_verified)
goto err;
- Assert(SizeOfXLogShortPHD <= readOff);
+ Assert(SizeOfXLogShortPHD <= state->readLen);
/* Check that the continuation on next page looks valid */
pageHeader = (XLogPageHeader) state->readBuf;
@@ -473,21 +487,14 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
/* Append the continuation from this page to the buffer */
pageHeaderSize = XLogPageHeaderSize(pageHeader);
- if (readOff < pageHeaderSize)
- readOff = ReadPageInternal(state, targetPagePtr,
- pageHeaderSize);
-
- Assert(pageHeaderSize <= readOff);
+ Assert(pageHeaderSize <= state->readLen);
contdata = (char *) state->readBuf + pageHeaderSize;
len = XLOG_BLCKSZ - pageHeaderSize;
if (pageHeader->xlp_rem_len < len)
len = pageHeader->xlp_rem_len;
- if (readOff < pageHeaderSize + len)
- readOff = ReadPageInternal(state, targetPagePtr,
- pageHeaderSize + len);
-
+ Assert (pageHeaderSize + len <= state->readLen);
memcpy(buffer, (char *) contdata, len);
buffer += len;
gotlen += len;
@@ -517,9 +524,16 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
else
{
/* Wait for the record data to become available */
- readOff = ReadPageInternal(state, targetPagePtr,
- Min(targetRecOff + total_len, XLOG_BLCKSZ));
- if (readOff < 0)
+ while (XLogNeedData(state, targetPagePtr,
+ Min(targetRecOff + total_len, XLOG_BLCKSZ), true))
+ {
+ if (!state->routine.page_read(state, state->readPagePtr,
+ state->readLen,
+ state->ReadRecPtr, state->readBuf))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
/* Record does not cross a page boundary */
@@ -562,109 +576,138 @@ err:
}
/*
- * Read a single xlog page including at least [pageptr, reqLen] of valid data
- * via the page_read() callback.
+ * Checks that an xlog page loaded in state->readBuf is including at least
+ * [pageptr, reqLen] and the page is valid. header_inclusive indicates that
+ * reqLen is calculated including page header length.
*
- * Returns -1 if the required page cannot be read for some reason; errormsg_buf
- * is set in that case (unless the error occurs in the page_read callback).
+ * Returns false if the buffer already contains the requested data, or found
+ * error. state->page_verified is set to true for the former and false for the
+ * latter.
*
- * We fetch the page from a reader-local cache if we know we have the required
- * data and if there hasn't been any error since caching the data.
+ * Otherwise returns true and requests data loaded onto state->readBuf by
+ * state->readPagePtr and state->readLen. The caller shall call this function
+ * again after filling the buffer at least with that portion of data and set
+ * state->readLen to the length of actually loaded data.
+ *
+ * If header_inclusive is false, corrects reqLen internally by adding the
+ * actual page header length and may request caller for new data.
*/
-static int
-ReadPageInternal(XLogReaderState *state, XLogRecPtr pageptr, int reqLen)
+static bool
+XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
+ bool header_inclusive)
{
- int readLen;
uint32 targetPageOff;
XLogSegNo targetSegNo;
- XLogPageHeader hdr;
+ uint32 addLen = 0;
- Assert((pageptr % XLOG_BLCKSZ) == 0);
+ /* Some data is loaded, but page header is not verified yet. */
+ if (!state->page_verified &&
+ !XLogRecPtrIsInvalid(state->readPagePtr) && state->readLen >= 0)
+ {
+ uint32 pageHeaderSize;
+ /* just loaded new data so needs to verify page header */
+
+ /* The caller must have loaded at least page header */
+ Assert (state->readLen >= SizeOfXLogShortPHD);
+
+ /*
+ * We have enough data to check the header length. Recheck the loaded
+ * length against the actual header length.
+ */
+ pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+
+ /* Request more data if we don't have the full header. */
+ if (state->readLen < pageHeaderSize)
+ {
+ state->readLen = pageHeaderSize;
+ return true;
+ }
+
+ /* Now that we know we have the full header, validate it. */
+ if (!XLogReaderValidatePageHeader(state, state->readPagePtr,
+ (char *) state->readBuf))
+ {
+ /* That's bad. Force reading the page again. */
+ XLogReaderInvalReadState(state);
+
+ return false;
+ }
+
+ state->page_verified = true;
+
+ XLByteToSeg(state->readPagePtr, state->seg.ws_segno,
+ state->segcxt.ws_segsize);
+ }
+
+ /*
+ * The loaded page may not be the one caller is supposing to read when we
+ * are verifying the first page of new segment. In that case, skip further
+ * verification and immediately load the target page.
+ */
+ if (state->page_verified && pageptr == state->readPagePtr)
+ {
+ /*
+ * calculate additional length for page header keeping the total
+ * length within the block size.
+ */
+ if (!header_inclusive)
+ {
+ uint32 pageHeaderSize =
+ XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+
+ addLen = pageHeaderSize;
+ if (reqLen + pageHeaderSize <= XLOG_BLCKSZ)
+ addLen = pageHeaderSize;
+ else
+ addLen = XLOG_BLCKSZ - reqLen;
+
+ Assert(addLen >= 0);
+ }
+
+ /* Return if we already have it. */
+ if (reqLen + addLen <= state->readLen)
+ return false;
+ }
+
+ /* Data is not in our buffer, request the caller for it. */
XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize);
targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize);
+ Assert((pageptr % XLOG_BLCKSZ) == 0);
- /* check whether we have all the requested data already */
- if (targetSegNo == state->seg.ws_segno &&
- targetPageOff == state->segoff && reqLen <= state->readLen)
- return state->readLen;
+ /*
+ * Every time we request to load new data of a page to the caller, even if
+ * we looked at a part of it before, we need to do verification on the next
+ * invocation as the caller might now be rereading data from a different
+ * source.
+ */
+ state->page_verified = false;
/*
- * Data is not in our buffer.
- *
- * Every time we actually read the segment, even if we looked at parts of
- * it before, we need to do verification as the page_read callback might
- * now be rereading data from a different source.
- *
* Whenever switching to a new WAL segment, we read the first page of the
* file and validate its header, even if that's not where the target
* record is. This is so that we can check the additional identification
* info that is present in the first page's "long" header.
+ * Don't do this if the caller requested the first page in the segment.
*/
if (targetSegNo != state->seg.ws_segno && targetPageOff != 0)
{
- XLogRecPtr targetSegmentPtr = pageptr - targetPageOff;
-
- readLen = state->routine.page_read(state, targetSegmentPtr, XLOG_BLCKSZ,
- state->currRecPtr,
- state->readBuf);
- if (readLen < 0)
- goto err;
-
- /* we can be sure to have enough WAL available, we scrolled back */
- Assert(readLen == XLOG_BLCKSZ);
-
- if (!XLogReaderValidatePageHeader(state, targetSegmentPtr,
- state->readBuf))
- goto err;
+ /*
+ * Then we'll see that the targetSegNo now matches the ws_segno, and
+ * will not come back here, but will request the actual target page.
+ */
+ state->readPagePtr = pageptr - targetPageOff;
+ state->readLen = XLOG_BLCKSZ;
+ return true;
}
/*
- * First, read the requested data length, but at least a short page header
- * so that we can validate it.
+ * Request the caller to load the page. We need at least a short page
+ * header so that we can validate it.
*/
- readLen = state->routine.page_read(state, pageptr, Max(reqLen, SizeOfXLogShortPHD),
- state->currRecPtr,
- state->readBuf);
- if (readLen < 0)
- goto err;
-
- Assert(readLen <= XLOG_BLCKSZ);
-
- /* Do we have enough data to check the header length? */
- if (readLen <= SizeOfXLogShortPHD)
- goto err;
-
- Assert(readLen >= reqLen);
-
- hdr = (XLogPageHeader) state->readBuf;
-
- /* still not enough */
- if (readLen < XLogPageHeaderSize(hdr))
- {
- readLen = state->routine.page_read(state, pageptr, XLogPageHeaderSize(hdr),
- state->currRecPtr,
- state->readBuf);
- if (readLen < 0)
- goto err;
- }
-
- /*
- * Now that we know we have the full header, validate it.
- */
- if (!XLogReaderValidatePageHeader(state, pageptr, (char *) hdr))
- goto err;
-
- /* update read state information */
- state->seg.ws_segno = targetSegNo;
- state->segoff = targetPageOff;
- state->readLen = readLen;
-
- return readLen;
-
-err:
- XLogReaderInvalReadState(state);
- return -1;
+ state->readPagePtr = pageptr;
+ state->readLen = Max(reqLen + addLen, SizeOfXLogShortPHD);
+ return true;
}
/*
@@ -673,9 +716,7 @@ err:
static void
XLogReaderInvalReadState(XLogReaderState *state)
{
- state->seg.ws_segno = 0;
- state->segoff = 0;
- state->readLen = 0;
+ state->readPagePtr = InvalidXLogRecPtr;
}
/*
@@ -953,7 +994,6 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
XLogRecPtr targetPagePtr;
int targetRecOff;
uint32 pageHeaderSize;
- int readLen;
/*
* Compute targetRecOff. It should typically be equal or greater than
@@ -961,7 +1001,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
* that, except when caller has explicitly specified the offset that
* falls somewhere there or when we are skipping multi-page
* continuation record. It doesn't matter though because
- * ReadPageInternal() is prepared to handle that and will read at
+ * XLogNeedData() is prepared to handle that and will read at
* least short page-header worth of data
*/
targetRecOff = tmpRecPtr % XLOG_BLCKSZ;
@@ -969,19 +1009,24 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
/* scroll back to page boundary */
targetPagePtr = tmpRecPtr - targetRecOff;
- /* Read the page containing the record */
- readLen = ReadPageInternal(state, targetPagePtr, targetRecOff);
- if (readLen < 0)
+ while(XLogNeedData(state, targetPagePtr, targetRecOff,
+ targetRecOff != 0))
+ {
+ if (!state->routine.page_read(state, state->readPagePtr,
+ state->readLen,
+ state->ReadRecPtr, state->readBuf))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
header = (XLogPageHeader) state->readBuf;
pageHeaderSize = XLogPageHeaderSize(header);
- /* make sure we have enough data for the page header */
- readLen = ReadPageInternal(state, targetPagePtr, pageHeaderSize);
- if (readLen < 0)
- goto err;
+ /* we should have read the page header */
+ Assert (state->readLen >= pageHeaderSize);
/* skip over potential continuation data */
if (header->xlp_info & XLP_FIRST_IS_CONTRECORD)
diff --git a/src/backend/access/transam/xlogutils.c b/src/backend/access/transam/xlogutils.c
index d17d660f46..46eda33f25 100644
--- a/src/backend/access/transam/xlogutils.c
+++ b/src/backend/access/transam/xlogutils.c
@@ -686,8 +686,8 @@ XLogTruncateRelation(RelFileNode rnode, ForkNumber forkNum,
void
XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wantLength)
{
- const XLogRecPtr lastReadPage = (state->seg.ws_segno *
- state->segcxt.ws_segsize + state->segoff);
+ const XLogRecPtr lastReadPage = state->seg.ws_segno *
+ state->segcxt.ws_segsize + state->readLen;
Assert(wantPage != InvalidXLogRecPtr && wantPage % XLOG_BLCKSZ == 0);
Assert(wantLength <= XLOG_BLCKSZ);
@@ -824,7 +824,7 @@ wal_segment_close(XLogReaderState *state)
* exists for normal backends, so we have to do a check/sleep/repeat style of
* loop for now.
*/
-int
+bool
read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *cur_page)
{
@@ -926,7 +926,8 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
else if (targetPagePtr + reqLen > read_upto)
{
/* not enough data there */
- return -1;
+ state->readLen = -1;
+ return false;
}
else
{
@@ -944,7 +945,8 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
WALReadRaiseError(&errinfo);
/* number of valid bytes in the buffer */
- return count;
+ state->readLen = count;
+ return true;
}
/*
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index 23baa4498a..c36b91ae5e 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -806,7 +806,7 @@ StartReplication(StartReplicationCmd *cmd)
* which has to do a plain sleep/busy loop, because the walsender's latch gets
* set every time WAL is flushed.
*/
-static int
+static bool
logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *cur_page)
{
@@ -826,7 +826,10 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
/* fail if not (implies we are going to shut down) */
if (flushptr < targetPagePtr + reqLen)
- return -1;
+ {
+ state->readLen = -1;
+ return false;
+ }
if (targetPagePtr + XLOG_BLCKSZ <= flushptr)
count = XLOG_BLCKSZ; /* more than one block available */
@@ -854,7 +857,8 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
XLByteToSeg(targetPagePtr, segno, state->segcxt.ws_segsize);
CheckXLogRemoved(segno, state->seg.ws_tli);
- return count;
+ state->readLen = count;
+ return true;
}
/*
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index 59ebac7d6a..cf119848b0 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -47,7 +47,7 @@ typedef struct XLogPageReadPrivate
int tliIndex;
} XLogPageReadPrivate;
-static int SimpleXLogPageRead(XLogReaderState *xlogreader,
+static bool SimpleXLogPageRead(XLogReaderState *xlogreader,
XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
@@ -246,7 +246,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
}
/* XLogReader callback function, to read a WAL page */
-static int
+static bool
SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf)
{
@@ -306,7 +306,8 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
if (private->restoreCommand == NULL)
{
pg_log_error("could not open file \"%s\": %m", xlogfpath);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
/*
@@ -319,7 +320,10 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
private->restoreCommand);
if (xlogreadfd < 0)
- return -1;
+ {
+ xlogreader->readLen = -1;
+ return false;
+ }
else
pg_log_debug("using file \"%s\" restored from archive",
xlogfpath);
@@ -335,7 +339,8 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
if (lseek(xlogreadfd, (off_t) targetPageOff, SEEK_SET) < 0)
{
pg_log_error("could not seek in file \"%s\": %m", xlogfpath);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
@@ -348,13 +353,15 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
pg_log_error("could not read file \"%s\": read %d of %zu",
xlogfpath, r, (Size) XLOG_BLCKSZ);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
Assert(targetSegNo == xlogreadsegno);
xlogreader->seg.ws_tli = targetHistory[private->tliIndex].tli;
- return XLOG_BLCKSZ;
+ xlogreader->readLen = XLOG_BLCKSZ;
+ return true;
}
/*
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index 610f65e471..0f0e0723a3 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -331,7 +331,7 @@ WALDumpCloseSegment(XLogReaderState *state)
}
/* pg_waldump's XLogReaderRoutine->page_read callback */
-static int
+static bool
WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetPtr, char *readBuff)
{
@@ -348,7 +348,8 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
else
{
private->endptr_reached = true;
- return -1;
+ state->readLen = -1;
+ return false;
}
}
@@ -373,7 +374,8 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
(Size) errinfo.wre_req);
}
- return count;
+ state->readLen = count;
+ return true;
}
/*
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index 21d200d3df..23d9e70a59 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -57,8 +57,8 @@ typedef struct WALSegmentContext
typedef struct XLogReaderState XLogReaderState;
-/* Function type definitions for various xlogreader interactions */
-typedef int (*XLogPageReadCB) (XLogReaderState *xlogreader,
+/* Function type definition for the read_page callback */
+typedef bool (*XLogPageReadCB) (XLogReaderState *xlogreader,
XLogRecPtr targetPagePtr,
int reqLen,
XLogRecPtr targetRecPtr,
@@ -175,6 +175,20 @@ struct XLogReaderState
XLogRecPtr ReadRecPtr; /* start of last record read */
XLogRecPtr EndRecPtr; /* end+1 of last record read */
+ /* ----------------------------------------
+ * Communication with page reader
+ * readBuf is XLOG_BLCKSZ bytes, valid up to at least readLen bytes.
+ * ----------------------------------------
+ */
+ /* variables to communicate with page reader */
+ XLogRecPtr readPagePtr; /* page pointer to read */
+ int32 readLen; /* bytes requested to reader, or actual bytes
+ * read by reader, which must be larger than
+ * the request, or -1 on error */
+ TimeLineID readPageTLI; /* TLI for data currently in readBuf */
+ char *readBuf; /* buffer to store data */
+ bool page_verified; /* is the page on the buffer verified? */
+
/* ----------------------------------------
* Decoded representation of current record
@@ -203,13 +217,6 @@ struct XLogReaderState
* ----------------------------------------
*/
- /*
- * Buffer for currently read page (XLOG_BLCKSZ bytes, valid up to at least
- * readLen bytes)
- */
- char *readBuf;
- uint32 readLen;
-
/* last read XLOG position for data currently in readBuf */
WALSegmentContext segcxt;
WALOpenSegment seg;
diff --git a/src/include/access/xlogutils.h b/src/include/access/xlogutils.h
index 9ac602b674..364a21c4ea 100644
--- a/src/include/access/xlogutils.h
+++ b/src/include/access/xlogutils.h
@@ -47,7 +47,7 @@ extern Buffer XLogReadBufferExtended(RelFileNode rnode, ForkNumber forknum,
extern Relation CreateFakeRelcacheEntry(RelFileNode rnode);
extern void FreeFakeRelcacheEntry(Relation fakerel);
-extern int read_local_xlog_page(XLogReaderState *state,
+extern bool read_local_xlog_page(XLogReaderState *state,
XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *cur_page);
extern void wal_segment_open(XLogReaderState *state,
--
2.27.0
----Next_Part(Thu_Mar__4_11_28_53_2021_014)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
filename="v17-0002-Move-page-reader-out-of-XLogReadRecord.patch"
^ permalink raw reply [nested|flat] 69+ messages in thread
* [PATCH v17 1/4] Move callback-call from ReadPageInternal to XLogReadRecord.
@ 2019-09-05 11:21 Kyotaro Horiguchi <[email protected]>
0 siblings, 0 replies; 69+ messages in thread
From: Kyotaro Horiguchi @ 2019-09-05 11:21 UTC (permalink / raw)
The current WAL record reader reads page data using a call back
function. Although it is not so problematic alone, it would be a
problem if we are going to do add tasks like encryption which is
performed on page data before WAL reader reads them. To avoid that the
record reader facility has to have a new code path corresponds to
every new callback, this patch separates page reader from WAL record
reading facility by modifying the current WAL record reader to a state
machine.
As the first step of that change, this patch moves the page reader
function out of ReadPageInternal, then the remaining tasks of the
function are taken over by the new function XLogNeedData. As the
result XLogPageRead directly calls the page reader callback function
according to the feedback from XLogNeedData.
---
src/backend/access/transam/xlog.c | 16 +-
src/backend/access/transam/xlogreader.c | 281 ++++++++++++++----------
src/backend/access/transam/xlogutils.c | 12 +-
src/backend/replication/walsender.c | 10 +-
src/bin/pg_rewind/parsexlog.c | 21 +-
src/bin/pg_waldump/pg_waldump.c | 8 +-
src/include/access/xlogreader.h | 25 ++-
src/include/access/xlogutils.h | 2 +-
8 files changed, 222 insertions(+), 153 deletions(-)
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 377afb8732..606328a65a 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -917,7 +917,7 @@ static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath,
static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli,
XLogSource source, bool notfoundOk);
static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, XLogSource source);
-static int XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
+static bool XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
bool fetching_ckpt, XLogRecPtr tliRecPtr);
@@ -4350,7 +4350,6 @@ ReadRecord(XLogReaderState *xlogreader, int emode,
XLogRecord *record;
XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
- /* Pass through parameters to XLogPageRead */
private->fetching_ckpt = fetching_ckpt;
private->emode = emode;
private->randAccess = (xlogreader->ReadRecPtr == InvalidXLogRecPtr);
@@ -11987,7 +11986,7 @@ CancelBackup(void)
* XLogPageRead() to try fetching the record from another source, or to
* sleep and retry.
*/
-static int
+static bool
XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *readBuf)
{
@@ -12046,7 +12045,8 @@ retry:
readLen = 0;
readSource = XLOG_FROM_ANY;
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
}
@@ -12141,7 +12141,8 @@ retry:
goto next_record_is_invalid;
}
- return readLen;
+ xlogreader->readLen = readLen;
+ return true;
next_record_is_invalid:
lastSourceFailed = true;
@@ -12155,8 +12156,9 @@ next_record_is_invalid:
/* In standby-mode, keep trying */
if (StandbyMode)
goto retry;
- else
- return -1;
+
+ xlogreader->readLen = -1;
+ return false;
}
/*
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index 42738eb940..7d04995b7b 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -36,8 +36,8 @@
static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
pg_attribute_printf(2, 3);
static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
-static int ReadPageInternal(XLogReaderState *state, XLogRecPtr pageptr,
- int reqLen);
+static bool XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr,
+ int reqLen, bool header_inclusive);
static void XLogReaderInvalReadState(XLogReaderState *state);
static bool ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
XLogRecPtr PrevRecPtr, XLogRecord *record, bool randAccess);
@@ -276,7 +276,6 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
uint32 targetRecOff;
uint32 pageHeaderSize;
bool gotheader;
- int readOff;
/*
* randAccess indicates whether to verify the previous-record pointer of
@@ -326,14 +325,20 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
* byte to cover the whole record header, or at least the part of it that
* fits on the same page.
*/
- readOff = ReadPageInternal(state, targetPagePtr,
- Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ));
- if (readOff < 0)
+ while (XLogNeedData(state, targetPagePtr,
+ Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+ targetRecOff != 0))
+ {
+ if (!state->routine.page_read(state, state->readPagePtr, state->readLen,
+ RecPtr, state->readBuf))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
/*
- * ReadPageInternal always returns at least the page header, so we can
- * examine it now.
+ * We have at least the page header, so we can examine it now.
*/
pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
if (targetRecOff == 0)
@@ -359,8 +364,8 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
goto err;
}
- /* ReadPageInternal has verified the page header */
- Assert(pageHeaderSize <= readOff);
+ /* XLogNeedData has verified the page header */
+ Assert(pageHeaderSize <= state->readLen);
/*
* Read the record length.
@@ -432,18 +437,27 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
do
{
+ int rest_len = total_len - gotlen;
+
/* Calculate pointer to beginning of next page */
targetPagePtr += XLOG_BLCKSZ;
/* Wait for the next page to become available */
- readOff = ReadPageInternal(state, targetPagePtr,
- Min(total_len - gotlen + SizeOfXLogShortPHD,
- XLOG_BLCKSZ));
+ while (XLogNeedData(state, targetPagePtr,
+ Min(rest_len, XLOG_BLCKSZ),
+ false))
+ {
+ if (!state->routine.page_read(state, state->readPagePtr,
+ state->readLen,
+ state->ReadRecPtr,
+ state->readBuf))
+ break;
+ }
- if (readOff < 0)
+ if (!state->page_verified)
goto err;
- Assert(SizeOfXLogShortPHD <= readOff);
+ Assert(SizeOfXLogShortPHD <= state->readLen);
/* Check that the continuation on next page looks valid */
pageHeader = (XLogPageHeader) state->readBuf;
@@ -473,21 +487,14 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
/* Append the continuation from this page to the buffer */
pageHeaderSize = XLogPageHeaderSize(pageHeader);
- if (readOff < pageHeaderSize)
- readOff = ReadPageInternal(state, targetPagePtr,
- pageHeaderSize);
-
- Assert(pageHeaderSize <= readOff);
+ Assert(pageHeaderSize <= state->readLen);
contdata = (char *) state->readBuf + pageHeaderSize;
len = XLOG_BLCKSZ - pageHeaderSize;
if (pageHeader->xlp_rem_len < len)
len = pageHeader->xlp_rem_len;
- if (readOff < pageHeaderSize + len)
- readOff = ReadPageInternal(state, targetPagePtr,
- pageHeaderSize + len);
-
+ Assert (pageHeaderSize + len <= state->readLen);
memcpy(buffer, (char *) contdata, len);
buffer += len;
gotlen += len;
@@ -517,9 +524,16 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
else
{
/* Wait for the record data to become available */
- readOff = ReadPageInternal(state, targetPagePtr,
- Min(targetRecOff + total_len, XLOG_BLCKSZ));
- if (readOff < 0)
+ while (XLogNeedData(state, targetPagePtr,
+ Min(targetRecOff + total_len, XLOG_BLCKSZ), true))
+ {
+ if (!state->routine.page_read(state, state->readPagePtr,
+ state->readLen,
+ state->ReadRecPtr, state->readBuf))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
/* Record does not cross a page boundary */
@@ -562,109 +576,138 @@ err:
}
/*
- * Read a single xlog page including at least [pageptr, reqLen] of valid data
- * via the page_read() callback.
+ * Checks that an xlog page loaded in state->readBuf is including at least
+ * [pageptr, reqLen] and the page is valid. header_inclusive indicates that
+ * reqLen is calculated including page header length.
*
- * Returns -1 if the required page cannot be read for some reason; errormsg_buf
- * is set in that case (unless the error occurs in the page_read callback).
+ * Returns false if the buffer already contains the requested data, or found
+ * error. state->page_verified is set to true for the former and false for the
+ * latter.
*
- * We fetch the page from a reader-local cache if we know we have the required
- * data and if there hasn't been any error since caching the data.
+ * Otherwise returns true and requests data loaded onto state->readBuf by
+ * state->readPagePtr and state->readLen. The caller shall call this function
+ * again after filling the buffer at least with that portion of data and set
+ * state->readLen to the length of actually loaded data.
+ *
+ * If header_inclusive is false, corrects reqLen internally by adding the
+ * actual page header length and may request caller for new data.
*/
-static int
-ReadPageInternal(XLogReaderState *state, XLogRecPtr pageptr, int reqLen)
+static bool
+XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
+ bool header_inclusive)
{
- int readLen;
uint32 targetPageOff;
XLogSegNo targetSegNo;
- XLogPageHeader hdr;
+ uint32 addLen = 0;
- Assert((pageptr % XLOG_BLCKSZ) == 0);
+ /* Some data is loaded, but page header is not verified yet. */
+ if (!state->page_verified &&
+ !XLogRecPtrIsInvalid(state->readPagePtr) && state->readLen >= 0)
+ {
+ uint32 pageHeaderSize;
+ /* just loaded new data so needs to verify page header */
+
+ /* The caller must have loaded at least page header */
+ Assert (state->readLen >= SizeOfXLogShortPHD);
+
+ /*
+ * We have enough data to check the header length. Recheck the loaded
+ * length against the actual header length.
+ */
+ pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+
+ /* Request more data if we don't have the full header. */
+ if (state->readLen < pageHeaderSize)
+ {
+ state->readLen = pageHeaderSize;
+ return true;
+ }
+
+ /* Now that we know we have the full header, validate it. */
+ if (!XLogReaderValidatePageHeader(state, state->readPagePtr,
+ (char *) state->readBuf))
+ {
+ /* That's bad. Force reading the page again. */
+ XLogReaderInvalReadState(state);
+
+ return false;
+ }
+
+ state->page_verified = true;
+
+ XLByteToSeg(state->readPagePtr, state->seg.ws_segno,
+ state->segcxt.ws_segsize);
+ }
+
+ /*
+ * The loaded page may not be the one caller is supposing to read when we
+ * are verifying the first page of new segment. In that case, skip further
+ * verification and immediately load the target page.
+ */
+ if (state->page_verified && pageptr == state->readPagePtr)
+ {
+ /*
+ * calculate additional length for page header keeping the total
+ * length within the block size.
+ */
+ if (!header_inclusive)
+ {
+ uint32 pageHeaderSize =
+ XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+
+ addLen = pageHeaderSize;
+ if (reqLen + pageHeaderSize <= XLOG_BLCKSZ)
+ addLen = pageHeaderSize;
+ else
+ addLen = XLOG_BLCKSZ - reqLen;
+
+ Assert(addLen >= 0);
+ }
+
+ /* Return if we already have it. */
+ if (reqLen + addLen <= state->readLen)
+ return false;
+ }
+
+ /* Data is not in our buffer, request the caller for it. */
XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize);
targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize);
+ Assert((pageptr % XLOG_BLCKSZ) == 0);
- /* check whether we have all the requested data already */
- if (targetSegNo == state->seg.ws_segno &&
- targetPageOff == state->segoff && reqLen <= state->readLen)
- return state->readLen;
+ /*
+ * Every time we request to load new data of a page to the caller, even if
+ * we looked at a part of it before, we need to do verification on the next
+ * invocation as the caller might now be rereading data from a different
+ * source.
+ */
+ state->page_verified = false;
/*
- * Data is not in our buffer.
- *
- * Every time we actually read the segment, even if we looked at parts of
- * it before, we need to do verification as the page_read callback might
- * now be rereading data from a different source.
- *
* Whenever switching to a new WAL segment, we read the first page of the
* file and validate its header, even if that's not where the target
* record is. This is so that we can check the additional identification
* info that is present in the first page's "long" header.
+ * Don't do this if the caller requested the first page in the segment.
*/
if (targetSegNo != state->seg.ws_segno && targetPageOff != 0)
{
- XLogRecPtr targetSegmentPtr = pageptr - targetPageOff;
-
- readLen = state->routine.page_read(state, targetSegmentPtr, XLOG_BLCKSZ,
- state->currRecPtr,
- state->readBuf);
- if (readLen < 0)
- goto err;
-
- /* we can be sure to have enough WAL available, we scrolled back */
- Assert(readLen == XLOG_BLCKSZ);
-
- if (!XLogReaderValidatePageHeader(state, targetSegmentPtr,
- state->readBuf))
- goto err;
+ /*
+ * Then we'll see that the targetSegNo now matches the ws_segno, and
+ * will not come back here, but will request the actual target page.
+ */
+ state->readPagePtr = pageptr - targetPageOff;
+ state->readLen = XLOG_BLCKSZ;
+ return true;
}
/*
- * First, read the requested data length, but at least a short page header
- * so that we can validate it.
+ * Request the caller to load the page. We need at least a short page
+ * header so that we can validate it.
*/
- readLen = state->routine.page_read(state, pageptr, Max(reqLen, SizeOfXLogShortPHD),
- state->currRecPtr,
- state->readBuf);
- if (readLen < 0)
- goto err;
-
- Assert(readLen <= XLOG_BLCKSZ);
-
- /* Do we have enough data to check the header length? */
- if (readLen <= SizeOfXLogShortPHD)
- goto err;
-
- Assert(readLen >= reqLen);
-
- hdr = (XLogPageHeader) state->readBuf;
-
- /* still not enough */
- if (readLen < XLogPageHeaderSize(hdr))
- {
- readLen = state->routine.page_read(state, pageptr, XLogPageHeaderSize(hdr),
- state->currRecPtr,
- state->readBuf);
- if (readLen < 0)
- goto err;
- }
-
- /*
- * Now that we know we have the full header, validate it.
- */
- if (!XLogReaderValidatePageHeader(state, pageptr, (char *) hdr))
- goto err;
-
- /* update read state information */
- state->seg.ws_segno = targetSegNo;
- state->segoff = targetPageOff;
- state->readLen = readLen;
-
- return readLen;
-
-err:
- XLogReaderInvalReadState(state);
- return -1;
+ state->readPagePtr = pageptr;
+ state->readLen = Max(reqLen + addLen, SizeOfXLogShortPHD);
+ return true;
}
/*
@@ -673,9 +716,7 @@ err:
static void
XLogReaderInvalReadState(XLogReaderState *state)
{
- state->seg.ws_segno = 0;
- state->segoff = 0;
- state->readLen = 0;
+ state->readPagePtr = InvalidXLogRecPtr;
}
/*
@@ -953,7 +994,6 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
XLogRecPtr targetPagePtr;
int targetRecOff;
uint32 pageHeaderSize;
- int readLen;
/*
* Compute targetRecOff. It should typically be equal or greater than
@@ -961,7 +1001,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
* that, except when caller has explicitly specified the offset that
* falls somewhere there or when we are skipping multi-page
* continuation record. It doesn't matter though because
- * ReadPageInternal() is prepared to handle that and will read at
+ * XLogNeedData() is prepared to handle that and will read at
* least short page-header worth of data
*/
targetRecOff = tmpRecPtr % XLOG_BLCKSZ;
@@ -969,19 +1009,24 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
/* scroll back to page boundary */
targetPagePtr = tmpRecPtr - targetRecOff;
- /* Read the page containing the record */
- readLen = ReadPageInternal(state, targetPagePtr, targetRecOff);
- if (readLen < 0)
+ while(XLogNeedData(state, targetPagePtr, targetRecOff,
+ targetRecOff != 0))
+ {
+ if (!state->routine.page_read(state, state->readPagePtr,
+ state->readLen,
+ state->ReadRecPtr, state->readBuf))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
header = (XLogPageHeader) state->readBuf;
pageHeaderSize = XLogPageHeaderSize(header);
- /* make sure we have enough data for the page header */
- readLen = ReadPageInternal(state, targetPagePtr, pageHeaderSize);
- if (readLen < 0)
- goto err;
+ /* we should have read the page header */
+ Assert (state->readLen >= pageHeaderSize);
/* skip over potential continuation data */
if (header->xlp_info & XLP_FIRST_IS_CONTRECORD)
diff --git a/src/backend/access/transam/xlogutils.c b/src/backend/access/transam/xlogutils.c
index d17d660f46..46eda33f25 100644
--- a/src/backend/access/transam/xlogutils.c
+++ b/src/backend/access/transam/xlogutils.c
@@ -686,8 +686,8 @@ XLogTruncateRelation(RelFileNode rnode, ForkNumber forkNum,
void
XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wantLength)
{
- const XLogRecPtr lastReadPage = (state->seg.ws_segno *
- state->segcxt.ws_segsize + state->segoff);
+ const XLogRecPtr lastReadPage = state->seg.ws_segno *
+ state->segcxt.ws_segsize + state->readLen;
Assert(wantPage != InvalidXLogRecPtr && wantPage % XLOG_BLCKSZ == 0);
Assert(wantLength <= XLOG_BLCKSZ);
@@ -824,7 +824,7 @@ wal_segment_close(XLogReaderState *state)
* exists for normal backends, so we have to do a check/sleep/repeat style of
* loop for now.
*/
-int
+bool
read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *cur_page)
{
@@ -926,7 +926,8 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
else if (targetPagePtr + reqLen > read_upto)
{
/* not enough data there */
- return -1;
+ state->readLen = -1;
+ return false;
}
else
{
@@ -944,7 +945,8 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
WALReadRaiseError(&errinfo);
/* number of valid bytes in the buffer */
- return count;
+ state->readLen = count;
+ return true;
}
/*
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index 23baa4498a..c36b91ae5e 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -806,7 +806,7 @@ StartReplication(StartReplicationCmd *cmd)
* which has to do a plain sleep/busy loop, because the walsender's latch gets
* set every time WAL is flushed.
*/
-static int
+static bool
logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *cur_page)
{
@@ -826,7 +826,10 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
/* fail if not (implies we are going to shut down) */
if (flushptr < targetPagePtr + reqLen)
- return -1;
+ {
+ state->readLen = -1;
+ return false;
+ }
if (targetPagePtr + XLOG_BLCKSZ <= flushptr)
count = XLOG_BLCKSZ; /* more than one block available */
@@ -854,7 +857,8 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
XLByteToSeg(targetPagePtr, segno, state->segcxt.ws_segsize);
CheckXLogRemoved(segno, state->seg.ws_tli);
- return count;
+ state->readLen = count;
+ return true;
}
/*
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index 59ebac7d6a..cf119848b0 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -47,7 +47,7 @@ typedef struct XLogPageReadPrivate
int tliIndex;
} XLogPageReadPrivate;
-static int SimpleXLogPageRead(XLogReaderState *xlogreader,
+static bool SimpleXLogPageRead(XLogReaderState *xlogreader,
XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
@@ -246,7 +246,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
}
/* XLogReader callback function, to read a WAL page */
-static int
+static bool
SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf)
{
@@ -306,7 +306,8 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
if (private->restoreCommand == NULL)
{
pg_log_error("could not open file \"%s\": %m", xlogfpath);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
/*
@@ -319,7 +320,10 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
private->restoreCommand);
if (xlogreadfd < 0)
- return -1;
+ {
+ xlogreader->readLen = -1;
+ return false;
+ }
else
pg_log_debug("using file \"%s\" restored from archive",
xlogfpath);
@@ -335,7 +339,8 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
if (lseek(xlogreadfd, (off_t) targetPageOff, SEEK_SET) < 0)
{
pg_log_error("could not seek in file \"%s\": %m", xlogfpath);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
@@ -348,13 +353,15 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
pg_log_error("could not read file \"%s\": read %d of %zu",
xlogfpath, r, (Size) XLOG_BLCKSZ);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
Assert(targetSegNo == xlogreadsegno);
xlogreader->seg.ws_tli = targetHistory[private->tliIndex].tli;
- return XLOG_BLCKSZ;
+ xlogreader->readLen = XLOG_BLCKSZ;
+ return true;
}
/*
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index 610f65e471..0f0e0723a3 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -331,7 +331,7 @@ WALDumpCloseSegment(XLogReaderState *state)
}
/* pg_waldump's XLogReaderRoutine->page_read callback */
-static int
+static bool
WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetPtr, char *readBuff)
{
@@ -348,7 +348,8 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
else
{
private->endptr_reached = true;
- return -1;
+ state->readLen = -1;
+ return false;
}
}
@@ -373,7 +374,8 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
(Size) errinfo.wre_req);
}
- return count;
+ state->readLen = count;
+ return true;
}
/*
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index 21d200d3df..23d9e70a59 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -57,8 +57,8 @@ typedef struct WALSegmentContext
typedef struct XLogReaderState XLogReaderState;
-/* Function type definitions for various xlogreader interactions */
-typedef int (*XLogPageReadCB) (XLogReaderState *xlogreader,
+/* Function type definition for the read_page callback */
+typedef bool (*XLogPageReadCB) (XLogReaderState *xlogreader,
XLogRecPtr targetPagePtr,
int reqLen,
XLogRecPtr targetRecPtr,
@@ -175,6 +175,20 @@ struct XLogReaderState
XLogRecPtr ReadRecPtr; /* start of last record read */
XLogRecPtr EndRecPtr; /* end+1 of last record read */
+ /* ----------------------------------------
+ * Communication with page reader
+ * readBuf is XLOG_BLCKSZ bytes, valid up to at least readLen bytes.
+ * ----------------------------------------
+ */
+ /* variables to communicate with page reader */
+ XLogRecPtr readPagePtr; /* page pointer to read */
+ int32 readLen; /* bytes requested to reader, or actual bytes
+ * read by reader, which must be larger than
+ * the request, or -1 on error */
+ TimeLineID readPageTLI; /* TLI for data currently in readBuf */
+ char *readBuf; /* buffer to store data */
+ bool page_verified; /* is the page on the buffer verified? */
+
/* ----------------------------------------
* Decoded representation of current record
@@ -203,13 +217,6 @@ struct XLogReaderState
* ----------------------------------------
*/
- /*
- * Buffer for currently read page (XLOG_BLCKSZ bytes, valid up to at least
- * readLen bytes)
- */
- char *readBuf;
- uint32 readLen;
-
/* last read XLOG position for data currently in readBuf */
WALSegmentContext segcxt;
WALOpenSegment seg;
diff --git a/src/include/access/xlogutils.h b/src/include/access/xlogutils.h
index 9ac602b674..364a21c4ea 100644
--- a/src/include/access/xlogutils.h
+++ b/src/include/access/xlogutils.h
@@ -47,7 +47,7 @@ extern Buffer XLogReadBufferExtended(RelFileNode rnode, ForkNumber forknum,
extern Relation CreateFakeRelcacheEntry(RelFileNode rnode);
extern void FreeFakeRelcacheEntry(Relation fakerel);
-extern int read_local_xlog_page(XLogReaderState *state,
+extern bool read_local_xlog_page(XLogReaderState *state,
XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *cur_page);
extern void wal_segment_open(XLogReaderState *state,
--
2.27.0
----Next_Part(Thu_Mar__4_11_28_53_2021_014)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
filename="v17-0002-Move-page-reader-out-of-XLogReadRecord.patch"
^ permalink raw reply [nested|flat] 69+ messages in thread
* [PATCH v15 1/4] Move callback-call from ReadPageInternal to XLogReadRecord.
@ 2019-09-05 11:21 Kyotaro Horiguchi <[email protected]>
0 siblings, 0 replies; 69+ messages in thread
From: Kyotaro Horiguchi @ 2019-09-05 11:21 UTC (permalink / raw)
The current WAL record reader reads page data using a call back
function. Although it is not so problematic alone, it would be a
problem if we are going to do add tasks like encryption which is
performed on page data before WAL reader reads them. To avoid that the
record reader facility has to have a new code path corresponds to
every new callback, this patch separates page reader from WAL record
reading facility by modifying the current WAL record reader to a state
machine.
As the first step of that change, this patch moves the page reader
function out of ReadPageInternal, then the remaining tasks of the
function are taken over by the new function XLogNeedData. As the
result XLogPageRead directly calls the page reader callback function
according to the feedback from XLogNeedData.
---
src/backend/access/transam/xlog.c | 16 +-
src/backend/access/transam/xlogreader.c | 281 ++++++++++++++----------
src/backend/access/transam/xlogutils.c | 12 +-
src/backend/replication/walsender.c | 10 +-
src/bin/pg_rewind/parsexlog.c | 21 +-
src/bin/pg_waldump/pg_waldump.c | 8 +-
src/include/access/xlogreader.h | 25 ++-
src/include/access/xlogutils.h | 2 +-
8 files changed, 222 insertions(+), 153 deletions(-)
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index fd93bcfaeb..e570e56a24 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -912,7 +912,7 @@ static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath,
static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli,
XLogSource source, bool notfoundOk);
static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, XLogSource source);
-static int XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
+static bool XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
bool fetching_ckpt, XLogRecPtr tliRecPtr);
@@ -4332,7 +4332,6 @@ ReadRecord(XLogReaderState *xlogreader, int emode,
XLogRecord *record;
XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
- /* Pass through parameters to XLogPageRead */
private->fetching_ckpt = fetching_ckpt;
private->emode = emode;
private->randAccess = (xlogreader->ReadRecPtr == InvalidXLogRecPtr);
@@ -11860,7 +11859,7 @@ CancelBackup(void)
* XLogPageRead() to try fetching the record from another source, or to
* sleep and retry.
*/
-static int
+static bool
XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *readBuf)
{
@@ -11919,7 +11918,8 @@ retry:
readLen = 0;
readSource = XLOG_FROM_ANY;
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
}
@@ -12014,7 +12014,8 @@ retry:
goto next_record_is_invalid;
}
- return readLen;
+ xlogreader->readLen = readLen;
+ return true;
next_record_is_invalid:
lastSourceFailed = true;
@@ -12028,8 +12029,9 @@ next_record_is_invalid:
/* In standby-mode, keep trying */
if (StandbyMode)
goto retry;
- else
- return -1;
+
+ xlogreader->readLen = -1;
+ return false;
}
/*
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index cb76be4f46..3d599325ee 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -36,8 +36,8 @@
static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
pg_attribute_printf(2, 3);
static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
-static int ReadPageInternal(XLogReaderState *state, XLogRecPtr pageptr,
- int reqLen);
+static bool XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr,
+ int reqLen, bool header_inclusive);
static void XLogReaderInvalReadState(XLogReaderState *state);
static bool ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
XLogRecPtr PrevRecPtr, XLogRecord *record, bool randAccess);
@@ -276,7 +276,6 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
uint32 targetRecOff;
uint32 pageHeaderSize;
bool gotheader;
- int readOff;
/*
* randAccess indicates whether to verify the previous-record pointer of
@@ -326,14 +325,20 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
* byte to cover the whole record header, or at least the part of it that
* fits on the same page.
*/
- readOff = ReadPageInternal(state, targetPagePtr,
- Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ));
- if (readOff < 0)
+ while (XLogNeedData(state, targetPagePtr,
+ Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+ targetRecOff != 0))
+ {
+ if (!state->routine.page_read(state, state->readPagePtr, state->readLen,
+ RecPtr, state->readBuf))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
/*
- * ReadPageInternal always returns at least the page header, so we can
- * examine it now.
+ * We have at least the page header, so we can examine it now.
*/
pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
if (targetRecOff == 0)
@@ -359,8 +364,8 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
goto err;
}
- /* ReadPageInternal has verified the page header */
- Assert(pageHeaderSize <= readOff);
+ /* XLogNeedData has verified the page header */
+ Assert(pageHeaderSize <= state->readLen);
/*
* Read the record length.
@@ -433,18 +438,27 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
do
{
+ int rest_len = total_len - gotlen;
+
/* Calculate pointer to beginning of next page */
targetPagePtr += XLOG_BLCKSZ;
/* Wait for the next page to become available */
- readOff = ReadPageInternal(state, targetPagePtr,
- Min(total_len - gotlen + SizeOfXLogShortPHD,
- XLOG_BLCKSZ));
+ while (XLogNeedData(state, targetPagePtr,
+ Min(rest_len, XLOG_BLCKSZ),
+ false))
+ {
+ if (!state->routine.page_read(state, state->readPagePtr,
+ state->readLen,
+ state->ReadRecPtr,
+ state->readBuf))
+ break;
+ }
- if (readOff < 0)
+ if (!state->page_verified)
goto err;
- Assert(SizeOfXLogShortPHD <= readOff);
+ Assert(SizeOfXLogShortPHD <= state->readLen);
/* Check that the continuation on next page looks valid */
pageHeader = (XLogPageHeader) state->readBuf;
@@ -473,21 +487,14 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
/* Append the continuation from this page to the buffer */
pageHeaderSize = XLogPageHeaderSize(pageHeader);
- if (readOff < pageHeaderSize)
- readOff = ReadPageInternal(state, targetPagePtr,
- pageHeaderSize);
-
- Assert(pageHeaderSize <= readOff);
+ Assert(pageHeaderSize <= state->readLen);
contdata = (char *) state->readBuf + pageHeaderSize;
len = XLOG_BLCKSZ - pageHeaderSize;
if (pageHeader->xlp_rem_len < len)
len = pageHeader->xlp_rem_len;
- if (readOff < pageHeaderSize + len)
- readOff = ReadPageInternal(state, targetPagePtr,
- pageHeaderSize + len);
-
+ Assert (pageHeaderSize + len <= state->readLen);
memcpy(buffer, (char *) contdata, len);
buffer += len;
gotlen += len;
@@ -517,9 +524,16 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
else
{
/* Wait for the record data to become available */
- readOff = ReadPageInternal(state, targetPagePtr,
- Min(targetRecOff + total_len, XLOG_BLCKSZ));
- if (readOff < 0)
+ while (XLogNeedData(state, targetPagePtr,
+ Min(targetRecOff + total_len, XLOG_BLCKSZ), true))
+ {
+ if (!state->routine.page_read(state, state->readPagePtr,
+ state->readLen,
+ state->ReadRecPtr, state->readBuf))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
/* Record does not cross a page boundary */
@@ -562,109 +576,138 @@ err:
}
/*
- * Read a single xlog page including at least [pageptr, reqLen] of valid data
- * via the page_read() callback.
+ * Checks that an xlog page loaded in state->readBuf is including at least
+ * [pageptr, reqLen] and the page is valid. header_inclusive indicates that
+ * reqLen is calculated including page header length.
*
- * Returns -1 if the required page cannot be read for some reason; errormsg_buf
- * is set in that case (unless the error occurs in the page_read callback).
+ * Returns false if the buffer already contains the requested data, or found
+ * error. state->page_verified is set to true for the former and false for the
+ * latter.
*
- * We fetch the page from a reader-local cache if we know we have the required
- * data and if there hasn't been any error since caching the data.
+ * Otherwise returns true and requests data loaded onto state->readBuf by
+ * state->readPagePtr and state->readLen. The caller shall call this function
+ * again after filling the buffer at least with that portion of data and set
+ * state->readLen to the length of actually loaded data.
+ *
+ * If header_inclusive is false, corrects reqLen internally by adding the
+ * actual page header length and may request caller for new data.
*/
-static int
-ReadPageInternal(XLogReaderState *state, XLogRecPtr pageptr, int reqLen)
+static bool
+XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
+ bool header_inclusive)
{
- int readLen;
uint32 targetPageOff;
XLogSegNo targetSegNo;
- XLogPageHeader hdr;
+ uint32 addLen = 0;
- Assert((pageptr % XLOG_BLCKSZ) == 0);
+ /* Some data is loaded, but page header is not verified yet. */
+ if (!state->page_verified &&
+ !XLogRecPtrIsInvalid(state->readPagePtr) && state->readLen >= 0)
+ {
+ uint32 pageHeaderSize;
+ /* just loaded new data so needs to verify page header */
+
+ /* The caller must have loaded at least page header */
+ Assert (state->readLen >= SizeOfXLogShortPHD);
+
+ /*
+ * We have enough data to check the header length. Recheck the loaded
+ * length against the actual header length.
+ */
+ pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+
+ /* Request more data if we don't have the full header. */
+ if (state->readLen < pageHeaderSize)
+ {
+ state->readLen = pageHeaderSize;
+ return true;
+ }
+
+ /* Now that we know we have the full header, validate it. */
+ if (!XLogReaderValidatePageHeader(state, state->readPagePtr,
+ (char *) state->readBuf))
+ {
+ /* That's bad. Force reading the page again. */
+ XLogReaderInvalReadState(state);
+
+ return false;
+ }
+
+ state->page_verified = true;
+
+ XLByteToSeg(state->readPagePtr, state->seg.ws_segno,
+ state->segcxt.ws_segsize);
+ }
+
+ /*
+ * The loaded page may not be the one caller is supposing to read when we
+ * are verifying the first page of new segment. In that case, skip further
+ * verification and immediately load the target page.
+ */
+ if (state->page_verified && pageptr == state->readPagePtr)
+ {
+ /*
+ * calculate additional length for page header keeping the total
+ * length within the block size.
+ */
+ if (!header_inclusive)
+ {
+ uint32 pageHeaderSize =
+ XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+
+ addLen = pageHeaderSize;
+ if (reqLen + pageHeaderSize <= XLOG_BLCKSZ)
+ addLen = pageHeaderSize;
+ else
+ addLen = XLOG_BLCKSZ - reqLen;
+
+ Assert(addLen >= 0);
+ }
+
+ /* Return if we already have it. */
+ if (reqLen + addLen <= state->readLen)
+ return false;
+ }
+
+ /* Data is not in our buffer, request the caller for it. */
XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize);
targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize);
+ Assert((pageptr % XLOG_BLCKSZ) == 0);
- /* check whether we have all the requested data already */
- if (targetSegNo == state->seg.ws_segno &&
- targetPageOff == state->segoff && reqLen <= state->readLen)
- return state->readLen;
+ /*
+ * Every time we request to load new data of a page to the caller, even if
+ * we looked at a part of it before, we need to do verification on the next
+ * invocation as the caller might now be rereading data from a different
+ * source.
+ */
+ state->page_verified = false;
/*
- * Data is not in our buffer.
- *
- * Every time we actually read the segment, even if we looked at parts of
- * it before, we need to do verification as the page_read callback might
- * now be rereading data from a different source.
- *
* Whenever switching to a new WAL segment, we read the first page of the
* file and validate its header, even if that's not where the target
* record is. This is so that we can check the additional identification
* info that is present in the first page's "long" header.
+ * Don't do this if the caller requested the first page in the segment.
*/
if (targetSegNo != state->seg.ws_segno && targetPageOff != 0)
{
- XLogRecPtr targetSegmentPtr = pageptr - targetPageOff;
-
- readLen = state->routine.page_read(state, targetSegmentPtr, XLOG_BLCKSZ,
- state->currRecPtr,
- state->readBuf);
- if (readLen < 0)
- goto err;
-
- /* we can be sure to have enough WAL available, we scrolled back */
- Assert(readLen == XLOG_BLCKSZ);
-
- if (!XLogReaderValidatePageHeader(state, targetSegmentPtr,
- state->readBuf))
- goto err;
+ /*
+ * Then we'll see that the targetSegNo now matches the ws_segno, and
+ * will not come back here, but will request the actual target page.
+ */
+ state->readPagePtr = pageptr - targetPageOff;
+ state->readLen = XLOG_BLCKSZ;
+ return true;
}
/*
- * First, read the requested data length, but at least a short page header
- * so that we can validate it.
+ * Request the caller to load the page. We need at least a short page
+ * header so that we can validate it.
*/
- readLen = state->routine.page_read(state, pageptr, Max(reqLen, SizeOfXLogShortPHD),
- state->currRecPtr,
- state->readBuf);
- if (readLen < 0)
- goto err;
-
- Assert(readLen <= XLOG_BLCKSZ);
-
- /* Do we have enough data to check the header length? */
- if (readLen <= SizeOfXLogShortPHD)
- goto err;
-
- Assert(readLen >= reqLen);
-
- hdr = (XLogPageHeader) state->readBuf;
-
- /* still not enough */
- if (readLen < XLogPageHeaderSize(hdr))
- {
- readLen = state->routine.page_read(state, pageptr, XLogPageHeaderSize(hdr),
- state->currRecPtr,
- state->readBuf);
- if (readLen < 0)
- goto err;
- }
-
- /*
- * Now that we know we have the full header, validate it.
- */
- if (!XLogReaderValidatePageHeader(state, pageptr, (char *) hdr))
- goto err;
-
- /* update read state information */
- state->seg.ws_segno = targetSegNo;
- state->segoff = targetPageOff;
- state->readLen = readLen;
-
- return readLen;
-
-err:
- XLogReaderInvalReadState(state);
- return -1;
+ state->readPagePtr = pageptr;
+ state->readLen = Max(reqLen + addLen, SizeOfXLogShortPHD);
+ return true;
}
/*
@@ -673,9 +716,7 @@ err:
static void
XLogReaderInvalReadState(XLogReaderState *state)
{
- state->seg.ws_segno = 0;
- state->segoff = 0;
- state->readLen = 0;
+ state->readPagePtr = InvalidXLogRecPtr;
}
/*
@@ -956,7 +997,6 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
XLogRecPtr targetPagePtr;
int targetRecOff;
uint32 pageHeaderSize;
- int readLen;
/*
* Compute targetRecOff. It should typically be equal or greater than
@@ -964,7 +1004,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
* that, except when caller has explicitly specified the offset that
* falls somewhere there or when we are skipping multi-page
* continuation record. It doesn't matter though because
- * ReadPageInternal() is prepared to handle that and will read at
+ * XLogNeedData() is prepared to handle that and will read at
* least short page-header worth of data
*/
targetRecOff = tmpRecPtr % XLOG_BLCKSZ;
@@ -972,19 +1012,24 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
/* scroll back to page boundary */
targetPagePtr = tmpRecPtr - targetRecOff;
- /* Read the page containing the record */
- readLen = ReadPageInternal(state, targetPagePtr, targetRecOff);
- if (readLen < 0)
+ while(XLogNeedData(state, targetPagePtr, targetRecOff,
+ targetRecOff != 0))
+ {
+ if (!state->routine.page_read(state, state->readPagePtr,
+ state->readLen,
+ state->ReadRecPtr, state->readBuf))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
header = (XLogPageHeader) state->readBuf;
pageHeaderSize = XLogPageHeaderSize(header);
- /* make sure we have enough data for the page header */
- readLen = ReadPageInternal(state, targetPagePtr, pageHeaderSize);
- if (readLen < 0)
- goto err;
+ /* we should have read the page header */
+ Assert (state->readLen >= pageHeaderSize);
/* skip over potential continuation data */
if (header->xlp_info & XLP_FIRST_IS_CONTRECORD)
diff --git a/src/backend/access/transam/xlogutils.c b/src/backend/access/transam/xlogutils.c
index 322b0e8ff5..1b10b6df20 100644
--- a/src/backend/access/transam/xlogutils.c
+++ b/src/backend/access/transam/xlogutils.c
@@ -689,8 +689,8 @@ XLogTruncateRelation(RelFileNode rnode, ForkNumber forkNum,
void
XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wantLength)
{
- const XLogRecPtr lastReadPage = (state->seg.ws_segno *
- state->segcxt.ws_segsize + state->segoff);
+ const XLogRecPtr lastReadPage = state->seg.ws_segno *
+ state->segcxt.ws_segsize + state->readLen;
Assert(wantPage != InvalidXLogRecPtr && wantPage % XLOG_BLCKSZ == 0);
Assert(wantLength <= XLOG_BLCKSZ);
@@ -828,7 +828,7 @@ wal_segment_close(XLogReaderState *state)
* exists for normal backends, so we have to do a check/sleep/repeat style of
* loop for now.
*/
-int
+bool
read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *cur_page)
{
@@ -930,7 +930,8 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
else if (targetPagePtr + reqLen > read_upto)
{
/* not enough data there */
- return -1;
+ state->readLen = -1;
+ return false;
}
else
{
@@ -948,7 +949,8 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
WALReadRaiseError(&errinfo);
/* number of valid bytes in the buffer */
- return count;
+ state->readLen = count;
+ return true;
}
/*
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index e2477c47e0..bf2711bddd 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -811,7 +811,7 @@ StartReplication(StartReplicationCmd *cmd)
* which has to do a plain sleep/busy loop, because the walsender's latch gets
* set every time WAL is flushed.
*/
-static int
+static bool
logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *cur_page)
{
@@ -831,7 +831,10 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
/* fail if not (implies we are going to shut down) */
if (flushptr < targetPagePtr + reqLen)
- return -1;
+ {
+ state->readLen = -1;
+ return false;
+ }
if (targetPagePtr + XLOG_BLCKSZ <= flushptr)
count = XLOG_BLCKSZ; /* more than one block available */
@@ -859,7 +862,8 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
XLByteToSeg(targetPagePtr, segno, state->segcxt.ws_segsize);
CheckXLogRemoved(segno, state->seg.ws_tli);
- return count;
+ state->readLen = count;
+ return true;
}
/*
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index bc6f976994..0b7e73ae79 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -46,7 +46,7 @@ typedef struct XLogPageReadPrivate
int tliIndex;
} XLogPageReadPrivate;
-static int SimpleXLogPageRead(XLogReaderState *xlogreader,
+static bool SimpleXLogPageRead(XLogReaderState *xlogreader,
XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
@@ -236,7 +236,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
}
/* XLogReader callback function, to read a WAL page */
-static int
+static bool
SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf)
{
@@ -296,7 +296,8 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
if (private->restoreCommand == NULL)
{
pg_log_error("could not open file \"%s\": %m", xlogfpath);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
/*
@@ -309,7 +310,10 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
private->restoreCommand);
if (xlogreadfd < 0)
- return -1;
+ {
+ xlogreader->readLen = -1;
+ return false;
+ }
else
pg_log_debug("using file \"%s\" restored from archive",
xlogfpath);
@@ -325,7 +329,8 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
if (lseek(xlogreadfd, (off_t) targetPageOff, SEEK_SET) < 0)
{
pg_log_error("could not seek in file \"%s\": %m", xlogfpath);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
@@ -338,13 +343,15 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
pg_log_error("could not read file \"%s\": read %d of %zu",
xlogfpath, r, (Size) XLOG_BLCKSZ);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
Assert(targetSegNo == xlogreadsegno);
xlogreader->seg.ws_tli = targetHistory[private->tliIndex].tli;
- return XLOG_BLCKSZ;
+ xlogreader->readLen = XLOG_BLCKSZ;
+ return true;
}
/*
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index d1a0678935..ab0c28d9b7 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -331,7 +331,7 @@ WALDumpCloseSegment(XLogReaderState *state)
}
/* pg_waldump's XLogReaderRoutine->page_read callback */
-static int
+static bool
WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetPtr, char *readBuff)
{
@@ -348,7 +348,8 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
else
{
private->endptr_reached = true;
- return -1;
+ state->readLen = -1;
+ return false;
}
}
@@ -373,7 +374,8 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
(Size) errinfo.wre_req);
}
- return count;
+ state->readLen = count;
+ return true;
}
/*
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index b0f2a6ed43..6c8848e14f 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -57,8 +57,8 @@ typedef struct WALSegmentContext
typedef struct XLogReaderState XLogReaderState;
-/* Function type definitions for various xlogreader interactions */
-typedef int (*XLogPageReadCB) (XLogReaderState *xlogreader,
+/* Function type definition for the read_page callback */
+typedef bool (*XLogPageReadCB) (XLogReaderState *xlogreader,
XLogRecPtr targetPagePtr,
int reqLen,
XLogRecPtr targetRecPtr,
@@ -175,6 +175,20 @@ struct XLogReaderState
XLogRecPtr ReadRecPtr; /* start of last record read */
XLogRecPtr EndRecPtr; /* end+1 of last record read */
+ /* ----------------------------------------
+ * Communication with page reader
+ * readBuf is XLOG_BLCKSZ bytes, valid up to at least readLen bytes.
+ * ----------------------------------------
+ */
+ /* variables to communicate with page reader */
+ XLogRecPtr readPagePtr; /* page pointer to read */
+ int32 readLen; /* bytes requested to reader, or actual bytes
+ * read by reader, which must be larger than
+ * the request, or -1 on error */
+ TimeLineID readPageTLI; /* TLI for data currently in readBuf */
+ char *readBuf; /* buffer to store data */
+ bool page_verified; /* is the page on the buffer verified? */
+
/* ----------------------------------------
* Decoded representation of current record
@@ -201,13 +215,6 @@ struct XLogReaderState
* ----------------------------------------
*/
- /*
- * Buffer for currently read page (XLOG_BLCKSZ bytes, valid up to at least
- * readLen bytes)
- */
- char *readBuf;
- uint32 readLen;
-
/* last read XLOG position for data currently in readBuf */
WALSegmentContext segcxt;
WALOpenSegment seg;
diff --git a/src/include/access/xlogutils.h b/src/include/access/xlogutils.h
index e59b6cf3a9..c287edf206 100644
--- a/src/include/access/xlogutils.h
+++ b/src/include/access/xlogutils.h
@@ -47,7 +47,7 @@ extern Buffer XLogReadBufferExtended(RelFileNode rnode, ForkNumber forknum,
extern Relation CreateFakeRelcacheEntry(RelFileNode rnode);
extern void FreeFakeRelcacheEntry(Relation fakerel);
-extern int read_local_xlog_page(XLogReaderState *state,
+extern bool read_local_xlog_page(XLogReaderState *state,
XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *cur_page);
extern void wal_segment_open(XLogReaderState *state,
--
2.18.4
----Next_Part(Thu_Jul__2_13_53_30_2020_072)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
filename="v15-0002-Move-page-reader-out-of-XLogReadRecord.patch"
^ permalink raw reply [nested|flat] 69+ messages in thread
* [PATCH v17 1/4] Move callback-call from ReadPageInternal to XLogReadRecord.
@ 2019-09-05 11:21 Kyotaro Horiguchi <[email protected]>
0 siblings, 0 replies; 69+ messages in thread
From: Kyotaro Horiguchi @ 2019-09-05 11:21 UTC (permalink / raw)
The current WAL record reader reads page data using a call back
function. Although it is not so problematic alone, it would be a
problem if we are going to do add tasks like encryption which is
performed on page data before WAL reader reads them. To avoid that the
record reader facility has to have a new code path corresponds to
every new callback, this patch separates page reader from WAL record
reading facility by modifying the current WAL record reader to a state
machine.
As the first step of that change, this patch moves the page reader
function out of ReadPageInternal, then the remaining tasks of the
function are taken over by the new function XLogNeedData. As the
result XLogPageRead directly calls the page reader callback function
according to the feedback from XLogNeedData.
---
src/backend/access/transam/xlog.c | 16 +-
src/backend/access/transam/xlogreader.c | 281 ++++++++++++++----------
src/backend/access/transam/xlogutils.c | 12 +-
src/backend/replication/walsender.c | 10 +-
src/bin/pg_rewind/parsexlog.c | 21 +-
src/bin/pg_waldump/pg_waldump.c | 8 +-
src/include/access/xlogreader.h | 25 ++-
src/include/access/xlogutils.h | 2 +-
8 files changed, 222 insertions(+), 153 deletions(-)
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 377afb8732..606328a65a 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -917,7 +917,7 @@ static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath,
static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli,
XLogSource source, bool notfoundOk);
static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, XLogSource source);
-static int XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
+static bool XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
bool fetching_ckpt, XLogRecPtr tliRecPtr);
@@ -4350,7 +4350,6 @@ ReadRecord(XLogReaderState *xlogreader, int emode,
XLogRecord *record;
XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
- /* Pass through parameters to XLogPageRead */
private->fetching_ckpt = fetching_ckpt;
private->emode = emode;
private->randAccess = (xlogreader->ReadRecPtr == InvalidXLogRecPtr);
@@ -11987,7 +11986,7 @@ CancelBackup(void)
* XLogPageRead() to try fetching the record from another source, or to
* sleep and retry.
*/
-static int
+static bool
XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *readBuf)
{
@@ -12046,7 +12045,8 @@ retry:
readLen = 0;
readSource = XLOG_FROM_ANY;
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
}
@@ -12141,7 +12141,8 @@ retry:
goto next_record_is_invalid;
}
- return readLen;
+ xlogreader->readLen = readLen;
+ return true;
next_record_is_invalid:
lastSourceFailed = true;
@@ -12155,8 +12156,9 @@ next_record_is_invalid:
/* In standby-mode, keep trying */
if (StandbyMode)
goto retry;
- else
- return -1;
+
+ xlogreader->readLen = -1;
+ return false;
}
/*
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index 42738eb940..7d04995b7b 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -36,8 +36,8 @@
static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
pg_attribute_printf(2, 3);
static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
-static int ReadPageInternal(XLogReaderState *state, XLogRecPtr pageptr,
- int reqLen);
+static bool XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr,
+ int reqLen, bool header_inclusive);
static void XLogReaderInvalReadState(XLogReaderState *state);
static bool ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
XLogRecPtr PrevRecPtr, XLogRecord *record, bool randAccess);
@@ -276,7 +276,6 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
uint32 targetRecOff;
uint32 pageHeaderSize;
bool gotheader;
- int readOff;
/*
* randAccess indicates whether to verify the previous-record pointer of
@@ -326,14 +325,20 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
* byte to cover the whole record header, or at least the part of it that
* fits on the same page.
*/
- readOff = ReadPageInternal(state, targetPagePtr,
- Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ));
- if (readOff < 0)
+ while (XLogNeedData(state, targetPagePtr,
+ Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+ targetRecOff != 0))
+ {
+ if (!state->routine.page_read(state, state->readPagePtr, state->readLen,
+ RecPtr, state->readBuf))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
/*
- * ReadPageInternal always returns at least the page header, so we can
- * examine it now.
+ * We have at least the page header, so we can examine it now.
*/
pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
if (targetRecOff == 0)
@@ -359,8 +364,8 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
goto err;
}
- /* ReadPageInternal has verified the page header */
- Assert(pageHeaderSize <= readOff);
+ /* XLogNeedData has verified the page header */
+ Assert(pageHeaderSize <= state->readLen);
/*
* Read the record length.
@@ -432,18 +437,27 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
do
{
+ int rest_len = total_len - gotlen;
+
/* Calculate pointer to beginning of next page */
targetPagePtr += XLOG_BLCKSZ;
/* Wait for the next page to become available */
- readOff = ReadPageInternal(state, targetPagePtr,
- Min(total_len - gotlen + SizeOfXLogShortPHD,
- XLOG_BLCKSZ));
+ while (XLogNeedData(state, targetPagePtr,
+ Min(rest_len, XLOG_BLCKSZ),
+ false))
+ {
+ if (!state->routine.page_read(state, state->readPagePtr,
+ state->readLen,
+ state->ReadRecPtr,
+ state->readBuf))
+ break;
+ }
- if (readOff < 0)
+ if (!state->page_verified)
goto err;
- Assert(SizeOfXLogShortPHD <= readOff);
+ Assert(SizeOfXLogShortPHD <= state->readLen);
/* Check that the continuation on next page looks valid */
pageHeader = (XLogPageHeader) state->readBuf;
@@ -473,21 +487,14 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
/* Append the continuation from this page to the buffer */
pageHeaderSize = XLogPageHeaderSize(pageHeader);
- if (readOff < pageHeaderSize)
- readOff = ReadPageInternal(state, targetPagePtr,
- pageHeaderSize);
-
- Assert(pageHeaderSize <= readOff);
+ Assert(pageHeaderSize <= state->readLen);
contdata = (char *) state->readBuf + pageHeaderSize;
len = XLOG_BLCKSZ - pageHeaderSize;
if (pageHeader->xlp_rem_len < len)
len = pageHeader->xlp_rem_len;
- if (readOff < pageHeaderSize + len)
- readOff = ReadPageInternal(state, targetPagePtr,
- pageHeaderSize + len);
-
+ Assert (pageHeaderSize + len <= state->readLen);
memcpy(buffer, (char *) contdata, len);
buffer += len;
gotlen += len;
@@ -517,9 +524,16 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
else
{
/* Wait for the record data to become available */
- readOff = ReadPageInternal(state, targetPagePtr,
- Min(targetRecOff + total_len, XLOG_BLCKSZ));
- if (readOff < 0)
+ while (XLogNeedData(state, targetPagePtr,
+ Min(targetRecOff + total_len, XLOG_BLCKSZ), true))
+ {
+ if (!state->routine.page_read(state, state->readPagePtr,
+ state->readLen,
+ state->ReadRecPtr, state->readBuf))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
/* Record does not cross a page boundary */
@@ -562,109 +576,138 @@ err:
}
/*
- * Read a single xlog page including at least [pageptr, reqLen] of valid data
- * via the page_read() callback.
+ * Checks that an xlog page loaded in state->readBuf is including at least
+ * [pageptr, reqLen] and the page is valid. header_inclusive indicates that
+ * reqLen is calculated including page header length.
*
- * Returns -1 if the required page cannot be read for some reason; errormsg_buf
- * is set in that case (unless the error occurs in the page_read callback).
+ * Returns false if the buffer already contains the requested data, or found
+ * error. state->page_verified is set to true for the former and false for the
+ * latter.
*
- * We fetch the page from a reader-local cache if we know we have the required
- * data and if there hasn't been any error since caching the data.
+ * Otherwise returns true and requests data loaded onto state->readBuf by
+ * state->readPagePtr and state->readLen. The caller shall call this function
+ * again after filling the buffer at least with that portion of data and set
+ * state->readLen to the length of actually loaded data.
+ *
+ * If header_inclusive is false, corrects reqLen internally by adding the
+ * actual page header length and may request caller for new data.
*/
-static int
-ReadPageInternal(XLogReaderState *state, XLogRecPtr pageptr, int reqLen)
+static bool
+XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
+ bool header_inclusive)
{
- int readLen;
uint32 targetPageOff;
XLogSegNo targetSegNo;
- XLogPageHeader hdr;
+ uint32 addLen = 0;
- Assert((pageptr % XLOG_BLCKSZ) == 0);
+ /* Some data is loaded, but page header is not verified yet. */
+ if (!state->page_verified &&
+ !XLogRecPtrIsInvalid(state->readPagePtr) && state->readLen >= 0)
+ {
+ uint32 pageHeaderSize;
+ /* just loaded new data so needs to verify page header */
+
+ /* The caller must have loaded at least page header */
+ Assert (state->readLen >= SizeOfXLogShortPHD);
+
+ /*
+ * We have enough data to check the header length. Recheck the loaded
+ * length against the actual header length.
+ */
+ pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+
+ /* Request more data if we don't have the full header. */
+ if (state->readLen < pageHeaderSize)
+ {
+ state->readLen = pageHeaderSize;
+ return true;
+ }
+
+ /* Now that we know we have the full header, validate it. */
+ if (!XLogReaderValidatePageHeader(state, state->readPagePtr,
+ (char *) state->readBuf))
+ {
+ /* That's bad. Force reading the page again. */
+ XLogReaderInvalReadState(state);
+
+ return false;
+ }
+
+ state->page_verified = true;
+
+ XLByteToSeg(state->readPagePtr, state->seg.ws_segno,
+ state->segcxt.ws_segsize);
+ }
+
+ /*
+ * The loaded page may not be the one caller is supposing to read when we
+ * are verifying the first page of new segment. In that case, skip further
+ * verification and immediately load the target page.
+ */
+ if (state->page_verified && pageptr == state->readPagePtr)
+ {
+ /*
+ * calculate additional length for page header keeping the total
+ * length within the block size.
+ */
+ if (!header_inclusive)
+ {
+ uint32 pageHeaderSize =
+ XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+
+ addLen = pageHeaderSize;
+ if (reqLen + pageHeaderSize <= XLOG_BLCKSZ)
+ addLen = pageHeaderSize;
+ else
+ addLen = XLOG_BLCKSZ - reqLen;
+
+ Assert(addLen >= 0);
+ }
+
+ /* Return if we already have it. */
+ if (reqLen + addLen <= state->readLen)
+ return false;
+ }
+
+ /* Data is not in our buffer, request the caller for it. */
XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize);
targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize);
+ Assert((pageptr % XLOG_BLCKSZ) == 0);
- /* check whether we have all the requested data already */
- if (targetSegNo == state->seg.ws_segno &&
- targetPageOff == state->segoff && reqLen <= state->readLen)
- return state->readLen;
+ /*
+ * Every time we request to load new data of a page to the caller, even if
+ * we looked at a part of it before, we need to do verification on the next
+ * invocation as the caller might now be rereading data from a different
+ * source.
+ */
+ state->page_verified = false;
/*
- * Data is not in our buffer.
- *
- * Every time we actually read the segment, even if we looked at parts of
- * it before, we need to do verification as the page_read callback might
- * now be rereading data from a different source.
- *
* Whenever switching to a new WAL segment, we read the first page of the
* file and validate its header, even if that's not where the target
* record is. This is so that we can check the additional identification
* info that is present in the first page's "long" header.
+ * Don't do this if the caller requested the first page in the segment.
*/
if (targetSegNo != state->seg.ws_segno && targetPageOff != 0)
{
- XLogRecPtr targetSegmentPtr = pageptr - targetPageOff;
-
- readLen = state->routine.page_read(state, targetSegmentPtr, XLOG_BLCKSZ,
- state->currRecPtr,
- state->readBuf);
- if (readLen < 0)
- goto err;
-
- /* we can be sure to have enough WAL available, we scrolled back */
- Assert(readLen == XLOG_BLCKSZ);
-
- if (!XLogReaderValidatePageHeader(state, targetSegmentPtr,
- state->readBuf))
- goto err;
+ /*
+ * Then we'll see that the targetSegNo now matches the ws_segno, and
+ * will not come back here, but will request the actual target page.
+ */
+ state->readPagePtr = pageptr - targetPageOff;
+ state->readLen = XLOG_BLCKSZ;
+ return true;
}
/*
- * First, read the requested data length, but at least a short page header
- * so that we can validate it.
+ * Request the caller to load the page. We need at least a short page
+ * header so that we can validate it.
*/
- readLen = state->routine.page_read(state, pageptr, Max(reqLen, SizeOfXLogShortPHD),
- state->currRecPtr,
- state->readBuf);
- if (readLen < 0)
- goto err;
-
- Assert(readLen <= XLOG_BLCKSZ);
-
- /* Do we have enough data to check the header length? */
- if (readLen <= SizeOfXLogShortPHD)
- goto err;
-
- Assert(readLen >= reqLen);
-
- hdr = (XLogPageHeader) state->readBuf;
-
- /* still not enough */
- if (readLen < XLogPageHeaderSize(hdr))
- {
- readLen = state->routine.page_read(state, pageptr, XLogPageHeaderSize(hdr),
- state->currRecPtr,
- state->readBuf);
- if (readLen < 0)
- goto err;
- }
-
- /*
- * Now that we know we have the full header, validate it.
- */
- if (!XLogReaderValidatePageHeader(state, pageptr, (char *) hdr))
- goto err;
-
- /* update read state information */
- state->seg.ws_segno = targetSegNo;
- state->segoff = targetPageOff;
- state->readLen = readLen;
-
- return readLen;
-
-err:
- XLogReaderInvalReadState(state);
- return -1;
+ state->readPagePtr = pageptr;
+ state->readLen = Max(reqLen + addLen, SizeOfXLogShortPHD);
+ return true;
}
/*
@@ -673,9 +716,7 @@ err:
static void
XLogReaderInvalReadState(XLogReaderState *state)
{
- state->seg.ws_segno = 0;
- state->segoff = 0;
- state->readLen = 0;
+ state->readPagePtr = InvalidXLogRecPtr;
}
/*
@@ -953,7 +994,6 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
XLogRecPtr targetPagePtr;
int targetRecOff;
uint32 pageHeaderSize;
- int readLen;
/*
* Compute targetRecOff. It should typically be equal or greater than
@@ -961,7 +1001,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
* that, except when caller has explicitly specified the offset that
* falls somewhere there or when we are skipping multi-page
* continuation record. It doesn't matter though because
- * ReadPageInternal() is prepared to handle that and will read at
+ * XLogNeedData() is prepared to handle that and will read at
* least short page-header worth of data
*/
targetRecOff = tmpRecPtr % XLOG_BLCKSZ;
@@ -969,19 +1009,24 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
/* scroll back to page boundary */
targetPagePtr = tmpRecPtr - targetRecOff;
- /* Read the page containing the record */
- readLen = ReadPageInternal(state, targetPagePtr, targetRecOff);
- if (readLen < 0)
+ while(XLogNeedData(state, targetPagePtr, targetRecOff,
+ targetRecOff != 0))
+ {
+ if (!state->routine.page_read(state, state->readPagePtr,
+ state->readLen,
+ state->ReadRecPtr, state->readBuf))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
header = (XLogPageHeader) state->readBuf;
pageHeaderSize = XLogPageHeaderSize(header);
- /* make sure we have enough data for the page header */
- readLen = ReadPageInternal(state, targetPagePtr, pageHeaderSize);
- if (readLen < 0)
- goto err;
+ /* we should have read the page header */
+ Assert (state->readLen >= pageHeaderSize);
/* skip over potential continuation data */
if (header->xlp_info & XLP_FIRST_IS_CONTRECORD)
diff --git a/src/backend/access/transam/xlogutils.c b/src/backend/access/transam/xlogutils.c
index d17d660f46..46eda33f25 100644
--- a/src/backend/access/transam/xlogutils.c
+++ b/src/backend/access/transam/xlogutils.c
@@ -686,8 +686,8 @@ XLogTruncateRelation(RelFileNode rnode, ForkNumber forkNum,
void
XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wantLength)
{
- const XLogRecPtr lastReadPage = (state->seg.ws_segno *
- state->segcxt.ws_segsize + state->segoff);
+ const XLogRecPtr lastReadPage = state->seg.ws_segno *
+ state->segcxt.ws_segsize + state->readLen;
Assert(wantPage != InvalidXLogRecPtr && wantPage % XLOG_BLCKSZ == 0);
Assert(wantLength <= XLOG_BLCKSZ);
@@ -824,7 +824,7 @@ wal_segment_close(XLogReaderState *state)
* exists for normal backends, so we have to do a check/sleep/repeat style of
* loop for now.
*/
-int
+bool
read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *cur_page)
{
@@ -926,7 +926,8 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
else if (targetPagePtr + reqLen > read_upto)
{
/* not enough data there */
- return -1;
+ state->readLen = -1;
+ return false;
}
else
{
@@ -944,7 +945,8 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
WALReadRaiseError(&errinfo);
/* number of valid bytes in the buffer */
- return count;
+ state->readLen = count;
+ return true;
}
/*
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index 23baa4498a..c36b91ae5e 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -806,7 +806,7 @@ StartReplication(StartReplicationCmd *cmd)
* which has to do a plain sleep/busy loop, because the walsender's latch gets
* set every time WAL is flushed.
*/
-static int
+static bool
logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *cur_page)
{
@@ -826,7 +826,10 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
/* fail if not (implies we are going to shut down) */
if (flushptr < targetPagePtr + reqLen)
- return -1;
+ {
+ state->readLen = -1;
+ return false;
+ }
if (targetPagePtr + XLOG_BLCKSZ <= flushptr)
count = XLOG_BLCKSZ; /* more than one block available */
@@ -854,7 +857,8 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
XLByteToSeg(targetPagePtr, segno, state->segcxt.ws_segsize);
CheckXLogRemoved(segno, state->seg.ws_tli);
- return count;
+ state->readLen = count;
+ return true;
}
/*
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index 59ebac7d6a..cf119848b0 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -47,7 +47,7 @@ typedef struct XLogPageReadPrivate
int tliIndex;
} XLogPageReadPrivate;
-static int SimpleXLogPageRead(XLogReaderState *xlogreader,
+static bool SimpleXLogPageRead(XLogReaderState *xlogreader,
XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
@@ -246,7 +246,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
}
/* XLogReader callback function, to read a WAL page */
-static int
+static bool
SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf)
{
@@ -306,7 +306,8 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
if (private->restoreCommand == NULL)
{
pg_log_error("could not open file \"%s\": %m", xlogfpath);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
/*
@@ -319,7 +320,10 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
private->restoreCommand);
if (xlogreadfd < 0)
- return -1;
+ {
+ xlogreader->readLen = -1;
+ return false;
+ }
else
pg_log_debug("using file \"%s\" restored from archive",
xlogfpath);
@@ -335,7 +339,8 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
if (lseek(xlogreadfd, (off_t) targetPageOff, SEEK_SET) < 0)
{
pg_log_error("could not seek in file \"%s\": %m", xlogfpath);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
@@ -348,13 +353,15 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
pg_log_error("could not read file \"%s\": read %d of %zu",
xlogfpath, r, (Size) XLOG_BLCKSZ);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
Assert(targetSegNo == xlogreadsegno);
xlogreader->seg.ws_tli = targetHistory[private->tliIndex].tli;
- return XLOG_BLCKSZ;
+ xlogreader->readLen = XLOG_BLCKSZ;
+ return true;
}
/*
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index 610f65e471..0f0e0723a3 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -331,7 +331,7 @@ WALDumpCloseSegment(XLogReaderState *state)
}
/* pg_waldump's XLogReaderRoutine->page_read callback */
-static int
+static bool
WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetPtr, char *readBuff)
{
@@ -348,7 +348,8 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
else
{
private->endptr_reached = true;
- return -1;
+ state->readLen = -1;
+ return false;
}
}
@@ -373,7 +374,8 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
(Size) errinfo.wre_req);
}
- return count;
+ state->readLen = count;
+ return true;
}
/*
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index 21d200d3df..23d9e70a59 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -57,8 +57,8 @@ typedef struct WALSegmentContext
typedef struct XLogReaderState XLogReaderState;
-/* Function type definitions for various xlogreader interactions */
-typedef int (*XLogPageReadCB) (XLogReaderState *xlogreader,
+/* Function type definition for the read_page callback */
+typedef bool (*XLogPageReadCB) (XLogReaderState *xlogreader,
XLogRecPtr targetPagePtr,
int reqLen,
XLogRecPtr targetRecPtr,
@@ -175,6 +175,20 @@ struct XLogReaderState
XLogRecPtr ReadRecPtr; /* start of last record read */
XLogRecPtr EndRecPtr; /* end+1 of last record read */
+ /* ----------------------------------------
+ * Communication with page reader
+ * readBuf is XLOG_BLCKSZ bytes, valid up to at least readLen bytes.
+ * ----------------------------------------
+ */
+ /* variables to communicate with page reader */
+ XLogRecPtr readPagePtr; /* page pointer to read */
+ int32 readLen; /* bytes requested to reader, or actual bytes
+ * read by reader, which must be larger than
+ * the request, or -1 on error */
+ TimeLineID readPageTLI; /* TLI for data currently in readBuf */
+ char *readBuf; /* buffer to store data */
+ bool page_verified; /* is the page on the buffer verified? */
+
/* ----------------------------------------
* Decoded representation of current record
@@ -203,13 +217,6 @@ struct XLogReaderState
* ----------------------------------------
*/
- /*
- * Buffer for currently read page (XLOG_BLCKSZ bytes, valid up to at least
- * readLen bytes)
- */
- char *readBuf;
- uint32 readLen;
-
/* last read XLOG position for data currently in readBuf */
WALSegmentContext segcxt;
WALOpenSegment seg;
diff --git a/src/include/access/xlogutils.h b/src/include/access/xlogutils.h
index 9ac602b674..364a21c4ea 100644
--- a/src/include/access/xlogutils.h
+++ b/src/include/access/xlogutils.h
@@ -47,7 +47,7 @@ extern Buffer XLogReadBufferExtended(RelFileNode rnode, ForkNumber forknum,
extern Relation CreateFakeRelcacheEntry(RelFileNode rnode);
extern void FreeFakeRelcacheEntry(Relation fakerel);
-extern int read_local_xlog_page(XLogReaderState *state,
+extern bool read_local_xlog_page(XLogReaderState *state,
XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *cur_page);
extern void wal_segment_open(XLogReaderState *state,
--
2.27.0
----Next_Part(Thu_Mar__4_11_28_53_2021_014)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
filename="v17-0002-Move-page-reader-out-of-XLogReadRecord.patch"
^ permalink raw reply [nested|flat] 69+ messages in thread
* [PATCH v15 1/4] Move callback-call from ReadPageInternal to XLogReadRecord.
@ 2019-09-05 11:21 Kyotaro Horiguchi <[email protected]>
0 siblings, 0 replies; 69+ messages in thread
From: Kyotaro Horiguchi @ 2019-09-05 11:21 UTC (permalink / raw)
The current WAL record reader reads page data using a call back
function. Although it is not so problematic alone, it would be a
problem if we are going to do add tasks like encryption which is
performed on page data before WAL reader reads them. To avoid that the
record reader facility has to have a new code path corresponds to
every new callback, this patch separates page reader from WAL record
reading facility by modifying the current WAL record reader to a state
machine.
As the first step of that change, this patch moves the page reader
function out of ReadPageInternal, then the remaining tasks of the
function are taken over by the new function XLogNeedData. As the
result XLogPageRead directly calls the page reader callback function
according to the feedback from XLogNeedData.
---
src/backend/access/transam/xlog.c | 16 +-
src/backend/access/transam/xlogreader.c | 281 ++++++++++++++----------
src/backend/access/transam/xlogutils.c | 12 +-
src/backend/replication/walsender.c | 10 +-
src/bin/pg_rewind/parsexlog.c | 21 +-
src/bin/pg_waldump/pg_waldump.c | 8 +-
src/include/access/xlogreader.h | 25 ++-
src/include/access/xlogutils.h | 2 +-
8 files changed, 222 insertions(+), 153 deletions(-)
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index fd93bcfaeb..e570e56a24 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -912,7 +912,7 @@ static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath,
static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli,
XLogSource source, bool notfoundOk);
static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, XLogSource source);
-static int XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
+static bool XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
bool fetching_ckpt, XLogRecPtr tliRecPtr);
@@ -4332,7 +4332,6 @@ ReadRecord(XLogReaderState *xlogreader, int emode,
XLogRecord *record;
XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
- /* Pass through parameters to XLogPageRead */
private->fetching_ckpt = fetching_ckpt;
private->emode = emode;
private->randAccess = (xlogreader->ReadRecPtr == InvalidXLogRecPtr);
@@ -11860,7 +11859,7 @@ CancelBackup(void)
* XLogPageRead() to try fetching the record from another source, or to
* sleep and retry.
*/
-static int
+static bool
XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *readBuf)
{
@@ -11919,7 +11918,8 @@ retry:
readLen = 0;
readSource = XLOG_FROM_ANY;
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
}
@@ -12014,7 +12014,8 @@ retry:
goto next_record_is_invalid;
}
- return readLen;
+ xlogreader->readLen = readLen;
+ return true;
next_record_is_invalid:
lastSourceFailed = true;
@@ -12028,8 +12029,9 @@ next_record_is_invalid:
/* In standby-mode, keep trying */
if (StandbyMode)
goto retry;
- else
- return -1;
+
+ xlogreader->readLen = -1;
+ return false;
}
/*
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index cb76be4f46..3d599325ee 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -36,8 +36,8 @@
static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
pg_attribute_printf(2, 3);
static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
-static int ReadPageInternal(XLogReaderState *state, XLogRecPtr pageptr,
- int reqLen);
+static bool XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr,
+ int reqLen, bool header_inclusive);
static void XLogReaderInvalReadState(XLogReaderState *state);
static bool ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
XLogRecPtr PrevRecPtr, XLogRecord *record, bool randAccess);
@@ -276,7 +276,6 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
uint32 targetRecOff;
uint32 pageHeaderSize;
bool gotheader;
- int readOff;
/*
* randAccess indicates whether to verify the previous-record pointer of
@@ -326,14 +325,20 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
* byte to cover the whole record header, or at least the part of it that
* fits on the same page.
*/
- readOff = ReadPageInternal(state, targetPagePtr,
- Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ));
- if (readOff < 0)
+ while (XLogNeedData(state, targetPagePtr,
+ Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+ targetRecOff != 0))
+ {
+ if (!state->routine.page_read(state, state->readPagePtr, state->readLen,
+ RecPtr, state->readBuf))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
/*
- * ReadPageInternal always returns at least the page header, so we can
- * examine it now.
+ * We have at least the page header, so we can examine it now.
*/
pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
if (targetRecOff == 0)
@@ -359,8 +364,8 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
goto err;
}
- /* ReadPageInternal has verified the page header */
- Assert(pageHeaderSize <= readOff);
+ /* XLogNeedData has verified the page header */
+ Assert(pageHeaderSize <= state->readLen);
/*
* Read the record length.
@@ -433,18 +438,27 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
do
{
+ int rest_len = total_len - gotlen;
+
/* Calculate pointer to beginning of next page */
targetPagePtr += XLOG_BLCKSZ;
/* Wait for the next page to become available */
- readOff = ReadPageInternal(state, targetPagePtr,
- Min(total_len - gotlen + SizeOfXLogShortPHD,
- XLOG_BLCKSZ));
+ while (XLogNeedData(state, targetPagePtr,
+ Min(rest_len, XLOG_BLCKSZ),
+ false))
+ {
+ if (!state->routine.page_read(state, state->readPagePtr,
+ state->readLen,
+ state->ReadRecPtr,
+ state->readBuf))
+ break;
+ }
- if (readOff < 0)
+ if (!state->page_verified)
goto err;
- Assert(SizeOfXLogShortPHD <= readOff);
+ Assert(SizeOfXLogShortPHD <= state->readLen);
/* Check that the continuation on next page looks valid */
pageHeader = (XLogPageHeader) state->readBuf;
@@ -473,21 +487,14 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
/* Append the continuation from this page to the buffer */
pageHeaderSize = XLogPageHeaderSize(pageHeader);
- if (readOff < pageHeaderSize)
- readOff = ReadPageInternal(state, targetPagePtr,
- pageHeaderSize);
-
- Assert(pageHeaderSize <= readOff);
+ Assert(pageHeaderSize <= state->readLen);
contdata = (char *) state->readBuf + pageHeaderSize;
len = XLOG_BLCKSZ - pageHeaderSize;
if (pageHeader->xlp_rem_len < len)
len = pageHeader->xlp_rem_len;
- if (readOff < pageHeaderSize + len)
- readOff = ReadPageInternal(state, targetPagePtr,
- pageHeaderSize + len);
-
+ Assert (pageHeaderSize + len <= state->readLen);
memcpy(buffer, (char *) contdata, len);
buffer += len;
gotlen += len;
@@ -517,9 +524,16 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
else
{
/* Wait for the record data to become available */
- readOff = ReadPageInternal(state, targetPagePtr,
- Min(targetRecOff + total_len, XLOG_BLCKSZ));
- if (readOff < 0)
+ while (XLogNeedData(state, targetPagePtr,
+ Min(targetRecOff + total_len, XLOG_BLCKSZ), true))
+ {
+ if (!state->routine.page_read(state, state->readPagePtr,
+ state->readLen,
+ state->ReadRecPtr, state->readBuf))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
/* Record does not cross a page boundary */
@@ -562,109 +576,138 @@ err:
}
/*
- * Read a single xlog page including at least [pageptr, reqLen] of valid data
- * via the page_read() callback.
+ * Checks that an xlog page loaded in state->readBuf is including at least
+ * [pageptr, reqLen] and the page is valid. header_inclusive indicates that
+ * reqLen is calculated including page header length.
*
- * Returns -1 if the required page cannot be read for some reason; errormsg_buf
- * is set in that case (unless the error occurs in the page_read callback).
+ * Returns false if the buffer already contains the requested data, or found
+ * error. state->page_verified is set to true for the former and false for the
+ * latter.
*
- * We fetch the page from a reader-local cache if we know we have the required
- * data and if there hasn't been any error since caching the data.
+ * Otherwise returns true and requests data loaded onto state->readBuf by
+ * state->readPagePtr and state->readLen. The caller shall call this function
+ * again after filling the buffer at least with that portion of data and set
+ * state->readLen to the length of actually loaded data.
+ *
+ * If header_inclusive is false, corrects reqLen internally by adding the
+ * actual page header length and may request caller for new data.
*/
-static int
-ReadPageInternal(XLogReaderState *state, XLogRecPtr pageptr, int reqLen)
+static bool
+XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
+ bool header_inclusive)
{
- int readLen;
uint32 targetPageOff;
XLogSegNo targetSegNo;
- XLogPageHeader hdr;
+ uint32 addLen = 0;
- Assert((pageptr % XLOG_BLCKSZ) == 0);
+ /* Some data is loaded, but page header is not verified yet. */
+ if (!state->page_verified &&
+ !XLogRecPtrIsInvalid(state->readPagePtr) && state->readLen >= 0)
+ {
+ uint32 pageHeaderSize;
+ /* just loaded new data so needs to verify page header */
+
+ /* The caller must have loaded at least page header */
+ Assert (state->readLen >= SizeOfXLogShortPHD);
+
+ /*
+ * We have enough data to check the header length. Recheck the loaded
+ * length against the actual header length.
+ */
+ pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+
+ /* Request more data if we don't have the full header. */
+ if (state->readLen < pageHeaderSize)
+ {
+ state->readLen = pageHeaderSize;
+ return true;
+ }
+
+ /* Now that we know we have the full header, validate it. */
+ if (!XLogReaderValidatePageHeader(state, state->readPagePtr,
+ (char *) state->readBuf))
+ {
+ /* That's bad. Force reading the page again. */
+ XLogReaderInvalReadState(state);
+
+ return false;
+ }
+
+ state->page_verified = true;
+
+ XLByteToSeg(state->readPagePtr, state->seg.ws_segno,
+ state->segcxt.ws_segsize);
+ }
+
+ /*
+ * The loaded page may not be the one caller is supposing to read when we
+ * are verifying the first page of new segment. In that case, skip further
+ * verification and immediately load the target page.
+ */
+ if (state->page_verified && pageptr == state->readPagePtr)
+ {
+ /*
+ * calculate additional length for page header keeping the total
+ * length within the block size.
+ */
+ if (!header_inclusive)
+ {
+ uint32 pageHeaderSize =
+ XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+
+ addLen = pageHeaderSize;
+ if (reqLen + pageHeaderSize <= XLOG_BLCKSZ)
+ addLen = pageHeaderSize;
+ else
+ addLen = XLOG_BLCKSZ - reqLen;
+
+ Assert(addLen >= 0);
+ }
+
+ /* Return if we already have it. */
+ if (reqLen + addLen <= state->readLen)
+ return false;
+ }
+
+ /* Data is not in our buffer, request the caller for it. */
XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize);
targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize);
+ Assert((pageptr % XLOG_BLCKSZ) == 0);
- /* check whether we have all the requested data already */
- if (targetSegNo == state->seg.ws_segno &&
- targetPageOff == state->segoff && reqLen <= state->readLen)
- return state->readLen;
+ /*
+ * Every time we request to load new data of a page to the caller, even if
+ * we looked at a part of it before, we need to do verification on the next
+ * invocation as the caller might now be rereading data from a different
+ * source.
+ */
+ state->page_verified = false;
/*
- * Data is not in our buffer.
- *
- * Every time we actually read the segment, even if we looked at parts of
- * it before, we need to do verification as the page_read callback might
- * now be rereading data from a different source.
- *
* Whenever switching to a new WAL segment, we read the first page of the
* file and validate its header, even if that's not where the target
* record is. This is so that we can check the additional identification
* info that is present in the first page's "long" header.
+ * Don't do this if the caller requested the first page in the segment.
*/
if (targetSegNo != state->seg.ws_segno && targetPageOff != 0)
{
- XLogRecPtr targetSegmentPtr = pageptr - targetPageOff;
-
- readLen = state->routine.page_read(state, targetSegmentPtr, XLOG_BLCKSZ,
- state->currRecPtr,
- state->readBuf);
- if (readLen < 0)
- goto err;
-
- /* we can be sure to have enough WAL available, we scrolled back */
- Assert(readLen == XLOG_BLCKSZ);
-
- if (!XLogReaderValidatePageHeader(state, targetSegmentPtr,
- state->readBuf))
- goto err;
+ /*
+ * Then we'll see that the targetSegNo now matches the ws_segno, and
+ * will not come back here, but will request the actual target page.
+ */
+ state->readPagePtr = pageptr - targetPageOff;
+ state->readLen = XLOG_BLCKSZ;
+ return true;
}
/*
- * First, read the requested data length, but at least a short page header
- * so that we can validate it.
+ * Request the caller to load the page. We need at least a short page
+ * header so that we can validate it.
*/
- readLen = state->routine.page_read(state, pageptr, Max(reqLen, SizeOfXLogShortPHD),
- state->currRecPtr,
- state->readBuf);
- if (readLen < 0)
- goto err;
-
- Assert(readLen <= XLOG_BLCKSZ);
-
- /* Do we have enough data to check the header length? */
- if (readLen <= SizeOfXLogShortPHD)
- goto err;
-
- Assert(readLen >= reqLen);
-
- hdr = (XLogPageHeader) state->readBuf;
-
- /* still not enough */
- if (readLen < XLogPageHeaderSize(hdr))
- {
- readLen = state->routine.page_read(state, pageptr, XLogPageHeaderSize(hdr),
- state->currRecPtr,
- state->readBuf);
- if (readLen < 0)
- goto err;
- }
-
- /*
- * Now that we know we have the full header, validate it.
- */
- if (!XLogReaderValidatePageHeader(state, pageptr, (char *) hdr))
- goto err;
-
- /* update read state information */
- state->seg.ws_segno = targetSegNo;
- state->segoff = targetPageOff;
- state->readLen = readLen;
-
- return readLen;
-
-err:
- XLogReaderInvalReadState(state);
- return -1;
+ state->readPagePtr = pageptr;
+ state->readLen = Max(reqLen + addLen, SizeOfXLogShortPHD);
+ return true;
}
/*
@@ -673,9 +716,7 @@ err:
static void
XLogReaderInvalReadState(XLogReaderState *state)
{
- state->seg.ws_segno = 0;
- state->segoff = 0;
- state->readLen = 0;
+ state->readPagePtr = InvalidXLogRecPtr;
}
/*
@@ -956,7 +997,6 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
XLogRecPtr targetPagePtr;
int targetRecOff;
uint32 pageHeaderSize;
- int readLen;
/*
* Compute targetRecOff. It should typically be equal or greater than
@@ -964,7 +1004,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
* that, except when caller has explicitly specified the offset that
* falls somewhere there or when we are skipping multi-page
* continuation record. It doesn't matter though because
- * ReadPageInternal() is prepared to handle that and will read at
+ * XLogNeedData() is prepared to handle that and will read at
* least short page-header worth of data
*/
targetRecOff = tmpRecPtr % XLOG_BLCKSZ;
@@ -972,19 +1012,24 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
/* scroll back to page boundary */
targetPagePtr = tmpRecPtr - targetRecOff;
- /* Read the page containing the record */
- readLen = ReadPageInternal(state, targetPagePtr, targetRecOff);
- if (readLen < 0)
+ while(XLogNeedData(state, targetPagePtr, targetRecOff,
+ targetRecOff != 0))
+ {
+ if (!state->routine.page_read(state, state->readPagePtr,
+ state->readLen,
+ state->ReadRecPtr, state->readBuf))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
header = (XLogPageHeader) state->readBuf;
pageHeaderSize = XLogPageHeaderSize(header);
- /* make sure we have enough data for the page header */
- readLen = ReadPageInternal(state, targetPagePtr, pageHeaderSize);
- if (readLen < 0)
- goto err;
+ /* we should have read the page header */
+ Assert (state->readLen >= pageHeaderSize);
/* skip over potential continuation data */
if (header->xlp_info & XLP_FIRST_IS_CONTRECORD)
diff --git a/src/backend/access/transam/xlogutils.c b/src/backend/access/transam/xlogutils.c
index 322b0e8ff5..1b10b6df20 100644
--- a/src/backend/access/transam/xlogutils.c
+++ b/src/backend/access/transam/xlogutils.c
@@ -689,8 +689,8 @@ XLogTruncateRelation(RelFileNode rnode, ForkNumber forkNum,
void
XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wantLength)
{
- const XLogRecPtr lastReadPage = (state->seg.ws_segno *
- state->segcxt.ws_segsize + state->segoff);
+ const XLogRecPtr lastReadPage = state->seg.ws_segno *
+ state->segcxt.ws_segsize + state->readLen;
Assert(wantPage != InvalidXLogRecPtr && wantPage % XLOG_BLCKSZ == 0);
Assert(wantLength <= XLOG_BLCKSZ);
@@ -828,7 +828,7 @@ wal_segment_close(XLogReaderState *state)
* exists for normal backends, so we have to do a check/sleep/repeat style of
* loop for now.
*/
-int
+bool
read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *cur_page)
{
@@ -930,7 +930,8 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
else if (targetPagePtr + reqLen > read_upto)
{
/* not enough data there */
- return -1;
+ state->readLen = -1;
+ return false;
}
else
{
@@ -948,7 +949,8 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
WALReadRaiseError(&errinfo);
/* number of valid bytes in the buffer */
- return count;
+ state->readLen = count;
+ return true;
}
/*
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index e2477c47e0..bf2711bddd 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -811,7 +811,7 @@ StartReplication(StartReplicationCmd *cmd)
* which has to do a plain sleep/busy loop, because the walsender's latch gets
* set every time WAL is flushed.
*/
-static int
+static bool
logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *cur_page)
{
@@ -831,7 +831,10 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
/* fail if not (implies we are going to shut down) */
if (flushptr < targetPagePtr + reqLen)
- return -1;
+ {
+ state->readLen = -1;
+ return false;
+ }
if (targetPagePtr + XLOG_BLCKSZ <= flushptr)
count = XLOG_BLCKSZ; /* more than one block available */
@@ -859,7 +862,8 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
XLByteToSeg(targetPagePtr, segno, state->segcxt.ws_segsize);
CheckXLogRemoved(segno, state->seg.ws_tli);
- return count;
+ state->readLen = count;
+ return true;
}
/*
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index bc6f976994..0b7e73ae79 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -46,7 +46,7 @@ typedef struct XLogPageReadPrivate
int tliIndex;
} XLogPageReadPrivate;
-static int SimpleXLogPageRead(XLogReaderState *xlogreader,
+static bool SimpleXLogPageRead(XLogReaderState *xlogreader,
XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
@@ -236,7 +236,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
}
/* XLogReader callback function, to read a WAL page */
-static int
+static bool
SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf)
{
@@ -296,7 +296,8 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
if (private->restoreCommand == NULL)
{
pg_log_error("could not open file \"%s\": %m", xlogfpath);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
/*
@@ -309,7 +310,10 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
private->restoreCommand);
if (xlogreadfd < 0)
- return -1;
+ {
+ xlogreader->readLen = -1;
+ return false;
+ }
else
pg_log_debug("using file \"%s\" restored from archive",
xlogfpath);
@@ -325,7 +329,8 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
if (lseek(xlogreadfd, (off_t) targetPageOff, SEEK_SET) < 0)
{
pg_log_error("could not seek in file \"%s\": %m", xlogfpath);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
@@ -338,13 +343,15 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
pg_log_error("could not read file \"%s\": read %d of %zu",
xlogfpath, r, (Size) XLOG_BLCKSZ);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
Assert(targetSegNo == xlogreadsegno);
xlogreader->seg.ws_tli = targetHistory[private->tliIndex].tli;
- return XLOG_BLCKSZ;
+ xlogreader->readLen = XLOG_BLCKSZ;
+ return true;
}
/*
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index d1a0678935..ab0c28d9b7 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -331,7 +331,7 @@ WALDumpCloseSegment(XLogReaderState *state)
}
/* pg_waldump's XLogReaderRoutine->page_read callback */
-static int
+static bool
WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetPtr, char *readBuff)
{
@@ -348,7 +348,8 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
else
{
private->endptr_reached = true;
- return -1;
+ state->readLen = -1;
+ return false;
}
}
@@ -373,7 +374,8 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
(Size) errinfo.wre_req);
}
- return count;
+ state->readLen = count;
+ return true;
}
/*
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index b0f2a6ed43..6c8848e14f 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -57,8 +57,8 @@ typedef struct WALSegmentContext
typedef struct XLogReaderState XLogReaderState;
-/* Function type definitions for various xlogreader interactions */
-typedef int (*XLogPageReadCB) (XLogReaderState *xlogreader,
+/* Function type definition for the read_page callback */
+typedef bool (*XLogPageReadCB) (XLogReaderState *xlogreader,
XLogRecPtr targetPagePtr,
int reqLen,
XLogRecPtr targetRecPtr,
@@ -175,6 +175,20 @@ struct XLogReaderState
XLogRecPtr ReadRecPtr; /* start of last record read */
XLogRecPtr EndRecPtr; /* end+1 of last record read */
+ /* ----------------------------------------
+ * Communication with page reader
+ * readBuf is XLOG_BLCKSZ bytes, valid up to at least readLen bytes.
+ * ----------------------------------------
+ */
+ /* variables to communicate with page reader */
+ XLogRecPtr readPagePtr; /* page pointer to read */
+ int32 readLen; /* bytes requested to reader, or actual bytes
+ * read by reader, which must be larger than
+ * the request, or -1 on error */
+ TimeLineID readPageTLI; /* TLI for data currently in readBuf */
+ char *readBuf; /* buffer to store data */
+ bool page_verified; /* is the page on the buffer verified? */
+
/* ----------------------------------------
* Decoded representation of current record
@@ -201,13 +215,6 @@ struct XLogReaderState
* ----------------------------------------
*/
- /*
- * Buffer for currently read page (XLOG_BLCKSZ bytes, valid up to at least
- * readLen bytes)
- */
- char *readBuf;
- uint32 readLen;
-
/* last read XLOG position for data currently in readBuf */
WALSegmentContext segcxt;
WALOpenSegment seg;
diff --git a/src/include/access/xlogutils.h b/src/include/access/xlogutils.h
index e59b6cf3a9..c287edf206 100644
--- a/src/include/access/xlogutils.h
+++ b/src/include/access/xlogutils.h
@@ -47,7 +47,7 @@ extern Buffer XLogReadBufferExtended(RelFileNode rnode, ForkNumber forknum,
extern Relation CreateFakeRelcacheEntry(RelFileNode rnode);
extern void FreeFakeRelcacheEntry(Relation fakerel);
-extern int read_local_xlog_page(XLogReaderState *state,
+extern bool read_local_xlog_page(XLogReaderState *state,
XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *cur_page);
extern void wal_segment_open(XLogReaderState *state,
--
2.18.4
----Next_Part(Thu_Jul__2_13_53_30_2020_072)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
filename="v15-0002-Move-page-reader-out-of-XLogReadRecord.patch"
^ permalink raw reply [nested|flat] 69+ messages in thread
* [PATCH v17 1/4] Move callback-call from ReadPageInternal to XLogReadRecord.
@ 2019-09-05 11:21 Kyotaro Horiguchi <[email protected]>
0 siblings, 0 replies; 69+ messages in thread
From: Kyotaro Horiguchi @ 2019-09-05 11:21 UTC (permalink / raw)
The current WAL record reader reads page data using a call back
function. Although it is not so problematic alone, it would be a
problem if we are going to do add tasks like encryption which is
performed on page data before WAL reader reads them. To avoid that the
record reader facility has to have a new code path corresponds to
every new callback, this patch separates page reader from WAL record
reading facility by modifying the current WAL record reader to a state
machine.
As the first step of that change, this patch moves the page reader
function out of ReadPageInternal, then the remaining tasks of the
function are taken over by the new function XLogNeedData. As the
result XLogPageRead directly calls the page reader callback function
according to the feedback from XLogNeedData.
---
src/backend/access/transam/xlog.c | 16 +-
src/backend/access/transam/xlogreader.c | 281 ++++++++++++++----------
src/backend/access/transam/xlogutils.c | 12 +-
src/backend/replication/walsender.c | 10 +-
src/bin/pg_rewind/parsexlog.c | 21 +-
src/bin/pg_waldump/pg_waldump.c | 8 +-
src/include/access/xlogreader.h | 25 ++-
src/include/access/xlogutils.h | 2 +-
8 files changed, 222 insertions(+), 153 deletions(-)
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 377afb8732..606328a65a 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -917,7 +917,7 @@ static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath,
static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli,
XLogSource source, bool notfoundOk);
static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, XLogSource source);
-static int XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
+static bool XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
bool fetching_ckpt, XLogRecPtr tliRecPtr);
@@ -4350,7 +4350,6 @@ ReadRecord(XLogReaderState *xlogreader, int emode,
XLogRecord *record;
XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
- /* Pass through parameters to XLogPageRead */
private->fetching_ckpt = fetching_ckpt;
private->emode = emode;
private->randAccess = (xlogreader->ReadRecPtr == InvalidXLogRecPtr);
@@ -11987,7 +11986,7 @@ CancelBackup(void)
* XLogPageRead() to try fetching the record from another source, or to
* sleep and retry.
*/
-static int
+static bool
XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *readBuf)
{
@@ -12046,7 +12045,8 @@ retry:
readLen = 0;
readSource = XLOG_FROM_ANY;
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
}
@@ -12141,7 +12141,8 @@ retry:
goto next_record_is_invalid;
}
- return readLen;
+ xlogreader->readLen = readLen;
+ return true;
next_record_is_invalid:
lastSourceFailed = true;
@@ -12155,8 +12156,9 @@ next_record_is_invalid:
/* In standby-mode, keep trying */
if (StandbyMode)
goto retry;
- else
- return -1;
+
+ xlogreader->readLen = -1;
+ return false;
}
/*
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index 42738eb940..7d04995b7b 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -36,8 +36,8 @@
static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
pg_attribute_printf(2, 3);
static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
-static int ReadPageInternal(XLogReaderState *state, XLogRecPtr pageptr,
- int reqLen);
+static bool XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr,
+ int reqLen, bool header_inclusive);
static void XLogReaderInvalReadState(XLogReaderState *state);
static bool ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
XLogRecPtr PrevRecPtr, XLogRecord *record, bool randAccess);
@@ -276,7 +276,6 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
uint32 targetRecOff;
uint32 pageHeaderSize;
bool gotheader;
- int readOff;
/*
* randAccess indicates whether to verify the previous-record pointer of
@@ -326,14 +325,20 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
* byte to cover the whole record header, or at least the part of it that
* fits on the same page.
*/
- readOff = ReadPageInternal(state, targetPagePtr,
- Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ));
- if (readOff < 0)
+ while (XLogNeedData(state, targetPagePtr,
+ Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+ targetRecOff != 0))
+ {
+ if (!state->routine.page_read(state, state->readPagePtr, state->readLen,
+ RecPtr, state->readBuf))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
/*
- * ReadPageInternal always returns at least the page header, so we can
- * examine it now.
+ * We have at least the page header, so we can examine it now.
*/
pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
if (targetRecOff == 0)
@@ -359,8 +364,8 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
goto err;
}
- /* ReadPageInternal has verified the page header */
- Assert(pageHeaderSize <= readOff);
+ /* XLogNeedData has verified the page header */
+ Assert(pageHeaderSize <= state->readLen);
/*
* Read the record length.
@@ -432,18 +437,27 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
do
{
+ int rest_len = total_len - gotlen;
+
/* Calculate pointer to beginning of next page */
targetPagePtr += XLOG_BLCKSZ;
/* Wait for the next page to become available */
- readOff = ReadPageInternal(state, targetPagePtr,
- Min(total_len - gotlen + SizeOfXLogShortPHD,
- XLOG_BLCKSZ));
+ while (XLogNeedData(state, targetPagePtr,
+ Min(rest_len, XLOG_BLCKSZ),
+ false))
+ {
+ if (!state->routine.page_read(state, state->readPagePtr,
+ state->readLen,
+ state->ReadRecPtr,
+ state->readBuf))
+ break;
+ }
- if (readOff < 0)
+ if (!state->page_verified)
goto err;
- Assert(SizeOfXLogShortPHD <= readOff);
+ Assert(SizeOfXLogShortPHD <= state->readLen);
/* Check that the continuation on next page looks valid */
pageHeader = (XLogPageHeader) state->readBuf;
@@ -473,21 +487,14 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
/* Append the continuation from this page to the buffer */
pageHeaderSize = XLogPageHeaderSize(pageHeader);
- if (readOff < pageHeaderSize)
- readOff = ReadPageInternal(state, targetPagePtr,
- pageHeaderSize);
-
- Assert(pageHeaderSize <= readOff);
+ Assert(pageHeaderSize <= state->readLen);
contdata = (char *) state->readBuf + pageHeaderSize;
len = XLOG_BLCKSZ - pageHeaderSize;
if (pageHeader->xlp_rem_len < len)
len = pageHeader->xlp_rem_len;
- if (readOff < pageHeaderSize + len)
- readOff = ReadPageInternal(state, targetPagePtr,
- pageHeaderSize + len);
-
+ Assert (pageHeaderSize + len <= state->readLen);
memcpy(buffer, (char *) contdata, len);
buffer += len;
gotlen += len;
@@ -517,9 +524,16 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
else
{
/* Wait for the record data to become available */
- readOff = ReadPageInternal(state, targetPagePtr,
- Min(targetRecOff + total_len, XLOG_BLCKSZ));
- if (readOff < 0)
+ while (XLogNeedData(state, targetPagePtr,
+ Min(targetRecOff + total_len, XLOG_BLCKSZ), true))
+ {
+ if (!state->routine.page_read(state, state->readPagePtr,
+ state->readLen,
+ state->ReadRecPtr, state->readBuf))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
/* Record does not cross a page boundary */
@@ -562,109 +576,138 @@ err:
}
/*
- * Read a single xlog page including at least [pageptr, reqLen] of valid data
- * via the page_read() callback.
+ * Checks that an xlog page loaded in state->readBuf is including at least
+ * [pageptr, reqLen] and the page is valid. header_inclusive indicates that
+ * reqLen is calculated including page header length.
*
- * Returns -1 if the required page cannot be read for some reason; errormsg_buf
- * is set in that case (unless the error occurs in the page_read callback).
+ * Returns false if the buffer already contains the requested data, or found
+ * error. state->page_verified is set to true for the former and false for the
+ * latter.
*
- * We fetch the page from a reader-local cache if we know we have the required
- * data and if there hasn't been any error since caching the data.
+ * Otherwise returns true and requests data loaded onto state->readBuf by
+ * state->readPagePtr and state->readLen. The caller shall call this function
+ * again after filling the buffer at least with that portion of data and set
+ * state->readLen to the length of actually loaded data.
+ *
+ * If header_inclusive is false, corrects reqLen internally by adding the
+ * actual page header length and may request caller for new data.
*/
-static int
-ReadPageInternal(XLogReaderState *state, XLogRecPtr pageptr, int reqLen)
+static bool
+XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
+ bool header_inclusive)
{
- int readLen;
uint32 targetPageOff;
XLogSegNo targetSegNo;
- XLogPageHeader hdr;
+ uint32 addLen = 0;
- Assert((pageptr % XLOG_BLCKSZ) == 0);
+ /* Some data is loaded, but page header is not verified yet. */
+ if (!state->page_verified &&
+ !XLogRecPtrIsInvalid(state->readPagePtr) && state->readLen >= 0)
+ {
+ uint32 pageHeaderSize;
+ /* just loaded new data so needs to verify page header */
+
+ /* The caller must have loaded at least page header */
+ Assert (state->readLen >= SizeOfXLogShortPHD);
+
+ /*
+ * We have enough data to check the header length. Recheck the loaded
+ * length against the actual header length.
+ */
+ pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+
+ /* Request more data if we don't have the full header. */
+ if (state->readLen < pageHeaderSize)
+ {
+ state->readLen = pageHeaderSize;
+ return true;
+ }
+
+ /* Now that we know we have the full header, validate it. */
+ if (!XLogReaderValidatePageHeader(state, state->readPagePtr,
+ (char *) state->readBuf))
+ {
+ /* That's bad. Force reading the page again. */
+ XLogReaderInvalReadState(state);
+
+ return false;
+ }
+
+ state->page_verified = true;
+
+ XLByteToSeg(state->readPagePtr, state->seg.ws_segno,
+ state->segcxt.ws_segsize);
+ }
+
+ /*
+ * The loaded page may not be the one caller is supposing to read when we
+ * are verifying the first page of new segment. In that case, skip further
+ * verification and immediately load the target page.
+ */
+ if (state->page_verified && pageptr == state->readPagePtr)
+ {
+ /*
+ * calculate additional length for page header keeping the total
+ * length within the block size.
+ */
+ if (!header_inclusive)
+ {
+ uint32 pageHeaderSize =
+ XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+
+ addLen = pageHeaderSize;
+ if (reqLen + pageHeaderSize <= XLOG_BLCKSZ)
+ addLen = pageHeaderSize;
+ else
+ addLen = XLOG_BLCKSZ - reqLen;
+
+ Assert(addLen >= 0);
+ }
+
+ /* Return if we already have it. */
+ if (reqLen + addLen <= state->readLen)
+ return false;
+ }
+
+ /* Data is not in our buffer, request the caller for it. */
XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize);
targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize);
+ Assert((pageptr % XLOG_BLCKSZ) == 0);
- /* check whether we have all the requested data already */
- if (targetSegNo == state->seg.ws_segno &&
- targetPageOff == state->segoff && reqLen <= state->readLen)
- return state->readLen;
+ /*
+ * Every time we request to load new data of a page to the caller, even if
+ * we looked at a part of it before, we need to do verification on the next
+ * invocation as the caller might now be rereading data from a different
+ * source.
+ */
+ state->page_verified = false;
/*
- * Data is not in our buffer.
- *
- * Every time we actually read the segment, even if we looked at parts of
- * it before, we need to do verification as the page_read callback might
- * now be rereading data from a different source.
- *
* Whenever switching to a new WAL segment, we read the first page of the
* file and validate its header, even if that's not where the target
* record is. This is so that we can check the additional identification
* info that is present in the first page's "long" header.
+ * Don't do this if the caller requested the first page in the segment.
*/
if (targetSegNo != state->seg.ws_segno && targetPageOff != 0)
{
- XLogRecPtr targetSegmentPtr = pageptr - targetPageOff;
-
- readLen = state->routine.page_read(state, targetSegmentPtr, XLOG_BLCKSZ,
- state->currRecPtr,
- state->readBuf);
- if (readLen < 0)
- goto err;
-
- /* we can be sure to have enough WAL available, we scrolled back */
- Assert(readLen == XLOG_BLCKSZ);
-
- if (!XLogReaderValidatePageHeader(state, targetSegmentPtr,
- state->readBuf))
- goto err;
+ /*
+ * Then we'll see that the targetSegNo now matches the ws_segno, and
+ * will not come back here, but will request the actual target page.
+ */
+ state->readPagePtr = pageptr - targetPageOff;
+ state->readLen = XLOG_BLCKSZ;
+ return true;
}
/*
- * First, read the requested data length, but at least a short page header
- * so that we can validate it.
+ * Request the caller to load the page. We need at least a short page
+ * header so that we can validate it.
*/
- readLen = state->routine.page_read(state, pageptr, Max(reqLen, SizeOfXLogShortPHD),
- state->currRecPtr,
- state->readBuf);
- if (readLen < 0)
- goto err;
-
- Assert(readLen <= XLOG_BLCKSZ);
-
- /* Do we have enough data to check the header length? */
- if (readLen <= SizeOfXLogShortPHD)
- goto err;
-
- Assert(readLen >= reqLen);
-
- hdr = (XLogPageHeader) state->readBuf;
-
- /* still not enough */
- if (readLen < XLogPageHeaderSize(hdr))
- {
- readLen = state->routine.page_read(state, pageptr, XLogPageHeaderSize(hdr),
- state->currRecPtr,
- state->readBuf);
- if (readLen < 0)
- goto err;
- }
-
- /*
- * Now that we know we have the full header, validate it.
- */
- if (!XLogReaderValidatePageHeader(state, pageptr, (char *) hdr))
- goto err;
-
- /* update read state information */
- state->seg.ws_segno = targetSegNo;
- state->segoff = targetPageOff;
- state->readLen = readLen;
-
- return readLen;
-
-err:
- XLogReaderInvalReadState(state);
- return -1;
+ state->readPagePtr = pageptr;
+ state->readLen = Max(reqLen + addLen, SizeOfXLogShortPHD);
+ return true;
}
/*
@@ -673,9 +716,7 @@ err:
static void
XLogReaderInvalReadState(XLogReaderState *state)
{
- state->seg.ws_segno = 0;
- state->segoff = 0;
- state->readLen = 0;
+ state->readPagePtr = InvalidXLogRecPtr;
}
/*
@@ -953,7 +994,6 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
XLogRecPtr targetPagePtr;
int targetRecOff;
uint32 pageHeaderSize;
- int readLen;
/*
* Compute targetRecOff. It should typically be equal or greater than
@@ -961,7 +1001,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
* that, except when caller has explicitly specified the offset that
* falls somewhere there or when we are skipping multi-page
* continuation record. It doesn't matter though because
- * ReadPageInternal() is prepared to handle that and will read at
+ * XLogNeedData() is prepared to handle that and will read at
* least short page-header worth of data
*/
targetRecOff = tmpRecPtr % XLOG_BLCKSZ;
@@ -969,19 +1009,24 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
/* scroll back to page boundary */
targetPagePtr = tmpRecPtr - targetRecOff;
- /* Read the page containing the record */
- readLen = ReadPageInternal(state, targetPagePtr, targetRecOff);
- if (readLen < 0)
+ while(XLogNeedData(state, targetPagePtr, targetRecOff,
+ targetRecOff != 0))
+ {
+ if (!state->routine.page_read(state, state->readPagePtr,
+ state->readLen,
+ state->ReadRecPtr, state->readBuf))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
header = (XLogPageHeader) state->readBuf;
pageHeaderSize = XLogPageHeaderSize(header);
- /* make sure we have enough data for the page header */
- readLen = ReadPageInternal(state, targetPagePtr, pageHeaderSize);
- if (readLen < 0)
- goto err;
+ /* we should have read the page header */
+ Assert (state->readLen >= pageHeaderSize);
/* skip over potential continuation data */
if (header->xlp_info & XLP_FIRST_IS_CONTRECORD)
diff --git a/src/backend/access/transam/xlogutils.c b/src/backend/access/transam/xlogutils.c
index d17d660f46..46eda33f25 100644
--- a/src/backend/access/transam/xlogutils.c
+++ b/src/backend/access/transam/xlogutils.c
@@ -686,8 +686,8 @@ XLogTruncateRelation(RelFileNode rnode, ForkNumber forkNum,
void
XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wantLength)
{
- const XLogRecPtr lastReadPage = (state->seg.ws_segno *
- state->segcxt.ws_segsize + state->segoff);
+ const XLogRecPtr lastReadPage = state->seg.ws_segno *
+ state->segcxt.ws_segsize + state->readLen;
Assert(wantPage != InvalidXLogRecPtr && wantPage % XLOG_BLCKSZ == 0);
Assert(wantLength <= XLOG_BLCKSZ);
@@ -824,7 +824,7 @@ wal_segment_close(XLogReaderState *state)
* exists for normal backends, so we have to do a check/sleep/repeat style of
* loop for now.
*/
-int
+bool
read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *cur_page)
{
@@ -926,7 +926,8 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
else if (targetPagePtr + reqLen > read_upto)
{
/* not enough data there */
- return -1;
+ state->readLen = -1;
+ return false;
}
else
{
@@ -944,7 +945,8 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
WALReadRaiseError(&errinfo);
/* number of valid bytes in the buffer */
- return count;
+ state->readLen = count;
+ return true;
}
/*
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index 23baa4498a..c36b91ae5e 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -806,7 +806,7 @@ StartReplication(StartReplicationCmd *cmd)
* which has to do a plain sleep/busy loop, because the walsender's latch gets
* set every time WAL is flushed.
*/
-static int
+static bool
logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *cur_page)
{
@@ -826,7 +826,10 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
/* fail if not (implies we are going to shut down) */
if (flushptr < targetPagePtr + reqLen)
- return -1;
+ {
+ state->readLen = -1;
+ return false;
+ }
if (targetPagePtr + XLOG_BLCKSZ <= flushptr)
count = XLOG_BLCKSZ; /* more than one block available */
@@ -854,7 +857,8 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
XLByteToSeg(targetPagePtr, segno, state->segcxt.ws_segsize);
CheckXLogRemoved(segno, state->seg.ws_tli);
- return count;
+ state->readLen = count;
+ return true;
}
/*
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index 59ebac7d6a..cf119848b0 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -47,7 +47,7 @@ typedef struct XLogPageReadPrivate
int tliIndex;
} XLogPageReadPrivate;
-static int SimpleXLogPageRead(XLogReaderState *xlogreader,
+static bool SimpleXLogPageRead(XLogReaderState *xlogreader,
XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
@@ -246,7 +246,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
}
/* XLogReader callback function, to read a WAL page */
-static int
+static bool
SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf)
{
@@ -306,7 +306,8 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
if (private->restoreCommand == NULL)
{
pg_log_error("could not open file \"%s\": %m", xlogfpath);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
/*
@@ -319,7 +320,10 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
private->restoreCommand);
if (xlogreadfd < 0)
- return -1;
+ {
+ xlogreader->readLen = -1;
+ return false;
+ }
else
pg_log_debug("using file \"%s\" restored from archive",
xlogfpath);
@@ -335,7 +339,8 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
if (lseek(xlogreadfd, (off_t) targetPageOff, SEEK_SET) < 0)
{
pg_log_error("could not seek in file \"%s\": %m", xlogfpath);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
@@ -348,13 +353,15 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
pg_log_error("could not read file \"%s\": read %d of %zu",
xlogfpath, r, (Size) XLOG_BLCKSZ);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
Assert(targetSegNo == xlogreadsegno);
xlogreader->seg.ws_tli = targetHistory[private->tliIndex].tli;
- return XLOG_BLCKSZ;
+ xlogreader->readLen = XLOG_BLCKSZ;
+ return true;
}
/*
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index 610f65e471..0f0e0723a3 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -331,7 +331,7 @@ WALDumpCloseSegment(XLogReaderState *state)
}
/* pg_waldump's XLogReaderRoutine->page_read callback */
-static int
+static bool
WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetPtr, char *readBuff)
{
@@ -348,7 +348,8 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
else
{
private->endptr_reached = true;
- return -1;
+ state->readLen = -1;
+ return false;
}
}
@@ -373,7 +374,8 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
(Size) errinfo.wre_req);
}
- return count;
+ state->readLen = count;
+ return true;
}
/*
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index 21d200d3df..23d9e70a59 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -57,8 +57,8 @@ typedef struct WALSegmentContext
typedef struct XLogReaderState XLogReaderState;
-/* Function type definitions for various xlogreader interactions */
-typedef int (*XLogPageReadCB) (XLogReaderState *xlogreader,
+/* Function type definition for the read_page callback */
+typedef bool (*XLogPageReadCB) (XLogReaderState *xlogreader,
XLogRecPtr targetPagePtr,
int reqLen,
XLogRecPtr targetRecPtr,
@@ -175,6 +175,20 @@ struct XLogReaderState
XLogRecPtr ReadRecPtr; /* start of last record read */
XLogRecPtr EndRecPtr; /* end+1 of last record read */
+ /* ----------------------------------------
+ * Communication with page reader
+ * readBuf is XLOG_BLCKSZ bytes, valid up to at least readLen bytes.
+ * ----------------------------------------
+ */
+ /* variables to communicate with page reader */
+ XLogRecPtr readPagePtr; /* page pointer to read */
+ int32 readLen; /* bytes requested to reader, or actual bytes
+ * read by reader, which must be larger than
+ * the request, or -1 on error */
+ TimeLineID readPageTLI; /* TLI for data currently in readBuf */
+ char *readBuf; /* buffer to store data */
+ bool page_verified; /* is the page on the buffer verified? */
+
/* ----------------------------------------
* Decoded representation of current record
@@ -203,13 +217,6 @@ struct XLogReaderState
* ----------------------------------------
*/
- /*
- * Buffer for currently read page (XLOG_BLCKSZ bytes, valid up to at least
- * readLen bytes)
- */
- char *readBuf;
- uint32 readLen;
-
/* last read XLOG position for data currently in readBuf */
WALSegmentContext segcxt;
WALOpenSegment seg;
diff --git a/src/include/access/xlogutils.h b/src/include/access/xlogutils.h
index 9ac602b674..364a21c4ea 100644
--- a/src/include/access/xlogutils.h
+++ b/src/include/access/xlogutils.h
@@ -47,7 +47,7 @@ extern Buffer XLogReadBufferExtended(RelFileNode rnode, ForkNumber forknum,
extern Relation CreateFakeRelcacheEntry(RelFileNode rnode);
extern void FreeFakeRelcacheEntry(Relation fakerel);
-extern int read_local_xlog_page(XLogReaderState *state,
+extern bool read_local_xlog_page(XLogReaderState *state,
XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *cur_page);
extern void wal_segment_open(XLogReaderState *state,
--
2.27.0
----Next_Part(Thu_Mar__4_11_28_53_2021_014)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
filename="v17-0002-Move-page-reader-out-of-XLogReadRecord.patch"
^ permalink raw reply [nested|flat] 69+ messages in thread
* [PATCH v15 1/4] Move callback-call from ReadPageInternal to XLogReadRecord.
@ 2019-09-05 11:21 Kyotaro Horiguchi <[email protected]>
0 siblings, 0 replies; 69+ messages in thread
From: Kyotaro Horiguchi @ 2019-09-05 11:21 UTC (permalink / raw)
The current WAL record reader reads page data using a call back
function. Although it is not so problematic alone, it would be a
problem if we are going to do add tasks like encryption which is
performed on page data before WAL reader reads them. To avoid that the
record reader facility has to have a new code path corresponds to
every new callback, this patch separates page reader from WAL record
reading facility by modifying the current WAL record reader to a state
machine.
As the first step of that change, this patch moves the page reader
function out of ReadPageInternal, then the remaining tasks of the
function are taken over by the new function XLogNeedData. As the
result XLogPageRead directly calls the page reader callback function
according to the feedback from XLogNeedData.
---
src/backend/access/transam/xlog.c | 16 +-
src/backend/access/transam/xlogreader.c | 281 ++++++++++++++----------
src/backend/access/transam/xlogutils.c | 12 +-
src/backend/replication/walsender.c | 10 +-
src/bin/pg_rewind/parsexlog.c | 21 +-
src/bin/pg_waldump/pg_waldump.c | 8 +-
src/include/access/xlogreader.h | 25 ++-
src/include/access/xlogutils.h | 2 +-
8 files changed, 222 insertions(+), 153 deletions(-)
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index fd93bcfaeb..e570e56a24 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -912,7 +912,7 @@ static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath,
static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli,
XLogSource source, bool notfoundOk);
static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, XLogSource source);
-static int XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
+static bool XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
bool fetching_ckpt, XLogRecPtr tliRecPtr);
@@ -4332,7 +4332,6 @@ ReadRecord(XLogReaderState *xlogreader, int emode,
XLogRecord *record;
XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
- /* Pass through parameters to XLogPageRead */
private->fetching_ckpt = fetching_ckpt;
private->emode = emode;
private->randAccess = (xlogreader->ReadRecPtr == InvalidXLogRecPtr);
@@ -11860,7 +11859,7 @@ CancelBackup(void)
* XLogPageRead() to try fetching the record from another source, or to
* sleep and retry.
*/
-static int
+static bool
XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *readBuf)
{
@@ -11919,7 +11918,8 @@ retry:
readLen = 0;
readSource = XLOG_FROM_ANY;
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
}
@@ -12014,7 +12014,8 @@ retry:
goto next_record_is_invalid;
}
- return readLen;
+ xlogreader->readLen = readLen;
+ return true;
next_record_is_invalid:
lastSourceFailed = true;
@@ -12028,8 +12029,9 @@ next_record_is_invalid:
/* In standby-mode, keep trying */
if (StandbyMode)
goto retry;
- else
- return -1;
+
+ xlogreader->readLen = -1;
+ return false;
}
/*
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index cb76be4f46..3d599325ee 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -36,8 +36,8 @@
static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
pg_attribute_printf(2, 3);
static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
-static int ReadPageInternal(XLogReaderState *state, XLogRecPtr pageptr,
- int reqLen);
+static bool XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr,
+ int reqLen, bool header_inclusive);
static void XLogReaderInvalReadState(XLogReaderState *state);
static bool ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
XLogRecPtr PrevRecPtr, XLogRecord *record, bool randAccess);
@@ -276,7 +276,6 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
uint32 targetRecOff;
uint32 pageHeaderSize;
bool gotheader;
- int readOff;
/*
* randAccess indicates whether to verify the previous-record pointer of
@@ -326,14 +325,20 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
* byte to cover the whole record header, or at least the part of it that
* fits on the same page.
*/
- readOff = ReadPageInternal(state, targetPagePtr,
- Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ));
- if (readOff < 0)
+ while (XLogNeedData(state, targetPagePtr,
+ Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+ targetRecOff != 0))
+ {
+ if (!state->routine.page_read(state, state->readPagePtr, state->readLen,
+ RecPtr, state->readBuf))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
/*
- * ReadPageInternal always returns at least the page header, so we can
- * examine it now.
+ * We have at least the page header, so we can examine it now.
*/
pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
if (targetRecOff == 0)
@@ -359,8 +364,8 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
goto err;
}
- /* ReadPageInternal has verified the page header */
- Assert(pageHeaderSize <= readOff);
+ /* XLogNeedData has verified the page header */
+ Assert(pageHeaderSize <= state->readLen);
/*
* Read the record length.
@@ -433,18 +438,27 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
do
{
+ int rest_len = total_len - gotlen;
+
/* Calculate pointer to beginning of next page */
targetPagePtr += XLOG_BLCKSZ;
/* Wait for the next page to become available */
- readOff = ReadPageInternal(state, targetPagePtr,
- Min(total_len - gotlen + SizeOfXLogShortPHD,
- XLOG_BLCKSZ));
+ while (XLogNeedData(state, targetPagePtr,
+ Min(rest_len, XLOG_BLCKSZ),
+ false))
+ {
+ if (!state->routine.page_read(state, state->readPagePtr,
+ state->readLen,
+ state->ReadRecPtr,
+ state->readBuf))
+ break;
+ }
- if (readOff < 0)
+ if (!state->page_verified)
goto err;
- Assert(SizeOfXLogShortPHD <= readOff);
+ Assert(SizeOfXLogShortPHD <= state->readLen);
/* Check that the continuation on next page looks valid */
pageHeader = (XLogPageHeader) state->readBuf;
@@ -473,21 +487,14 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
/* Append the continuation from this page to the buffer */
pageHeaderSize = XLogPageHeaderSize(pageHeader);
- if (readOff < pageHeaderSize)
- readOff = ReadPageInternal(state, targetPagePtr,
- pageHeaderSize);
-
- Assert(pageHeaderSize <= readOff);
+ Assert(pageHeaderSize <= state->readLen);
contdata = (char *) state->readBuf + pageHeaderSize;
len = XLOG_BLCKSZ - pageHeaderSize;
if (pageHeader->xlp_rem_len < len)
len = pageHeader->xlp_rem_len;
- if (readOff < pageHeaderSize + len)
- readOff = ReadPageInternal(state, targetPagePtr,
- pageHeaderSize + len);
-
+ Assert (pageHeaderSize + len <= state->readLen);
memcpy(buffer, (char *) contdata, len);
buffer += len;
gotlen += len;
@@ -517,9 +524,16 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
else
{
/* Wait for the record data to become available */
- readOff = ReadPageInternal(state, targetPagePtr,
- Min(targetRecOff + total_len, XLOG_BLCKSZ));
- if (readOff < 0)
+ while (XLogNeedData(state, targetPagePtr,
+ Min(targetRecOff + total_len, XLOG_BLCKSZ), true))
+ {
+ if (!state->routine.page_read(state, state->readPagePtr,
+ state->readLen,
+ state->ReadRecPtr, state->readBuf))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
/* Record does not cross a page boundary */
@@ -562,109 +576,138 @@ err:
}
/*
- * Read a single xlog page including at least [pageptr, reqLen] of valid data
- * via the page_read() callback.
+ * Checks that an xlog page loaded in state->readBuf is including at least
+ * [pageptr, reqLen] and the page is valid. header_inclusive indicates that
+ * reqLen is calculated including page header length.
*
- * Returns -1 if the required page cannot be read for some reason; errormsg_buf
- * is set in that case (unless the error occurs in the page_read callback).
+ * Returns false if the buffer already contains the requested data, or found
+ * error. state->page_verified is set to true for the former and false for the
+ * latter.
*
- * We fetch the page from a reader-local cache if we know we have the required
- * data and if there hasn't been any error since caching the data.
+ * Otherwise returns true and requests data loaded onto state->readBuf by
+ * state->readPagePtr and state->readLen. The caller shall call this function
+ * again after filling the buffer at least with that portion of data and set
+ * state->readLen to the length of actually loaded data.
+ *
+ * If header_inclusive is false, corrects reqLen internally by adding the
+ * actual page header length and may request caller for new data.
*/
-static int
-ReadPageInternal(XLogReaderState *state, XLogRecPtr pageptr, int reqLen)
+static bool
+XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
+ bool header_inclusive)
{
- int readLen;
uint32 targetPageOff;
XLogSegNo targetSegNo;
- XLogPageHeader hdr;
+ uint32 addLen = 0;
- Assert((pageptr % XLOG_BLCKSZ) == 0);
+ /* Some data is loaded, but page header is not verified yet. */
+ if (!state->page_verified &&
+ !XLogRecPtrIsInvalid(state->readPagePtr) && state->readLen >= 0)
+ {
+ uint32 pageHeaderSize;
+ /* just loaded new data so needs to verify page header */
+
+ /* The caller must have loaded at least page header */
+ Assert (state->readLen >= SizeOfXLogShortPHD);
+
+ /*
+ * We have enough data to check the header length. Recheck the loaded
+ * length against the actual header length.
+ */
+ pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+
+ /* Request more data if we don't have the full header. */
+ if (state->readLen < pageHeaderSize)
+ {
+ state->readLen = pageHeaderSize;
+ return true;
+ }
+
+ /* Now that we know we have the full header, validate it. */
+ if (!XLogReaderValidatePageHeader(state, state->readPagePtr,
+ (char *) state->readBuf))
+ {
+ /* That's bad. Force reading the page again. */
+ XLogReaderInvalReadState(state);
+
+ return false;
+ }
+
+ state->page_verified = true;
+
+ XLByteToSeg(state->readPagePtr, state->seg.ws_segno,
+ state->segcxt.ws_segsize);
+ }
+
+ /*
+ * The loaded page may not be the one caller is supposing to read when we
+ * are verifying the first page of new segment. In that case, skip further
+ * verification and immediately load the target page.
+ */
+ if (state->page_verified && pageptr == state->readPagePtr)
+ {
+ /*
+ * calculate additional length for page header keeping the total
+ * length within the block size.
+ */
+ if (!header_inclusive)
+ {
+ uint32 pageHeaderSize =
+ XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+
+ addLen = pageHeaderSize;
+ if (reqLen + pageHeaderSize <= XLOG_BLCKSZ)
+ addLen = pageHeaderSize;
+ else
+ addLen = XLOG_BLCKSZ - reqLen;
+
+ Assert(addLen >= 0);
+ }
+
+ /* Return if we already have it. */
+ if (reqLen + addLen <= state->readLen)
+ return false;
+ }
+
+ /* Data is not in our buffer, request the caller for it. */
XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize);
targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize);
+ Assert((pageptr % XLOG_BLCKSZ) == 0);
- /* check whether we have all the requested data already */
- if (targetSegNo == state->seg.ws_segno &&
- targetPageOff == state->segoff && reqLen <= state->readLen)
- return state->readLen;
+ /*
+ * Every time we request to load new data of a page to the caller, even if
+ * we looked at a part of it before, we need to do verification on the next
+ * invocation as the caller might now be rereading data from a different
+ * source.
+ */
+ state->page_verified = false;
/*
- * Data is not in our buffer.
- *
- * Every time we actually read the segment, even if we looked at parts of
- * it before, we need to do verification as the page_read callback might
- * now be rereading data from a different source.
- *
* Whenever switching to a new WAL segment, we read the first page of the
* file and validate its header, even if that's not where the target
* record is. This is so that we can check the additional identification
* info that is present in the first page's "long" header.
+ * Don't do this if the caller requested the first page in the segment.
*/
if (targetSegNo != state->seg.ws_segno && targetPageOff != 0)
{
- XLogRecPtr targetSegmentPtr = pageptr - targetPageOff;
-
- readLen = state->routine.page_read(state, targetSegmentPtr, XLOG_BLCKSZ,
- state->currRecPtr,
- state->readBuf);
- if (readLen < 0)
- goto err;
-
- /* we can be sure to have enough WAL available, we scrolled back */
- Assert(readLen == XLOG_BLCKSZ);
-
- if (!XLogReaderValidatePageHeader(state, targetSegmentPtr,
- state->readBuf))
- goto err;
+ /*
+ * Then we'll see that the targetSegNo now matches the ws_segno, and
+ * will not come back here, but will request the actual target page.
+ */
+ state->readPagePtr = pageptr - targetPageOff;
+ state->readLen = XLOG_BLCKSZ;
+ return true;
}
/*
- * First, read the requested data length, but at least a short page header
- * so that we can validate it.
+ * Request the caller to load the page. We need at least a short page
+ * header so that we can validate it.
*/
- readLen = state->routine.page_read(state, pageptr, Max(reqLen, SizeOfXLogShortPHD),
- state->currRecPtr,
- state->readBuf);
- if (readLen < 0)
- goto err;
-
- Assert(readLen <= XLOG_BLCKSZ);
-
- /* Do we have enough data to check the header length? */
- if (readLen <= SizeOfXLogShortPHD)
- goto err;
-
- Assert(readLen >= reqLen);
-
- hdr = (XLogPageHeader) state->readBuf;
-
- /* still not enough */
- if (readLen < XLogPageHeaderSize(hdr))
- {
- readLen = state->routine.page_read(state, pageptr, XLogPageHeaderSize(hdr),
- state->currRecPtr,
- state->readBuf);
- if (readLen < 0)
- goto err;
- }
-
- /*
- * Now that we know we have the full header, validate it.
- */
- if (!XLogReaderValidatePageHeader(state, pageptr, (char *) hdr))
- goto err;
-
- /* update read state information */
- state->seg.ws_segno = targetSegNo;
- state->segoff = targetPageOff;
- state->readLen = readLen;
-
- return readLen;
-
-err:
- XLogReaderInvalReadState(state);
- return -1;
+ state->readPagePtr = pageptr;
+ state->readLen = Max(reqLen + addLen, SizeOfXLogShortPHD);
+ return true;
}
/*
@@ -673,9 +716,7 @@ err:
static void
XLogReaderInvalReadState(XLogReaderState *state)
{
- state->seg.ws_segno = 0;
- state->segoff = 0;
- state->readLen = 0;
+ state->readPagePtr = InvalidXLogRecPtr;
}
/*
@@ -956,7 +997,6 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
XLogRecPtr targetPagePtr;
int targetRecOff;
uint32 pageHeaderSize;
- int readLen;
/*
* Compute targetRecOff. It should typically be equal or greater than
@@ -964,7 +1004,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
* that, except when caller has explicitly specified the offset that
* falls somewhere there or when we are skipping multi-page
* continuation record. It doesn't matter though because
- * ReadPageInternal() is prepared to handle that and will read at
+ * XLogNeedData() is prepared to handle that and will read at
* least short page-header worth of data
*/
targetRecOff = tmpRecPtr % XLOG_BLCKSZ;
@@ -972,19 +1012,24 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
/* scroll back to page boundary */
targetPagePtr = tmpRecPtr - targetRecOff;
- /* Read the page containing the record */
- readLen = ReadPageInternal(state, targetPagePtr, targetRecOff);
- if (readLen < 0)
+ while(XLogNeedData(state, targetPagePtr, targetRecOff,
+ targetRecOff != 0))
+ {
+ if (!state->routine.page_read(state, state->readPagePtr,
+ state->readLen,
+ state->ReadRecPtr, state->readBuf))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
header = (XLogPageHeader) state->readBuf;
pageHeaderSize = XLogPageHeaderSize(header);
- /* make sure we have enough data for the page header */
- readLen = ReadPageInternal(state, targetPagePtr, pageHeaderSize);
- if (readLen < 0)
- goto err;
+ /* we should have read the page header */
+ Assert (state->readLen >= pageHeaderSize);
/* skip over potential continuation data */
if (header->xlp_info & XLP_FIRST_IS_CONTRECORD)
diff --git a/src/backend/access/transam/xlogutils.c b/src/backend/access/transam/xlogutils.c
index 322b0e8ff5..1b10b6df20 100644
--- a/src/backend/access/transam/xlogutils.c
+++ b/src/backend/access/transam/xlogutils.c
@@ -689,8 +689,8 @@ XLogTruncateRelation(RelFileNode rnode, ForkNumber forkNum,
void
XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wantLength)
{
- const XLogRecPtr lastReadPage = (state->seg.ws_segno *
- state->segcxt.ws_segsize + state->segoff);
+ const XLogRecPtr lastReadPage = state->seg.ws_segno *
+ state->segcxt.ws_segsize + state->readLen;
Assert(wantPage != InvalidXLogRecPtr && wantPage % XLOG_BLCKSZ == 0);
Assert(wantLength <= XLOG_BLCKSZ);
@@ -828,7 +828,7 @@ wal_segment_close(XLogReaderState *state)
* exists for normal backends, so we have to do a check/sleep/repeat style of
* loop for now.
*/
-int
+bool
read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *cur_page)
{
@@ -930,7 +930,8 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
else if (targetPagePtr + reqLen > read_upto)
{
/* not enough data there */
- return -1;
+ state->readLen = -1;
+ return false;
}
else
{
@@ -948,7 +949,8 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
WALReadRaiseError(&errinfo);
/* number of valid bytes in the buffer */
- return count;
+ state->readLen = count;
+ return true;
}
/*
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index e2477c47e0..bf2711bddd 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -811,7 +811,7 @@ StartReplication(StartReplicationCmd *cmd)
* which has to do a plain sleep/busy loop, because the walsender's latch gets
* set every time WAL is flushed.
*/
-static int
+static bool
logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *cur_page)
{
@@ -831,7 +831,10 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
/* fail if not (implies we are going to shut down) */
if (flushptr < targetPagePtr + reqLen)
- return -1;
+ {
+ state->readLen = -1;
+ return false;
+ }
if (targetPagePtr + XLOG_BLCKSZ <= flushptr)
count = XLOG_BLCKSZ; /* more than one block available */
@@ -859,7 +862,8 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
XLByteToSeg(targetPagePtr, segno, state->segcxt.ws_segsize);
CheckXLogRemoved(segno, state->seg.ws_tli);
- return count;
+ state->readLen = count;
+ return true;
}
/*
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index bc6f976994..0b7e73ae79 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -46,7 +46,7 @@ typedef struct XLogPageReadPrivate
int tliIndex;
} XLogPageReadPrivate;
-static int SimpleXLogPageRead(XLogReaderState *xlogreader,
+static bool SimpleXLogPageRead(XLogReaderState *xlogreader,
XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
@@ -236,7 +236,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
}
/* XLogReader callback function, to read a WAL page */
-static int
+static bool
SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf)
{
@@ -296,7 +296,8 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
if (private->restoreCommand == NULL)
{
pg_log_error("could not open file \"%s\": %m", xlogfpath);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
/*
@@ -309,7 +310,10 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
private->restoreCommand);
if (xlogreadfd < 0)
- return -1;
+ {
+ xlogreader->readLen = -1;
+ return false;
+ }
else
pg_log_debug("using file \"%s\" restored from archive",
xlogfpath);
@@ -325,7 +329,8 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
if (lseek(xlogreadfd, (off_t) targetPageOff, SEEK_SET) < 0)
{
pg_log_error("could not seek in file \"%s\": %m", xlogfpath);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
@@ -338,13 +343,15 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
pg_log_error("could not read file \"%s\": read %d of %zu",
xlogfpath, r, (Size) XLOG_BLCKSZ);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
Assert(targetSegNo == xlogreadsegno);
xlogreader->seg.ws_tli = targetHistory[private->tliIndex].tli;
- return XLOG_BLCKSZ;
+ xlogreader->readLen = XLOG_BLCKSZ;
+ return true;
}
/*
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index d1a0678935..ab0c28d9b7 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -331,7 +331,7 @@ WALDumpCloseSegment(XLogReaderState *state)
}
/* pg_waldump's XLogReaderRoutine->page_read callback */
-static int
+static bool
WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetPtr, char *readBuff)
{
@@ -348,7 +348,8 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
else
{
private->endptr_reached = true;
- return -1;
+ state->readLen = -1;
+ return false;
}
}
@@ -373,7 +374,8 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
(Size) errinfo.wre_req);
}
- return count;
+ state->readLen = count;
+ return true;
}
/*
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index b0f2a6ed43..6c8848e14f 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -57,8 +57,8 @@ typedef struct WALSegmentContext
typedef struct XLogReaderState XLogReaderState;
-/* Function type definitions for various xlogreader interactions */
-typedef int (*XLogPageReadCB) (XLogReaderState *xlogreader,
+/* Function type definition for the read_page callback */
+typedef bool (*XLogPageReadCB) (XLogReaderState *xlogreader,
XLogRecPtr targetPagePtr,
int reqLen,
XLogRecPtr targetRecPtr,
@@ -175,6 +175,20 @@ struct XLogReaderState
XLogRecPtr ReadRecPtr; /* start of last record read */
XLogRecPtr EndRecPtr; /* end+1 of last record read */
+ /* ----------------------------------------
+ * Communication with page reader
+ * readBuf is XLOG_BLCKSZ bytes, valid up to at least readLen bytes.
+ * ----------------------------------------
+ */
+ /* variables to communicate with page reader */
+ XLogRecPtr readPagePtr; /* page pointer to read */
+ int32 readLen; /* bytes requested to reader, or actual bytes
+ * read by reader, which must be larger than
+ * the request, or -1 on error */
+ TimeLineID readPageTLI; /* TLI for data currently in readBuf */
+ char *readBuf; /* buffer to store data */
+ bool page_verified; /* is the page on the buffer verified? */
+
/* ----------------------------------------
* Decoded representation of current record
@@ -201,13 +215,6 @@ struct XLogReaderState
* ----------------------------------------
*/
- /*
- * Buffer for currently read page (XLOG_BLCKSZ bytes, valid up to at least
- * readLen bytes)
- */
- char *readBuf;
- uint32 readLen;
-
/* last read XLOG position for data currently in readBuf */
WALSegmentContext segcxt;
WALOpenSegment seg;
diff --git a/src/include/access/xlogutils.h b/src/include/access/xlogutils.h
index e59b6cf3a9..c287edf206 100644
--- a/src/include/access/xlogutils.h
+++ b/src/include/access/xlogutils.h
@@ -47,7 +47,7 @@ extern Buffer XLogReadBufferExtended(RelFileNode rnode, ForkNumber forknum,
extern Relation CreateFakeRelcacheEntry(RelFileNode rnode);
extern void FreeFakeRelcacheEntry(Relation fakerel);
-extern int read_local_xlog_page(XLogReaderState *state,
+extern bool read_local_xlog_page(XLogReaderState *state,
XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *cur_page);
extern void wal_segment_open(XLogReaderState *state,
--
2.18.4
----Next_Part(Thu_Jul__2_13_53_30_2020_072)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
filename="v15-0002-Move-page-reader-out-of-XLogReadRecord.patch"
^ permalink raw reply [nested|flat] 69+ messages in thread
* [PATCH v17 1/4] Move callback-call from ReadPageInternal to XLogReadRecord.
@ 2019-09-05 11:21 Kyotaro Horiguchi <[email protected]>
0 siblings, 0 replies; 69+ messages in thread
From: Kyotaro Horiguchi @ 2019-09-05 11:21 UTC (permalink / raw)
The current WAL record reader reads page data using a call back
function. Although it is not so problematic alone, it would be a
problem if we are going to do add tasks like encryption which is
performed on page data before WAL reader reads them. To avoid that the
record reader facility has to have a new code path corresponds to
every new callback, this patch separates page reader from WAL record
reading facility by modifying the current WAL record reader to a state
machine.
As the first step of that change, this patch moves the page reader
function out of ReadPageInternal, then the remaining tasks of the
function are taken over by the new function XLogNeedData. As the
result XLogPageRead directly calls the page reader callback function
according to the feedback from XLogNeedData.
---
src/backend/access/transam/xlog.c | 16 +-
src/backend/access/transam/xlogreader.c | 281 ++++++++++++++----------
src/backend/access/transam/xlogutils.c | 12 +-
src/backend/replication/walsender.c | 10 +-
src/bin/pg_rewind/parsexlog.c | 21 +-
src/bin/pg_waldump/pg_waldump.c | 8 +-
src/include/access/xlogreader.h | 25 ++-
src/include/access/xlogutils.h | 2 +-
8 files changed, 222 insertions(+), 153 deletions(-)
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 377afb8732..606328a65a 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -917,7 +917,7 @@ static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath,
static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli,
XLogSource source, bool notfoundOk);
static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, XLogSource source);
-static int XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
+static bool XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
bool fetching_ckpt, XLogRecPtr tliRecPtr);
@@ -4350,7 +4350,6 @@ ReadRecord(XLogReaderState *xlogreader, int emode,
XLogRecord *record;
XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
- /* Pass through parameters to XLogPageRead */
private->fetching_ckpt = fetching_ckpt;
private->emode = emode;
private->randAccess = (xlogreader->ReadRecPtr == InvalidXLogRecPtr);
@@ -11987,7 +11986,7 @@ CancelBackup(void)
* XLogPageRead() to try fetching the record from another source, or to
* sleep and retry.
*/
-static int
+static bool
XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *readBuf)
{
@@ -12046,7 +12045,8 @@ retry:
readLen = 0;
readSource = XLOG_FROM_ANY;
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
}
@@ -12141,7 +12141,8 @@ retry:
goto next_record_is_invalid;
}
- return readLen;
+ xlogreader->readLen = readLen;
+ return true;
next_record_is_invalid:
lastSourceFailed = true;
@@ -12155,8 +12156,9 @@ next_record_is_invalid:
/* In standby-mode, keep trying */
if (StandbyMode)
goto retry;
- else
- return -1;
+
+ xlogreader->readLen = -1;
+ return false;
}
/*
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index 42738eb940..7d04995b7b 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -36,8 +36,8 @@
static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
pg_attribute_printf(2, 3);
static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
-static int ReadPageInternal(XLogReaderState *state, XLogRecPtr pageptr,
- int reqLen);
+static bool XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr,
+ int reqLen, bool header_inclusive);
static void XLogReaderInvalReadState(XLogReaderState *state);
static bool ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
XLogRecPtr PrevRecPtr, XLogRecord *record, bool randAccess);
@@ -276,7 +276,6 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
uint32 targetRecOff;
uint32 pageHeaderSize;
bool gotheader;
- int readOff;
/*
* randAccess indicates whether to verify the previous-record pointer of
@@ -326,14 +325,20 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
* byte to cover the whole record header, or at least the part of it that
* fits on the same page.
*/
- readOff = ReadPageInternal(state, targetPagePtr,
- Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ));
- if (readOff < 0)
+ while (XLogNeedData(state, targetPagePtr,
+ Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+ targetRecOff != 0))
+ {
+ if (!state->routine.page_read(state, state->readPagePtr, state->readLen,
+ RecPtr, state->readBuf))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
/*
- * ReadPageInternal always returns at least the page header, so we can
- * examine it now.
+ * We have at least the page header, so we can examine it now.
*/
pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
if (targetRecOff == 0)
@@ -359,8 +364,8 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
goto err;
}
- /* ReadPageInternal has verified the page header */
- Assert(pageHeaderSize <= readOff);
+ /* XLogNeedData has verified the page header */
+ Assert(pageHeaderSize <= state->readLen);
/*
* Read the record length.
@@ -432,18 +437,27 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
do
{
+ int rest_len = total_len - gotlen;
+
/* Calculate pointer to beginning of next page */
targetPagePtr += XLOG_BLCKSZ;
/* Wait for the next page to become available */
- readOff = ReadPageInternal(state, targetPagePtr,
- Min(total_len - gotlen + SizeOfXLogShortPHD,
- XLOG_BLCKSZ));
+ while (XLogNeedData(state, targetPagePtr,
+ Min(rest_len, XLOG_BLCKSZ),
+ false))
+ {
+ if (!state->routine.page_read(state, state->readPagePtr,
+ state->readLen,
+ state->ReadRecPtr,
+ state->readBuf))
+ break;
+ }
- if (readOff < 0)
+ if (!state->page_verified)
goto err;
- Assert(SizeOfXLogShortPHD <= readOff);
+ Assert(SizeOfXLogShortPHD <= state->readLen);
/* Check that the continuation on next page looks valid */
pageHeader = (XLogPageHeader) state->readBuf;
@@ -473,21 +487,14 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
/* Append the continuation from this page to the buffer */
pageHeaderSize = XLogPageHeaderSize(pageHeader);
- if (readOff < pageHeaderSize)
- readOff = ReadPageInternal(state, targetPagePtr,
- pageHeaderSize);
-
- Assert(pageHeaderSize <= readOff);
+ Assert(pageHeaderSize <= state->readLen);
contdata = (char *) state->readBuf + pageHeaderSize;
len = XLOG_BLCKSZ - pageHeaderSize;
if (pageHeader->xlp_rem_len < len)
len = pageHeader->xlp_rem_len;
- if (readOff < pageHeaderSize + len)
- readOff = ReadPageInternal(state, targetPagePtr,
- pageHeaderSize + len);
-
+ Assert (pageHeaderSize + len <= state->readLen);
memcpy(buffer, (char *) contdata, len);
buffer += len;
gotlen += len;
@@ -517,9 +524,16 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
else
{
/* Wait for the record data to become available */
- readOff = ReadPageInternal(state, targetPagePtr,
- Min(targetRecOff + total_len, XLOG_BLCKSZ));
- if (readOff < 0)
+ while (XLogNeedData(state, targetPagePtr,
+ Min(targetRecOff + total_len, XLOG_BLCKSZ), true))
+ {
+ if (!state->routine.page_read(state, state->readPagePtr,
+ state->readLen,
+ state->ReadRecPtr, state->readBuf))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
/* Record does not cross a page boundary */
@@ -562,109 +576,138 @@ err:
}
/*
- * Read a single xlog page including at least [pageptr, reqLen] of valid data
- * via the page_read() callback.
+ * Checks that an xlog page loaded in state->readBuf is including at least
+ * [pageptr, reqLen] and the page is valid. header_inclusive indicates that
+ * reqLen is calculated including page header length.
*
- * Returns -1 if the required page cannot be read for some reason; errormsg_buf
- * is set in that case (unless the error occurs in the page_read callback).
+ * Returns false if the buffer already contains the requested data, or found
+ * error. state->page_verified is set to true for the former and false for the
+ * latter.
*
- * We fetch the page from a reader-local cache if we know we have the required
- * data and if there hasn't been any error since caching the data.
+ * Otherwise returns true and requests data loaded onto state->readBuf by
+ * state->readPagePtr and state->readLen. The caller shall call this function
+ * again after filling the buffer at least with that portion of data and set
+ * state->readLen to the length of actually loaded data.
+ *
+ * If header_inclusive is false, corrects reqLen internally by adding the
+ * actual page header length and may request caller for new data.
*/
-static int
-ReadPageInternal(XLogReaderState *state, XLogRecPtr pageptr, int reqLen)
+static bool
+XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
+ bool header_inclusive)
{
- int readLen;
uint32 targetPageOff;
XLogSegNo targetSegNo;
- XLogPageHeader hdr;
+ uint32 addLen = 0;
- Assert((pageptr % XLOG_BLCKSZ) == 0);
+ /* Some data is loaded, but page header is not verified yet. */
+ if (!state->page_verified &&
+ !XLogRecPtrIsInvalid(state->readPagePtr) && state->readLen >= 0)
+ {
+ uint32 pageHeaderSize;
+ /* just loaded new data so needs to verify page header */
+
+ /* The caller must have loaded at least page header */
+ Assert (state->readLen >= SizeOfXLogShortPHD);
+
+ /*
+ * We have enough data to check the header length. Recheck the loaded
+ * length against the actual header length.
+ */
+ pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+
+ /* Request more data if we don't have the full header. */
+ if (state->readLen < pageHeaderSize)
+ {
+ state->readLen = pageHeaderSize;
+ return true;
+ }
+
+ /* Now that we know we have the full header, validate it. */
+ if (!XLogReaderValidatePageHeader(state, state->readPagePtr,
+ (char *) state->readBuf))
+ {
+ /* That's bad. Force reading the page again. */
+ XLogReaderInvalReadState(state);
+
+ return false;
+ }
+
+ state->page_verified = true;
+
+ XLByteToSeg(state->readPagePtr, state->seg.ws_segno,
+ state->segcxt.ws_segsize);
+ }
+
+ /*
+ * The loaded page may not be the one caller is supposing to read when we
+ * are verifying the first page of new segment. In that case, skip further
+ * verification and immediately load the target page.
+ */
+ if (state->page_verified && pageptr == state->readPagePtr)
+ {
+ /*
+ * calculate additional length for page header keeping the total
+ * length within the block size.
+ */
+ if (!header_inclusive)
+ {
+ uint32 pageHeaderSize =
+ XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+
+ addLen = pageHeaderSize;
+ if (reqLen + pageHeaderSize <= XLOG_BLCKSZ)
+ addLen = pageHeaderSize;
+ else
+ addLen = XLOG_BLCKSZ - reqLen;
+
+ Assert(addLen >= 0);
+ }
+
+ /* Return if we already have it. */
+ if (reqLen + addLen <= state->readLen)
+ return false;
+ }
+
+ /* Data is not in our buffer, request the caller for it. */
XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize);
targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize);
+ Assert((pageptr % XLOG_BLCKSZ) == 0);
- /* check whether we have all the requested data already */
- if (targetSegNo == state->seg.ws_segno &&
- targetPageOff == state->segoff && reqLen <= state->readLen)
- return state->readLen;
+ /*
+ * Every time we request to load new data of a page to the caller, even if
+ * we looked at a part of it before, we need to do verification on the next
+ * invocation as the caller might now be rereading data from a different
+ * source.
+ */
+ state->page_verified = false;
/*
- * Data is not in our buffer.
- *
- * Every time we actually read the segment, even if we looked at parts of
- * it before, we need to do verification as the page_read callback might
- * now be rereading data from a different source.
- *
* Whenever switching to a new WAL segment, we read the first page of the
* file and validate its header, even if that's not where the target
* record is. This is so that we can check the additional identification
* info that is present in the first page's "long" header.
+ * Don't do this if the caller requested the first page in the segment.
*/
if (targetSegNo != state->seg.ws_segno && targetPageOff != 0)
{
- XLogRecPtr targetSegmentPtr = pageptr - targetPageOff;
-
- readLen = state->routine.page_read(state, targetSegmentPtr, XLOG_BLCKSZ,
- state->currRecPtr,
- state->readBuf);
- if (readLen < 0)
- goto err;
-
- /* we can be sure to have enough WAL available, we scrolled back */
- Assert(readLen == XLOG_BLCKSZ);
-
- if (!XLogReaderValidatePageHeader(state, targetSegmentPtr,
- state->readBuf))
- goto err;
+ /*
+ * Then we'll see that the targetSegNo now matches the ws_segno, and
+ * will not come back here, but will request the actual target page.
+ */
+ state->readPagePtr = pageptr - targetPageOff;
+ state->readLen = XLOG_BLCKSZ;
+ return true;
}
/*
- * First, read the requested data length, but at least a short page header
- * so that we can validate it.
+ * Request the caller to load the page. We need at least a short page
+ * header so that we can validate it.
*/
- readLen = state->routine.page_read(state, pageptr, Max(reqLen, SizeOfXLogShortPHD),
- state->currRecPtr,
- state->readBuf);
- if (readLen < 0)
- goto err;
-
- Assert(readLen <= XLOG_BLCKSZ);
-
- /* Do we have enough data to check the header length? */
- if (readLen <= SizeOfXLogShortPHD)
- goto err;
-
- Assert(readLen >= reqLen);
-
- hdr = (XLogPageHeader) state->readBuf;
-
- /* still not enough */
- if (readLen < XLogPageHeaderSize(hdr))
- {
- readLen = state->routine.page_read(state, pageptr, XLogPageHeaderSize(hdr),
- state->currRecPtr,
- state->readBuf);
- if (readLen < 0)
- goto err;
- }
-
- /*
- * Now that we know we have the full header, validate it.
- */
- if (!XLogReaderValidatePageHeader(state, pageptr, (char *) hdr))
- goto err;
-
- /* update read state information */
- state->seg.ws_segno = targetSegNo;
- state->segoff = targetPageOff;
- state->readLen = readLen;
-
- return readLen;
-
-err:
- XLogReaderInvalReadState(state);
- return -1;
+ state->readPagePtr = pageptr;
+ state->readLen = Max(reqLen + addLen, SizeOfXLogShortPHD);
+ return true;
}
/*
@@ -673,9 +716,7 @@ err:
static void
XLogReaderInvalReadState(XLogReaderState *state)
{
- state->seg.ws_segno = 0;
- state->segoff = 0;
- state->readLen = 0;
+ state->readPagePtr = InvalidXLogRecPtr;
}
/*
@@ -953,7 +994,6 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
XLogRecPtr targetPagePtr;
int targetRecOff;
uint32 pageHeaderSize;
- int readLen;
/*
* Compute targetRecOff. It should typically be equal or greater than
@@ -961,7 +1001,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
* that, except when caller has explicitly specified the offset that
* falls somewhere there or when we are skipping multi-page
* continuation record. It doesn't matter though because
- * ReadPageInternal() is prepared to handle that and will read at
+ * XLogNeedData() is prepared to handle that and will read at
* least short page-header worth of data
*/
targetRecOff = tmpRecPtr % XLOG_BLCKSZ;
@@ -969,19 +1009,24 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
/* scroll back to page boundary */
targetPagePtr = tmpRecPtr - targetRecOff;
- /* Read the page containing the record */
- readLen = ReadPageInternal(state, targetPagePtr, targetRecOff);
- if (readLen < 0)
+ while(XLogNeedData(state, targetPagePtr, targetRecOff,
+ targetRecOff != 0))
+ {
+ if (!state->routine.page_read(state, state->readPagePtr,
+ state->readLen,
+ state->ReadRecPtr, state->readBuf))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
header = (XLogPageHeader) state->readBuf;
pageHeaderSize = XLogPageHeaderSize(header);
- /* make sure we have enough data for the page header */
- readLen = ReadPageInternal(state, targetPagePtr, pageHeaderSize);
- if (readLen < 0)
- goto err;
+ /* we should have read the page header */
+ Assert (state->readLen >= pageHeaderSize);
/* skip over potential continuation data */
if (header->xlp_info & XLP_FIRST_IS_CONTRECORD)
diff --git a/src/backend/access/transam/xlogutils.c b/src/backend/access/transam/xlogutils.c
index d17d660f46..46eda33f25 100644
--- a/src/backend/access/transam/xlogutils.c
+++ b/src/backend/access/transam/xlogutils.c
@@ -686,8 +686,8 @@ XLogTruncateRelation(RelFileNode rnode, ForkNumber forkNum,
void
XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wantLength)
{
- const XLogRecPtr lastReadPage = (state->seg.ws_segno *
- state->segcxt.ws_segsize + state->segoff);
+ const XLogRecPtr lastReadPage = state->seg.ws_segno *
+ state->segcxt.ws_segsize + state->readLen;
Assert(wantPage != InvalidXLogRecPtr && wantPage % XLOG_BLCKSZ == 0);
Assert(wantLength <= XLOG_BLCKSZ);
@@ -824,7 +824,7 @@ wal_segment_close(XLogReaderState *state)
* exists for normal backends, so we have to do a check/sleep/repeat style of
* loop for now.
*/
-int
+bool
read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *cur_page)
{
@@ -926,7 +926,8 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
else if (targetPagePtr + reqLen > read_upto)
{
/* not enough data there */
- return -1;
+ state->readLen = -1;
+ return false;
}
else
{
@@ -944,7 +945,8 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
WALReadRaiseError(&errinfo);
/* number of valid bytes in the buffer */
- return count;
+ state->readLen = count;
+ return true;
}
/*
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index 23baa4498a..c36b91ae5e 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -806,7 +806,7 @@ StartReplication(StartReplicationCmd *cmd)
* which has to do a plain sleep/busy loop, because the walsender's latch gets
* set every time WAL is flushed.
*/
-static int
+static bool
logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *cur_page)
{
@@ -826,7 +826,10 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
/* fail if not (implies we are going to shut down) */
if (flushptr < targetPagePtr + reqLen)
- return -1;
+ {
+ state->readLen = -1;
+ return false;
+ }
if (targetPagePtr + XLOG_BLCKSZ <= flushptr)
count = XLOG_BLCKSZ; /* more than one block available */
@@ -854,7 +857,8 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
XLByteToSeg(targetPagePtr, segno, state->segcxt.ws_segsize);
CheckXLogRemoved(segno, state->seg.ws_tli);
- return count;
+ state->readLen = count;
+ return true;
}
/*
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index 59ebac7d6a..cf119848b0 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -47,7 +47,7 @@ typedef struct XLogPageReadPrivate
int tliIndex;
} XLogPageReadPrivate;
-static int SimpleXLogPageRead(XLogReaderState *xlogreader,
+static bool SimpleXLogPageRead(XLogReaderState *xlogreader,
XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
@@ -246,7 +246,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
}
/* XLogReader callback function, to read a WAL page */
-static int
+static bool
SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf)
{
@@ -306,7 +306,8 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
if (private->restoreCommand == NULL)
{
pg_log_error("could not open file \"%s\": %m", xlogfpath);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
/*
@@ -319,7 +320,10 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
private->restoreCommand);
if (xlogreadfd < 0)
- return -1;
+ {
+ xlogreader->readLen = -1;
+ return false;
+ }
else
pg_log_debug("using file \"%s\" restored from archive",
xlogfpath);
@@ -335,7 +339,8 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
if (lseek(xlogreadfd, (off_t) targetPageOff, SEEK_SET) < 0)
{
pg_log_error("could not seek in file \"%s\": %m", xlogfpath);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
@@ -348,13 +353,15 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
pg_log_error("could not read file \"%s\": read %d of %zu",
xlogfpath, r, (Size) XLOG_BLCKSZ);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
Assert(targetSegNo == xlogreadsegno);
xlogreader->seg.ws_tli = targetHistory[private->tliIndex].tli;
- return XLOG_BLCKSZ;
+ xlogreader->readLen = XLOG_BLCKSZ;
+ return true;
}
/*
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index 610f65e471..0f0e0723a3 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -331,7 +331,7 @@ WALDumpCloseSegment(XLogReaderState *state)
}
/* pg_waldump's XLogReaderRoutine->page_read callback */
-static int
+static bool
WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetPtr, char *readBuff)
{
@@ -348,7 +348,8 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
else
{
private->endptr_reached = true;
- return -1;
+ state->readLen = -1;
+ return false;
}
}
@@ -373,7 +374,8 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
(Size) errinfo.wre_req);
}
- return count;
+ state->readLen = count;
+ return true;
}
/*
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index 21d200d3df..23d9e70a59 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -57,8 +57,8 @@ typedef struct WALSegmentContext
typedef struct XLogReaderState XLogReaderState;
-/* Function type definitions for various xlogreader interactions */
-typedef int (*XLogPageReadCB) (XLogReaderState *xlogreader,
+/* Function type definition for the read_page callback */
+typedef bool (*XLogPageReadCB) (XLogReaderState *xlogreader,
XLogRecPtr targetPagePtr,
int reqLen,
XLogRecPtr targetRecPtr,
@@ -175,6 +175,20 @@ struct XLogReaderState
XLogRecPtr ReadRecPtr; /* start of last record read */
XLogRecPtr EndRecPtr; /* end+1 of last record read */
+ /* ----------------------------------------
+ * Communication with page reader
+ * readBuf is XLOG_BLCKSZ bytes, valid up to at least readLen bytes.
+ * ----------------------------------------
+ */
+ /* variables to communicate with page reader */
+ XLogRecPtr readPagePtr; /* page pointer to read */
+ int32 readLen; /* bytes requested to reader, or actual bytes
+ * read by reader, which must be larger than
+ * the request, or -1 on error */
+ TimeLineID readPageTLI; /* TLI for data currently in readBuf */
+ char *readBuf; /* buffer to store data */
+ bool page_verified; /* is the page on the buffer verified? */
+
/* ----------------------------------------
* Decoded representation of current record
@@ -203,13 +217,6 @@ struct XLogReaderState
* ----------------------------------------
*/
- /*
- * Buffer for currently read page (XLOG_BLCKSZ bytes, valid up to at least
- * readLen bytes)
- */
- char *readBuf;
- uint32 readLen;
-
/* last read XLOG position for data currently in readBuf */
WALSegmentContext segcxt;
WALOpenSegment seg;
diff --git a/src/include/access/xlogutils.h b/src/include/access/xlogutils.h
index 9ac602b674..364a21c4ea 100644
--- a/src/include/access/xlogutils.h
+++ b/src/include/access/xlogutils.h
@@ -47,7 +47,7 @@ extern Buffer XLogReadBufferExtended(RelFileNode rnode, ForkNumber forknum,
extern Relation CreateFakeRelcacheEntry(RelFileNode rnode);
extern void FreeFakeRelcacheEntry(Relation fakerel);
-extern int read_local_xlog_page(XLogReaderState *state,
+extern bool read_local_xlog_page(XLogReaderState *state,
XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *cur_page);
extern void wal_segment_open(XLogReaderState *state,
--
2.27.0
----Next_Part(Thu_Mar__4_11_28_53_2021_014)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
filename="v17-0002-Move-page-reader-out-of-XLogReadRecord.patch"
^ permalink raw reply [nested|flat] 69+ messages in thread
* [PATCH v15 1/4] Move callback-call from ReadPageInternal to XLogReadRecord.
@ 2019-09-05 11:21 Kyotaro Horiguchi <[email protected]>
0 siblings, 0 replies; 69+ messages in thread
From: Kyotaro Horiguchi @ 2019-09-05 11:21 UTC (permalink / raw)
The current WAL record reader reads page data using a call back
function. Although it is not so problematic alone, it would be a
problem if we are going to do add tasks like encryption which is
performed on page data before WAL reader reads them. To avoid that the
record reader facility has to have a new code path corresponds to
every new callback, this patch separates page reader from WAL record
reading facility by modifying the current WAL record reader to a state
machine.
As the first step of that change, this patch moves the page reader
function out of ReadPageInternal, then the remaining tasks of the
function are taken over by the new function XLogNeedData. As the
result XLogPageRead directly calls the page reader callback function
according to the feedback from XLogNeedData.
---
src/backend/access/transam/xlog.c | 16 +-
src/backend/access/transam/xlogreader.c | 281 ++++++++++++++----------
src/backend/access/transam/xlogutils.c | 12 +-
src/backend/replication/walsender.c | 10 +-
src/bin/pg_rewind/parsexlog.c | 21 +-
src/bin/pg_waldump/pg_waldump.c | 8 +-
src/include/access/xlogreader.h | 25 ++-
src/include/access/xlogutils.h | 2 +-
8 files changed, 222 insertions(+), 153 deletions(-)
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index fd93bcfaeb..e570e56a24 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -912,7 +912,7 @@ static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath,
static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli,
XLogSource source, bool notfoundOk);
static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, XLogSource source);
-static int XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
+static bool XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
bool fetching_ckpt, XLogRecPtr tliRecPtr);
@@ -4332,7 +4332,6 @@ ReadRecord(XLogReaderState *xlogreader, int emode,
XLogRecord *record;
XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
- /* Pass through parameters to XLogPageRead */
private->fetching_ckpt = fetching_ckpt;
private->emode = emode;
private->randAccess = (xlogreader->ReadRecPtr == InvalidXLogRecPtr);
@@ -11860,7 +11859,7 @@ CancelBackup(void)
* XLogPageRead() to try fetching the record from another source, or to
* sleep and retry.
*/
-static int
+static bool
XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *readBuf)
{
@@ -11919,7 +11918,8 @@ retry:
readLen = 0;
readSource = XLOG_FROM_ANY;
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
}
@@ -12014,7 +12014,8 @@ retry:
goto next_record_is_invalid;
}
- return readLen;
+ xlogreader->readLen = readLen;
+ return true;
next_record_is_invalid:
lastSourceFailed = true;
@@ -12028,8 +12029,9 @@ next_record_is_invalid:
/* In standby-mode, keep trying */
if (StandbyMode)
goto retry;
- else
- return -1;
+
+ xlogreader->readLen = -1;
+ return false;
}
/*
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index cb76be4f46..3d599325ee 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -36,8 +36,8 @@
static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
pg_attribute_printf(2, 3);
static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
-static int ReadPageInternal(XLogReaderState *state, XLogRecPtr pageptr,
- int reqLen);
+static bool XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr,
+ int reqLen, bool header_inclusive);
static void XLogReaderInvalReadState(XLogReaderState *state);
static bool ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
XLogRecPtr PrevRecPtr, XLogRecord *record, bool randAccess);
@@ -276,7 +276,6 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
uint32 targetRecOff;
uint32 pageHeaderSize;
bool gotheader;
- int readOff;
/*
* randAccess indicates whether to verify the previous-record pointer of
@@ -326,14 +325,20 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
* byte to cover the whole record header, or at least the part of it that
* fits on the same page.
*/
- readOff = ReadPageInternal(state, targetPagePtr,
- Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ));
- if (readOff < 0)
+ while (XLogNeedData(state, targetPagePtr,
+ Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+ targetRecOff != 0))
+ {
+ if (!state->routine.page_read(state, state->readPagePtr, state->readLen,
+ RecPtr, state->readBuf))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
/*
- * ReadPageInternal always returns at least the page header, so we can
- * examine it now.
+ * We have at least the page header, so we can examine it now.
*/
pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
if (targetRecOff == 0)
@@ -359,8 +364,8 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
goto err;
}
- /* ReadPageInternal has verified the page header */
- Assert(pageHeaderSize <= readOff);
+ /* XLogNeedData has verified the page header */
+ Assert(pageHeaderSize <= state->readLen);
/*
* Read the record length.
@@ -433,18 +438,27 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
do
{
+ int rest_len = total_len - gotlen;
+
/* Calculate pointer to beginning of next page */
targetPagePtr += XLOG_BLCKSZ;
/* Wait for the next page to become available */
- readOff = ReadPageInternal(state, targetPagePtr,
- Min(total_len - gotlen + SizeOfXLogShortPHD,
- XLOG_BLCKSZ));
+ while (XLogNeedData(state, targetPagePtr,
+ Min(rest_len, XLOG_BLCKSZ),
+ false))
+ {
+ if (!state->routine.page_read(state, state->readPagePtr,
+ state->readLen,
+ state->ReadRecPtr,
+ state->readBuf))
+ break;
+ }
- if (readOff < 0)
+ if (!state->page_verified)
goto err;
- Assert(SizeOfXLogShortPHD <= readOff);
+ Assert(SizeOfXLogShortPHD <= state->readLen);
/* Check that the continuation on next page looks valid */
pageHeader = (XLogPageHeader) state->readBuf;
@@ -473,21 +487,14 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
/* Append the continuation from this page to the buffer */
pageHeaderSize = XLogPageHeaderSize(pageHeader);
- if (readOff < pageHeaderSize)
- readOff = ReadPageInternal(state, targetPagePtr,
- pageHeaderSize);
-
- Assert(pageHeaderSize <= readOff);
+ Assert(pageHeaderSize <= state->readLen);
contdata = (char *) state->readBuf + pageHeaderSize;
len = XLOG_BLCKSZ - pageHeaderSize;
if (pageHeader->xlp_rem_len < len)
len = pageHeader->xlp_rem_len;
- if (readOff < pageHeaderSize + len)
- readOff = ReadPageInternal(state, targetPagePtr,
- pageHeaderSize + len);
-
+ Assert (pageHeaderSize + len <= state->readLen);
memcpy(buffer, (char *) contdata, len);
buffer += len;
gotlen += len;
@@ -517,9 +524,16 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
else
{
/* Wait for the record data to become available */
- readOff = ReadPageInternal(state, targetPagePtr,
- Min(targetRecOff + total_len, XLOG_BLCKSZ));
- if (readOff < 0)
+ while (XLogNeedData(state, targetPagePtr,
+ Min(targetRecOff + total_len, XLOG_BLCKSZ), true))
+ {
+ if (!state->routine.page_read(state, state->readPagePtr,
+ state->readLen,
+ state->ReadRecPtr, state->readBuf))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
/* Record does not cross a page boundary */
@@ -562,109 +576,138 @@ err:
}
/*
- * Read a single xlog page including at least [pageptr, reqLen] of valid data
- * via the page_read() callback.
+ * Checks that an xlog page loaded in state->readBuf is including at least
+ * [pageptr, reqLen] and the page is valid. header_inclusive indicates that
+ * reqLen is calculated including page header length.
*
- * Returns -1 if the required page cannot be read for some reason; errormsg_buf
- * is set in that case (unless the error occurs in the page_read callback).
+ * Returns false if the buffer already contains the requested data, or found
+ * error. state->page_verified is set to true for the former and false for the
+ * latter.
*
- * We fetch the page from a reader-local cache if we know we have the required
- * data and if there hasn't been any error since caching the data.
+ * Otherwise returns true and requests data loaded onto state->readBuf by
+ * state->readPagePtr and state->readLen. The caller shall call this function
+ * again after filling the buffer at least with that portion of data and set
+ * state->readLen to the length of actually loaded data.
+ *
+ * If header_inclusive is false, corrects reqLen internally by adding the
+ * actual page header length and may request caller for new data.
*/
-static int
-ReadPageInternal(XLogReaderState *state, XLogRecPtr pageptr, int reqLen)
+static bool
+XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
+ bool header_inclusive)
{
- int readLen;
uint32 targetPageOff;
XLogSegNo targetSegNo;
- XLogPageHeader hdr;
+ uint32 addLen = 0;
- Assert((pageptr % XLOG_BLCKSZ) == 0);
+ /* Some data is loaded, but page header is not verified yet. */
+ if (!state->page_verified &&
+ !XLogRecPtrIsInvalid(state->readPagePtr) && state->readLen >= 0)
+ {
+ uint32 pageHeaderSize;
+ /* just loaded new data so needs to verify page header */
+
+ /* The caller must have loaded at least page header */
+ Assert (state->readLen >= SizeOfXLogShortPHD);
+
+ /*
+ * We have enough data to check the header length. Recheck the loaded
+ * length against the actual header length.
+ */
+ pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+
+ /* Request more data if we don't have the full header. */
+ if (state->readLen < pageHeaderSize)
+ {
+ state->readLen = pageHeaderSize;
+ return true;
+ }
+
+ /* Now that we know we have the full header, validate it. */
+ if (!XLogReaderValidatePageHeader(state, state->readPagePtr,
+ (char *) state->readBuf))
+ {
+ /* That's bad. Force reading the page again. */
+ XLogReaderInvalReadState(state);
+
+ return false;
+ }
+
+ state->page_verified = true;
+
+ XLByteToSeg(state->readPagePtr, state->seg.ws_segno,
+ state->segcxt.ws_segsize);
+ }
+
+ /*
+ * The loaded page may not be the one caller is supposing to read when we
+ * are verifying the first page of new segment. In that case, skip further
+ * verification and immediately load the target page.
+ */
+ if (state->page_verified && pageptr == state->readPagePtr)
+ {
+ /*
+ * calculate additional length for page header keeping the total
+ * length within the block size.
+ */
+ if (!header_inclusive)
+ {
+ uint32 pageHeaderSize =
+ XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+
+ addLen = pageHeaderSize;
+ if (reqLen + pageHeaderSize <= XLOG_BLCKSZ)
+ addLen = pageHeaderSize;
+ else
+ addLen = XLOG_BLCKSZ - reqLen;
+
+ Assert(addLen >= 0);
+ }
+
+ /* Return if we already have it. */
+ if (reqLen + addLen <= state->readLen)
+ return false;
+ }
+
+ /* Data is not in our buffer, request the caller for it. */
XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize);
targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize);
+ Assert((pageptr % XLOG_BLCKSZ) == 0);
- /* check whether we have all the requested data already */
- if (targetSegNo == state->seg.ws_segno &&
- targetPageOff == state->segoff && reqLen <= state->readLen)
- return state->readLen;
+ /*
+ * Every time we request to load new data of a page to the caller, even if
+ * we looked at a part of it before, we need to do verification on the next
+ * invocation as the caller might now be rereading data from a different
+ * source.
+ */
+ state->page_verified = false;
/*
- * Data is not in our buffer.
- *
- * Every time we actually read the segment, even if we looked at parts of
- * it before, we need to do verification as the page_read callback might
- * now be rereading data from a different source.
- *
* Whenever switching to a new WAL segment, we read the first page of the
* file and validate its header, even if that's not where the target
* record is. This is so that we can check the additional identification
* info that is present in the first page's "long" header.
+ * Don't do this if the caller requested the first page in the segment.
*/
if (targetSegNo != state->seg.ws_segno && targetPageOff != 0)
{
- XLogRecPtr targetSegmentPtr = pageptr - targetPageOff;
-
- readLen = state->routine.page_read(state, targetSegmentPtr, XLOG_BLCKSZ,
- state->currRecPtr,
- state->readBuf);
- if (readLen < 0)
- goto err;
-
- /* we can be sure to have enough WAL available, we scrolled back */
- Assert(readLen == XLOG_BLCKSZ);
-
- if (!XLogReaderValidatePageHeader(state, targetSegmentPtr,
- state->readBuf))
- goto err;
+ /*
+ * Then we'll see that the targetSegNo now matches the ws_segno, and
+ * will not come back here, but will request the actual target page.
+ */
+ state->readPagePtr = pageptr - targetPageOff;
+ state->readLen = XLOG_BLCKSZ;
+ return true;
}
/*
- * First, read the requested data length, but at least a short page header
- * so that we can validate it.
+ * Request the caller to load the page. We need at least a short page
+ * header so that we can validate it.
*/
- readLen = state->routine.page_read(state, pageptr, Max(reqLen, SizeOfXLogShortPHD),
- state->currRecPtr,
- state->readBuf);
- if (readLen < 0)
- goto err;
-
- Assert(readLen <= XLOG_BLCKSZ);
-
- /* Do we have enough data to check the header length? */
- if (readLen <= SizeOfXLogShortPHD)
- goto err;
-
- Assert(readLen >= reqLen);
-
- hdr = (XLogPageHeader) state->readBuf;
-
- /* still not enough */
- if (readLen < XLogPageHeaderSize(hdr))
- {
- readLen = state->routine.page_read(state, pageptr, XLogPageHeaderSize(hdr),
- state->currRecPtr,
- state->readBuf);
- if (readLen < 0)
- goto err;
- }
-
- /*
- * Now that we know we have the full header, validate it.
- */
- if (!XLogReaderValidatePageHeader(state, pageptr, (char *) hdr))
- goto err;
-
- /* update read state information */
- state->seg.ws_segno = targetSegNo;
- state->segoff = targetPageOff;
- state->readLen = readLen;
-
- return readLen;
-
-err:
- XLogReaderInvalReadState(state);
- return -1;
+ state->readPagePtr = pageptr;
+ state->readLen = Max(reqLen + addLen, SizeOfXLogShortPHD);
+ return true;
}
/*
@@ -673,9 +716,7 @@ err:
static void
XLogReaderInvalReadState(XLogReaderState *state)
{
- state->seg.ws_segno = 0;
- state->segoff = 0;
- state->readLen = 0;
+ state->readPagePtr = InvalidXLogRecPtr;
}
/*
@@ -956,7 +997,6 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
XLogRecPtr targetPagePtr;
int targetRecOff;
uint32 pageHeaderSize;
- int readLen;
/*
* Compute targetRecOff. It should typically be equal or greater than
@@ -964,7 +1004,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
* that, except when caller has explicitly specified the offset that
* falls somewhere there or when we are skipping multi-page
* continuation record. It doesn't matter though because
- * ReadPageInternal() is prepared to handle that and will read at
+ * XLogNeedData() is prepared to handle that and will read at
* least short page-header worth of data
*/
targetRecOff = tmpRecPtr % XLOG_BLCKSZ;
@@ -972,19 +1012,24 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
/* scroll back to page boundary */
targetPagePtr = tmpRecPtr - targetRecOff;
- /* Read the page containing the record */
- readLen = ReadPageInternal(state, targetPagePtr, targetRecOff);
- if (readLen < 0)
+ while(XLogNeedData(state, targetPagePtr, targetRecOff,
+ targetRecOff != 0))
+ {
+ if (!state->routine.page_read(state, state->readPagePtr,
+ state->readLen,
+ state->ReadRecPtr, state->readBuf))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
header = (XLogPageHeader) state->readBuf;
pageHeaderSize = XLogPageHeaderSize(header);
- /* make sure we have enough data for the page header */
- readLen = ReadPageInternal(state, targetPagePtr, pageHeaderSize);
- if (readLen < 0)
- goto err;
+ /* we should have read the page header */
+ Assert (state->readLen >= pageHeaderSize);
/* skip over potential continuation data */
if (header->xlp_info & XLP_FIRST_IS_CONTRECORD)
diff --git a/src/backend/access/transam/xlogutils.c b/src/backend/access/transam/xlogutils.c
index 322b0e8ff5..1b10b6df20 100644
--- a/src/backend/access/transam/xlogutils.c
+++ b/src/backend/access/transam/xlogutils.c
@@ -689,8 +689,8 @@ XLogTruncateRelation(RelFileNode rnode, ForkNumber forkNum,
void
XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wantLength)
{
- const XLogRecPtr lastReadPage = (state->seg.ws_segno *
- state->segcxt.ws_segsize + state->segoff);
+ const XLogRecPtr lastReadPage = state->seg.ws_segno *
+ state->segcxt.ws_segsize + state->readLen;
Assert(wantPage != InvalidXLogRecPtr && wantPage % XLOG_BLCKSZ == 0);
Assert(wantLength <= XLOG_BLCKSZ);
@@ -828,7 +828,7 @@ wal_segment_close(XLogReaderState *state)
* exists for normal backends, so we have to do a check/sleep/repeat style of
* loop for now.
*/
-int
+bool
read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *cur_page)
{
@@ -930,7 +930,8 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
else if (targetPagePtr + reqLen > read_upto)
{
/* not enough data there */
- return -1;
+ state->readLen = -1;
+ return false;
}
else
{
@@ -948,7 +949,8 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
WALReadRaiseError(&errinfo);
/* number of valid bytes in the buffer */
- return count;
+ state->readLen = count;
+ return true;
}
/*
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index e2477c47e0..bf2711bddd 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -811,7 +811,7 @@ StartReplication(StartReplicationCmd *cmd)
* which has to do a plain sleep/busy loop, because the walsender's latch gets
* set every time WAL is flushed.
*/
-static int
+static bool
logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *cur_page)
{
@@ -831,7 +831,10 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
/* fail if not (implies we are going to shut down) */
if (flushptr < targetPagePtr + reqLen)
- return -1;
+ {
+ state->readLen = -1;
+ return false;
+ }
if (targetPagePtr + XLOG_BLCKSZ <= flushptr)
count = XLOG_BLCKSZ; /* more than one block available */
@@ -859,7 +862,8 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
XLByteToSeg(targetPagePtr, segno, state->segcxt.ws_segsize);
CheckXLogRemoved(segno, state->seg.ws_tli);
- return count;
+ state->readLen = count;
+ return true;
}
/*
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index bc6f976994..0b7e73ae79 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -46,7 +46,7 @@ typedef struct XLogPageReadPrivate
int tliIndex;
} XLogPageReadPrivate;
-static int SimpleXLogPageRead(XLogReaderState *xlogreader,
+static bool SimpleXLogPageRead(XLogReaderState *xlogreader,
XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
@@ -236,7 +236,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
}
/* XLogReader callback function, to read a WAL page */
-static int
+static bool
SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf)
{
@@ -296,7 +296,8 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
if (private->restoreCommand == NULL)
{
pg_log_error("could not open file \"%s\": %m", xlogfpath);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
/*
@@ -309,7 +310,10 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
private->restoreCommand);
if (xlogreadfd < 0)
- return -1;
+ {
+ xlogreader->readLen = -1;
+ return false;
+ }
else
pg_log_debug("using file \"%s\" restored from archive",
xlogfpath);
@@ -325,7 +329,8 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
if (lseek(xlogreadfd, (off_t) targetPageOff, SEEK_SET) < 0)
{
pg_log_error("could not seek in file \"%s\": %m", xlogfpath);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
@@ -338,13 +343,15 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
pg_log_error("could not read file \"%s\": read %d of %zu",
xlogfpath, r, (Size) XLOG_BLCKSZ);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
Assert(targetSegNo == xlogreadsegno);
xlogreader->seg.ws_tli = targetHistory[private->tliIndex].tli;
- return XLOG_BLCKSZ;
+ xlogreader->readLen = XLOG_BLCKSZ;
+ return true;
}
/*
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index d1a0678935..ab0c28d9b7 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -331,7 +331,7 @@ WALDumpCloseSegment(XLogReaderState *state)
}
/* pg_waldump's XLogReaderRoutine->page_read callback */
-static int
+static bool
WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetPtr, char *readBuff)
{
@@ -348,7 +348,8 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
else
{
private->endptr_reached = true;
- return -1;
+ state->readLen = -1;
+ return false;
}
}
@@ -373,7 +374,8 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
(Size) errinfo.wre_req);
}
- return count;
+ state->readLen = count;
+ return true;
}
/*
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index b0f2a6ed43..6c8848e14f 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -57,8 +57,8 @@ typedef struct WALSegmentContext
typedef struct XLogReaderState XLogReaderState;
-/* Function type definitions for various xlogreader interactions */
-typedef int (*XLogPageReadCB) (XLogReaderState *xlogreader,
+/* Function type definition for the read_page callback */
+typedef bool (*XLogPageReadCB) (XLogReaderState *xlogreader,
XLogRecPtr targetPagePtr,
int reqLen,
XLogRecPtr targetRecPtr,
@@ -175,6 +175,20 @@ struct XLogReaderState
XLogRecPtr ReadRecPtr; /* start of last record read */
XLogRecPtr EndRecPtr; /* end+1 of last record read */
+ /* ----------------------------------------
+ * Communication with page reader
+ * readBuf is XLOG_BLCKSZ bytes, valid up to at least readLen bytes.
+ * ----------------------------------------
+ */
+ /* variables to communicate with page reader */
+ XLogRecPtr readPagePtr; /* page pointer to read */
+ int32 readLen; /* bytes requested to reader, or actual bytes
+ * read by reader, which must be larger than
+ * the request, or -1 on error */
+ TimeLineID readPageTLI; /* TLI for data currently in readBuf */
+ char *readBuf; /* buffer to store data */
+ bool page_verified; /* is the page on the buffer verified? */
+
/* ----------------------------------------
* Decoded representation of current record
@@ -201,13 +215,6 @@ struct XLogReaderState
* ----------------------------------------
*/
- /*
- * Buffer for currently read page (XLOG_BLCKSZ bytes, valid up to at least
- * readLen bytes)
- */
- char *readBuf;
- uint32 readLen;
-
/* last read XLOG position for data currently in readBuf */
WALSegmentContext segcxt;
WALOpenSegment seg;
diff --git a/src/include/access/xlogutils.h b/src/include/access/xlogutils.h
index e59b6cf3a9..c287edf206 100644
--- a/src/include/access/xlogutils.h
+++ b/src/include/access/xlogutils.h
@@ -47,7 +47,7 @@ extern Buffer XLogReadBufferExtended(RelFileNode rnode, ForkNumber forknum,
extern Relation CreateFakeRelcacheEntry(RelFileNode rnode);
extern void FreeFakeRelcacheEntry(Relation fakerel);
-extern int read_local_xlog_page(XLogReaderState *state,
+extern bool read_local_xlog_page(XLogReaderState *state,
XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *cur_page);
extern void wal_segment_open(XLogReaderState *state,
--
2.18.4
----Next_Part(Thu_Jul__2_13_53_30_2020_072)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
filename="v15-0002-Move-page-reader-out-of-XLogReadRecord.patch"
^ permalink raw reply [nested|flat] 69+ messages in thread
* [PATCH v17 1/4] Move callback-call from ReadPageInternal to XLogReadRecord.
@ 2019-09-05 11:21 Kyotaro Horiguchi <[email protected]>
0 siblings, 0 replies; 69+ messages in thread
From: Kyotaro Horiguchi @ 2019-09-05 11:21 UTC (permalink / raw)
The current WAL record reader reads page data using a call back
function. Although it is not so problematic alone, it would be a
problem if we are going to do add tasks like encryption which is
performed on page data before WAL reader reads them. To avoid that the
record reader facility has to have a new code path corresponds to
every new callback, this patch separates page reader from WAL record
reading facility by modifying the current WAL record reader to a state
machine.
As the first step of that change, this patch moves the page reader
function out of ReadPageInternal, then the remaining tasks of the
function are taken over by the new function XLogNeedData. As the
result XLogPageRead directly calls the page reader callback function
according to the feedback from XLogNeedData.
---
src/backend/access/transam/xlog.c | 16 +-
src/backend/access/transam/xlogreader.c | 281 ++++++++++++++----------
src/backend/access/transam/xlogutils.c | 12 +-
src/backend/replication/walsender.c | 10 +-
src/bin/pg_rewind/parsexlog.c | 21 +-
src/bin/pg_waldump/pg_waldump.c | 8 +-
src/include/access/xlogreader.h | 25 ++-
src/include/access/xlogutils.h | 2 +-
8 files changed, 222 insertions(+), 153 deletions(-)
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 377afb8732..606328a65a 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -917,7 +917,7 @@ static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath,
static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli,
XLogSource source, bool notfoundOk);
static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, XLogSource source);
-static int XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
+static bool XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
bool fetching_ckpt, XLogRecPtr tliRecPtr);
@@ -4350,7 +4350,6 @@ ReadRecord(XLogReaderState *xlogreader, int emode,
XLogRecord *record;
XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
- /* Pass through parameters to XLogPageRead */
private->fetching_ckpt = fetching_ckpt;
private->emode = emode;
private->randAccess = (xlogreader->ReadRecPtr == InvalidXLogRecPtr);
@@ -11987,7 +11986,7 @@ CancelBackup(void)
* XLogPageRead() to try fetching the record from another source, or to
* sleep and retry.
*/
-static int
+static bool
XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *readBuf)
{
@@ -12046,7 +12045,8 @@ retry:
readLen = 0;
readSource = XLOG_FROM_ANY;
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
}
@@ -12141,7 +12141,8 @@ retry:
goto next_record_is_invalid;
}
- return readLen;
+ xlogreader->readLen = readLen;
+ return true;
next_record_is_invalid:
lastSourceFailed = true;
@@ -12155,8 +12156,9 @@ next_record_is_invalid:
/* In standby-mode, keep trying */
if (StandbyMode)
goto retry;
- else
- return -1;
+
+ xlogreader->readLen = -1;
+ return false;
}
/*
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index 42738eb940..7d04995b7b 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -36,8 +36,8 @@
static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
pg_attribute_printf(2, 3);
static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
-static int ReadPageInternal(XLogReaderState *state, XLogRecPtr pageptr,
- int reqLen);
+static bool XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr,
+ int reqLen, bool header_inclusive);
static void XLogReaderInvalReadState(XLogReaderState *state);
static bool ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
XLogRecPtr PrevRecPtr, XLogRecord *record, bool randAccess);
@@ -276,7 +276,6 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
uint32 targetRecOff;
uint32 pageHeaderSize;
bool gotheader;
- int readOff;
/*
* randAccess indicates whether to verify the previous-record pointer of
@@ -326,14 +325,20 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
* byte to cover the whole record header, or at least the part of it that
* fits on the same page.
*/
- readOff = ReadPageInternal(state, targetPagePtr,
- Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ));
- if (readOff < 0)
+ while (XLogNeedData(state, targetPagePtr,
+ Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+ targetRecOff != 0))
+ {
+ if (!state->routine.page_read(state, state->readPagePtr, state->readLen,
+ RecPtr, state->readBuf))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
/*
- * ReadPageInternal always returns at least the page header, so we can
- * examine it now.
+ * We have at least the page header, so we can examine it now.
*/
pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
if (targetRecOff == 0)
@@ -359,8 +364,8 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
goto err;
}
- /* ReadPageInternal has verified the page header */
- Assert(pageHeaderSize <= readOff);
+ /* XLogNeedData has verified the page header */
+ Assert(pageHeaderSize <= state->readLen);
/*
* Read the record length.
@@ -432,18 +437,27 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
do
{
+ int rest_len = total_len - gotlen;
+
/* Calculate pointer to beginning of next page */
targetPagePtr += XLOG_BLCKSZ;
/* Wait for the next page to become available */
- readOff = ReadPageInternal(state, targetPagePtr,
- Min(total_len - gotlen + SizeOfXLogShortPHD,
- XLOG_BLCKSZ));
+ while (XLogNeedData(state, targetPagePtr,
+ Min(rest_len, XLOG_BLCKSZ),
+ false))
+ {
+ if (!state->routine.page_read(state, state->readPagePtr,
+ state->readLen,
+ state->ReadRecPtr,
+ state->readBuf))
+ break;
+ }
- if (readOff < 0)
+ if (!state->page_verified)
goto err;
- Assert(SizeOfXLogShortPHD <= readOff);
+ Assert(SizeOfXLogShortPHD <= state->readLen);
/* Check that the continuation on next page looks valid */
pageHeader = (XLogPageHeader) state->readBuf;
@@ -473,21 +487,14 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
/* Append the continuation from this page to the buffer */
pageHeaderSize = XLogPageHeaderSize(pageHeader);
- if (readOff < pageHeaderSize)
- readOff = ReadPageInternal(state, targetPagePtr,
- pageHeaderSize);
-
- Assert(pageHeaderSize <= readOff);
+ Assert(pageHeaderSize <= state->readLen);
contdata = (char *) state->readBuf + pageHeaderSize;
len = XLOG_BLCKSZ - pageHeaderSize;
if (pageHeader->xlp_rem_len < len)
len = pageHeader->xlp_rem_len;
- if (readOff < pageHeaderSize + len)
- readOff = ReadPageInternal(state, targetPagePtr,
- pageHeaderSize + len);
-
+ Assert (pageHeaderSize + len <= state->readLen);
memcpy(buffer, (char *) contdata, len);
buffer += len;
gotlen += len;
@@ -517,9 +524,16 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
else
{
/* Wait for the record data to become available */
- readOff = ReadPageInternal(state, targetPagePtr,
- Min(targetRecOff + total_len, XLOG_BLCKSZ));
- if (readOff < 0)
+ while (XLogNeedData(state, targetPagePtr,
+ Min(targetRecOff + total_len, XLOG_BLCKSZ), true))
+ {
+ if (!state->routine.page_read(state, state->readPagePtr,
+ state->readLen,
+ state->ReadRecPtr, state->readBuf))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
/* Record does not cross a page boundary */
@@ -562,109 +576,138 @@ err:
}
/*
- * Read a single xlog page including at least [pageptr, reqLen] of valid data
- * via the page_read() callback.
+ * Checks that an xlog page loaded in state->readBuf is including at least
+ * [pageptr, reqLen] and the page is valid. header_inclusive indicates that
+ * reqLen is calculated including page header length.
*
- * Returns -1 if the required page cannot be read for some reason; errormsg_buf
- * is set in that case (unless the error occurs in the page_read callback).
+ * Returns false if the buffer already contains the requested data, or found
+ * error. state->page_verified is set to true for the former and false for the
+ * latter.
*
- * We fetch the page from a reader-local cache if we know we have the required
- * data and if there hasn't been any error since caching the data.
+ * Otherwise returns true and requests data loaded onto state->readBuf by
+ * state->readPagePtr and state->readLen. The caller shall call this function
+ * again after filling the buffer at least with that portion of data and set
+ * state->readLen to the length of actually loaded data.
+ *
+ * If header_inclusive is false, corrects reqLen internally by adding the
+ * actual page header length and may request caller for new data.
*/
-static int
-ReadPageInternal(XLogReaderState *state, XLogRecPtr pageptr, int reqLen)
+static bool
+XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
+ bool header_inclusive)
{
- int readLen;
uint32 targetPageOff;
XLogSegNo targetSegNo;
- XLogPageHeader hdr;
+ uint32 addLen = 0;
- Assert((pageptr % XLOG_BLCKSZ) == 0);
+ /* Some data is loaded, but page header is not verified yet. */
+ if (!state->page_verified &&
+ !XLogRecPtrIsInvalid(state->readPagePtr) && state->readLen >= 0)
+ {
+ uint32 pageHeaderSize;
+ /* just loaded new data so needs to verify page header */
+
+ /* The caller must have loaded at least page header */
+ Assert (state->readLen >= SizeOfXLogShortPHD);
+
+ /*
+ * We have enough data to check the header length. Recheck the loaded
+ * length against the actual header length.
+ */
+ pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+
+ /* Request more data if we don't have the full header. */
+ if (state->readLen < pageHeaderSize)
+ {
+ state->readLen = pageHeaderSize;
+ return true;
+ }
+
+ /* Now that we know we have the full header, validate it. */
+ if (!XLogReaderValidatePageHeader(state, state->readPagePtr,
+ (char *) state->readBuf))
+ {
+ /* That's bad. Force reading the page again. */
+ XLogReaderInvalReadState(state);
+
+ return false;
+ }
+
+ state->page_verified = true;
+
+ XLByteToSeg(state->readPagePtr, state->seg.ws_segno,
+ state->segcxt.ws_segsize);
+ }
+
+ /*
+ * The loaded page may not be the one caller is supposing to read when we
+ * are verifying the first page of new segment. In that case, skip further
+ * verification and immediately load the target page.
+ */
+ if (state->page_verified && pageptr == state->readPagePtr)
+ {
+ /*
+ * calculate additional length for page header keeping the total
+ * length within the block size.
+ */
+ if (!header_inclusive)
+ {
+ uint32 pageHeaderSize =
+ XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+
+ addLen = pageHeaderSize;
+ if (reqLen + pageHeaderSize <= XLOG_BLCKSZ)
+ addLen = pageHeaderSize;
+ else
+ addLen = XLOG_BLCKSZ - reqLen;
+
+ Assert(addLen >= 0);
+ }
+
+ /* Return if we already have it. */
+ if (reqLen + addLen <= state->readLen)
+ return false;
+ }
+
+ /* Data is not in our buffer, request the caller for it. */
XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize);
targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize);
+ Assert((pageptr % XLOG_BLCKSZ) == 0);
- /* check whether we have all the requested data already */
- if (targetSegNo == state->seg.ws_segno &&
- targetPageOff == state->segoff && reqLen <= state->readLen)
- return state->readLen;
+ /*
+ * Every time we request to load new data of a page to the caller, even if
+ * we looked at a part of it before, we need to do verification on the next
+ * invocation as the caller might now be rereading data from a different
+ * source.
+ */
+ state->page_verified = false;
/*
- * Data is not in our buffer.
- *
- * Every time we actually read the segment, even if we looked at parts of
- * it before, we need to do verification as the page_read callback might
- * now be rereading data from a different source.
- *
* Whenever switching to a new WAL segment, we read the first page of the
* file and validate its header, even if that's not where the target
* record is. This is so that we can check the additional identification
* info that is present in the first page's "long" header.
+ * Don't do this if the caller requested the first page in the segment.
*/
if (targetSegNo != state->seg.ws_segno && targetPageOff != 0)
{
- XLogRecPtr targetSegmentPtr = pageptr - targetPageOff;
-
- readLen = state->routine.page_read(state, targetSegmentPtr, XLOG_BLCKSZ,
- state->currRecPtr,
- state->readBuf);
- if (readLen < 0)
- goto err;
-
- /* we can be sure to have enough WAL available, we scrolled back */
- Assert(readLen == XLOG_BLCKSZ);
-
- if (!XLogReaderValidatePageHeader(state, targetSegmentPtr,
- state->readBuf))
- goto err;
+ /*
+ * Then we'll see that the targetSegNo now matches the ws_segno, and
+ * will not come back here, but will request the actual target page.
+ */
+ state->readPagePtr = pageptr - targetPageOff;
+ state->readLen = XLOG_BLCKSZ;
+ return true;
}
/*
- * First, read the requested data length, but at least a short page header
- * so that we can validate it.
+ * Request the caller to load the page. We need at least a short page
+ * header so that we can validate it.
*/
- readLen = state->routine.page_read(state, pageptr, Max(reqLen, SizeOfXLogShortPHD),
- state->currRecPtr,
- state->readBuf);
- if (readLen < 0)
- goto err;
-
- Assert(readLen <= XLOG_BLCKSZ);
-
- /* Do we have enough data to check the header length? */
- if (readLen <= SizeOfXLogShortPHD)
- goto err;
-
- Assert(readLen >= reqLen);
-
- hdr = (XLogPageHeader) state->readBuf;
-
- /* still not enough */
- if (readLen < XLogPageHeaderSize(hdr))
- {
- readLen = state->routine.page_read(state, pageptr, XLogPageHeaderSize(hdr),
- state->currRecPtr,
- state->readBuf);
- if (readLen < 0)
- goto err;
- }
-
- /*
- * Now that we know we have the full header, validate it.
- */
- if (!XLogReaderValidatePageHeader(state, pageptr, (char *) hdr))
- goto err;
-
- /* update read state information */
- state->seg.ws_segno = targetSegNo;
- state->segoff = targetPageOff;
- state->readLen = readLen;
-
- return readLen;
-
-err:
- XLogReaderInvalReadState(state);
- return -1;
+ state->readPagePtr = pageptr;
+ state->readLen = Max(reqLen + addLen, SizeOfXLogShortPHD);
+ return true;
}
/*
@@ -673,9 +716,7 @@ err:
static void
XLogReaderInvalReadState(XLogReaderState *state)
{
- state->seg.ws_segno = 0;
- state->segoff = 0;
- state->readLen = 0;
+ state->readPagePtr = InvalidXLogRecPtr;
}
/*
@@ -953,7 +994,6 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
XLogRecPtr targetPagePtr;
int targetRecOff;
uint32 pageHeaderSize;
- int readLen;
/*
* Compute targetRecOff. It should typically be equal or greater than
@@ -961,7 +1001,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
* that, except when caller has explicitly specified the offset that
* falls somewhere there or when we are skipping multi-page
* continuation record. It doesn't matter though because
- * ReadPageInternal() is prepared to handle that and will read at
+ * XLogNeedData() is prepared to handle that and will read at
* least short page-header worth of data
*/
targetRecOff = tmpRecPtr % XLOG_BLCKSZ;
@@ -969,19 +1009,24 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
/* scroll back to page boundary */
targetPagePtr = tmpRecPtr - targetRecOff;
- /* Read the page containing the record */
- readLen = ReadPageInternal(state, targetPagePtr, targetRecOff);
- if (readLen < 0)
+ while(XLogNeedData(state, targetPagePtr, targetRecOff,
+ targetRecOff != 0))
+ {
+ if (!state->routine.page_read(state, state->readPagePtr,
+ state->readLen,
+ state->ReadRecPtr, state->readBuf))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
header = (XLogPageHeader) state->readBuf;
pageHeaderSize = XLogPageHeaderSize(header);
- /* make sure we have enough data for the page header */
- readLen = ReadPageInternal(state, targetPagePtr, pageHeaderSize);
- if (readLen < 0)
- goto err;
+ /* we should have read the page header */
+ Assert (state->readLen >= pageHeaderSize);
/* skip over potential continuation data */
if (header->xlp_info & XLP_FIRST_IS_CONTRECORD)
diff --git a/src/backend/access/transam/xlogutils.c b/src/backend/access/transam/xlogutils.c
index d17d660f46..46eda33f25 100644
--- a/src/backend/access/transam/xlogutils.c
+++ b/src/backend/access/transam/xlogutils.c
@@ -686,8 +686,8 @@ XLogTruncateRelation(RelFileNode rnode, ForkNumber forkNum,
void
XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wantLength)
{
- const XLogRecPtr lastReadPage = (state->seg.ws_segno *
- state->segcxt.ws_segsize + state->segoff);
+ const XLogRecPtr lastReadPage = state->seg.ws_segno *
+ state->segcxt.ws_segsize + state->readLen;
Assert(wantPage != InvalidXLogRecPtr && wantPage % XLOG_BLCKSZ == 0);
Assert(wantLength <= XLOG_BLCKSZ);
@@ -824,7 +824,7 @@ wal_segment_close(XLogReaderState *state)
* exists for normal backends, so we have to do a check/sleep/repeat style of
* loop for now.
*/
-int
+bool
read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *cur_page)
{
@@ -926,7 +926,8 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
else if (targetPagePtr + reqLen > read_upto)
{
/* not enough data there */
- return -1;
+ state->readLen = -1;
+ return false;
}
else
{
@@ -944,7 +945,8 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
WALReadRaiseError(&errinfo);
/* number of valid bytes in the buffer */
- return count;
+ state->readLen = count;
+ return true;
}
/*
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index 23baa4498a..c36b91ae5e 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -806,7 +806,7 @@ StartReplication(StartReplicationCmd *cmd)
* which has to do a plain sleep/busy loop, because the walsender's latch gets
* set every time WAL is flushed.
*/
-static int
+static bool
logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *cur_page)
{
@@ -826,7 +826,10 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
/* fail if not (implies we are going to shut down) */
if (flushptr < targetPagePtr + reqLen)
- return -1;
+ {
+ state->readLen = -1;
+ return false;
+ }
if (targetPagePtr + XLOG_BLCKSZ <= flushptr)
count = XLOG_BLCKSZ; /* more than one block available */
@@ -854,7 +857,8 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
XLByteToSeg(targetPagePtr, segno, state->segcxt.ws_segsize);
CheckXLogRemoved(segno, state->seg.ws_tli);
- return count;
+ state->readLen = count;
+ return true;
}
/*
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index 59ebac7d6a..cf119848b0 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -47,7 +47,7 @@ typedef struct XLogPageReadPrivate
int tliIndex;
} XLogPageReadPrivate;
-static int SimpleXLogPageRead(XLogReaderState *xlogreader,
+static bool SimpleXLogPageRead(XLogReaderState *xlogreader,
XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
@@ -246,7 +246,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
}
/* XLogReader callback function, to read a WAL page */
-static int
+static bool
SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf)
{
@@ -306,7 +306,8 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
if (private->restoreCommand == NULL)
{
pg_log_error("could not open file \"%s\": %m", xlogfpath);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
/*
@@ -319,7 +320,10 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
private->restoreCommand);
if (xlogreadfd < 0)
- return -1;
+ {
+ xlogreader->readLen = -1;
+ return false;
+ }
else
pg_log_debug("using file \"%s\" restored from archive",
xlogfpath);
@@ -335,7 +339,8 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
if (lseek(xlogreadfd, (off_t) targetPageOff, SEEK_SET) < 0)
{
pg_log_error("could not seek in file \"%s\": %m", xlogfpath);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
@@ -348,13 +353,15 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
pg_log_error("could not read file \"%s\": read %d of %zu",
xlogfpath, r, (Size) XLOG_BLCKSZ);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
Assert(targetSegNo == xlogreadsegno);
xlogreader->seg.ws_tli = targetHistory[private->tliIndex].tli;
- return XLOG_BLCKSZ;
+ xlogreader->readLen = XLOG_BLCKSZ;
+ return true;
}
/*
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index 610f65e471..0f0e0723a3 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -331,7 +331,7 @@ WALDumpCloseSegment(XLogReaderState *state)
}
/* pg_waldump's XLogReaderRoutine->page_read callback */
-static int
+static bool
WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetPtr, char *readBuff)
{
@@ -348,7 +348,8 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
else
{
private->endptr_reached = true;
- return -1;
+ state->readLen = -1;
+ return false;
}
}
@@ -373,7 +374,8 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
(Size) errinfo.wre_req);
}
- return count;
+ state->readLen = count;
+ return true;
}
/*
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index 21d200d3df..23d9e70a59 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -57,8 +57,8 @@ typedef struct WALSegmentContext
typedef struct XLogReaderState XLogReaderState;
-/* Function type definitions for various xlogreader interactions */
-typedef int (*XLogPageReadCB) (XLogReaderState *xlogreader,
+/* Function type definition for the read_page callback */
+typedef bool (*XLogPageReadCB) (XLogReaderState *xlogreader,
XLogRecPtr targetPagePtr,
int reqLen,
XLogRecPtr targetRecPtr,
@@ -175,6 +175,20 @@ struct XLogReaderState
XLogRecPtr ReadRecPtr; /* start of last record read */
XLogRecPtr EndRecPtr; /* end+1 of last record read */
+ /* ----------------------------------------
+ * Communication with page reader
+ * readBuf is XLOG_BLCKSZ bytes, valid up to at least readLen bytes.
+ * ----------------------------------------
+ */
+ /* variables to communicate with page reader */
+ XLogRecPtr readPagePtr; /* page pointer to read */
+ int32 readLen; /* bytes requested to reader, or actual bytes
+ * read by reader, which must be larger than
+ * the request, or -1 on error */
+ TimeLineID readPageTLI; /* TLI for data currently in readBuf */
+ char *readBuf; /* buffer to store data */
+ bool page_verified; /* is the page on the buffer verified? */
+
/* ----------------------------------------
* Decoded representation of current record
@@ -203,13 +217,6 @@ struct XLogReaderState
* ----------------------------------------
*/
- /*
- * Buffer for currently read page (XLOG_BLCKSZ bytes, valid up to at least
- * readLen bytes)
- */
- char *readBuf;
- uint32 readLen;
-
/* last read XLOG position for data currently in readBuf */
WALSegmentContext segcxt;
WALOpenSegment seg;
diff --git a/src/include/access/xlogutils.h b/src/include/access/xlogutils.h
index 9ac602b674..364a21c4ea 100644
--- a/src/include/access/xlogutils.h
+++ b/src/include/access/xlogutils.h
@@ -47,7 +47,7 @@ extern Buffer XLogReadBufferExtended(RelFileNode rnode, ForkNumber forknum,
extern Relation CreateFakeRelcacheEntry(RelFileNode rnode);
extern void FreeFakeRelcacheEntry(Relation fakerel);
-extern int read_local_xlog_page(XLogReaderState *state,
+extern bool read_local_xlog_page(XLogReaderState *state,
XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *cur_page);
extern void wal_segment_open(XLogReaderState *state,
--
2.27.0
----Next_Part(Thu_Mar__4_11_28_53_2021_014)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
filename="v17-0002-Move-page-reader-out-of-XLogReadRecord.patch"
^ permalink raw reply [nested|flat] 69+ messages in thread
* [PATCH v15 1/4] Move callback-call from ReadPageInternal to XLogReadRecord.
@ 2019-09-05 11:21 Kyotaro Horiguchi <[email protected]>
0 siblings, 0 replies; 69+ messages in thread
From: Kyotaro Horiguchi @ 2019-09-05 11:21 UTC (permalink / raw)
The current WAL record reader reads page data using a call back
function. Although it is not so problematic alone, it would be a
problem if we are going to do add tasks like encryption which is
performed on page data before WAL reader reads them. To avoid that the
record reader facility has to have a new code path corresponds to
every new callback, this patch separates page reader from WAL record
reading facility by modifying the current WAL record reader to a state
machine.
As the first step of that change, this patch moves the page reader
function out of ReadPageInternal, then the remaining tasks of the
function are taken over by the new function XLogNeedData. As the
result XLogPageRead directly calls the page reader callback function
according to the feedback from XLogNeedData.
---
src/backend/access/transam/xlog.c | 16 +-
src/backend/access/transam/xlogreader.c | 281 ++++++++++++++----------
src/backend/access/transam/xlogutils.c | 12 +-
src/backend/replication/walsender.c | 10 +-
src/bin/pg_rewind/parsexlog.c | 21 +-
src/bin/pg_waldump/pg_waldump.c | 8 +-
src/include/access/xlogreader.h | 25 ++-
src/include/access/xlogutils.h | 2 +-
8 files changed, 222 insertions(+), 153 deletions(-)
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index fd93bcfaeb..e570e56a24 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -912,7 +912,7 @@ static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath,
static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli,
XLogSource source, bool notfoundOk);
static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, XLogSource source);
-static int XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
+static bool XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
bool fetching_ckpt, XLogRecPtr tliRecPtr);
@@ -4332,7 +4332,6 @@ ReadRecord(XLogReaderState *xlogreader, int emode,
XLogRecord *record;
XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
- /* Pass through parameters to XLogPageRead */
private->fetching_ckpt = fetching_ckpt;
private->emode = emode;
private->randAccess = (xlogreader->ReadRecPtr == InvalidXLogRecPtr);
@@ -11860,7 +11859,7 @@ CancelBackup(void)
* XLogPageRead() to try fetching the record from another source, or to
* sleep and retry.
*/
-static int
+static bool
XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *readBuf)
{
@@ -11919,7 +11918,8 @@ retry:
readLen = 0;
readSource = XLOG_FROM_ANY;
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
}
@@ -12014,7 +12014,8 @@ retry:
goto next_record_is_invalid;
}
- return readLen;
+ xlogreader->readLen = readLen;
+ return true;
next_record_is_invalid:
lastSourceFailed = true;
@@ -12028,8 +12029,9 @@ next_record_is_invalid:
/* In standby-mode, keep trying */
if (StandbyMode)
goto retry;
- else
- return -1;
+
+ xlogreader->readLen = -1;
+ return false;
}
/*
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index cb76be4f46..3d599325ee 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -36,8 +36,8 @@
static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
pg_attribute_printf(2, 3);
static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
-static int ReadPageInternal(XLogReaderState *state, XLogRecPtr pageptr,
- int reqLen);
+static bool XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr,
+ int reqLen, bool header_inclusive);
static void XLogReaderInvalReadState(XLogReaderState *state);
static bool ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
XLogRecPtr PrevRecPtr, XLogRecord *record, bool randAccess);
@@ -276,7 +276,6 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
uint32 targetRecOff;
uint32 pageHeaderSize;
bool gotheader;
- int readOff;
/*
* randAccess indicates whether to verify the previous-record pointer of
@@ -326,14 +325,20 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
* byte to cover the whole record header, or at least the part of it that
* fits on the same page.
*/
- readOff = ReadPageInternal(state, targetPagePtr,
- Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ));
- if (readOff < 0)
+ while (XLogNeedData(state, targetPagePtr,
+ Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+ targetRecOff != 0))
+ {
+ if (!state->routine.page_read(state, state->readPagePtr, state->readLen,
+ RecPtr, state->readBuf))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
/*
- * ReadPageInternal always returns at least the page header, so we can
- * examine it now.
+ * We have at least the page header, so we can examine it now.
*/
pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
if (targetRecOff == 0)
@@ -359,8 +364,8 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
goto err;
}
- /* ReadPageInternal has verified the page header */
- Assert(pageHeaderSize <= readOff);
+ /* XLogNeedData has verified the page header */
+ Assert(pageHeaderSize <= state->readLen);
/*
* Read the record length.
@@ -433,18 +438,27 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
do
{
+ int rest_len = total_len - gotlen;
+
/* Calculate pointer to beginning of next page */
targetPagePtr += XLOG_BLCKSZ;
/* Wait for the next page to become available */
- readOff = ReadPageInternal(state, targetPagePtr,
- Min(total_len - gotlen + SizeOfXLogShortPHD,
- XLOG_BLCKSZ));
+ while (XLogNeedData(state, targetPagePtr,
+ Min(rest_len, XLOG_BLCKSZ),
+ false))
+ {
+ if (!state->routine.page_read(state, state->readPagePtr,
+ state->readLen,
+ state->ReadRecPtr,
+ state->readBuf))
+ break;
+ }
- if (readOff < 0)
+ if (!state->page_verified)
goto err;
- Assert(SizeOfXLogShortPHD <= readOff);
+ Assert(SizeOfXLogShortPHD <= state->readLen);
/* Check that the continuation on next page looks valid */
pageHeader = (XLogPageHeader) state->readBuf;
@@ -473,21 +487,14 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
/* Append the continuation from this page to the buffer */
pageHeaderSize = XLogPageHeaderSize(pageHeader);
- if (readOff < pageHeaderSize)
- readOff = ReadPageInternal(state, targetPagePtr,
- pageHeaderSize);
-
- Assert(pageHeaderSize <= readOff);
+ Assert(pageHeaderSize <= state->readLen);
contdata = (char *) state->readBuf + pageHeaderSize;
len = XLOG_BLCKSZ - pageHeaderSize;
if (pageHeader->xlp_rem_len < len)
len = pageHeader->xlp_rem_len;
- if (readOff < pageHeaderSize + len)
- readOff = ReadPageInternal(state, targetPagePtr,
- pageHeaderSize + len);
-
+ Assert (pageHeaderSize + len <= state->readLen);
memcpy(buffer, (char *) contdata, len);
buffer += len;
gotlen += len;
@@ -517,9 +524,16 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
else
{
/* Wait for the record data to become available */
- readOff = ReadPageInternal(state, targetPagePtr,
- Min(targetRecOff + total_len, XLOG_BLCKSZ));
- if (readOff < 0)
+ while (XLogNeedData(state, targetPagePtr,
+ Min(targetRecOff + total_len, XLOG_BLCKSZ), true))
+ {
+ if (!state->routine.page_read(state, state->readPagePtr,
+ state->readLen,
+ state->ReadRecPtr, state->readBuf))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
/* Record does not cross a page boundary */
@@ -562,109 +576,138 @@ err:
}
/*
- * Read a single xlog page including at least [pageptr, reqLen] of valid data
- * via the page_read() callback.
+ * Checks that an xlog page loaded in state->readBuf is including at least
+ * [pageptr, reqLen] and the page is valid. header_inclusive indicates that
+ * reqLen is calculated including page header length.
*
- * Returns -1 if the required page cannot be read for some reason; errormsg_buf
- * is set in that case (unless the error occurs in the page_read callback).
+ * Returns false if the buffer already contains the requested data, or found
+ * error. state->page_verified is set to true for the former and false for the
+ * latter.
*
- * We fetch the page from a reader-local cache if we know we have the required
- * data and if there hasn't been any error since caching the data.
+ * Otherwise returns true and requests data loaded onto state->readBuf by
+ * state->readPagePtr and state->readLen. The caller shall call this function
+ * again after filling the buffer at least with that portion of data and set
+ * state->readLen to the length of actually loaded data.
+ *
+ * If header_inclusive is false, corrects reqLen internally by adding the
+ * actual page header length and may request caller for new data.
*/
-static int
-ReadPageInternal(XLogReaderState *state, XLogRecPtr pageptr, int reqLen)
+static bool
+XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
+ bool header_inclusive)
{
- int readLen;
uint32 targetPageOff;
XLogSegNo targetSegNo;
- XLogPageHeader hdr;
+ uint32 addLen = 0;
- Assert((pageptr % XLOG_BLCKSZ) == 0);
+ /* Some data is loaded, but page header is not verified yet. */
+ if (!state->page_verified &&
+ !XLogRecPtrIsInvalid(state->readPagePtr) && state->readLen >= 0)
+ {
+ uint32 pageHeaderSize;
+ /* just loaded new data so needs to verify page header */
+
+ /* The caller must have loaded at least page header */
+ Assert (state->readLen >= SizeOfXLogShortPHD);
+
+ /*
+ * We have enough data to check the header length. Recheck the loaded
+ * length against the actual header length.
+ */
+ pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+
+ /* Request more data if we don't have the full header. */
+ if (state->readLen < pageHeaderSize)
+ {
+ state->readLen = pageHeaderSize;
+ return true;
+ }
+
+ /* Now that we know we have the full header, validate it. */
+ if (!XLogReaderValidatePageHeader(state, state->readPagePtr,
+ (char *) state->readBuf))
+ {
+ /* That's bad. Force reading the page again. */
+ XLogReaderInvalReadState(state);
+
+ return false;
+ }
+
+ state->page_verified = true;
+
+ XLByteToSeg(state->readPagePtr, state->seg.ws_segno,
+ state->segcxt.ws_segsize);
+ }
+
+ /*
+ * The loaded page may not be the one caller is supposing to read when we
+ * are verifying the first page of new segment. In that case, skip further
+ * verification and immediately load the target page.
+ */
+ if (state->page_verified && pageptr == state->readPagePtr)
+ {
+ /*
+ * calculate additional length for page header keeping the total
+ * length within the block size.
+ */
+ if (!header_inclusive)
+ {
+ uint32 pageHeaderSize =
+ XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+
+ addLen = pageHeaderSize;
+ if (reqLen + pageHeaderSize <= XLOG_BLCKSZ)
+ addLen = pageHeaderSize;
+ else
+ addLen = XLOG_BLCKSZ - reqLen;
+
+ Assert(addLen >= 0);
+ }
+
+ /* Return if we already have it. */
+ if (reqLen + addLen <= state->readLen)
+ return false;
+ }
+
+ /* Data is not in our buffer, request the caller for it. */
XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize);
targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize);
+ Assert((pageptr % XLOG_BLCKSZ) == 0);
- /* check whether we have all the requested data already */
- if (targetSegNo == state->seg.ws_segno &&
- targetPageOff == state->segoff && reqLen <= state->readLen)
- return state->readLen;
+ /*
+ * Every time we request to load new data of a page to the caller, even if
+ * we looked at a part of it before, we need to do verification on the next
+ * invocation as the caller might now be rereading data from a different
+ * source.
+ */
+ state->page_verified = false;
/*
- * Data is not in our buffer.
- *
- * Every time we actually read the segment, even if we looked at parts of
- * it before, we need to do verification as the page_read callback might
- * now be rereading data from a different source.
- *
* Whenever switching to a new WAL segment, we read the first page of the
* file and validate its header, even if that's not where the target
* record is. This is so that we can check the additional identification
* info that is present in the first page's "long" header.
+ * Don't do this if the caller requested the first page in the segment.
*/
if (targetSegNo != state->seg.ws_segno && targetPageOff != 0)
{
- XLogRecPtr targetSegmentPtr = pageptr - targetPageOff;
-
- readLen = state->routine.page_read(state, targetSegmentPtr, XLOG_BLCKSZ,
- state->currRecPtr,
- state->readBuf);
- if (readLen < 0)
- goto err;
-
- /* we can be sure to have enough WAL available, we scrolled back */
- Assert(readLen == XLOG_BLCKSZ);
-
- if (!XLogReaderValidatePageHeader(state, targetSegmentPtr,
- state->readBuf))
- goto err;
+ /*
+ * Then we'll see that the targetSegNo now matches the ws_segno, and
+ * will not come back here, but will request the actual target page.
+ */
+ state->readPagePtr = pageptr - targetPageOff;
+ state->readLen = XLOG_BLCKSZ;
+ return true;
}
/*
- * First, read the requested data length, but at least a short page header
- * so that we can validate it.
+ * Request the caller to load the page. We need at least a short page
+ * header so that we can validate it.
*/
- readLen = state->routine.page_read(state, pageptr, Max(reqLen, SizeOfXLogShortPHD),
- state->currRecPtr,
- state->readBuf);
- if (readLen < 0)
- goto err;
-
- Assert(readLen <= XLOG_BLCKSZ);
-
- /* Do we have enough data to check the header length? */
- if (readLen <= SizeOfXLogShortPHD)
- goto err;
-
- Assert(readLen >= reqLen);
-
- hdr = (XLogPageHeader) state->readBuf;
-
- /* still not enough */
- if (readLen < XLogPageHeaderSize(hdr))
- {
- readLen = state->routine.page_read(state, pageptr, XLogPageHeaderSize(hdr),
- state->currRecPtr,
- state->readBuf);
- if (readLen < 0)
- goto err;
- }
-
- /*
- * Now that we know we have the full header, validate it.
- */
- if (!XLogReaderValidatePageHeader(state, pageptr, (char *) hdr))
- goto err;
-
- /* update read state information */
- state->seg.ws_segno = targetSegNo;
- state->segoff = targetPageOff;
- state->readLen = readLen;
-
- return readLen;
-
-err:
- XLogReaderInvalReadState(state);
- return -1;
+ state->readPagePtr = pageptr;
+ state->readLen = Max(reqLen + addLen, SizeOfXLogShortPHD);
+ return true;
}
/*
@@ -673,9 +716,7 @@ err:
static void
XLogReaderInvalReadState(XLogReaderState *state)
{
- state->seg.ws_segno = 0;
- state->segoff = 0;
- state->readLen = 0;
+ state->readPagePtr = InvalidXLogRecPtr;
}
/*
@@ -956,7 +997,6 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
XLogRecPtr targetPagePtr;
int targetRecOff;
uint32 pageHeaderSize;
- int readLen;
/*
* Compute targetRecOff. It should typically be equal or greater than
@@ -964,7 +1004,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
* that, except when caller has explicitly specified the offset that
* falls somewhere there or when we are skipping multi-page
* continuation record. It doesn't matter though because
- * ReadPageInternal() is prepared to handle that and will read at
+ * XLogNeedData() is prepared to handle that and will read at
* least short page-header worth of data
*/
targetRecOff = tmpRecPtr % XLOG_BLCKSZ;
@@ -972,19 +1012,24 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
/* scroll back to page boundary */
targetPagePtr = tmpRecPtr - targetRecOff;
- /* Read the page containing the record */
- readLen = ReadPageInternal(state, targetPagePtr, targetRecOff);
- if (readLen < 0)
+ while(XLogNeedData(state, targetPagePtr, targetRecOff,
+ targetRecOff != 0))
+ {
+ if (!state->routine.page_read(state, state->readPagePtr,
+ state->readLen,
+ state->ReadRecPtr, state->readBuf))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
header = (XLogPageHeader) state->readBuf;
pageHeaderSize = XLogPageHeaderSize(header);
- /* make sure we have enough data for the page header */
- readLen = ReadPageInternal(state, targetPagePtr, pageHeaderSize);
- if (readLen < 0)
- goto err;
+ /* we should have read the page header */
+ Assert (state->readLen >= pageHeaderSize);
/* skip over potential continuation data */
if (header->xlp_info & XLP_FIRST_IS_CONTRECORD)
diff --git a/src/backend/access/transam/xlogutils.c b/src/backend/access/transam/xlogutils.c
index 322b0e8ff5..1b10b6df20 100644
--- a/src/backend/access/transam/xlogutils.c
+++ b/src/backend/access/transam/xlogutils.c
@@ -689,8 +689,8 @@ XLogTruncateRelation(RelFileNode rnode, ForkNumber forkNum,
void
XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wantLength)
{
- const XLogRecPtr lastReadPage = (state->seg.ws_segno *
- state->segcxt.ws_segsize + state->segoff);
+ const XLogRecPtr lastReadPage = state->seg.ws_segno *
+ state->segcxt.ws_segsize + state->readLen;
Assert(wantPage != InvalidXLogRecPtr && wantPage % XLOG_BLCKSZ == 0);
Assert(wantLength <= XLOG_BLCKSZ);
@@ -828,7 +828,7 @@ wal_segment_close(XLogReaderState *state)
* exists for normal backends, so we have to do a check/sleep/repeat style of
* loop for now.
*/
-int
+bool
read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *cur_page)
{
@@ -930,7 +930,8 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
else if (targetPagePtr + reqLen > read_upto)
{
/* not enough data there */
- return -1;
+ state->readLen = -1;
+ return false;
}
else
{
@@ -948,7 +949,8 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
WALReadRaiseError(&errinfo);
/* number of valid bytes in the buffer */
- return count;
+ state->readLen = count;
+ return true;
}
/*
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index e2477c47e0..bf2711bddd 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -811,7 +811,7 @@ StartReplication(StartReplicationCmd *cmd)
* which has to do a plain sleep/busy loop, because the walsender's latch gets
* set every time WAL is flushed.
*/
-static int
+static bool
logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *cur_page)
{
@@ -831,7 +831,10 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
/* fail if not (implies we are going to shut down) */
if (flushptr < targetPagePtr + reqLen)
- return -1;
+ {
+ state->readLen = -1;
+ return false;
+ }
if (targetPagePtr + XLOG_BLCKSZ <= flushptr)
count = XLOG_BLCKSZ; /* more than one block available */
@@ -859,7 +862,8 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
XLByteToSeg(targetPagePtr, segno, state->segcxt.ws_segsize);
CheckXLogRemoved(segno, state->seg.ws_tli);
- return count;
+ state->readLen = count;
+ return true;
}
/*
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index bc6f976994..0b7e73ae79 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -46,7 +46,7 @@ typedef struct XLogPageReadPrivate
int tliIndex;
} XLogPageReadPrivate;
-static int SimpleXLogPageRead(XLogReaderState *xlogreader,
+static bool SimpleXLogPageRead(XLogReaderState *xlogreader,
XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
@@ -236,7 +236,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
}
/* XLogReader callback function, to read a WAL page */
-static int
+static bool
SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf)
{
@@ -296,7 +296,8 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
if (private->restoreCommand == NULL)
{
pg_log_error("could not open file \"%s\": %m", xlogfpath);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
/*
@@ -309,7 +310,10 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
private->restoreCommand);
if (xlogreadfd < 0)
- return -1;
+ {
+ xlogreader->readLen = -1;
+ return false;
+ }
else
pg_log_debug("using file \"%s\" restored from archive",
xlogfpath);
@@ -325,7 +329,8 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
if (lseek(xlogreadfd, (off_t) targetPageOff, SEEK_SET) < 0)
{
pg_log_error("could not seek in file \"%s\": %m", xlogfpath);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
@@ -338,13 +343,15 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
pg_log_error("could not read file \"%s\": read %d of %zu",
xlogfpath, r, (Size) XLOG_BLCKSZ);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
Assert(targetSegNo == xlogreadsegno);
xlogreader->seg.ws_tli = targetHistory[private->tliIndex].tli;
- return XLOG_BLCKSZ;
+ xlogreader->readLen = XLOG_BLCKSZ;
+ return true;
}
/*
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index d1a0678935..ab0c28d9b7 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -331,7 +331,7 @@ WALDumpCloseSegment(XLogReaderState *state)
}
/* pg_waldump's XLogReaderRoutine->page_read callback */
-static int
+static bool
WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetPtr, char *readBuff)
{
@@ -348,7 +348,8 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
else
{
private->endptr_reached = true;
- return -1;
+ state->readLen = -1;
+ return false;
}
}
@@ -373,7 +374,8 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
(Size) errinfo.wre_req);
}
- return count;
+ state->readLen = count;
+ return true;
}
/*
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index b0f2a6ed43..6c8848e14f 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -57,8 +57,8 @@ typedef struct WALSegmentContext
typedef struct XLogReaderState XLogReaderState;
-/* Function type definitions for various xlogreader interactions */
-typedef int (*XLogPageReadCB) (XLogReaderState *xlogreader,
+/* Function type definition for the read_page callback */
+typedef bool (*XLogPageReadCB) (XLogReaderState *xlogreader,
XLogRecPtr targetPagePtr,
int reqLen,
XLogRecPtr targetRecPtr,
@@ -175,6 +175,20 @@ struct XLogReaderState
XLogRecPtr ReadRecPtr; /* start of last record read */
XLogRecPtr EndRecPtr; /* end+1 of last record read */
+ /* ----------------------------------------
+ * Communication with page reader
+ * readBuf is XLOG_BLCKSZ bytes, valid up to at least readLen bytes.
+ * ----------------------------------------
+ */
+ /* variables to communicate with page reader */
+ XLogRecPtr readPagePtr; /* page pointer to read */
+ int32 readLen; /* bytes requested to reader, or actual bytes
+ * read by reader, which must be larger than
+ * the request, or -1 on error */
+ TimeLineID readPageTLI; /* TLI for data currently in readBuf */
+ char *readBuf; /* buffer to store data */
+ bool page_verified; /* is the page on the buffer verified? */
+
/* ----------------------------------------
* Decoded representation of current record
@@ -201,13 +215,6 @@ struct XLogReaderState
* ----------------------------------------
*/
- /*
- * Buffer for currently read page (XLOG_BLCKSZ bytes, valid up to at least
- * readLen bytes)
- */
- char *readBuf;
- uint32 readLen;
-
/* last read XLOG position for data currently in readBuf */
WALSegmentContext segcxt;
WALOpenSegment seg;
diff --git a/src/include/access/xlogutils.h b/src/include/access/xlogutils.h
index e59b6cf3a9..c287edf206 100644
--- a/src/include/access/xlogutils.h
+++ b/src/include/access/xlogutils.h
@@ -47,7 +47,7 @@ extern Buffer XLogReadBufferExtended(RelFileNode rnode, ForkNumber forknum,
extern Relation CreateFakeRelcacheEntry(RelFileNode rnode);
extern void FreeFakeRelcacheEntry(Relation fakerel);
-extern int read_local_xlog_page(XLogReaderState *state,
+extern bool read_local_xlog_page(XLogReaderState *state,
XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *cur_page);
extern void wal_segment_open(XLogReaderState *state,
--
2.18.4
----Next_Part(Thu_Jul__2_13_53_30_2020_072)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
filename="v15-0002-Move-page-reader-out-of-XLogReadRecord.patch"
^ permalink raw reply [nested|flat] 69+ messages in thread
* [PATCH v17 1/4] Move callback-call from ReadPageInternal to XLogReadRecord.
@ 2019-09-05 11:21 Kyotaro Horiguchi <[email protected]>
0 siblings, 0 replies; 69+ messages in thread
From: Kyotaro Horiguchi @ 2019-09-05 11:21 UTC (permalink / raw)
The current WAL record reader reads page data using a call back
function. Although it is not so problematic alone, it would be a
problem if we are going to do add tasks like encryption which is
performed on page data before WAL reader reads them. To avoid that the
record reader facility has to have a new code path corresponds to
every new callback, this patch separates page reader from WAL record
reading facility by modifying the current WAL record reader to a state
machine.
As the first step of that change, this patch moves the page reader
function out of ReadPageInternal, then the remaining tasks of the
function are taken over by the new function XLogNeedData. As the
result XLogPageRead directly calls the page reader callback function
according to the feedback from XLogNeedData.
---
src/backend/access/transam/xlog.c | 16 +-
src/backend/access/transam/xlogreader.c | 281 ++++++++++++++----------
src/backend/access/transam/xlogutils.c | 12 +-
src/backend/replication/walsender.c | 10 +-
src/bin/pg_rewind/parsexlog.c | 21 +-
src/bin/pg_waldump/pg_waldump.c | 8 +-
src/include/access/xlogreader.h | 25 ++-
src/include/access/xlogutils.h | 2 +-
8 files changed, 222 insertions(+), 153 deletions(-)
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 377afb8732..606328a65a 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -917,7 +917,7 @@ static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath,
static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli,
XLogSource source, bool notfoundOk);
static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, XLogSource source);
-static int XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
+static bool XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
bool fetching_ckpt, XLogRecPtr tliRecPtr);
@@ -4350,7 +4350,6 @@ ReadRecord(XLogReaderState *xlogreader, int emode,
XLogRecord *record;
XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
- /* Pass through parameters to XLogPageRead */
private->fetching_ckpt = fetching_ckpt;
private->emode = emode;
private->randAccess = (xlogreader->ReadRecPtr == InvalidXLogRecPtr);
@@ -11987,7 +11986,7 @@ CancelBackup(void)
* XLogPageRead() to try fetching the record from another source, or to
* sleep and retry.
*/
-static int
+static bool
XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *readBuf)
{
@@ -12046,7 +12045,8 @@ retry:
readLen = 0;
readSource = XLOG_FROM_ANY;
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
}
@@ -12141,7 +12141,8 @@ retry:
goto next_record_is_invalid;
}
- return readLen;
+ xlogreader->readLen = readLen;
+ return true;
next_record_is_invalid:
lastSourceFailed = true;
@@ -12155,8 +12156,9 @@ next_record_is_invalid:
/* In standby-mode, keep trying */
if (StandbyMode)
goto retry;
- else
- return -1;
+
+ xlogreader->readLen = -1;
+ return false;
}
/*
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index 42738eb940..7d04995b7b 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -36,8 +36,8 @@
static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
pg_attribute_printf(2, 3);
static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
-static int ReadPageInternal(XLogReaderState *state, XLogRecPtr pageptr,
- int reqLen);
+static bool XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr,
+ int reqLen, bool header_inclusive);
static void XLogReaderInvalReadState(XLogReaderState *state);
static bool ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
XLogRecPtr PrevRecPtr, XLogRecord *record, bool randAccess);
@@ -276,7 +276,6 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
uint32 targetRecOff;
uint32 pageHeaderSize;
bool gotheader;
- int readOff;
/*
* randAccess indicates whether to verify the previous-record pointer of
@@ -326,14 +325,20 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
* byte to cover the whole record header, or at least the part of it that
* fits on the same page.
*/
- readOff = ReadPageInternal(state, targetPagePtr,
- Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ));
- if (readOff < 0)
+ while (XLogNeedData(state, targetPagePtr,
+ Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+ targetRecOff != 0))
+ {
+ if (!state->routine.page_read(state, state->readPagePtr, state->readLen,
+ RecPtr, state->readBuf))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
/*
- * ReadPageInternal always returns at least the page header, so we can
- * examine it now.
+ * We have at least the page header, so we can examine it now.
*/
pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
if (targetRecOff == 0)
@@ -359,8 +364,8 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
goto err;
}
- /* ReadPageInternal has verified the page header */
- Assert(pageHeaderSize <= readOff);
+ /* XLogNeedData has verified the page header */
+ Assert(pageHeaderSize <= state->readLen);
/*
* Read the record length.
@@ -432,18 +437,27 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
do
{
+ int rest_len = total_len - gotlen;
+
/* Calculate pointer to beginning of next page */
targetPagePtr += XLOG_BLCKSZ;
/* Wait for the next page to become available */
- readOff = ReadPageInternal(state, targetPagePtr,
- Min(total_len - gotlen + SizeOfXLogShortPHD,
- XLOG_BLCKSZ));
+ while (XLogNeedData(state, targetPagePtr,
+ Min(rest_len, XLOG_BLCKSZ),
+ false))
+ {
+ if (!state->routine.page_read(state, state->readPagePtr,
+ state->readLen,
+ state->ReadRecPtr,
+ state->readBuf))
+ break;
+ }
- if (readOff < 0)
+ if (!state->page_verified)
goto err;
- Assert(SizeOfXLogShortPHD <= readOff);
+ Assert(SizeOfXLogShortPHD <= state->readLen);
/* Check that the continuation on next page looks valid */
pageHeader = (XLogPageHeader) state->readBuf;
@@ -473,21 +487,14 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
/* Append the continuation from this page to the buffer */
pageHeaderSize = XLogPageHeaderSize(pageHeader);
- if (readOff < pageHeaderSize)
- readOff = ReadPageInternal(state, targetPagePtr,
- pageHeaderSize);
-
- Assert(pageHeaderSize <= readOff);
+ Assert(pageHeaderSize <= state->readLen);
contdata = (char *) state->readBuf + pageHeaderSize;
len = XLOG_BLCKSZ - pageHeaderSize;
if (pageHeader->xlp_rem_len < len)
len = pageHeader->xlp_rem_len;
- if (readOff < pageHeaderSize + len)
- readOff = ReadPageInternal(state, targetPagePtr,
- pageHeaderSize + len);
-
+ Assert (pageHeaderSize + len <= state->readLen);
memcpy(buffer, (char *) contdata, len);
buffer += len;
gotlen += len;
@@ -517,9 +524,16 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
else
{
/* Wait for the record data to become available */
- readOff = ReadPageInternal(state, targetPagePtr,
- Min(targetRecOff + total_len, XLOG_BLCKSZ));
- if (readOff < 0)
+ while (XLogNeedData(state, targetPagePtr,
+ Min(targetRecOff + total_len, XLOG_BLCKSZ), true))
+ {
+ if (!state->routine.page_read(state, state->readPagePtr,
+ state->readLen,
+ state->ReadRecPtr, state->readBuf))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
/* Record does not cross a page boundary */
@@ -562,109 +576,138 @@ err:
}
/*
- * Read a single xlog page including at least [pageptr, reqLen] of valid data
- * via the page_read() callback.
+ * Checks that an xlog page loaded in state->readBuf is including at least
+ * [pageptr, reqLen] and the page is valid. header_inclusive indicates that
+ * reqLen is calculated including page header length.
*
- * Returns -1 if the required page cannot be read for some reason; errormsg_buf
- * is set in that case (unless the error occurs in the page_read callback).
+ * Returns false if the buffer already contains the requested data, or found
+ * error. state->page_verified is set to true for the former and false for the
+ * latter.
*
- * We fetch the page from a reader-local cache if we know we have the required
- * data and if there hasn't been any error since caching the data.
+ * Otherwise returns true and requests data loaded onto state->readBuf by
+ * state->readPagePtr and state->readLen. The caller shall call this function
+ * again after filling the buffer at least with that portion of data and set
+ * state->readLen to the length of actually loaded data.
+ *
+ * If header_inclusive is false, corrects reqLen internally by adding the
+ * actual page header length and may request caller for new data.
*/
-static int
-ReadPageInternal(XLogReaderState *state, XLogRecPtr pageptr, int reqLen)
+static bool
+XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
+ bool header_inclusive)
{
- int readLen;
uint32 targetPageOff;
XLogSegNo targetSegNo;
- XLogPageHeader hdr;
+ uint32 addLen = 0;
- Assert((pageptr % XLOG_BLCKSZ) == 0);
+ /* Some data is loaded, but page header is not verified yet. */
+ if (!state->page_verified &&
+ !XLogRecPtrIsInvalid(state->readPagePtr) && state->readLen >= 0)
+ {
+ uint32 pageHeaderSize;
+ /* just loaded new data so needs to verify page header */
+
+ /* The caller must have loaded at least page header */
+ Assert (state->readLen >= SizeOfXLogShortPHD);
+
+ /*
+ * We have enough data to check the header length. Recheck the loaded
+ * length against the actual header length.
+ */
+ pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+
+ /* Request more data if we don't have the full header. */
+ if (state->readLen < pageHeaderSize)
+ {
+ state->readLen = pageHeaderSize;
+ return true;
+ }
+
+ /* Now that we know we have the full header, validate it. */
+ if (!XLogReaderValidatePageHeader(state, state->readPagePtr,
+ (char *) state->readBuf))
+ {
+ /* That's bad. Force reading the page again. */
+ XLogReaderInvalReadState(state);
+
+ return false;
+ }
+
+ state->page_verified = true;
+
+ XLByteToSeg(state->readPagePtr, state->seg.ws_segno,
+ state->segcxt.ws_segsize);
+ }
+
+ /*
+ * The loaded page may not be the one caller is supposing to read when we
+ * are verifying the first page of new segment. In that case, skip further
+ * verification and immediately load the target page.
+ */
+ if (state->page_verified && pageptr == state->readPagePtr)
+ {
+ /*
+ * calculate additional length for page header keeping the total
+ * length within the block size.
+ */
+ if (!header_inclusive)
+ {
+ uint32 pageHeaderSize =
+ XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+
+ addLen = pageHeaderSize;
+ if (reqLen + pageHeaderSize <= XLOG_BLCKSZ)
+ addLen = pageHeaderSize;
+ else
+ addLen = XLOG_BLCKSZ - reqLen;
+
+ Assert(addLen >= 0);
+ }
+
+ /* Return if we already have it. */
+ if (reqLen + addLen <= state->readLen)
+ return false;
+ }
+
+ /* Data is not in our buffer, request the caller for it. */
XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize);
targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize);
+ Assert((pageptr % XLOG_BLCKSZ) == 0);
- /* check whether we have all the requested data already */
- if (targetSegNo == state->seg.ws_segno &&
- targetPageOff == state->segoff && reqLen <= state->readLen)
- return state->readLen;
+ /*
+ * Every time we request to load new data of a page to the caller, even if
+ * we looked at a part of it before, we need to do verification on the next
+ * invocation as the caller might now be rereading data from a different
+ * source.
+ */
+ state->page_verified = false;
/*
- * Data is not in our buffer.
- *
- * Every time we actually read the segment, even if we looked at parts of
- * it before, we need to do verification as the page_read callback might
- * now be rereading data from a different source.
- *
* Whenever switching to a new WAL segment, we read the first page of the
* file and validate its header, even if that's not where the target
* record is. This is so that we can check the additional identification
* info that is present in the first page's "long" header.
+ * Don't do this if the caller requested the first page in the segment.
*/
if (targetSegNo != state->seg.ws_segno && targetPageOff != 0)
{
- XLogRecPtr targetSegmentPtr = pageptr - targetPageOff;
-
- readLen = state->routine.page_read(state, targetSegmentPtr, XLOG_BLCKSZ,
- state->currRecPtr,
- state->readBuf);
- if (readLen < 0)
- goto err;
-
- /* we can be sure to have enough WAL available, we scrolled back */
- Assert(readLen == XLOG_BLCKSZ);
-
- if (!XLogReaderValidatePageHeader(state, targetSegmentPtr,
- state->readBuf))
- goto err;
+ /*
+ * Then we'll see that the targetSegNo now matches the ws_segno, and
+ * will not come back here, but will request the actual target page.
+ */
+ state->readPagePtr = pageptr - targetPageOff;
+ state->readLen = XLOG_BLCKSZ;
+ return true;
}
/*
- * First, read the requested data length, but at least a short page header
- * so that we can validate it.
+ * Request the caller to load the page. We need at least a short page
+ * header so that we can validate it.
*/
- readLen = state->routine.page_read(state, pageptr, Max(reqLen, SizeOfXLogShortPHD),
- state->currRecPtr,
- state->readBuf);
- if (readLen < 0)
- goto err;
-
- Assert(readLen <= XLOG_BLCKSZ);
-
- /* Do we have enough data to check the header length? */
- if (readLen <= SizeOfXLogShortPHD)
- goto err;
-
- Assert(readLen >= reqLen);
-
- hdr = (XLogPageHeader) state->readBuf;
-
- /* still not enough */
- if (readLen < XLogPageHeaderSize(hdr))
- {
- readLen = state->routine.page_read(state, pageptr, XLogPageHeaderSize(hdr),
- state->currRecPtr,
- state->readBuf);
- if (readLen < 0)
- goto err;
- }
-
- /*
- * Now that we know we have the full header, validate it.
- */
- if (!XLogReaderValidatePageHeader(state, pageptr, (char *) hdr))
- goto err;
-
- /* update read state information */
- state->seg.ws_segno = targetSegNo;
- state->segoff = targetPageOff;
- state->readLen = readLen;
-
- return readLen;
-
-err:
- XLogReaderInvalReadState(state);
- return -1;
+ state->readPagePtr = pageptr;
+ state->readLen = Max(reqLen + addLen, SizeOfXLogShortPHD);
+ return true;
}
/*
@@ -673,9 +716,7 @@ err:
static void
XLogReaderInvalReadState(XLogReaderState *state)
{
- state->seg.ws_segno = 0;
- state->segoff = 0;
- state->readLen = 0;
+ state->readPagePtr = InvalidXLogRecPtr;
}
/*
@@ -953,7 +994,6 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
XLogRecPtr targetPagePtr;
int targetRecOff;
uint32 pageHeaderSize;
- int readLen;
/*
* Compute targetRecOff. It should typically be equal or greater than
@@ -961,7 +1001,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
* that, except when caller has explicitly specified the offset that
* falls somewhere there or when we are skipping multi-page
* continuation record. It doesn't matter though because
- * ReadPageInternal() is prepared to handle that and will read at
+ * XLogNeedData() is prepared to handle that and will read at
* least short page-header worth of data
*/
targetRecOff = tmpRecPtr % XLOG_BLCKSZ;
@@ -969,19 +1009,24 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
/* scroll back to page boundary */
targetPagePtr = tmpRecPtr - targetRecOff;
- /* Read the page containing the record */
- readLen = ReadPageInternal(state, targetPagePtr, targetRecOff);
- if (readLen < 0)
+ while(XLogNeedData(state, targetPagePtr, targetRecOff,
+ targetRecOff != 0))
+ {
+ if (!state->routine.page_read(state, state->readPagePtr,
+ state->readLen,
+ state->ReadRecPtr, state->readBuf))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
header = (XLogPageHeader) state->readBuf;
pageHeaderSize = XLogPageHeaderSize(header);
- /* make sure we have enough data for the page header */
- readLen = ReadPageInternal(state, targetPagePtr, pageHeaderSize);
- if (readLen < 0)
- goto err;
+ /* we should have read the page header */
+ Assert (state->readLen >= pageHeaderSize);
/* skip over potential continuation data */
if (header->xlp_info & XLP_FIRST_IS_CONTRECORD)
diff --git a/src/backend/access/transam/xlogutils.c b/src/backend/access/transam/xlogutils.c
index d17d660f46..46eda33f25 100644
--- a/src/backend/access/transam/xlogutils.c
+++ b/src/backend/access/transam/xlogutils.c
@@ -686,8 +686,8 @@ XLogTruncateRelation(RelFileNode rnode, ForkNumber forkNum,
void
XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wantLength)
{
- const XLogRecPtr lastReadPage = (state->seg.ws_segno *
- state->segcxt.ws_segsize + state->segoff);
+ const XLogRecPtr lastReadPage = state->seg.ws_segno *
+ state->segcxt.ws_segsize + state->readLen;
Assert(wantPage != InvalidXLogRecPtr && wantPage % XLOG_BLCKSZ == 0);
Assert(wantLength <= XLOG_BLCKSZ);
@@ -824,7 +824,7 @@ wal_segment_close(XLogReaderState *state)
* exists for normal backends, so we have to do a check/sleep/repeat style of
* loop for now.
*/
-int
+bool
read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *cur_page)
{
@@ -926,7 +926,8 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
else if (targetPagePtr + reqLen > read_upto)
{
/* not enough data there */
- return -1;
+ state->readLen = -1;
+ return false;
}
else
{
@@ -944,7 +945,8 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
WALReadRaiseError(&errinfo);
/* number of valid bytes in the buffer */
- return count;
+ state->readLen = count;
+ return true;
}
/*
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index 23baa4498a..c36b91ae5e 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -806,7 +806,7 @@ StartReplication(StartReplicationCmd *cmd)
* which has to do a plain sleep/busy loop, because the walsender's latch gets
* set every time WAL is flushed.
*/
-static int
+static bool
logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *cur_page)
{
@@ -826,7 +826,10 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
/* fail if not (implies we are going to shut down) */
if (flushptr < targetPagePtr + reqLen)
- return -1;
+ {
+ state->readLen = -1;
+ return false;
+ }
if (targetPagePtr + XLOG_BLCKSZ <= flushptr)
count = XLOG_BLCKSZ; /* more than one block available */
@@ -854,7 +857,8 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
XLByteToSeg(targetPagePtr, segno, state->segcxt.ws_segsize);
CheckXLogRemoved(segno, state->seg.ws_tli);
- return count;
+ state->readLen = count;
+ return true;
}
/*
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index 59ebac7d6a..cf119848b0 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -47,7 +47,7 @@ typedef struct XLogPageReadPrivate
int tliIndex;
} XLogPageReadPrivate;
-static int SimpleXLogPageRead(XLogReaderState *xlogreader,
+static bool SimpleXLogPageRead(XLogReaderState *xlogreader,
XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
@@ -246,7 +246,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
}
/* XLogReader callback function, to read a WAL page */
-static int
+static bool
SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf)
{
@@ -306,7 +306,8 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
if (private->restoreCommand == NULL)
{
pg_log_error("could not open file \"%s\": %m", xlogfpath);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
/*
@@ -319,7 +320,10 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
private->restoreCommand);
if (xlogreadfd < 0)
- return -1;
+ {
+ xlogreader->readLen = -1;
+ return false;
+ }
else
pg_log_debug("using file \"%s\" restored from archive",
xlogfpath);
@@ -335,7 +339,8 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
if (lseek(xlogreadfd, (off_t) targetPageOff, SEEK_SET) < 0)
{
pg_log_error("could not seek in file \"%s\": %m", xlogfpath);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
@@ -348,13 +353,15 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
pg_log_error("could not read file \"%s\": read %d of %zu",
xlogfpath, r, (Size) XLOG_BLCKSZ);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
Assert(targetSegNo == xlogreadsegno);
xlogreader->seg.ws_tli = targetHistory[private->tliIndex].tli;
- return XLOG_BLCKSZ;
+ xlogreader->readLen = XLOG_BLCKSZ;
+ return true;
}
/*
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index 610f65e471..0f0e0723a3 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -331,7 +331,7 @@ WALDumpCloseSegment(XLogReaderState *state)
}
/* pg_waldump's XLogReaderRoutine->page_read callback */
-static int
+static bool
WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetPtr, char *readBuff)
{
@@ -348,7 +348,8 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
else
{
private->endptr_reached = true;
- return -1;
+ state->readLen = -1;
+ return false;
}
}
@@ -373,7 +374,8 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
(Size) errinfo.wre_req);
}
- return count;
+ state->readLen = count;
+ return true;
}
/*
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index 21d200d3df..23d9e70a59 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -57,8 +57,8 @@ typedef struct WALSegmentContext
typedef struct XLogReaderState XLogReaderState;
-/* Function type definitions for various xlogreader interactions */
-typedef int (*XLogPageReadCB) (XLogReaderState *xlogreader,
+/* Function type definition for the read_page callback */
+typedef bool (*XLogPageReadCB) (XLogReaderState *xlogreader,
XLogRecPtr targetPagePtr,
int reqLen,
XLogRecPtr targetRecPtr,
@@ -175,6 +175,20 @@ struct XLogReaderState
XLogRecPtr ReadRecPtr; /* start of last record read */
XLogRecPtr EndRecPtr; /* end+1 of last record read */
+ /* ----------------------------------------
+ * Communication with page reader
+ * readBuf is XLOG_BLCKSZ bytes, valid up to at least readLen bytes.
+ * ----------------------------------------
+ */
+ /* variables to communicate with page reader */
+ XLogRecPtr readPagePtr; /* page pointer to read */
+ int32 readLen; /* bytes requested to reader, or actual bytes
+ * read by reader, which must be larger than
+ * the request, or -1 on error */
+ TimeLineID readPageTLI; /* TLI for data currently in readBuf */
+ char *readBuf; /* buffer to store data */
+ bool page_verified; /* is the page on the buffer verified? */
+
/* ----------------------------------------
* Decoded representation of current record
@@ -203,13 +217,6 @@ struct XLogReaderState
* ----------------------------------------
*/
- /*
- * Buffer for currently read page (XLOG_BLCKSZ bytes, valid up to at least
- * readLen bytes)
- */
- char *readBuf;
- uint32 readLen;
-
/* last read XLOG position for data currently in readBuf */
WALSegmentContext segcxt;
WALOpenSegment seg;
diff --git a/src/include/access/xlogutils.h b/src/include/access/xlogutils.h
index 9ac602b674..364a21c4ea 100644
--- a/src/include/access/xlogutils.h
+++ b/src/include/access/xlogutils.h
@@ -47,7 +47,7 @@ extern Buffer XLogReadBufferExtended(RelFileNode rnode, ForkNumber forknum,
extern Relation CreateFakeRelcacheEntry(RelFileNode rnode);
extern void FreeFakeRelcacheEntry(Relation fakerel);
-extern int read_local_xlog_page(XLogReaderState *state,
+extern bool read_local_xlog_page(XLogReaderState *state,
XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *cur_page);
extern void wal_segment_open(XLogReaderState *state,
--
2.27.0
----Next_Part(Thu_Mar__4_11_28_53_2021_014)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
filename="v17-0002-Move-page-reader-out-of-XLogReadRecord.patch"
^ permalink raw reply [nested|flat] 69+ messages in thread
* [PATCH v10 1/4] Move callback-call from ReadPageInternal to XLogReadRecord.
@ 2019-09-05 11:21 Kyotaro Horiguchi <[email protected]>
0 siblings, 0 replies; 69+ messages in thread
From: Kyotaro Horiguchi @ 2019-09-05 11:21 UTC (permalink / raw)
The current WAL record reader reads page data using a call back
function. Although it is not so problematic alone, it would be a
problem if we are going to do add tasks like encryption which is
performed on page data before WAL reader reads them. To avoid that the
record reader facility has to have a new code path corresponds to
every new callback, this patch separates page reader from WAL record
reading facility by modifying the current WAL record reader to a state
machine.
As the first step of that change, this patch moves the page reader
function out of ReadPageInternal, then the remaining tasks of the
function are taken over by the new function XLogNeedData. As the
result XLogPageRead directly calls the page reader callback function
according to the feedback from XLogNeedData.
---
src/backend/access/transam/xlog.c | 16 +-
src/backend/access/transam/xlogreader.c | 281 ++++++++++++++----------
src/backend/access/transam/xlogutils.c | 12 +-
src/backend/replication/walsender.c | 10 +-
src/bin/pg_rewind/parsexlog.c | 21 +-
src/bin/pg_waldump/pg_waldump.c | 8 +-
src/include/access/xlogreader.h | 25 ++-
src/include/access/xlogutils.h | 2 +-
8 files changed, 222 insertions(+), 153 deletions(-)
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index ca09d81b08..da468598e4 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -909,7 +909,7 @@ static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath,
static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli,
XLogSource source, bool notfoundOk);
static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, XLogSource source);
-static int XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
+static bool XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
bool fetching_ckpt, XLogRecPtr tliRecPtr);
@@ -4329,7 +4329,6 @@ ReadRecord(XLogReaderState *xlogreader, int emode,
XLogRecord *record;
XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
- /* Pass through parameters to XLogPageRead */
private->fetching_ckpt = fetching_ckpt;
private->emode = emode;
private->randAccess = (xlogreader->ReadRecPtr == InvalidXLogRecPtr);
@@ -11854,7 +11853,7 @@ CancelBackup(void)
* XLogPageRead() to try fetching the record from another source, or to
* sleep and retry.
*/
-static int
+static bool
XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *readBuf)
{
@@ -11913,7 +11912,8 @@ retry:
readLen = 0;
readSource = XLOG_FROM_ANY;
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
}
@@ -12008,7 +12008,8 @@ retry:
goto next_record_is_invalid;
}
- return readLen;
+ xlogreader->readLen = readLen;
+ return true;
next_record_is_invalid:
lastSourceFailed = true;
@@ -12022,8 +12023,9 @@ next_record_is_invalid:
/* In standby-mode, keep trying */
if (StandbyMode)
goto retry;
- else
- return -1;
+
+ xlogreader->readLen = -1;
+ return false;
}
/*
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index 5995798b58..399bad0603 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -36,8 +36,8 @@
static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
pg_attribute_printf(2, 3);
static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
-static int ReadPageInternal(XLogReaderState *state, XLogRecPtr pageptr,
- int reqLen);
+static bool XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr,
+ int reqLen, bool header_inclusive);
static void XLogReaderInvalReadState(XLogReaderState *state);
static bool ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
XLogRecPtr PrevRecPtr, XLogRecord *record, bool randAccess);
@@ -274,7 +274,6 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
uint32 targetRecOff;
uint32 pageHeaderSize;
bool gotheader;
- int readOff;
/*
* randAccess indicates whether to verify the previous-record pointer of
@@ -324,14 +323,20 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
* byte to cover the whole record header, or at least the part of it that
* fits on the same page.
*/
- readOff = ReadPageInternal(state, targetPagePtr,
- Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ));
- if (readOff < 0)
+ while (XLogNeedData(state, targetPagePtr,
+ Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+ targetRecOff != 0))
+ {
+ if (!state->routine.page_read(state, state->readPagePtr, state->readLen,
+ RecPtr, state->readBuf))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
/*
- * ReadPageInternal always returns at least the page header, so we can
- * examine it now.
+ * We have at least the page header, so we can examine it now.
*/
pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
if (targetRecOff == 0)
@@ -357,8 +362,8 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
goto err;
}
- /* ReadPageInternal has verified the page header */
- Assert(pageHeaderSize <= readOff);
+ /* XLogNeedData has verified the page header */
+ Assert(pageHeaderSize <= state->readLen);
/*
* Read the record length.
@@ -431,18 +436,27 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
do
{
+ int rest_len = total_len - gotlen;
+
/* Calculate pointer to beginning of next page */
targetPagePtr += XLOG_BLCKSZ;
/* Wait for the next page to become available */
- readOff = ReadPageInternal(state, targetPagePtr,
- Min(total_len - gotlen + SizeOfXLogShortPHD,
- XLOG_BLCKSZ));
+ while (XLogNeedData(state, targetPagePtr,
+ Min(rest_len, XLOG_BLCKSZ),
+ false))
+ {
+ if (!state->routine.page_read(state, state->readPagePtr,
+ state->readLen,
+ state->ReadRecPtr,
+ state->readBuf))
+ break;
+ }
- if (readOff < 0)
+ if (!state->page_verified)
goto err;
- Assert(SizeOfXLogShortPHD <= readOff);
+ Assert(SizeOfXLogShortPHD <= state->readLen);
/* Check that the continuation on next page looks valid */
pageHeader = (XLogPageHeader) state->readBuf;
@@ -471,21 +485,14 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
/* Append the continuation from this page to the buffer */
pageHeaderSize = XLogPageHeaderSize(pageHeader);
- if (readOff < pageHeaderSize)
- readOff = ReadPageInternal(state, targetPagePtr,
- pageHeaderSize);
-
- Assert(pageHeaderSize <= readOff);
+ Assert(pageHeaderSize <= state->readLen);
contdata = (char *) state->readBuf + pageHeaderSize;
len = XLOG_BLCKSZ - pageHeaderSize;
if (pageHeader->xlp_rem_len < len)
len = pageHeader->xlp_rem_len;
- if (readOff < pageHeaderSize + len)
- readOff = ReadPageInternal(state, targetPagePtr,
- pageHeaderSize + len);
-
+ Assert (pageHeaderSize + len <= state->readLen);
memcpy(buffer, (char *) contdata, len);
buffer += len;
gotlen += len;
@@ -515,9 +522,16 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
else
{
/* Wait for the record data to become available */
- readOff = ReadPageInternal(state, targetPagePtr,
- Min(targetRecOff + total_len, XLOG_BLCKSZ));
- if (readOff < 0)
+ while (XLogNeedData(state, targetPagePtr,
+ Min(targetRecOff + total_len, XLOG_BLCKSZ), true))
+ {
+ if (!state->routine.page_read(state, state->readPagePtr,
+ state->readLen,
+ state->ReadRecPtr, state->readBuf))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
/* Record does not cross a page boundary */
@@ -560,109 +574,138 @@ err:
}
/*
- * Read a single xlog page including at least [pageptr, reqLen] of valid data
- * via the page_read() callback.
+ * Checks that an xlog page loaded in state->readBuf is including at least
+ * [pageptr, reqLen] and the page is valid. header_inclusive indicates that
+ * reqLen is calculated including page header length.
*
- * Returns -1 if the required page cannot be read for some reason; errormsg_buf
- * is set in that case (unless the error occurs in the page_read callback).
+ * Returns false if the buffer already contains the requested data, or found
+ * error. state->page_verified is set to true for the former and false for the
+ * latter.
*
- * We fetch the page from a reader-local cache if we know we have the required
- * data and if there hasn't been any error since caching the data.
+ * Otherwise returns true and requests data loaded onto state->readBuf by
+ * state->readPagePtr and state->readLen. The caller shall call this function
+ * again after filling the buffer at least with that portion of data and set
+ * state->readLen to the length of actually loaded data.
+ *
+ * If header_inclusive is false, corrects reqLen internally by adding the
+ * actual page header length and may request caller for new data.
*/
-static int
-ReadPageInternal(XLogReaderState *state, XLogRecPtr pageptr, int reqLen)
+static bool
+XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
+ bool header_inclusive)
{
- int readLen;
uint32 targetPageOff;
XLogSegNo targetSegNo;
- XLogPageHeader hdr;
+ uint32 addLen = 0;
- Assert((pageptr % XLOG_BLCKSZ) == 0);
+ /* Some data is loaded, but page header is not verified yet. */
+ if (!state->page_verified &&
+ !XLogRecPtrIsInvalid(state->readPagePtr) && state->readLen >= 0)
+ {
+ uint32 pageHeaderSize;
+ /* just loaded new data so needs to verify page header */
+
+ /* The caller must have loaded at least page header */
+ Assert (state->readLen >= SizeOfXLogShortPHD);
+
+ /*
+ * We have enough data to check the header length. Recheck the loaded
+ * length against the actual header length.
+ */
+ pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+
+ /* Request more data if we don't have the full header. */
+ if (state->readLen < pageHeaderSize)
+ {
+ state->readLen = pageHeaderSize;
+ return true;
+ }
+
+ /* Now that we know we have the full header, validate it. */
+ if (!XLogReaderValidatePageHeader(state, state->readPagePtr,
+ (char *) state->readBuf))
+ {
+ /* That's bad. Force reading the page again. */
+ XLogReaderInvalReadState(state);
+
+ return false;
+ }
+
+ state->page_verified = true;
+
+ XLByteToSeg(state->readPagePtr, state->seg.ws_segno,
+ state->segcxt.ws_segsize);
+ }
+
+ /*
+ * The loaded page may not be the one caller is supposing to read when we
+ * are verifying the first page of new segment. In that case, skip further
+ * verification and immediately load the target page.
+ */
+ if (state->page_verified && pageptr == state->readPagePtr)
+ {
+ /*
+ * calculate additional length for page header keeping the total
+ * length within the block size.
+ */
+ if (!header_inclusive)
+ {
+ uint32 pageHeaderSize =
+ XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+
+ addLen = pageHeaderSize;
+ if (reqLen + pageHeaderSize <= XLOG_BLCKSZ)
+ addLen = pageHeaderSize;
+ else
+ addLen = XLOG_BLCKSZ - reqLen;
+
+ Assert(addLen >= 0);
+ }
+
+ /* Return if we already have it. */
+ if (reqLen + addLen <= state->readLen)
+ return false;
+ }
+
+ /* Data is not in our buffer, request the caller for it. */
XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize);
targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize);
+ Assert((pageptr % XLOG_BLCKSZ) == 0);
- /* check whether we have all the requested data already */
- if (targetSegNo == state->seg.ws_segno &&
- targetPageOff == state->segoff && reqLen <= state->readLen)
- return state->readLen;
+ /*
+ * Every time we request to load new data of a page to the caller, even if
+ * we looked at a part of it before, we need to do verification on the next
+ * invocation as the caller might now be rereading data from a different
+ * source.
+ */
+ state->page_verified = false;
/*
- * Data is not in our buffer.
- *
- * Every time we actually read the segment, even if we looked at parts of
- * it before, we need to do verification as the page_read callback might
- * now be rereading data from a different source.
- *
* Whenever switching to a new WAL segment, we read the first page of the
* file and validate its header, even if that's not where the target
* record is. This is so that we can check the additional identification
* info that is present in the first page's "long" header.
+ * Don't do this if the caller requested the first page in the segment.
*/
if (targetSegNo != state->seg.ws_segno && targetPageOff != 0)
{
- XLogRecPtr targetSegmentPtr = pageptr - targetPageOff;
-
- readLen = state->routine.page_read(state, targetSegmentPtr, XLOG_BLCKSZ,
- state->currRecPtr,
- state->readBuf);
- if (readLen < 0)
- goto err;
-
- /* we can be sure to have enough WAL available, we scrolled back */
- Assert(readLen == XLOG_BLCKSZ);
-
- if (!XLogReaderValidatePageHeader(state, targetSegmentPtr,
- state->readBuf))
- goto err;
+ /*
+ * Then we'll see that the targetSegNo now matches the ws_segno, and
+ * will not come back here, but will request the actual target page.
+ */
+ state->readPagePtr = pageptr - targetPageOff;
+ state->readLen = XLOG_BLCKSZ;
+ return true;
}
/*
- * First, read the requested data length, but at least a short page header
- * so that we can validate it.
+ * Request the caller to load the page. We need at least a short page
+ * header so that we can validate it.
*/
- readLen = state->routine.page_read(state, pageptr, Max(reqLen, SizeOfXLogShortPHD),
- state->currRecPtr,
- state->readBuf);
- if (readLen < 0)
- goto err;
-
- Assert(readLen <= XLOG_BLCKSZ);
-
- /* Do we have enough data to check the header length? */
- if (readLen <= SizeOfXLogShortPHD)
- goto err;
-
- Assert(readLen >= reqLen);
-
- hdr = (XLogPageHeader) state->readBuf;
-
- /* still not enough */
- if (readLen < XLogPageHeaderSize(hdr))
- {
- readLen = state->routine.page_read(state, pageptr, XLogPageHeaderSize(hdr),
- state->currRecPtr,
- state->readBuf);
- if (readLen < 0)
- goto err;
- }
-
- /*
- * Now that we know we have the full header, validate it.
- */
- if (!XLogReaderValidatePageHeader(state, pageptr, (char *) hdr))
- goto err;
-
- /* update read state information */
- state->seg.ws_segno = targetSegNo;
- state->segoff = targetPageOff;
- state->readLen = readLen;
-
- return readLen;
-
-err:
- XLogReaderInvalReadState(state);
- return -1;
+ state->readPagePtr = pageptr;
+ state->readLen = Max(reqLen + addLen, SizeOfXLogShortPHD);
+ return true;
}
/*
@@ -671,9 +714,7 @@ err:
static void
XLogReaderInvalReadState(XLogReaderState *state)
{
- state->seg.ws_segno = 0;
- state->segoff = 0;
- state->readLen = 0;
+ state->readPagePtr = InvalidXLogRecPtr;
}
/*
@@ -954,7 +995,6 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
XLogRecPtr targetPagePtr;
int targetRecOff;
uint32 pageHeaderSize;
- int readLen;
/*
* Compute targetRecOff. It should typically be equal or greater than
@@ -962,7 +1002,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
* that, except when caller has explicitly specified the offset that
* falls somewhere there or when we are skipping multi-page
* continuation record. It doesn't matter though because
- * ReadPageInternal() is prepared to handle that and will read at
+ * XLogNeedData() is prepared to handle that and will read at
* least short page-header worth of data
*/
targetRecOff = tmpRecPtr % XLOG_BLCKSZ;
@@ -970,19 +1010,24 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
/* scroll back to page boundary */
targetPagePtr = tmpRecPtr - targetRecOff;
- /* Read the page containing the record */
- readLen = ReadPageInternal(state, targetPagePtr, targetRecOff);
- if (readLen < 0)
+ while(XLogNeedData(state, targetPagePtr, targetRecOff,
+ targetRecOff != 0))
+ {
+ if (!state->routine.page_read(state, state->readPagePtr,
+ state->readLen,
+ state->ReadRecPtr, state->readBuf))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
header = (XLogPageHeader) state->readBuf;
pageHeaderSize = XLogPageHeaderSize(header);
- /* make sure we have enough data for the page header */
- readLen = ReadPageInternal(state, targetPagePtr, pageHeaderSize);
- if (readLen < 0)
- goto err;
+ /* we should have read the page header */
+ Assert (state->readLen >= pageHeaderSize);
/* skip over potential continuation data */
if (header->xlp_info & XLP_FIRST_IS_CONTRECORD)
diff --git a/src/backend/access/transam/xlogutils.c b/src/backend/access/transam/xlogutils.c
index 322b0e8ff5..1b10b6df20 100644
--- a/src/backend/access/transam/xlogutils.c
+++ b/src/backend/access/transam/xlogutils.c
@@ -689,8 +689,8 @@ XLogTruncateRelation(RelFileNode rnode, ForkNumber forkNum,
void
XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wantLength)
{
- const XLogRecPtr lastReadPage = (state->seg.ws_segno *
- state->segcxt.ws_segsize + state->segoff);
+ const XLogRecPtr lastReadPage = state->seg.ws_segno *
+ state->segcxt.ws_segsize + state->readLen;
Assert(wantPage != InvalidXLogRecPtr && wantPage % XLOG_BLCKSZ == 0);
Assert(wantLength <= XLOG_BLCKSZ);
@@ -828,7 +828,7 @@ wal_segment_close(XLogReaderState *state)
* exists for normal backends, so we have to do a check/sleep/repeat style of
* loop for now.
*/
-int
+bool
read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *cur_page)
{
@@ -930,7 +930,8 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
else if (targetPagePtr + reqLen > read_upto)
{
/* not enough data there */
- return -1;
+ state->readLen = -1;
+ return false;
}
else
{
@@ -948,7 +949,8 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
WALReadRaiseError(&errinfo);
/* number of valid bytes in the buffer */
- return count;
+ state->readLen = count;
+ return true;
}
/*
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index 86847cbb54..d3a94d611f 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -815,7 +815,7 @@ StartReplication(StartReplicationCmd *cmd)
* which has to do a plain sleep/busy loop, because the walsender's latch gets
* set every time WAL is flushed.
*/
-static int
+static bool
logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *cur_page)
{
@@ -835,7 +835,10 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
/* fail if not (implies we are going to shut down) */
if (flushptr < targetPagePtr + reqLen)
- return -1;
+ {
+ state->readLen = -1;
+ return false;
+ }
if (targetPagePtr + XLOG_BLCKSZ <= flushptr)
count = XLOG_BLCKSZ; /* more than one block available */
@@ -863,7 +866,8 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
XLByteToSeg(targetPagePtr, segno, state->segcxt.ws_segsize);
CheckXLogRemoved(segno, state->seg.ws_tli);
- return count;
+ state->readLen = count;
+ return true;
}
/*
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index d637f5eb77..907db33b9e 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -46,7 +46,7 @@ typedef struct XLogPageReadPrivate
int tliIndex;
} XLogPageReadPrivate;
-static int SimpleXLogPageRead(XLogReaderState *xlogreader,
+static bool SimpleXLogPageRead(XLogReaderState *xlogreader,
XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
@@ -236,7 +236,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
}
/* XLogReader callback function, to read a WAL page */
-static int
+static bool
SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf)
{
@@ -296,7 +296,8 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
if (private->restoreCommand == NULL)
{
pg_log_error("could not open file \"%s\": %m", xlogfpath);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
/*
@@ -309,7 +310,10 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
private->restoreCommand);
if (xlogreadfd < 0)
- return -1;
+ {
+ xlogreader->readLen = -1;
+ return false;
+ }
else
pg_log_debug("using file \"%s\" restored from archive",
xlogfpath);
@@ -325,7 +329,8 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
if (lseek(xlogreadfd, (off_t) targetPageOff, SEEK_SET) < 0)
{
pg_log_error("could not seek in file \"%s\": %m", xlogfpath);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
@@ -338,13 +343,15 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
pg_log_error("could not read file \"%s\": read %d of %zu",
xlogfpath, r, (Size) XLOG_BLCKSZ);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
Assert(targetSegNo == xlogreadsegno);
xlogreader->seg.ws_tli = targetHistory[private->tliIndex].tli;
- return XLOG_BLCKSZ;
+ xlogreader->readLen = XLOG_BLCKSZ;
+ return true;
}
/*
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index d1a0678935..ab0c28d9b7 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -331,7 +331,7 @@ WALDumpCloseSegment(XLogReaderState *state)
}
/* pg_waldump's XLogReaderRoutine->page_read callback */
-static int
+static bool
WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetPtr, char *readBuff)
{
@@ -348,7 +348,8 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
else
{
private->endptr_reached = true;
- return -1;
+ state->readLen = -1;
+ return false;
}
}
@@ -373,7 +374,8 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
(Size) errinfo.wre_req);
}
- return count;
+ state->readLen = count;
+ return true;
}
/*
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index c21b0ba972..53e4212ac4 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -57,8 +57,8 @@ typedef struct WALSegmentContext
typedef struct XLogReaderState XLogReaderState;
-/* Function type definitions for various xlogreader interactions */
-typedef int (*XLogPageReadCB) (XLogReaderState *xlogreader,
+/* Function type definition for the read_page callback */
+typedef bool (*XLogPageReadCB) (XLogReaderState *xlogreader,
XLogRecPtr targetPagePtr,
int reqLen,
XLogRecPtr targetRecPtr,
@@ -175,6 +175,20 @@ struct XLogReaderState
XLogRecPtr ReadRecPtr; /* start of last record read */
XLogRecPtr EndRecPtr; /* end+1 of last record read */
+ /* ----------------------------------------
+ * Communication with page reader
+ * readBuf is XLOG_BLCKSZ bytes, valid up to at least readLen bytes.
+ * ----------------------------------------
+ */
+ /* variables to communicate with page reader */
+ XLogRecPtr readPagePtr; /* page pointer to read */
+ int32 readLen; /* bytes requested to reader, or actual bytes
+ * read by reader, which must be larger than
+ * the request, or -1 on error */
+ TimeLineID readPageTLI; /* TLI for data currently in readBuf */
+ char *readBuf; /* buffer to store data */
+ bool page_verified; /* is the page on the buffer verified? */
+
/* ----------------------------------------
* Decoded representation of current record
@@ -201,13 +215,6 @@ struct XLogReaderState
* ----------------------------------------
*/
- /*
- * Buffer for currently read page (XLOG_BLCKSZ bytes, valid up to at least
- * readLen bytes)
- */
- char *readBuf;
- uint32 readLen;
-
/* last read XLOG position for data currently in readBuf */
WALSegmentContext segcxt;
WALOpenSegment seg;
diff --git a/src/include/access/xlogutils.h b/src/include/access/xlogutils.h
index e59b6cf3a9..c287edf206 100644
--- a/src/include/access/xlogutils.h
+++ b/src/include/access/xlogutils.h
@@ -47,7 +47,7 @@ extern Buffer XLogReadBufferExtended(RelFileNode rnode, ForkNumber forknum,
extern Relation CreateFakeRelcacheEntry(RelFileNode rnode);
extern void FreeFakeRelcacheEntry(Relation fakerel);
-extern int read_local_xlog_page(XLogReaderState *state,
+extern bool read_local_xlog_page(XLogReaderState *state,
XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *cur_page);
extern void wal_segment_open(XLogReaderState *state,
--
2.18.2
----Next_Part(Tue_May_26_16_40_02_2020_373)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
filename="v10-0002-Move-page-reader-out-of-XLogReadRecord.patch"
^ permalink raw reply [nested|flat] 69+ messages in thread
* [PATCH v15 1/4] Move callback-call from ReadPageInternal to XLogReadRecord.
@ 2019-09-05 11:21 Kyotaro Horiguchi <[email protected]>
0 siblings, 0 replies; 69+ messages in thread
From: Kyotaro Horiguchi @ 2019-09-05 11:21 UTC (permalink / raw)
The current WAL record reader reads page data using a call back
function. Although it is not so problematic alone, it would be a
problem if we are going to do add tasks like encryption which is
performed on page data before WAL reader reads them. To avoid that the
record reader facility has to have a new code path corresponds to
every new callback, this patch separates page reader from WAL record
reading facility by modifying the current WAL record reader to a state
machine.
As the first step of that change, this patch moves the page reader
function out of ReadPageInternal, then the remaining tasks of the
function are taken over by the new function XLogNeedData. As the
result XLogPageRead directly calls the page reader callback function
according to the feedback from XLogNeedData.
---
src/backend/access/transam/xlog.c | 16 +-
src/backend/access/transam/xlogreader.c | 281 ++++++++++++++----------
src/backend/access/transam/xlogutils.c | 12 +-
src/backend/replication/walsender.c | 10 +-
src/bin/pg_rewind/parsexlog.c | 21 +-
src/bin/pg_waldump/pg_waldump.c | 8 +-
src/include/access/xlogreader.h | 25 ++-
src/include/access/xlogutils.h | 2 +-
8 files changed, 222 insertions(+), 153 deletions(-)
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index fd93bcfaeb..e570e56a24 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -912,7 +912,7 @@ static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath,
static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli,
XLogSource source, bool notfoundOk);
static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, XLogSource source);
-static int XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
+static bool XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
bool fetching_ckpt, XLogRecPtr tliRecPtr);
@@ -4332,7 +4332,6 @@ ReadRecord(XLogReaderState *xlogreader, int emode,
XLogRecord *record;
XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
- /* Pass through parameters to XLogPageRead */
private->fetching_ckpt = fetching_ckpt;
private->emode = emode;
private->randAccess = (xlogreader->ReadRecPtr == InvalidXLogRecPtr);
@@ -11860,7 +11859,7 @@ CancelBackup(void)
* XLogPageRead() to try fetching the record from another source, or to
* sleep and retry.
*/
-static int
+static bool
XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *readBuf)
{
@@ -11919,7 +11918,8 @@ retry:
readLen = 0;
readSource = XLOG_FROM_ANY;
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
}
@@ -12014,7 +12014,8 @@ retry:
goto next_record_is_invalid;
}
- return readLen;
+ xlogreader->readLen = readLen;
+ return true;
next_record_is_invalid:
lastSourceFailed = true;
@@ -12028,8 +12029,9 @@ next_record_is_invalid:
/* In standby-mode, keep trying */
if (StandbyMode)
goto retry;
- else
- return -1;
+
+ xlogreader->readLen = -1;
+ return false;
}
/*
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index cb76be4f46..3d599325ee 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -36,8 +36,8 @@
static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
pg_attribute_printf(2, 3);
static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
-static int ReadPageInternal(XLogReaderState *state, XLogRecPtr pageptr,
- int reqLen);
+static bool XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr,
+ int reqLen, bool header_inclusive);
static void XLogReaderInvalReadState(XLogReaderState *state);
static bool ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
XLogRecPtr PrevRecPtr, XLogRecord *record, bool randAccess);
@@ -276,7 +276,6 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
uint32 targetRecOff;
uint32 pageHeaderSize;
bool gotheader;
- int readOff;
/*
* randAccess indicates whether to verify the previous-record pointer of
@@ -326,14 +325,20 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
* byte to cover the whole record header, or at least the part of it that
* fits on the same page.
*/
- readOff = ReadPageInternal(state, targetPagePtr,
- Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ));
- if (readOff < 0)
+ while (XLogNeedData(state, targetPagePtr,
+ Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+ targetRecOff != 0))
+ {
+ if (!state->routine.page_read(state, state->readPagePtr, state->readLen,
+ RecPtr, state->readBuf))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
/*
- * ReadPageInternal always returns at least the page header, so we can
- * examine it now.
+ * We have at least the page header, so we can examine it now.
*/
pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
if (targetRecOff == 0)
@@ -359,8 +364,8 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
goto err;
}
- /* ReadPageInternal has verified the page header */
- Assert(pageHeaderSize <= readOff);
+ /* XLogNeedData has verified the page header */
+ Assert(pageHeaderSize <= state->readLen);
/*
* Read the record length.
@@ -433,18 +438,27 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
do
{
+ int rest_len = total_len - gotlen;
+
/* Calculate pointer to beginning of next page */
targetPagePtr += XLOG_BLCKSZ;
/* Wait for the next page to become available */
- readOff = ReadPageInternal(state, targetPagePtr,
- Min(total_len - gotlen + SizeOfXLogShortPHD,
- XLOG_BLCKSZ));
+ while (XLogNeedData(state, targetPagePtr,
+ Min(rest_len, XLOG_BLCKSZ),
+ false))
+ {
+ if (!state->routine.page_read(state, state->readPagePtr,
+ state->readLen,
+ state->ReadRecPtr,
+ state->readBuf))
+ break;
+ }
- if (readOff < 0)
+ if (!state->page_verified)
goto err;
- Assert(SizeOfXLogShortPHD <= readOff);
+ Assert(SizeOfXLogShortPHD <= state->readLen);
/* Check that the continuation on next page looks valid */
pageHeader = (XLogPageHeader) state->readBuf;
@@ -473,21 +487,14 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
/* Append the continuation from this page to the buffer */
pageHeaderSize = XLogPageHeaderSize(pageHeader);
- if (readOff < pageHeaderSize)
- readOff = ReadPageInternal(state, targetPagePtr,
- pageHeaderSize);
-
- Assert(pageHeaderSize <= readOff);
+ Assert(pageHeaderSize <= state->readLen);
contdata = (char *) state->readBuf + pageHeaderSize;
len = XLOG_BLCKSZ - pageHeaderSize;
if (pageHeader->xlp_rem_len < len)
len = pageHeader->xlp_rem_len;
- if (readOff < pageHeaderSize + len)
- readOff = ReadPageInternal(state, targetPagePtr,
- pageHeaderSize + len);
-
+ Assert (pageHeaderSize + len <= state->readLen);
memcpy(buffer, (char *) contdata, len);
buffer += len;
gotlen += len;
@@ -517,9 +524,16 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
else
{
/* Wait for the record data to become available */
- readOff = ReadPageInternal(state, targetPagePtr,
- Min(targetRecOff + total_len, XLOG_BLCKSZ));
- if (readOff < 0)
+ while (XLogNeedData(state, targetPagePtr,
+ Min(targetRecOff + total_len, XLOG_BLCKSZ), true))
+ {
+ if (!state->routine.page_read(state, state->readPagePtr,
+ state->readLen,
+ state->ReadRecPtr, state->readBuf))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
/* Record does not cross a page boundary */
@@ -562,109 +576,138 @@ err:
}
/*
- * Read a single xlog page including at least [pageptr, reqLen] of valid data
- * via the page_read() callback.
+ * Checks that an xlog page loaded in state->readBuf is including at least
+ * [pageptr, reqLen] and the page is valid. header_inclusive indicates that
+ * reqLen is calculated including page header length.
*
- * Returns -1 if the required page cannot be read for some reason; errormsg_buf
- * is set in that case (unless the error occurs in the page_read callback).
+ * Returns false if the buffer already contains the requested data, or found
+ * error. state->page_verified is set to true for the former and false for the
+ * latter.
*
- * We fetch the page from a reader-local cache if we know we have the required
- * data and if there hasn't been any error since caching the data.
+ * Otherwise returns true and requests data loaded onto state->readBuf by
+ * state->readPagePtr and state->readLen. The caller shall call this function
+ * again after filling the buffer at least with that portion of data and set
+ * state->readLen to the length of actually loaded data.
+ *
+ * If header_inclusive is false, corrects reqLen internally by adding the
+ * actual page header length and may request caller for new data.
*/
-static int
-ReadPageInternal(XLogReaderState *state, XLogRecPtr pageptr, int reqLen)
+static bool
+XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
+ bool header_inclusive)
{
- int readLen;
uint32 targetPageOff;
XLogSegNo targetSegNo;
- XLogPageHeader hdr;
+ uint32 addLen = 0;
- Assert((pageptr % XLOG_BLCKSZ) == 0);
+ /* Some data is loaded, but page header is not verified yet. */
+ if (!state->page_verified &&
+ !XLogRecPtrIsInvalid(state->readPagePtr) && state->readLen >= 0)
+ {
+ uint32 pageHeaderSize;
+ /* just loaded new data so needs to verify page header */
+
+ /* The caller must have loaded at least page header */
+ Assert (state->readLen >= SizeOfXLogShortPHD);
+
+ /*
+ * We have enough data to check the header length. Recheck the loaded
+ * length against the actual header length.
+ */
+ pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+
+ /* Request more data if we don't have the full header. */
+ if (state->readLen < pageHeaderSize)
+ {
+ state->readLen = pageHeaderSize;
+ return true;
+ }
+
+ /* Now that we know we have the full header, validate it. */
+ if (!XLogReaderValidatePageHeader(state, state->readPagePtr,
+ (char *) state->readBuf))
+ {
+ /* That's bad. Force reading the page again. */
+ XLogReaderInvalReadState(state);
+
+ return false;
+ }
+
+ state->page_verified = true;
+
+ XLByteToSeg(state->readPagePtr, state->seg.ws_segno,
+ state->segcxt.ws_segsize);
+ }
+
+ /*
+ * The loaded page may not be the one caller is supposing to read when we
+ * are verifying the first page of new segment. In that case, skip further
+ * verification and immediately load the target page.
+ */
+ if (state->page_verified && pageptr == state->readPagePtr)
+ {
+ /*
+ * calculate additional length for page header keeping the total
+ * length within the block size.
+ */
+ if (!header_inclusive)
+ {
+ uint32 pageHeaderSize =
+ XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+
+ addLen = pageHeaderSize;
+ if (reqLen + pageHeaderSize <= XLOG_BLCKSZ)
+ addLen = pageHeaderSize;
+ else
+ addLen = XLOG_BLCKSZ - reqLen;
+
+ Assert(addLen >= 0);
+ }
+
+ /* Return if we already have it. */
+ if (reqLen + addLen <= state->readLen)
+ return false;
+ }
+
+ /* Data is not in our buffer, request the caller for it. */
XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize);
targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize);
+ Assert((pageptr % XLOG_BLCKSZ) == 0);
- /* check whether we have all the requested data already */
- if (targetSegNo == state->seg.ws_segno &&
- targetPageOff == state->segoff && reqLen <= state->readLen)
- return state->readLen;
+ /*
+ * Every time we request to load new data of a page to the caller, even if
+ * we looked at a part of it before, we need to do verification on the next
+ * invocation as the caller might now be rereading data from a different
+ * source.
+ */
+ state->page_verified = false;
/*
- * Data is not in our buffer.
- *
- * Every time we actually read the segment, even if we looked at parts of
- * it before, we need to do verification as the page_read callback might
- * now be rereading data from a different source.
- *
* Whenever switching to a new WAL segment, we read the first page of the
* file and validate its header, even if that's not where the target
* record is. This is so that we can check the additional identification
* info that is present in the first page's "long" header.
+ * Don't do this if the caller requested the first page in the segment.
*/
if (targetSegNo != state->seg.ws_segno && targetPageOff != 0)
{
- XLogRecPtr targetSegmentPtr = pageptr - targetPageOff;
-
- readLen = state->routine.page_read(state, targetSegmentPtr, XLOG_BLCKSZ,
- state->currRecPtr,
- state->readBuf);
- if (readLen < 0)
- goto err;
-
- /* we can be sure to have enough WAL available, we scrolled back */
- Assert(readLen == XLOG_BLCKSZ);
-
- if (!XLogReaderValidatePageHeader(state, targetSegmentPtr,
- state->readBuf))
- goto err;
+ /*
+ * Then we'll see that the targetSegNo now matches the ws_segno, and
+ * will not come back here, but will request the actual target page.
+ */
+ state->readPagePtr = pageptr - targetPageOff;
+ state->readLen = XLOG_BLCKSZ;
+ return true;
}
/*
- * First, read the requested data length, but at least a short page header
- * so that we can validate it.
+ * Request the caller to load the page. We need at least a short page
+ * header so that we can validate it.
*/
- readLen = state->routine.page_read(state, pageptr, Max(reqLen, SizeOfXLogShortPHD),
- state->currRecPtr,
- state->readBuf);
- if (readLen < 0)
- goto err;
-
- Assert(readLen <= XLOG_BLCKSZ);
-
- /* Do we have enough data to check the header length? */
- if (readLen <= SizeOfXLogShortPHD)
- goto err;
-
- Assert(readLen >= reqLen);
-
- hdr = (XLogPageHeader) state->readBuf;
-
- /* still not enough */
- if (readLen < XLogPageHeaderSize(hdr))
- {
- readLen = state->routine.page_read(state, pageptr, XLogPageHeaderSize(hdr),
- state->currRecPtr,
- state->readBuf);
- if (readLen < 0)
- goto err;
- }
-
- /*
- * Now that we know we have the full header, validate it.
- */
- if (!XLogReaderValidatePageHeader(state, pageptr, (char *) hdr))
- goto err;
-
- /* update read state information */
- state->seg.ws_segno = targetSegNo;
- state->segoff = targetPageOff;
- state->readLen = readLen;
-
- return readLen;
-
-err:
- XLogReaderInvalReadState(state);
- return -1;
+ state->readPagePtr = pageptr;
+ state->readLen = Max(reqLen + addLen, SizeOfXLogShortPHD);
+ return true;
}
/*
@@ -673,9 +716,7 @@ err:
static void
XLogReaderInvalReadState(XLogReaderState *state)
{
- state->seg.ws_segno = 0;
- state->segoff = 0;
- state->readLen = 0;
+ state->readPagePtr = InvalidXLogRecPtr;
}
/*
@@ -956,7 +997,6 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
XLogRecPtr targetPagePtr;
int targetRecOff;
uint32 pageHeaderSize;
- int readLen;
/*
* Compute targetRecOff. It should typically be equal or greater than
@@ -964,7 +1004,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
* that, except when caller has explicitly specified the offset that
* falls somewhere there or when we are skipping multi-page
* continuation record. It doesn't matter though because
- * ReadPageInternal() is prepared to handle that and will read at
+ * XLogNeedData() is prepared to handle that and will read at
* least short page-header worth of data
*/
targetRecOff = tmpRecPtr % XLOG_BLCKSZ;
@@ -972,19 +1012,24 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
/* scroll back to page boundary */
targetPagePtr = tmpRecPtr - targetRecOff;
- /* Read the page containing the record */
- readLen = ReadPageInternal(state, targetPagePtr, targetRecOff);
- if (readLen < 0)
+ while(XLogNeedData(state, targetPagePtr, targetRecOff,
+ targetRecOff != 0))
+ {
+ if (!state->routine.page_read(state, state->readPagePtr,
+ state->readLen,
+ state->ReadRecPtr, state->readBuf))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
header = (XLogPageHeader) state->readBuf;
pageHeaderSize = XLogPageHeaderSize(header);
- /* make sure we have enough data for the page header */
- readLen = ReadPageInternal(state, targetPagePtr, pageHeaderSize);
- if (readLen < 0)
- goto err;
+ /* we should have read the page header */
+ Assert (state->readLen >= pageHeaderSize);
/* skip over potential continuation data */
if (header->xlp_info & XLP_FIRST_IS_CONTRECORD)
diff --git a/src/backend/access/transam/xlogutils.c b/src/backend/access/transam/xlogutils.c
index 322b0e8ff5..1b10b6df20 100644
--- a/src/backend/access/transam/xlogutils.c
+++ b/src/backend/access/transam/xlogutils.c
@@ -689,8 +689,8 @@ XLogTruncateRelation(RelFileNode rnode, ForkNumber forkNum,
void
XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wantLength)
{
- const XLogRecPtr lastReadPage = (state->seg.ws_segno *
- state->segcxt.ws_segsize + state->segoff);
+ const XLogRecPtr lastReadPage = state->seg.ws_segno *
+ state->segcxt.ws_segsize + state->readLen;
Assert(wantPage != InvalidXLogRecPtr && wantPage % XLOG_BLCKSZ == 0);
Assert(wantLength <= XLOG_BLCKSZ);
@@ -828,7 +828,7 @@ wal_segment_close(XLogReaderState *state)
* exists for normal backends, so we have to do a check/sleep/repeat style of
* loop for now.
*/
-int
+bool
read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *cur_page)
{
@@ -930,7 +930,8 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
else if (targetPagePtr + reqLen > read_upto)
{
/* not enough data there */
- return -1;
+ state->readLen = -1;
+ return false;
}
else
{
@@ -948,7 +949,8 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
WALReadRaiseError(&errinfo);
/* number of valid bytes in the buffer */
- return count;
+ state->readLen = count;
+ return true;
}
/*
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index e2477c47e0..bf2711bddd 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -811,7 +811,7 @@ StartReplication(StartReplicationCmd *cmd)
* which has to do a plain sleep/busy loop, because the walsender's latch gets
* set every time WAL is flushed.
*/
-static int
+static bool
logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *cur_page)
{
@@ -831,7 +831,10 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
/* fail if not (implies we are going to shut down) */
if (flushptr < targetPagePtr + reqLen)
- return -1;
+ {
+ state->readLen = -1;
+ return false;
+ }
if (targetPagePtr + XLOG_BLCKSZ <= flushptr)
count = XLOG_BLCKSZ; /* more than one block available */
@@ -859,7 +862,8 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
XLByteToSeg(targetPagePtr, segno, state->segcxt.ws_segsize);
CheckXLogRemoved(segno, state->seg.ws_tli);
- return count;
+ state->readLen = count;
+ return true;
}
/*
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index bc6f976994..0b7e73ae79 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -46,7 +46,7 @@ typedef struct XLogPageReadPrivate
int tliIndex;
} XLogPageReadPrivate;
-static int SimpleXLogPageRead(XLogReaderState *xlogreader,
+static bool SimpleXLogPageRead(XLogReaderState *xlogreader,
XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
@@ -236,7 +236,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
}
/* XLogReader callback function, to read a WAL page */
-static int
+static bool
SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf)
{
@@ -296,7 +296,8 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
if (private->restoreCommand == NULL)
{
pg_log_error("could not open file \"%s\": %m", xlogfpath);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
/*
@@ -309,7 +310,10 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
private->restoreCommand);
if (xlogreadfd < 0)
- return -1;
+ {
+ xlogreader->readLen = -1;
+ return false;
+ }
else
pg_log_debug("using file \"%s\" restored from archive",
xlogfpath);
@@ -325,7 +329,8 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
if (lseek(xlogreadfd, (off_t) targetPageOff, SEEK_SET) < 0)
{
pg_log_error("could not seek in file \"%s\": %m", xlogfpath);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
@@ -338,13 +343,15 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
pg_log_error("could not read file \"%s\": read %d of %zu",
xlogfpath, r, (Size) XLOG_BLCKSZ);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
Assert(targetSegNo == xlogreadsegno);
xlogreader->seg.ws_tli = targetHistory[private->tliIndex].tli;
- return XLOG_BLCKSZ;
+ xlogreader->readLen = XLOG_BLCKSZ;
+ return true;
}
/*
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index d1a0678935..ab0c28d9b7 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -331,7 +331,7 @@ WALDumpCloseSegment(XLogReaderState *state)
}
/* pg_waldump's XLogReaderRoutine->page_read callback */
-static int
+static bool
WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetPtr, char *readBuff)
{
@@ -348,7 +348,8 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
else
{
private->endptr_reached = true;
- return -1;
+ state->readLen = -1;
+ return false;
}
}
@@ -373,7 +374,8 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
(Size) errinfo.wre_req);
}
- return count;
+ state->readLen = count;
+ return true;
}
/*
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index b0f2a6ed43..6c8848e14f 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -57,8 +57,8 @@ typedef struct WALSegmentContext
typedef struct XLogReaderState XLogReaderState;
-/* Function type definitions for various xlogreader interactions */
-typedef int (*XLogPageReadCB) (XLogReaderState *xlogreader,
+/* Function type definition for the read_page callback */
+typedef bool (*XLogPageReadCB) (XLogReaderState *xlogreader,
XLogRecPtr targetPagePtr,
int reqLen,
XLogRecPtr targetRecPtr,
@@ -175,6 +175,20 @@ struct XLogReaderState
XLogRecPtr ReadRecPtr; /* start of last record read */
XLogRecPtr EndRecPtr; /* end+1 of last record read */
+ /* ----------------------------------------
+ * Communication with page reader
+ * readBuf is XLOG_BLCKSZ bytes, valid up to at least readLen bytes.
+ * ----------------------------------------
+ */
+ /* variables to communicate with page reader */
+ XLogRecPtr readPagePtr; /* page pointer to read */
+ int32 readLen; /* bytes requested to reader, or actual bytes
+ * read by reader, which must be larger than
+ * the request, or -1 on error */
+ TimeLineID readPageTLI; /* TLI for data currently in readBuf */
+ char *readBuf; /* buffer to store data */
+ bool page_verified; /* is the page on the buffer verified? */
+
/* ----------------------------------------
* Decoded representation of current record
@@ -201,13 +215,6 @@ struct XLogReaderState
* ----------------------------------------
*/
- /*
- * Buffer for currently read page (XLOG_BLCKSZ bytes, valid up to at least
- * readLen bytes)
- */
- char *readBuf;
- uint32 readLen;
-
/* last read XLOG position for data currently in readBuf */
WALSegmentContext segcxt;
WALOpenSegment seg;
diff --git a/src/include/access/xlogutils.h b/src/include/access/xlogutils.h
index e59b6cf3a9..c287edf206 100644
--- a/src/include/access/xlogutils.h
+++ b/src/include/access/xlogutils.h
@@ -47,7 +47,7 @@ extern Buffer XLogReadBufferExtended(RelFileNode rnode, ForkNumber forknum,
extern Relation CreateFakeRelcacheEntry(RelFileNode rnode);
extern void FreeFakeRelcacheEntry(Relation fakerel);
-extern int read_local_xlog_page(XLogReaderState *state,
+extern bool read_local_xlog_page(XLogReaderState *state,
XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *cur_page);
extern void wal_segment_open(XLogReaderState *state,
--
2.18.4
----Next_Part(Thu_Jul__2_13_53_30_2020_072)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
filename="v15-0002-Move-page-reader-out-of-XLogReadRecord.patch"
^ permalink raw reply [nested|flat] 69+ messages in thread
* [PATCH v17 1/4] Move callback-call from ReadPageInternal to XLogReadRecord.
@ 2019-09-05 11:21 Kyotaro Horiguchi <[email protected]>
0 siblings, 0 replies; 69+ messages in thread
From: Kyotaro Horiguchi @ 2019-09-05 11:21 UTC (permalink / raw)
The current WAL record reader reads page data using a call back
function. Although it is not so problematic alone, it would be a
problem if we are going to do add tasks like encryption which is
performed on page data before WAL reader reads them. To avoid that the
record reader facility has to have a new code path corresponds to
every new callback, this patch separates page reader from WAL record
reading facility by modifying the current WAL record reader to a state
machine.
As the first step of that change, this patch moves the page reader
function out of ReadPageInternal, then the remaining tasks of the
function are taken over by the new function XLogNeedData. As the
result XLogPageRead directly calls the page reader callback function
according to the feedback from XLogNeedData.
---
src/backend/access/transam/xlog.c | 16 +-
src/backend/access/transam/xlogreader.c | 281 ++++++++++++++----------
src/backend/access/transam/xlogutils.c | 12 +-
src/backend/replication/walsender.c | 10 +-
src/bin/pg_rewind/parsexlog.c | 21 +-
src/bin/pg_waldump/pg_waldump.c | 8 +-
src/include/access/xlogreader.h | 25 ++-
src/include/access/xlogutils.h | 2 +-
8 files changed, 222 insertions(+), 153 deletions(-)
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 377afb8732..606328a65a 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -917,7 +917,7 @@ static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath,
static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli,
XLogSource source, bool notfoundOk);
static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, XLogSource source);
-static int XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
+static bool XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
bool fetching_ckpt, XLogRecPtr tliRecPtr);
@@ -4350,7 +4350,6 @@ ReadRecord(XLogReaderState *xlogreader, int emode,
XLogRecord *record;
XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
- /* Pass through parameters to XLogPageRead */
private->fetching_ckpt = fetching_ckpt;
private->emode = emode;
private->randAccess = (xlogreader->ReadRecPtr == InvalidXLogRecPtr);
@@ -11987,7 +11986,7 @@ CancelBackup(void)
* XLogPageRead() to try fetching the record from another source, or to
* sleep and retry.
*/
-static int
+static bool
XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *readBuf)
{
@@ -12046,7 +12045,8 @@ retry:
readLen = 0;
readSource = XLOG_FROM_ANY;
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
}
@@ -12141,7 +12141,8 @@ retry:
goto next_record_is_invalid;
}
- return readLen;
+ xlogreader->readLen = readLen;
+ return true;
next_record_is_invalid:
lastSourceFailed = true;
@@ -12155,8 +12156,9 @@ next_record_is_invalid:
/* In standby-mode, keep trying */
if (StandbyMode)
goto retry;
- else
- return -1;
+
+ xlogreader->readLen = -1;
+ return false;
}
/*
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index 42738eb940..7d04995b7b 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -36,8 +36,8 @@
static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
pg_attribute_printf(2, 3);
static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
-static int ReadPageInternal(XLogReaderState *state, XLogRecPtr pageptr,
- int reqLen);
+static bool XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr,
+ int reqLen, bool header_inclusive);
static void XLogReaderInvalReadState(XLogReaderState *state);
static bool ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
XLogRecPtr PrevRecPtr, XLogRecord *record, bool randAccess);
@@ -276,7 +276,6 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
uint32 targetRecOff;
uint32 pageHeaderSize;
bool gotheader;
- int readOff;
/*
* randAccess indicates whether to verify the previous-record pointer of
@@ -326,14 +325,20 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
* byte to cover the whole record header, or at least the part of it that
* fits on the same page.
*/
- readOff = ReadPageInternal(state, targetPagePtr,
- Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ));
- if (readOff < 0)
+ while (XLogNeedData(state, targetPagePtr,
+ Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+ targetRecOff != 0))
+ {
+ if (!state->routine.page_read(state, state->readPagePtr, state->readLen,
+ RecPtr, state->readBuf))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
/*
- * ReadPageInternal always returns at least the page header, so we can
- * examine it now.
+ * We have at least the page header, so we can examine it now.
*/
pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
if (targetRecOff == 0)
@@ -359,8 +364,8 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
goto err;
}
- /* ReadPageInternal has verified the page header */
- Assert(pageHeaderSize <= readOff);
+ /* XLogNeedData has verified the page header */
+ Assert(pageHeaderSize <= state->readLen);
/*
* Read the record length.
@@ -432,18 +437,27 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
do
{
+ int rest_len = total_len - gotlen;
+
/* Calculate pointer to beginning of next page */
targetPagePtr += XLOG_BLCKSZ;
/* Wait for the next page to become available */
- readOff = ReadPageInternal(state, targetPagePtr,
- Min(total_len - gotlen + SizeOfXLogShortPHD,
- XLOG_BLCKSZ));
+ while (XLogNeedData(state, targetPagePtr,
+ Min(rest_len, XLOG_BLCKSZ),
+ false))
+ {
+ if (!state->routine.page_read(state, state->readPagePtr,
+ state->readLen,
+ state->ReadRecPtr,
+ state->readBuf))
+ break;
+ }
- if (readOff < 0)
+ if (!state->page_verified)
goto err;
- Assert(SizeOfXLogShortPHD <= readOff);
+ Assert(SizeOfXLogShortPHD <= state->readLen);
/* Check that the continuation on next page looks valid */
pageHeader = (XLogPageHeader) state->readBuf;
@@ -473,21 +487,14 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
/* Append the continuation from this page to the buffer */
pageHeaderSize = XLogPageHeaderSize(pageHeader);
- if (readOff < pageHeaderSize)
- readOff = ReadPageInternal(state, targetPagePtr,
- pageHeaderSize);
-
- Assert(pageHeaderSize <= readOff);
+ Assert(pageHeaderSize <= state->readLen);
contdata = (char *) state->readBuf + pageHeaderSize;
len = XLOG_BLCKSZ - pageHeaderSize;
if (pageHeader->xlp_rem_len < len)
len = pageHeader->xlp_rem_len;
- if (readOff < pageHeaderSize + len)
- readOff = ReadPageInternal(state, targetPagePtr,
- pageHeaderSize + len);
-
+ Assert (pageHeaderSize + len <= state->readLen);
memcpy(buffer, (char *) contdata, len);
buffer += len;
gotlen += len;
@@ -517,9 +524,16 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
else
{
/* Wait for the record data to become available */
- readOff = ReadPageInternal(state, targetPagePtr,
- Min(targetRecOff + total_len, XLOG_BLCKSZ));
- if (readOff < 0)
+ while (XLogNeedData(state, targetPagePtr,
+ Min(targetRecOff + total_len, XLOG_BLCKSZ), true))
+ {
+ if (!state->routine.page_read(state, state->readPagePtr,
+ state->readLen,
+ state->ReadRecPtr, state->readBuf))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
/* Record does not cross a page boundary */
@@ -562,109 +576,138 @@ err:
}
/*
- * Read a single xlog page including at least [pageptr, reqLen] of valid data
- * via the page_read() callback.
+ * Checks that an xlog page loaded in state->readBuf is including at least
+ * [pageptr, reqLen] and the page is valid. header_inclusive indicates that
+ * reqLen is calculated including page header length.
*
- * Returns -1 if the required page cannot be read for some reason; errormsg_buf
- * is set in that case (unless the error occurs in the page_read callback).
+ * Returns false if the buffer already contains the requested data, or found
+ * error. state->page_verified is set to true for the former and false for the
+ * latter.
*
- * We fetch the page from a reader-local cache if we know we have the required
- * data and if there hasn't been any error since caching the data.
+ * Otherwise returns true and requests data loaded onto state->readBuf by
+ * state->readPagePtr and state->readLen. The caller shall call this function
+ * again after filling the buffer at least with that portion of data and set
+ * state->readLen to the length of actually loaded data.
+ *
+ * If header_inclusive is false, corrects reqLen internally by adding the
+ * actual page header length and may request caller for new data.
*/
-static int
-ReadPageInternal(XLogReaderState *state, XLogRecPtr pageptr, int reqLen)
+static bool
+XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
+ bool header_inclusive)
{
- int readLen;
uint32 targetPageOff;
XLogSegNo targetSegNo;
- XLogPageHeader hdr;
+ uint32 addLen = 0;
- Assert((pageptr % XLOG_BLCKSZ) == 0);
+ /* Some data is loaded, but page header is not verified yet. */
+ if (!state->page_verified &&
+ !XLogRecPtrIsInvalid(state->readPagePtr) && state->readLen >= 0)
+ {
+ uint32 pageHeaderSize;
+ /* just loaded new data so needs to verify page header */
+
+ /* The caller must have loaded at least page header */
+ Assert (state->readLen >= SizeOfXLogShortPHD);
+
+ /*
+ * We have enough data to check the header length. Recheck the loaded
+ * length against the actual header length.
+ */
+ pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+
+ /* Request more data if we don't have the full header. */
+ if (state->readLen < pageHeaderSize)
+ {
+ state->readLen = pageHeaderSize;
+ return true;
+ }
+
+ /* Now that we know we have the full header, validate it. */
+ if (!XLogReaderValidatePageHeader(state, state->readPagePtr,
+ (char *) state->readBuf))
+ {
+ /* That's bad. Force reading the page again. */
+ XLogReaderInvalReadState(state);
+
+ return false;
+ }
+
+ state->page_verified = true;
+
+ XLByteToSeg(state->readPagePtr, state->seg.ws_segno,
+ state->segcxt.ws_segsize);
+ }
+
+ /*
+ * The loaded page may not be the one caller is supposing to read when we
+ * are verifying the first page of new segment. In that case, skip further
+ * verification and immediately load the target page.
+ */
+ if (state->page_verified && pageptr == state->readPagePtr)
+ {
+ /*
+ * calculate additional length for page header keeping the total
+ * length within the block size.
+ */
+ if (!header_inclusive)
+ {
+ uint32 pageHeaderSize =
+ XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+
+ addLen = pageHeaderSize;
+ if (reqLen + pageHeaderSize <= XLOG_BLCKSZ)
+ addLen = pageHeaderSize;
+ else
+ addLen = XLOG_BLCKSZ - reqLen;
+
+ Assert(addLen >= 0);
+ }
+
+ /* Return if we already have it. */
+ if (reqLen + addLen <= state->readLen)
+ return false;
+ }
+
+ /* Data is not in our buffer, request the caller for it. */
XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize);
targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize);
+ Assert((pageptr % XLOG_BLCKSZ) == 0);
- /* check whether we have all the requested data already */
- if (targetSegNo == state->seg.ws_segno &&
- targetPageOff == state->segoff && reqLen <= state->readLen)
- return state->readLen;
+ /*
+ * Every time we request to load new data of a page to the caller, even if
+ * we looked at a part of it before, we need to do verification on the next
+ * invocation as the caller might now be rereading data from a different
+ * source.
+ */
+ state->page_verified = false;
/*
- * Data is not in our buffer.
- *
- * Every time we actually read the segment, even if we looked at parts of
- * it before, we need to do verification as the page_read callback might
- * now be rereading data from a different source.
- *
* Whenever switching to a new WAL segment, we read the first page of the
* file and validate its header, even if that's not where the target
* record is. This is so that we can check the additional identification
* info that is present in the first page's "long" header.
+ * Don't do this if the caller requested the first page in the segment.
*/
if (targetSegNo != state->seg.ws_segno && targetPageOff != 0)
{
- XLogRecPtr targetSegmentPtr = pageptr - targetPageOff;
-
- readLen = state->routine.page_read(state, targetSegmentPtr, XLOG_BLCKSZ,
- state->currRecPtr,
- state->readBuf);
- if (readLen < 0)
- goto err;
-
- /* we can be sure to have enough WAL available, we scrolled back */
- Assert(readLen == XLOG_BLCKSZ);
-
- if (!XLogReaderValidatePageHeader(state, targetSegmentPtr,
- state->readBuf))
- goto err;
+ /*
+ * Then we'll see that the targetSegNo now matches the ws_segno, and
+ * will not come back here, but will request the actual target page.
+ */
+ state->readPagePtr = pageptr - targetPageOff;
+ state->readLen = XLOG_BLCKSZ;
+ return true;
}
/*
- * First, read the requested data length, but at least a short page header
- * so that we can validate it.
+ * Request the caller to load the page. We need at least a short page
+ * header so that we can validate it.
*/
- readLen = state->routine.page_read(state, pageptr, Max(reqLen, SizeOfXLogShortPHD),
- state->currRecPtr,
- state->readBuf);
- if (readLen < 0)
- goto err;
-
- Assert(readLen <= XLOG_BLCKSZ);
-
- /* Do we have enough data to check the header length? */
- if (readLen <= SizeOfXLogShortPHD)
- goto err;
-
- Assert(readLen >= reqLen);
-
- hdr = (XLogPageHeader) state->readBuf;
-
- /* still not enough */
- if (readLen < XLogPageHeaderSize(hdr))
- {
- readLen = state->routine.page_read(state, pageptr, XLogPageHeaderSize(hdr),
- state->currRecPtr,
- state->readBuf);
- if (readLen < 0)
- goto err;
- }
-
- /*
- * Now that we know we have the full header, validate it.
- */
- if (!XLogReaderValidatePageHeader(state, pageptr, (char *) hdr))
- goto err;
-
- /* update read state information */
- state->seg.ws_segno = targetSegNo;
- state->segoff = targetPageOff;
- state->readLen = readLen;
-
- return readLen;
-
-err:
- XLogReaderInvalReadState(state);
- return -1;
+ state->readPagePtr = pageptr;
+ state->readLen = Max(reqLen + addLen, SizeOfXLogShortPHD);
+ return true;
}
/*
@@ -673,9 +716,7 @@ err:
static void
XLogReaderInvalReadState(XLogReaderState *state)
{
- state->seg.ws_segno = 0;
- state->segoff = 0;
- state->readLen = 0;
+ state->readPagePtr = InvalidXLogRecPtr;
}
/*
@@ -953,7 +994,6 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
XLogRecPtr targetPagePtr;
int targetRecOff;
uint32 pageHeaderSize;
- int readLen;
/*
* Compute targetRecOff. It should typically be equal or greater than
@@ -961,7 +1001,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
* that, except when caller has explicitly specified the offset that
* falls somewhere there or when we are skipping multi-page
* continuation record. It doesn't matter though because
- * ReadPageInternal() is prepared to handle that and will read at
+ * XLogNeedData() is prepared to handle that and will read at
* least short page-header worth of data
*/
targetRecOff = tmpRecPtr % XLOG_BLCKSZ;
@@ -969,19 +1009,24 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
/* scroll back to page boundary */
targetPagePtr = tmpRecPtr - targetRecOff;
- /* Read the page containing the record */
- readLen = ReadPageInternal(state, targetPagePtr, targetRecOff);
- if (readLen < 0)
+ while(XLogNeedData(state, targetPagePtr, targetRecOff,
+ targetRecOff != 0))
+ {
+ if (!state->routine.page_read(state, state->readPagePtr,
+ state->readLen,
+ state->ReadRecPtr, state->readBuf))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
header = (XLogPageHeader) state->readBuf;
pageHeaderSize = XLogPageHeaderSize(header);
- /* make sure we have enough data for the page header */
- readLen = ReadPageInternal(state, targetPagePtr, pageHeaderSize);
- if (readLen < 0)
- goto err;
+ /* we should have read the page header */
+ Assert (state->readLen >= pageHeaderSize);
/* skip over potential continuation data */
if (header->xlp_info & XLP_FIRST_IS_CONTRECORD)
diff --git a/src/backend/access/transam/xlogutils.c b/src/backend/access/transam/xlogutils.c
index d17d660f46..46eda33f25 100644
--- a/src/backend/access/transam/xlogutils.c
+++ b/src/backend/access/transam/xlogutils.c
@@ -686,8 +686,8 @@ XLogTruncateRelation(RelFileNode rnode, ForkNumber forkNum,
void
XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wantLength)
{
- const XLogRecPtr lastReadPage = (state->seg.ws_segno *
- state->segcxt.ws_segsize + state->segoff);
+ const XLogRecPtr lastReadPage = state->seg.ws_segno *
+ state->segcxt.ws_segsize + state->readLen;
Assert(wantPage != InvalidXLogRecPtr && wantPage % XLOG_BLCKSZ == 0);
Assert(wantLength <= XLOG_BLCKSZ);
@@ -824,7 +824,7 @@ wal_segment_close(XLogReaderState *state)
* exists for normal backends, so we have to do a check/sleep/repeat style of
* loop for now.
*/
-int
+bool
read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *cur_page)
{
@@ -926,7 +926,8 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
else if (targetPagePtr + reqLen > read_upto)
{
/* not enough data there */
- return -1;
+ state->readLen = -1;
+ return false;
}
else
{
@@ -944,7 +945,8 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
WALReadRaiseError(&errinfo);
/* number of valid bytes in the buffer */
- return count;
+ state->readLen = count;
+ return true;
}
/*
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index 23baa4498a..c36b91ae5e 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -806,7 +806,7 @@ StartReplication(StartReplicationCmd *cmd)
* which has to do a plain sleep/busy loop, because the walsender's latch gets
* set every time WAL is flushed.
*/
-static int
+static bool
logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *cur_page)
{
@@ -826,7 +826,10 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
/* fail if not (implies we are going to shut down) */
if (flushptr < targetPagePtr + reqLen)
- return -1;
+ {
+ state->readLen = -1;
+ return false;
+ }
if (targetPagePtr + XLOG_BLCKSZ <= flushptr)
count = XLOG_BLCKSZ; /* more than one block available */
@@ -854,7 +857,8 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
XLByteToSeg(targetPagePtr, segno, state->segcxt.ws_segsize);
CheckXLogRemoved(segno, state->seg.ws_tli);
- return count;
+ state->readLen = count;
+ return true;
}
/*
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index 59ebac7d6a..cf119848b0 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -47,7 +47,7 @@ typedef struct XLogPageReadPrivate
int tliIndex;
} XLogPageReadPrivate;
-static int SimpleXLogPageRead(XLogReaderState *xlogreader,
+static bool SimpleXLogPageRead(XLogReaderState *xlogreader,
XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
@@ -246,7 +246,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
}
/* XLogReader callback function, to read a WAL page */
-static int
+static bool
SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf)
{
@@ -306,7 +306,8 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
if (private->restoreCommand == NULL)
{
pg_log_error("could not open file \"%s\": %m", xlogfpath);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
/*
@@ -319,7 +320,10 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
private->restoreCommand);
if (xlogreadfd < 0)
- return -1;
+ {
+ xlogreader->readLen = -1;
+ return false;
+ }
else
pg_log_debug("using file \"%s\" restored from archive",
xlogfpath);
@@ -335,7 +339,8 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
if (lseek(xlogreadfd, (off_t) targetPageOff, SEEK_SET) < 0)
{
pg_log_error("could not seek in file \"%s\": %m", xlogfpath);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
@@ -348,13 +353,15 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
pg_log_error("could not read file \"%s\": read %d of %zu",
xlogfpath, r, (Size) XLOG_BLCKSZ);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
Assert(targetSegNo == xlogreadsegno);
xlogreader->seg.ws_tli = targetHistory[private->tliIndex].tli;
- return XLOG_BLCKSZ;
+ xlogreader->readLen = XLOG_BLCKSZ;
+ return true;
}
/*
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index 610f65e471..0f0e0723a3 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -331,7 +331,7 @@ WALDumpCloseSegment(XLogReaderState *state)
}
/* pg_waldump's XLogReaderRoutine->page_read callback */
-static int
+static bool
WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetPtr, char *readBuff)
{
@@ -348,7 +348,8 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
else
{
private->endptr_reached = true;
- return -1;
+ state->readLen = -1;
+ return false;
}
}
@@ -373,7 +374,8 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
(Size) errinfo.wre_req);
}
- return count;
+ state->readLen = count;
+ return true;
}
/*
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index 21d200d3df..23d9e70a59 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -57,8 +57,8 @@ typedef struct WALSegmentContext
typedef struct XLogReaderState XLogReaderState;
-/* Function type definitions for various xlogreader interactions */
-typedef int (*XLogPageReadCB) (XLogReaderState *xlogreader,
+/* Function type definition for the read_page callback */
+typedef bool (*XLogPageReadCB) (XLogReaderState *xlogreader,
XLogRecPtr targetPagePtr,
int reqLen,
XLogRecPtr targetRecPtr,
@@ -175,6 +175,20 @@ struct XLogReaderState
XLogRecPtr ReadRecPtr; /* start of last record read */
XLogRecPtr EndRecPtr; /* end+1 of last record read */
+ /* ----------------------------------------
+ * Communication with page reader
+ * readBuf is XLOG_BLCKSZ bytes, valid up to at least readLen bytes.
+ * ----------------------------------------
+ */
+ /* variables to communicate with page reader */
+ XLogRecPtr readPagePtr; /* page pointer to read */
+ int32 readLen; /* bytes requested to reader, or actual bytes
+ * read by reader, which must be larger than
+ * the request, or -1 on error */
+ TimeLineID readPageTLI; /* TLI for data currently in readBuf */
+ char *readBuf; /* buffer to store data */
+ bool page_verified; /* is the page on the buffer verified? */
+
/* ----------------------------------------
* Decoded representation of current record
@@ -203,13 +217,6 @@ struct XLogReaderState
* ----------------------------------------
*/
- /*
- * Buffer for currently read page (XLOG_BLCKSZ bytes, valid up to at least
- * readLen bytes)
- */
- char *readBuf;
- uint32 readLen;
-
/* last read XLOG position for data currently in readBuf */
WALSegmentContext segcxt;
WALOpenSegment seg;
diff --git a/src/include/access/xlogutils.h b/src/include/access/xlogutils.h
index 9ac602b674..364a21c4ea 100644
--- a/src/include/access/xlogutils.h
+++ b/src/include/access/xlogutils.h
@@ -47,7 +47,7 @@ extern Buffer XLogReadBufferExtended(RelFileNode rnode, ForkNumber forknum,
extern Relation CreateFakeRelcacheEntry(RelFileNode rnode);
extern void FreeFakeRelcacheEntry(Relation fakerel);
-extern int read_local_xlog_page(XLogReaderState *state,
+extern bool read_local_xlog_page(XLogReaderState *state,
XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *cur_page);
extern void wal_segment_open(XLogReaderState *state,
--
2.27.0
----Next_Part(Thu_Mar__4_11_28_53_2021_014)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
filename="v17-0002-Move-page-reader-out-of-XLogReadRecord.patch"
^ permalink raw reply [nested|flat] 69+ messages in thread
* [PATCH v17 1/4] Move callback-call from ReadPageInternal to XLogReadRecord.
@ 2019-09-05 11:21 Kyotaro Horiguchi <[email protected]>
0 siblings, 0 replies; 69+ messages in thread
From: Kyotaro Horiguchi @ 2019-09-05 11:21 UTC (permalink / raw)
The current WAL record reader reads page data using a call back
function. Although it is not so problematic alone, it would be a
problem if we are going to do add tasks like encryption which is
performed on page data before WAL reader reads them. To avoid that the
record reader facility has to have a new code path corresponds to
every new callback, this patch separates page reader from WAL record
reading facility by modifying the current WAL record reader to a state
machine.
As the first step of that change, this patch moves the page reader
function out of ReadPageInternal, then the remaining tasks of the
function are taken over by the new function XLogNeedData. As the
result XLogPageRead directly calls the page reader callback function
according to the feedback from XLogNeedData.
---
src/backend/access/transam/xlog.c | 16 +-
src/backend/access/transam/xlogreader.c | 281 ++++++++++++++----------
src/backend/access/transam/xlogutils.c | 12 +-
src/backend/replication/walsender.c | 10 +-
src/bin/pg_rewind/parsexlog.c | 21 +-
src/bin/pg_waldump/pg_waldump.c | 8 +-
src/include/access/xlogreader.h | 25 ++-
src/include/access/xlogutils.h | 2 +-
8 files changed, 222 insertions(+), 153 deletions(-)
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 377afb8732..606328a65a 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -917,7 +917,7 @@ static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath,
static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli,
XLogSource source, bool notfoundOk);
static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, XLogSource source);
-static int XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
+static bool XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
bool fetching_ckpt, XLogRecPtr tliRecPtr);
@@ -4350,7 +4350,6 @@ ReadRecord(XLogReaderState *xlogreader, int emode,
XLogRecord *record;
XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
- /* Pass through parameters to XLogPageRead */
private->fetching_ckpt = fetching_ckpt;
private->emode = emode;
private->randAccess = (xlogreader->ReadRecPtr == InvalidXLogRecPtr);
@@ -11987,7 +11986,7 @@ CancelBackup(void)
* XLogPageRead() to try fetching the record from another source, or to
* sleep and retry.
*/
-static int
+static bool
XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *readBuf)
{
@@ -12046,7 +12045,8 @@ retry:
readLen = 0;
readSource = XLOG_FROM_ANY;
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
}
@@ -12141,7 +12141,8 @@ retry:
goto next_record_is_invalid;
}
- return readLen;
+ xlogreader->readLen = readLen;
+ return true;
next_record_is_invalid:
lastSourceFailed = true;
@@ -12155,8 +12156,9 @@ next_record_is_invalid:
/* In standby-mode, keep trying */
if (StandbyMode)
goto retry;
- else
- return -1;
+
+ xlogreader->readLen = -1;
+ return false;
}
/*
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index 42738eb940..7d04995b7b 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -36,8 +36,8 @@
static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
pg_attribute_printf(2, 3);
static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
-static int ReadPageInternal(XLogReaderState *state, XLogRecPtr pageptr,
- int reqLen);
+static bool XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr,
+ int reqLen, bool header_inclusive);
static void XLogReaderInvalReadState(XLogReaderState *state);
static bool ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
XLogRecPtr PrevRecPtr, XLogRecord *record, bool randAccess);
@@ -276,7 +276,6 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
uint32 targetRecOff;
uint32 pageHeaderSize;
bool gotheader;
- int readOff;
/*
* randAccess indicates whether to verify the previous-record pointer of
@@ -326,14 +325,20 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
* byte to cover the whole record header, or at least the part of it that
* fits on the same page.
*/
- readOff = ReadPageInternal(state, targetPagePtr,
- Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ));
- if (readOff < 0)
+ while (XLogNeedData(state, targetPagePtr,
+ Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+ targetRecOff != 0))
+ {
+ if (!state->routine.page_read(state, state->readPagePtr, state->readLen,
+ RecPtr, state->readBuf))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
/*
- * ReadPageInternal always returns at least the page header, so we can
- * examine it now.
+ * We have at least the page header, so we can examine it now.
*/
pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
if (targetRecOff == 0)
@@ -359,8 +364,8 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
goto err;
}
- /* ReadPageInternal has verified the page header */
- Assert(pageHeaderSize <= readOff);
+ /* XLogNeedData has verified the page header */
+ Assert(pageHeaderSize <= state->readLen);
/*
* Read the record length.
@@ -432,18 +437,27 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
do
{
+ int rest_len = total_len - gotlen;
+
/* Calculate pointer to beginning of next page */
targetPagePtr += XLOG_BLCKSZ;
/* Wait for the next page to become available */
- readOff = ReadPageInternal(state, targetPagePtr,
- Min(total_len - gotlen + SizeOfXLogShortPHD,
- XLOG_BLCKSZ));
+ while (XLogNeedData(state, targetPagePtr,
+ Min(rest_len, XLOG_BLCKSZ),
+ false))
+ {
+ if (!state->routine.page_read(state, state->readPagePtr,
+ state->readLen,
+ state->ReadRecPtr,
+ state->readBuf))
+ break;
+ }
- if (readOff < 0)
+ if (!state->page_verified)
goto err;
- Assert(SizeOfXLogShortPHD <= readOff);
+ Assert(SizeOfXLogShortPHD <= state->readLen);
/* Check that the continuation on next page looks valid */
pageHeader = (XLogPageHeader) state->readBuf;
@@ -473,21 +487,14 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
/* Append the continuation from this page to the buffer */
pageHeaderSize = XLogPageHeaderSize(pageHeader);
- if (readOff < pageHeaderSize)
- readOff = ReadPageInternal(state, targetPagePtr,
- pageHeaderSize);
-
- Assert(pageHeaderSize <= readOff);
+ Assert(pageHeaderSize <= state->readLen);
contdata = (char *) state->readBuf + pageHeaderSize;
len = XLOG_BLCKSZ - pageHeaderSize;
if (pageHeader->xlp_rem_len < len)
len = pageHeader->xlp_rem_len;
- if (readOff < pageHeaderSize + len)
- readOff = ReadPageInternal(state, targetPagePtr,
- pageHeaderSize + len);
-
+ Assert (pageHeaderSize + len <= state->readLen);
memcpy(buffer, (char *) contdata, len);
buffer += len;
gotlen += len;
@@ -517,9 +524,16 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
else
{
/* Wait for the record data to become available */
- readOff = ReadPageInternal(state, targetPagePtr,
- Min(targetRecOff + total_len, XLOG_BLCKSZ));
- if (readOff < 0)
+ while (XLogNeedData(state, targetPagePtr,
+ Min(targetRecOff + total_len, XLOG_BLCKSZ), true))
+ {
+ if (!state->routine.page_read(state, state->readPagePtr,
+ state->readLen,
+ state->ReadRecPtr, state->readBuf))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
/* Record does not cross a page boundary */
@@ -562,109 +576,138 @@ err:
}
/*
- * Read a single xlog page including at least [pageptr, reqLen] of valid data
- * via the page_read() callback.
+ * Checks that an xlog page loaded in state->readBuf is including at least
+ * [pageptr, reqLen] and the page is valid. header_inclusive indicates that
+ * reqLen is calculated including page header length.
*
- * Returns -1 if the required page cannot be read for some reason; errormsg_buf
- * is set in that case (unless the error occurs in the page_read callback).
+ * Returns false if the buffer already contains the requested data, or found
+ * error. state->page_verified is set to true for the former and false for the
+ * latter.
*
- * We fetch the page from a reader-local cache if we know we have the required
- * data and if there hasn't been any error since caching the data.
+ * Otherwise returns true and requests data loaded onto state->readBuf by
+ * state->readPagePtr and state->readLen. The caller shall call this function
+ * again after filling the buffer at least with that portion of data and set
+ * state->readLen to the length of actually loaded data.
+ *
+ * If header_inclusive is false, corrects reqLen internally by adding the
+ * actual page header length and may request caller for new data.
*/
-static int
-ReadPageInternal(XLogReaderState *state, XLogRecPtr pageptr, int reqLen)
+static bool
+XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
+ bool header_inclusive)
{
- int readLen;
uint32 targetPageOff;
XLogSegNo targetSegNo;
- XLogPageHeader hdr;
+ uint32 addLen = 0;
- Assert((pageptr % XLOG_BLCKSZ) == 0);
+ /* Some data is loaded, but page header is not verified yet. */
+ if (!state->page_verified &&
+ !XLogRecPtrIsInvalid(state->readPagePtr) && state->readLen >= 0)
+ {
+ uint32 pageHeaderSize;
+ /* just loaded new data so needs to verify page header */
+
+ /* The caller must have loaded at least page header */
+ Assert (state->readLen >= SizeOfXLogShortPHD);
+
+ /*
+ * We have enough data to check the header length. Recheck the loaded
+ * length against the actual header length.
+ */
+ pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+
+ /* Request more data if we don't have the full header. */
+ if (state->readLen < pageHeaderSize)
+ {
+ state->readLen = pageHeaderSize;
+ return true;
+ }
+
+ /* Now that we know we have the full header, validate it. */
+ if (!XLogReaderValidatePageHeader(state, state->readPagePtr,
+ (char *) state->readBuf))
+ {
+ /* That's bad. Force reading the page again. */
+ XLogReaderInvalReadState(state);
+
+ return false;
+ }
+
+ state->page_verified = true;
+
+ XLByteToSeg(state->readPagePtr, state->seg.ws_segno,
+ state->segcxt.ws_segsize);
+ }
+
+ /*
+ * The loaded page may not be the one caller is supposing to read when we
+ * are verifying the first page of new segment. In that case, skip further
+ * verification and immediately load the target page.
+ */
+ if (state->page_verified && pageptr == state->readPagePtr)
+ {
+ /*
+ * calculate additional length for page header keeping the total
+ * length within the block size.
+ */
+ if (!header_inclusive)
+ {
+ uint32 pageHeaderSize =
+ XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+
+ addLen = pageHeaderSize;
+ if (reqLen + pageHeaderSize <= XLOG_BLCKSZ)
+ addLen = pageHeaderSize;
+ else
+ addLen = XLOG_BLCKSZ - reqLen;
+
+ Assert(addLen >= 0);
+ }
+
+ /* Return if we already have it. */
+ if (reqLen + addLen <= state->readLen)
+ return false;
+ }
+
+ /* Data is not in our buffer, request the caller for it. */
XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize);
targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize);
+ Assert((pageptr % XLOG_BLCKSZ) == 0);
- /* check whether we have all the requested data already */
- if (targetSegNo == state->seg.ws_segno &&
- targetPageOff == state->segoff && reqLen <= state->readLen)
- return state->readLen;
+ /*
+ * Every time we request to load new data of a page to the caller, even if
+ * we looked at a part of it before, we need to do verification on the next
+ * invocation as the caller might now be rereading data from a different
+ * source.
+ */
+ state->page_verified = false;
/*
- * Data is not in our buffer.
- *
- * Every time we actually read the segment, even if we looked at parts of
- * it before, we need to do verification as the page_read callback might
- * now be rereading data from a different source.
- *
* Whenever switching to a new WAL segment, we read the first page of the
* file and validate its header, even if that's not where the target
* record is. This is so that we can check the additional identification
* info that is present in the first page's "long" header.
+ * Don't do this if the caller requested the first page in the segment.
*/
if (targetSegNo != state->seg.ws_segno && targetPageOff != 0)
{
- XLogRecPtr targetSegmentPtr = pageptr - targetPageOff;
-
- readLen = state->routine.page_read(state, targetSegmentPtr, XLOG_BLCKSZ,
- state->currRecPtr,
- state->readBuf);
- if (readLen < 0)
- goto err;
-
- /* we can be sure to have enough WAL available, we scrolled back */
- Assert(readLen == XLOG_BLCKSZ);
-
- if (!XLogReaderValidatePageHeader(state, targetSegmentPtr,
- state->readBuf))
- goto err;
+ /*
+ * Then we'll see that the targetSegNo now matches the ws_segno, and
+ * will not come back here, but will request the actual target page.
+ */
+ state->readPagePtr = pageptr - targetPageOff;
+ state->readLen = XLOG_BLCKSZ;
+ return true;
}
/*
- * First, read the requested data length, but at least a short page header
- * so that we can validate it.
+ * Request the caller to load the page. We need at least a short page
+ * header so that we can validate it.
*/
- readLen = state->routine.page_read(state, pageptr, Max(reqLen, SizeOfXLogShortPHD),
- state->currRecPtr,
- state->readBuf);
- if (readLen < 0)
- goto err;
-
- Assert(readLen <= XLOG_BLCKSZ);
-
- /* Do we have enough data to check the header length? */
- if (readLen <= SizeOfXLogShortPHD)
- goto err;
-
- Assert(readLen >= reqLen);
-
- hdr = (XLogPageHeader) state->readBuf;
-
- /* still not enough */
- if (readLen < XLogPageHeaderSize(hdr))
- {
- readLen = state->routine.page_read(state, pageptr, XLogPageHeaderSize(hdr),
- state->currRecPtr,
- state->readBuf);
- if (readLen < 0)
- goto err;
- }
-
- /*
- * Now that we know we have the full header, validate it.
- */
- if (!XLogReaderValidatePageHeader(state, pageptr, (char *) hdr))
- goto err;
-
- /* update read state information */
- state->seg.ws_segno = targetSegNo;
- state->segoff = targetPageOff;
- state->readLen = readLen;
-
- return readLen;
-
-err:
- XLogReaderInvalReadState(state);
- return -1;
+ state->readPagePtr = pageptr;
+ state->readLen = Max(reqLen + addLen, SizeOfXLogShortPHD);
+ return true;
}
/*
@@ -673,9 +716,7 @@ err:
static void
XLogReaderInvalReadState(XLogReaderState *state)
{
- state->seg.ws_segno = 0;
- state->segoff = 0;
- state->readLen = 0;
+ state->readPagePtr = InvalidXLogRecPtr;
}
/*
@@ -953,7 +994,6 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
XLogRecPtr targetPagePtr;
int targetRecOff;
uint32 pageHeaderSize;
- int readLen;
/*
* Compute targetRecOff. It should typically be equal or greater than
@@ -961,7 +1001,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
* that, except when caller has explicitly specified the offset that
* falls somewhere there or when we are skipping multi-page
* continuation record. It doesn't matter though because
- * ReadPageInternal() is prepared to handle that and will read at
+ * XLogNeedData() is prepared to handle that and will read at
* least short page-header worth of data
*/
targetRecOff = tmpRecPtr % XLOG_BLCKSZ;
@@ -969,19 +1009,24 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
/* scroll back to page boundary */
targetPagePtr = tmpRecPtr - targetRecOff;
- /* Read the page containing the record */
- readLen = ReadPageInternal(state, targetPagePtr, targetRecOff);
- if (readLen < 0)
+ while(XLogNeedData(state, targetPagePtr, targetRecOff,
+ targetRecOff != 0))
+ {
+ if (!state->routine.page_read(state, state->readPagePtr,
+ state->readLen,
+ state->ReadRecPtr, state->readBuf))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
header = (XLogPageHeader) state->readBuf;
pageHeaderSize = XLogPageHeaderSize(header);
- /* make sure we have enough data for the page header */
- readLen = ReadPageInternal(state, targetPagePtr, pageHeaderSize);
- if (readLen < 0)
- goto err;
+ /* we should have read the page header */
+ Assert (state->readLen >= pageHeaderSize);
/* skip over potential continuation data */
if (header->xlp_info & XLP_FIRST_IS_CONTRECORD)
diff --git a/src/backend/access/transam/xlogutils.c b/src/backend/access/transam/xlogutils.c
index d17d660f46..46eda33f25 100644
--- a/src/backend/access/transam/xlogutils.c
+++ b/src/backend/access/transam/xlogutils.c
@@ -686,8 +686,8 @@ XLogTruncateRelation(RelFileNode rnode, ForkNumber forkNum,
void
XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wantLength)
{
- const XLogRecPtr lastReadPage = (state->seg.ws_segno *
- state->segcxt.ws_segsize + state->segoff);
+ const XLogRecPtr lastReadPage = state->seg.ws_segno *
+ state->segcxt.ws_segsize + state->readLen;
Assert(wantPage != InvalidXLogRecPtr && wantPage % XLOG_BLCKSZ == 0);
Assert(wantLength <= XLOG_BLCKSZ);
@@ -824,7 +824,7 @@ wal_segment_close(XLogReaderState *state)
* exists for normal backends, so we have to do a check/sleep/repeat style of
* loop for now.
*/
-int
+bool
read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *cur_page)
{
@@ -926,7 +926,8 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
else if (targetPagePtr + reqLen > read_upto)
{
/* not enough data there */
- return -1;
+ state->readLen = -1;
+ return false;
}
else
{
@@ -944,7 +945,8 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
WALReadRaiseError(&errinfo);
/* number of valid bytes in the buffer */
- return count;
+ state->readLen = count;
+ return true;
}
/*
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index 23baa4498a..c36b91ae5e 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -806,7 +806,7 @@ StartReplication(StartReplicationCmd *cmd)
* which has to do a plain sleep/busy loop, because the walsender's latch gets
* set every time WAL is flushed.
*/
-static int
+static bool
logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *cur_page)
{
@@ -826,7 +826,10 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
/* fail if not (implies we are going to shut down) */
if (flushptr < targetPagePtr + reqLen)
- return -1;
+ {
+ state->readLen = -1;
+ return false;
+ }
if (targetPagePtr + XLOG_BLCKSZ <= flushptr)
count = XLOG_BLCKSZ; /* more than one block available */
@@ -854,7 +857,8 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
XLByteToSeg(targetPagePtr, segno, state->segcxt.ws_segsize);
CheckXLogRemoved(segno, state->seg.ws_tli);
- return count;
+ state->readLen = count;
+ return true;
}
/*
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index 59ebac7d6a..cf119848b0 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -47,7 +47,7 @@ typedef struct XLogPageReadPrivate
int tliIndex;
} XLogPageReadPrivate;
-static int SimpleXLogPageRead(XLogReaderState *xlogreader,
+static bool SimpleXLogPageRead(XLogReaderState *xlogreader,
XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
@@ -246,7 +246,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
}
/* XLogReader callback function, to read a WAL page */
-static int
+static bool
SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf)
{
@@ -306,7 +306,8 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
if (private->restoreCommand == NULL)
{
pg_log_error("could not open file \"%s\": %m", xlogfpath);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
/*
@@ -319,7 +320,10 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
private->restoreCommand);
if (xlogreadfd < 0)
- return -1;
+ {
+ xlogreader->readLen = -1;
+ return false;
+ }
else
pg_log_debug("using file \"%s\" restored from archive",
xlogfpath);
@@ -335,7 +339,8 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
if (lseek(xlogreadfd, (off_t) targetPageOff, SEEK_SET) < 0)
{
pg_log_error("could not seek in file \"%s\": %m", xlogfpath);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
@@ -348,13 +353,15 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
pg_log_error("could not read file \"%s\": read %d of %zu",
xlogfpath, r, (Size) XLOG_BLCKSZ);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
Assert(targetSegNo == xlogreadsegno);
xlogreader->seg.ws_tli = targetHistory[private->tliIndex].tli;
- return XLOG_BLCKSZ;
+ xlogreader->readLen = XLOG_BLCKSZ;
+ return true;
}
/*
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index 610f65e471..0f0e0723a3 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -331,7 +331,7 @@ WALDumpCloseSegment(XLogReaderState *state)
}
/* pg_waldump's XLogReaderRoutine->page_read callback */
-static int
+static bool
WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetPtr, char *readBuff)
{
@@ -348,7 +348,8 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
else
{
private->endptr_reached = true;
- return -1;
+ state->readLen = -1;
+ return false;
}
}
@@ -373,7 +374,8 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
(Size) errinfo.wre_req);
}
- return count;
+ state->readLen = count;
+ return true;
}
/*
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index 21d200d3df..23d9e70a59 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -57,8 +57,8 @@ typedef struct WALSegmentContext
typedef struct XLogReaderState XLogReaderState;
-/* Function type definitions for various xlogreader interactions */
-typedef int (*XLogPageReadCB) (XLogReaderState *xlogreader,
+/* Function type definition for the read_page callback */
+typedef bool (*XLogPageReadCB) (XLogReaderState *xlogreader,
XLogRecPtr targetPagePtr,
int reqLen,
XLogRecPtr targetRecPtr,
@@ -175,6 +175,20 @@ struct XLogReaderState
XLogRecPtr ReadRecPtr; /* start of last record read */
XLogRecPtr EndRecPtr; /* end+1 of last record read */
+ /* ----------------------------------------
+ * Communication with page reader
+ * readBuf is XLOG_BLCKSZ bytes, valid up to at least readLen bytes.
+ * ----------------------------------------
+ */
+ /* variables to communicate with page reader */
+ XLogRecPtr readPagePtr; /* page pointer to read */
+ int32 readLen; /* bytes requested to reader, or actual bytes
+ * read by reader, which must be larger than
+ * the request, or -1 on error */
+ TimeLineID readPageTLI; /* TLI for data currently in readBuf */
+ char *readBuf; /* buffer to store data */
+ bool page_verified; /* is the page on the buffer verified? */
+
/* ----------------------------------------
* Decoded representation of current record
@@ -203,13 +217,6 @@ struct XLogReaderState
* ----------------------------------------
*/
- /*
- * Buffer for currently read page (XLOG_BLCKSZ bytes, valid up to at least
- * readLen bytes)
- */
- char *readBuf;
- uint32 readLen;
-
/* last read XLOG position for data currently in readBuf */
WALSegmentContext segcxt;
WALOpenSegment seg;
diff --git a/src/include/access/xlogutils.h b/src/include/access/xlogutils.h
index 9ac602b674..364a21c4ea 100644
--- a/src/include/access/xlogutils.h
+++ b/src/include/access/xlogutils.h
@@ -47,7 +47,7 @@ extern Buffer XLogReadBufferExtended(RelFileNode rnode, ForkNumber forknum,
extern Relation CreateFakeRelcacheEntry(RelFileNode rnode);
extern void FreeFakeRelcacheEntry(Relation fakerel);
-extern int read_local_xlog_page(XLogReaderState *state,
+extern bool read_local_xlog_page(XLogReaderState *state,
XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *cur_page);
extern void wal_segment_open(XLogReaderState *state,
--
2.27.0
----Next_Part(Thu_Mar__4_11_28_53_2021_014)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
filename="v17-0002-Move-page-reader-out-of-XLogReadRecord.patch"
^ permalink raw reply [nested|flat] 69+ messages in thread
* [PATCH v13 1/4] Move callback-call from ReadPageInternal to XLogReadRecord.
@ 2019-09-05 11:21 Kyotaro Horiguchi <[email protected]>
0 siblings, 0 replies; 69+ messages in thread
From: Kyotaro Horiguchi @ 2019-09-05 11:21 UTC (permalink / raw)
The current WAL record reader reads page data using a call back
function. Although it is not so problematic alone, it would be a
problem if we are going to do add tasks like encryption which is
performed on page data before WAL reader reads them. To avoid that the
record reader facility has to have a new code path corresponds to
every new callback, this patch separates page reader from WAL record
reading facility by modifying the current WAL record reader to a state
machine.
As the first step of that change, this patch moves the page reader
function out of ReadPageInternal, then the remaining tasks of the
function are taken over by the new function XLogNeedData. As the
result XLogPageRead directly calls the page reader callback function
according to the feedback from XLogNeedData.
---
src/backend/access/transam/xlog.c | 16 +-
src/backend/access/transam/xlogreader.c | 282 ++++++++++--------
src/backend/access/transam/xlogutils.c | 12 +-
.../replication/logical/logicalfuncs.c | 2 +-
src/backend/replication/walsender.c | 10 +-
src/bin/pg_rewind/parsexlog.c | 16 +-
src/bin/pg_waldump/pg_waldump.c | 8 +-
src/include/access/xlogreader.h | 23 +-
src/include/access/xlogutils.h | 2 +-
src/include/replication/logicalfuncs.h | 2 +-
10 files changed, 218 insertions(+), 155 deletions(-)
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 6e09ded597..357432ffa2 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -883,7 +883,7 @@ static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath,
static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli,
int source, bool notfoundOk);
static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, int source);
-static int XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
+static bool XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
bool fetching_ckpt, XLogRecPtr tliRecPtr);
@@ -4262,7 +4262,6 @@ ReadRecord(XLogReaderState *xlogreader, int emode,
XLogRecord *record;
XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
- /* Pass through parameters to XLogPageRead */
private->fetching_ckpt = fetching_ckpt;
private->emode = emode;
private->randAccess = (xlogreader->ReadRecPtr == InvalidXLogRecPtr);
@@ -11586,7 +11585,7 @@ CancelBackup(void)
* XLogPageRead() to try fetching the record from another source, or to
* sleep and retry.
*/
-static int
+static bool
XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *readBuf)
{
@@ -11645,7 +11644,8 @@ retry:
readLen = 0;
readSource = 0;
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
}
@@ -11740,7 +11740,8 @@ retry:
goto next_record_is_invalid;
}
- return readLen;
+ xlogreader->readLen = readLen;
+ return true;
next_record_is_invalid:
lastSourceFailed = true;
@@ -11754,8 +11755,9 @@ next_record_is_invalid:
/* In standby-mode, keep trying */
if (StandbyMode)
goto retry;
- else
- return -1;
+
+ xlogreader->readLen = -1;
+ return false;
}
/*
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index 32f02256ed..2c1500443e 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -36,8 +36,8 @@
static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
pg_attribute_printf(2, 3);
static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
-static int ReadPageInternal(XLogReaderState *state, XLogRecPtr pageptr,
- int reqLen);
+static bool XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr,
+ int reqLen, bool header_inclusive);
static void XLogReaderInvalReadState(XLogReaderState *state);
static bool ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
XLogRecPtr PrevRecPtr, XLogRecord *record, bool randAccess);
@@ -269,7 +269,6 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
uint32 targetRecOff;
uint32 pageHeaderSize;
bool gotheader;
- int readOff;
/*
* randAccess indicates whether to verify the previous-record pointer of
@@ -319,14 +318,20 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
* byte to cover the whole record header, or at least the part of it that
* fits on the same page.
*/
- readOff = ReadPageInternal(state, targetPagePtr,
- Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ));
- if (readOff < 0)
+ while (XLogNeedData(state, targetPagePtr,
+ Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+ targetRecOff != 0))
+ {
+ if (!state->read_page(state, state->readPagePtr, state->readLen,
+ RecPtr, state->readBuf))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
/*
- * ReadPageInternal always returns at least the page header, so we can
- * examine it now.
+ * We have at least the page header, so we can examine it now.
*/
pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
if (targetRecOff == 0)
@@ -352,8 +357,8 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
goto err;
}
- /* ReadPageInternal has verified the page header */
- Assert(pageHeaderSize <= readOff);
+ /* XLogNeedData has verified the page header */
+ Assert(pageHeaderSize <= state->readLen);
/*
* Read the record length.
@@ -426,18 +431,25 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
do
{
+ int rest_len = total_len - gotlen;
+
/* Calculate pointer to beginning of next page */
targetPagePtr += XLOG_BLCKSZ;
/* Wait for the next page to become available */
- readOff = ReadPageInternal(state, targetPagePtr,
- Min(total_len - gotlen + SizeOfXLogShortPHD,
- XLOG_BLCKSZ));
+ while (XLogNeedData(state, targetPagePtr,
+ Min(rest_len, XLOG_BLCKSZ),
+ false))
+ {
+ if (!state->read_page(state, state->readPagePtr, state->readLen,
+ state->ReadRecPtr, state->readBuf))
+ break;
+ }
- if (readOff < 0)
+ if (!state->page_verified)
goto err;
- Assert(SizeOfXLogShortPHD <= readOff);
+ Assert(SizeOfXLogShortPHD <= state->readLen);
/* Check that the continuation on next page looks valid */
pageHeader = (XLogPageHeader) state->readBuf;
@@ -466,21 +478,14 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
/* Append the continuation from this page to the buffer */
pageHeaderSize = XLogPageHeaderSize(pageHeader);
- if (readOff < pageHeaderSize)
- readOff = ReadPageInternal(state, targetPagePtr,
- pageHeaderSize);
-
- Assert(pageHeaderSize <= readOff);
+ Assert(pageHeaderSize <= state->readLen);
contdata = (char *) state->readBuf + pageHeaderSize;
len = XLOG_BLCKSZ - pageHeaderSize;
if (pageHeader->xlp_rem_len < len)
len = pageHeader->xlp_rem_len;
- if (readOff < pageHeaderSize + len)
- readOff = ReadPageInternal(state, targetPagePtr,
- pageHeaderSize + len);
-
+ Assert (pageHeaderSize + len <= state->readLen);
memcpy(buffer, (char *) contdata, len);
buffer += len;
gotlen += len;
@@ -510,9 +515,15 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
else
{
/* Wait for the record data to become available */
- readOff = ReadPageInternal(state, targetPagePtr,
- Min(targetRecOff + total_len, XLOG_BLCKSZ));
- if (readOff < 0)
+ while (XLogNeedData(state, targetPagePtr,
+ Min(targetRecOff + total_len, XLOG_BLCKSZ), true))
+ {
+ if (!state->read_page(state, state->readPagePtr, state->readLen,
+ state->ReadRecPtr, state->readBuf))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
/* Record does not cross a page boundary */
@@ -555,109 +566,139 @@ err:
}
/*
- * Read a single xlog page including at least [pageptr, reqLen] of valid data
- * via the read_page() callback.
+ * Checks that an xlog page loaded in state->readBuf is including at least
+ * [pageptr, reqLen] and the page is valid. header_inclusive indicates that
+ * reqLen is calculated including page header length.
*
- * Returns -1 if the required page cannot be read for some reason; errormsg_buf
- * is set in that case (unless the error occurs in the read_page callback).
+ * Returns false if the buffer already contains the requested data, or found
+ * error. state->page_verified is set to true for the former and false for the
+ * latter.
*
- * We fetch the page from a reader-local cache if we know we have the required
- * data and if there hasn't been any error since caching the data.
+ * Otherwise returns true and requests data loaded onto state->readBuf by
+ * state->readPagePtr and state->readLen. The caller shall call this function
+ * again after filling the buffer at least with that portion of data and set
+ * state->readLen to the length of actually loaded data.
+ *
+ * If header_inclusive is false, corrects reqLen internally by adding the
+ * actual page header length and may request caller for new data.
*/
-static int
-ReadPageInternal(XLogReaderState *state, XLogRecPtr pageptr, int reqLen)
+static bool
+XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
+ bool header_inclusive)
{
- int readLen;
uint32 targetPageOff;
XLogSegNo targetSegNo;
- XLogPageHeader hdr;
+ uint32 addLen = 0;
- Assert((pageptr % XLOG_BLCKSZ) == 0);
+ /* Some data is loaded, but page header is not verified yet. */
+ if (!state->page_verified &&
+ !XLogRecPtrIsInvalid(state->readPagePtr) && state->readLen >= 0)
+ {
+ uint32 pageHeaderSize;
- XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize);
- targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize);
+ /* just loaded new data so needs to verify page header */
- /* check whether we have all the requested data already */
- if (targetSegNo == state->seg.ws_segno &&
- targetPageOff == state->segoff && reqLen <= state->readLen)
- return state->readLen;
+ /* The caller must have loaded at least page header */
+ Assert (state->readLen >= SizeOfXLogShortPHD);
+
+ /*
+ * We have enough data to check the header length. Recheck the loaded
+ * length against the actual header length.
+ */
+ pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+
+ /* Request more data if we don't have the full header. */
+ if (state->readLen < pageHeaderSize)
+ {
+ state->readLen = pageHeaderSize;
+ return true;
+ }
+
+ /* Now that we know we have the full header, validate it. */
+ if (!XLogReaderValidatePageHeader(state, state->readPagePtr,
+ (char *) state->readBuf))
+ {
+ /* That's bad. Force reading the page again. */
+ XLogReaderInvalReadState(state);
+
+ return false;
+ }
+
+ state->page_verified = true;
+
+ XLByteToSeg(state->readPagePtr, state->seg.ws_segno,
+ state->segcxt.ws_segsize);
+ }
+
+ /*
+ * The loaded page may not be the one caller is supposing to read when we
+ * are verifying the first page of new segment. In that case, skip further
+ * verification and immediately load the target page.
+ */
+ if (state->page_verified && pageptr == state->readPagePtr)
+ {
+
+ /*
+ * calculate additional length for page header keeping the total
+ * length within the block size.
+ */
+ if (!header_inclusive)
+ {
+ uint32 pageHeaderSize =
+ XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+
+ addLen = pageHeaderSize;
+ if (reqLen + pageHeaderSize <= XLOG_BLCKSZ)
+ addLen = pageHeaderSize;
+ else
+ addLen = XLOG_BLCKSZ - reqLen;
+
+ Assert(addLen >= 0);
+ }
+
+ /* Return if we already have it. */
+ if (reqLen + addLen <= state->readLen)
+ return false;
+ }
+
+ /* Data is not in our buffer, request the caller for it. */
+ XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize);
+ targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize);
+ Assert((pageptr % XLOG_BLCKSZ) == 0);
+
+ /*
+ * Every time we request to load new data of a page to the caller, even if
+ * we looked at a part of it before, we need to do verification on the next
+ * invocation as the caller might now be rereading data from a different
+ * source.
+ */
+ state->page_verified = false;
/*
- * Data is not in our buffer.
- *
- * Every time we actually read the page, even if we looked at parts of it
- * before, we need to do verification as the read_page callback might now
- * be rereading data from a different source.
- *
* Whenever switching to a new WAL segment, we read the first page of the
* file and validate its header, even if that's not where the target
* record is. This is so that we can check the additional identification
* info that is present in the first page's "long" header.
+ * Don't do this if the caller requested the first page in the segment.
*/
if (targetSegNo != state->seg.ws_segno && targetPageOff != 0)
{
- XLogRecPtr targetSegmentPtr = pageptr - targetPageOff;
-
- readLen = state->read_page(state, targetSegmentPtr, XLOG_BLCKSZ,
- state->currRecPtr,
- state->readBuf);
- if (readLen < 0)
- goto err;
-
- /* we can be sure to have enough WAL available, we scrolled back */
- Assert(readLen == XLOG_BLCKSZ);
-
- if (!XLogReaderValidatePageHeader(state, targetSegmentPtr,
- state->readBuf))
- goto err;
+ /*
+ * Then we'll see that the targetSegNo now matches the ws_segno, and
+ * will not come back here, but will request the actual target page.
+ */
+ state->readPagePtr = pageptr - targetPageOff;
+ state->readLen = XLOG_BLCKSZ;
+ return true;
}
/*
- * First, read the requested data length, but at least a short page header
- * so that we can validate it.
+ * Request the caller to load the page. We need at least a short page
+ * header so that we can validate it.
*/
- readLen = state->read_page(state, pageptr, Max(reqLen, SizeOfXLogShortPHD),
- state->currRecPtr,
- state->readBuf);
- if (readLen < 0)
- goto err;
-
- Assert(readLen <= XLOG_BLCKSZ);
-
- /* Do we have enough data to check the header length? */
- if (readLen <= SizeOfXLogShortPHD)
- goto err;
-
- Assert(readLen >= reqLen);
-
- hdr = (XLogPageHeader) state->readBuf;
-
- /* still not enough */
- if (readLen < XLogPageHeaderSize(hdr))
- {
- readLen = state->read_page(state, pageptr, XLogPageHeaderSize(hdr),
- state->currRecPtr,
- state->readBuf);
- if (readLen < 0)
- goto err;
- }
-
- /*
- * Now that we know we have the full header, validate it.
- */
- if (!XLogReaderValidatePageHeader(state, pageptr, (char *) hdr))
- goto err;
-
- /* update read state information */
- state->seg.ws_segno = targetSegNo;
- state->segoff = targetPageOff;
- state->readLen = readLen;
-
- return readLen;
-
-err:
- XLogReaderInvalReadState(state);
- return -1;
+ state->readPagePtr = pageptr;
+ state->readLen = Max(reqLen + addLen, SizeOfXLogShortPHD);
+ return true;
}
/*
@@ -666,9 +707,7 @@ err:
static void
XLogReaderInvalReadState(XLogReaderState *state)
{
- state->seg.ws_segno = 0;
- state->segoff = 0;
- state->readLen = 0;
+ state->readPagePtr = InvalidXLogRecPtr;
}
/*
@@ -949,7 +988,6 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
XLogRecPtr targetPagePtr;
int targetRecOff;
uint32 pageHeaderSize;
- int readLen;
/*
* Compute targetRecOff. It should typically be equal or greater than
@@ -957,7 +995,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
* that, except when caller has explicitly specified the offset that
* falls somewhere there or when we are skipping multi-page
* continuation record. It doesn't matter though because
- * ReadPageInternal() is prepared to handle that and will read at
+ * XLogNeedData() is prepared to handle that and will read at
* least short page-header worth of data
*/
targetRecOff = tmpRecPtr % XLOG_BLCKSZ;
@@ -965,19 +1003,23 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
/* scroll back to page boundary */
targetPagePtr = tmpRecPtr - targetRecOff;
- /* Read the page containing the record */
- readLen = ReadPageInternal(state, targetPagePtr, targetRecOff);
- if (readLen < 0)
+ while(XLogNeedData(state, targetPagePtr, targetRecOff,
+ targetRecOff != 0))
+ {
+ if (!state->read_page(state, state->readPagePtr, state->readLen,
+ state->ReadRecPtr, state->readBuf))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
header = (XLogPageHeader) state->readBuf;
pageHeaderSize = XLogPageHeaderSize(header);
- /* make sure we have enough data for the page header */
- readLen = ReadPageInternal(state, targetPagePtr, pageHeaderSize);
- if (readLen < 0)
- goto err;
+ /* we should have read the page header */
+ Assert (state->readLen >= pageHeaderSize);
/* skip over potential continuation data */
if (header->xlp_info & XLP_FIRST_IS_CONTRECORD)
diff --git a/src/backend/access/transam/xlogutils.c b/src/backend/access/transam/xlogutils.c
index b217ffa52f..47676bf800 100644
--- a/src/backend/access/transam/xlogutils.c
+++ b/src/backend/access/transam/xlogutils.c
@@ -685,8 +685,8 @@ XLogTruncateRelation(RelFileNode rnode, ForkNumber forkNum,
void
XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wantLength)
{
- const XLogRecPtr lastReadPage = (state->seg.ws_segno *
- state->segcxt.ws_segsize + state->segoff);
+ const XLogRecPtr lastReadPage = state->seg.ws_segno *
+ state->segcxt.ws_segsize + state->readLen;
Assert(wantPage != InvalidXLogRecPtr && wantPage % XLOG_BLCKSZ == 0);
Assert(wantLength <= XLOG_BLCKSZ);
@@ -818,7 +818,7 @@ wal_segment_open(XLogSegNo nextSegNo, WALSegmentContext * segcxt,
* exists for normal backends, so we have to do a check/sleep/repeat style of
* loop for now.
*/
-int
+bool
read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *cur_page)
{
@@ -920,7 +920,8 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
else if (targetPagePtr + reqLen > read_upto)
{
/* not enough data there */
- return -1;
+ state->readLen = -1;
+ return false;
}
else
{
@@ -938,7 +939,8 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
WALReadRaiseError(&errinfo);
/* number of valid bytes in the buffer */
- return count;
+ state->readLen = count;
+ return true;
}
/*
diff --git a/src/backend/replication/logical/logicalfuncs.c b/src/backend/replication/logical/logicalfuncs.c
index 25b89e5616..acc8ef73be 100644
--- a/src/backend/replication/logical/logicalfuncs.c
+++ b/src/backend/replication/logical/logicalfuncs.c
@@ -106,7 +106,7 @@ check_permissions(void)
(errmsg("must be superuser or replication role to use replication slots"))));
}
-int
+bool
logical_read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *cur_page)
{
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index 0c65f1660b..9f8e5a592d 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -762,7 +762,7 @@ StartReplication(StartReplicationCmd *cmd)
* which has to do a plain sleep/busy loop, because the walsender's latch gets
* set every time WAL is flushed.
*/
-static int
+static bool
logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *cur_page)
{
@@ -782,7 +782,10 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
/* fail if not (implies we are going to shut down) */
if (flushptr < targetPagePtr + reqLen)
- return -1;
+ {
+ state->readLen = -1;
+ return false;
+ }
if (targetPagePtr + XLOG_BLCKSZ <= flushptr)
count = XLOG_BLCKSZ; /* more than one block available */
@@ -812,7 +815,8 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
XLByteToSeg(targetPagePtr, segno, sendCxt->ws_segsize);
CheckXLogRemoved(segno, sendSeg->ws_tli);
- return count;
+ state->readLen = count;
+ return true;
}
/*
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index eb61cb8803..78ee9f3faa 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -44,7 +44,7 @@ typedef struct XLogPageReadPrivate
int tliIndex;
} XLogPageReadPrivate;
-static int SimpleXLogPageRead(XLogReaderState *xlogreader,
+static bool SimpleXLogPageRead(XLogReaderState *xlogreader,
XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
@@ -227,7 +227,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
}
/* XLogReader callback function, to read a WAL page */
-static int
+static bool
SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf)
{
@@ -282,7 +282,8 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
if (xlogreadfd < 0)
{
pg_log_error("could not open file \"%s\": %m", xlogfpath);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
}
@@ -295,7 +296,8 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
if (lseek(xlogreadfd, (off_t) targetPageOff, SEEK_SET) < 0)
{
pg_log_error("could not seek in file \"%s\": %m", xlogfpath);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
@@ -308,13 +310,15 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
pg_log_error("could not read file \"%s\": read %d of %zu",
xlogfpath, r, (Size) XLOG_BLCKSZ);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
Assert(targetSegNo == xlogreadsegno);
xlogreader->seg.ws_tli = targetHistory[private->tliIndex].tli;
- return XLOG_BLCKSZ;
+ xlogreader->readLen = XLOG_BLCKSZ;
+ return true;
}
/*
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index 83202b5b87..fcfe80730c 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -322,7 +322,7 @@ WALDumpOpenSegment(XLogSegNo nextSegNo, WALSegmentContext *segcxt,
/*
* XLogReader read_page callback
*/
-static int
+static bool
WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetPtr, char *readBuff)
{
@@ -339,7 +339,8 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
else
{
private->endptr_reached = true;
- return -1;
+ state->readLen = -1;
+ return false;
}
}
@@ -364,7 +365,8 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
(Size) errinfo.wre_req);
}
- return count;
+ state->readLen = count;
+ return true;
}
/*
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index 4582196e18..6ad953eea3 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -51,7 +51,7 @@ typedef struct WALSegmentContext
typedef struct XLogReaderState XLogReaderState;
/* Function type definition for the read_page callback */
-typedef int (*XLogPageReadCB) (XLogReaderState *xlogreader,
+typedef bool (*XLogPageReadCB) (XLogReaderState *xlogreader,
XLogRecPtr targetPagePtr,
int reqLen,
XLogRecPtr targetRecPtr,
@@ -134,6 +134,20 @@ struct XLogReaderState
XLogRecPtr ReadRecPtr; /* start of last record read */
XLogRecPtr EndRecPtr; /* end+1 of last record read */
+ /* ----------------------------------------
+ * Communication with page reader
+ * readBuf is XLOG_BLCKSZ bytes, valid up to at least readLen bytes.
+ * ----------------------------------------
+ */
+ /* variables to communicate with page reader */
+ XLogRecPtr readPagePtr; /* page pointer to read */
+ int32 readLen; /* bytes requested to reader, or actual bytes
+ * read by reader, which must be larger than
+ * the request, or -1 on error */
+ TimeLineID readPageTLI; /* TLI for data currently in readBuf */
+ char *readBuf; /* buffer to store data */
+ bool page_verified; /* is the page on the buffer verified? */
+
/* ----------------------------------------
* Decoded representation of current record
@@ -160,13 +174,6 @@ struct XLogReaderState
* ----------------------------------------
*/
- /*
- * Buffer for currently read page (XLOG_BLCKSZ bytes, valid up to at least
- * readLen bytes)
- */
- char *readBuf;
- uint32 readLen;
-
/* last read XLOG position for data currently in readBuf */
WALSegmentContext segcxt;
WALOpenSegment seg;
diff --git a/src/include/access/xlogutils.h b/src/include/access/xlogutils.h
index 5181a077d9..dc7d894e5d 100644
--- a/src/include/access/xlogutils.h
+++ b/src/include/access/xlogutils.h
@@ -47,7 +47,7 @@ extern Buffer XLogReadBufferExtended(RelFileNode rnode, ForkNumber forknum,
extern Relation CreateFakeRelcacheEntry(RelFileNode rnode);
extern void FreeFakeRelcacheEntry(Relation fakerel);
-extern int read_local_xlog_page(XLogReaderState *state,
+extern bool read_local_xlog_page(XLogReaderState *state,
XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *cur_page);
diff --git a/src/include/replication/logicalfuncs.h b/src/include/replication/logicalfuncs.h
index 67c3fa12d9..ab7941a155 100644
--- a/src/include/replication/logicalfuncs.h
+++ b/src/include/replication/logicalfuncs.h
@@ -11,7 +11,7 @@
#include "replication/logical.h"
-extern int logical_read_local_xlog_page(XLogReaderState *state,
+extern bool logical_read_local_xlog_page(XLogReaderState *state,
XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr,
char *cur_page);
--
2.18.2
----Next_Part(Tue_Jan_28_21_20_20_2020_871)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
filename="v13-0002-Move-page-reader-out-of-XLogReadRecord.patch"
^ permalink raw reply [nested|flat] 69+ messages in thread
* [PATCH v8 1/4] Move callback-call from ReadPageInternal to XLogReadRecord.
@ 2019-09-05 11:21 Kyotaro Horiguchi <[email protected]>
0 siblings, 0 replies; 69+ messages in thread
From: Kyotaro Horiguchi @ 2019-09-05 11:21 UTC (permalink / raw)
The current WAL record reader reads page data using a call back
function. Although it is not so problematic alone, it would be a
problem if we are going to do add tasks like encryption which is
performed on page data before WAL reader reads them. To avoid that the
record reader facility has to have a new code path corresponds to
every new callback, this patch separates page reader from WAL record
reading facility by modifying the current WAL record reader to a state
machine.
As the first step of that change, this patch moves the page reader
function out of ReadPageInternal, then the remaining tasks of the
function are taken over by the new function XLogNeedData. As the
result XLogPageRead directly calls the page reader callback function
according to the feedback from XLogNeedData.
---
src/backend/access/transam/xlog.c | 16 +-
src/backend/access/transam/xlogreader.c | 278 ++++++++++++++----------
src/backend/access/transam/xlogutils.c | 12 +-
src/backend/replication/walsender.c | 10 +-
src/bin/pg_rewind/parsexlog.c | 21 +-
src/bin/pg_waldump/pg_waldump.c | 8 +-
src/include/access/xlogreader.h | 23 +-
src/include/access/xlogutils.h | 2 +-
8 files changed, 218 insertions(+), 152 deletions(-)
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 11e32733c4..b0ad9376d6 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -908,7 +908,7 @@ static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath,
static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli,
XLogSource source, bool notfoundOk);
static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, XLogSource source);
-static int XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
+static bool XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
bool fetching_ckpt, XLogRecPtr tliRecPtr);
@@ -4328,7 +4328,6 @@ ReadRecord(XLogReaderState *xlogreader, int emode,
XLogRecord *record;
XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
- /* Pass through parameters to XLogPageRead */
private->fetching_ckpt = fetching_ckpt;
private->emode = emode;
private->randAccess = (xlogreader->ReadRecPtr == InvalidXLogRecPtr);
@@ -11810,7 +11809,7 @@ CancelBackup(void)
* XLogPageRead() to try fetching the record from another source, or to
* sleep and retry.
*/
-static int
+static bool
XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *readBuf)
{
@@ -11869,7 +11868,8 @@ retry:
readLen = 0;
readSource = XLOG_FROM_ANY;
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
}
@@ -11964,7 +11964,8 @@ retry:
goto next_record_is_invalid;
}
- return readLen;
+ xlogreader->readLen = readLen;
+ return true;
next_record_is_invalid:
lastSourceFailed = true;
@@ -11978,8 +11979,9 @@ next_record_is_invalid:
/* In standby-mode, keep trying */
if (StandbyMode)
goto retry;
- else
- return -1;
+
+ xlogreader->readLen = -1;
+ return false;
}
/*
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index 79ff976474..6250093dd9 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -36,8 +36,8 @@
static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
pg_attribute_printf(2, 3);
static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
-static int ReadPageInternal(XLogReaderState *state, XLogRecPtr pageptr,
- int reqLen);
+static bool XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr,
+ int reqLen, bool header_inclusive);
static void XLogReaderInvalReadState(XLogReaderState *state);
static bool ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
XLogRecPtr PrevRecPtr, XLogRecord *record, bool randAccess);
@@ -272,7 +272,6 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
uint32 targetRecOff;
uint32 pageHeaderSize;
bool gotheader;
- int readOff;
/*
* randAccess indicates whether to verify the previous-record pointer of
@@ -322,14 +321,20 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
* byte to cover the whole record header, or at least the part of it that
* fits on the same page.
*/
- readOff = ReadPageInternal(state, targetPagePtr,
- Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ));
- if (readOff < 0)
+ while (XLogNeedData(state, targetPagePtr,
+ Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+ targetRecOff != 0))
+ {
+ if (!state->read_page(state, state->readPagePtr, state->readLen,
+ RecPtr, state->readBuf))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
/*
- * ReadPageInternal always returns at least the page header, so we can
- * examine it now.
+ * We have at least the page header, so we can examine it now.
*/
pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
if (targetRecOff == 0)
@@ -355,8 +360,8 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
goto err;
}
- /* ReadPageInternal has verified the page header */
- Assert(pageHeaderSize <= readOff);
+ /* XLogNeedData has verified the page header */
+ Assert(pageHeaderSize <= state->readLen);
/*
* Read the record length.
@@ -429,18 +434,25 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
do
{
+ int rest_len = total_len - gotlen;
+
/* Calculate pointer to beginning of next page */
targetPagePtr += XLOG_BLCKSZ;
/* Wait for the next page to become available */
- readOff = ReadPageInternal(state, targetPagePtr,
- Min(total_len - gotlen + SizeOfXLogShortPHD,
- XLOG_BLCKSZ));
+ while (XLogNeedData(state, targetPagePtr,
+ Min(rest_len, XLOG_BLCKSZ),
+ false))
+ {
+ if (!state->read_page(state, state->readPagePtr, state->readLen,
+ state->ReadRecPtr, state->readBuf))
+ break;
+ }
- if (readOff < 0)
+ if (!state->page_verified)
goto err;
- Assert(SizeOfXLogShortPHD <= readOff);
+ Assert(SizeOfXLogShortPHD <= state->readLen);
/* Check that the continuation on next page looks valid */
pageHeader = (XLogPageHeader) state->readBuf;
@@ -469,21 +481,14 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
/* Append the continuation from this page to the buffer */
pageHeaderSize = XLogPageHeaderSize(pageHeader);
- if (readOff < pageHeaderSize)
- readOff = ReadPageInternal(state, targetPagePtr,
- pageHeaderSize);
-
- Assert(pageHeaderSize <= readOff);
+ Assert(pageHeaderSize <= state->readLen);
contdata = (char *) state->readBuf + pageHeaderSize;
len = XLOG_BLCKSZ - pageHeaderSize;
if (pageHeader->xlp_rem_len < len)
len = pageHeader->xlp_rem_len;
- if (readOff < pageHeaderSize + len)
- readOff = ReadPageInternal(state, targetPagePtr,
- pageHeaderSize + len);
-
+ Assert (pageHeaderSize + len <= state->readLen);
memcpy(buffer, (char *) contdata, len);
buffer += len;
gotlen += len;
@@ -513,9 +518,15 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
else
{
/* Wait for the record data to become available */
- readOff = ReadPageInternal(state, targetPagePtr,
- Min(targetRecOff + total_len, XLOG_BLCKSZ));
- if (readOff < 0)
+ while (XLogNeedData(state, targetPagePtr,
+ Min(targetRecOff + total_len, XLOG_BLCKSZ), true))
+ {
+ if (!state->read_page(state, state->readPagePtr, state->readLen,
+ state->ReadRecPtr, state->readBuf))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
/* Record does not cross a page boundary */
@@ -558,109 +569,139 @@ err:
}
/*
- * Read a single xlog page including at least [pageptr, reqLen] of valid data
- * via the read_page() callback.
+ * Checks that an xlog page loaded in state->readBuf is including at least
+ * [pageptr, reqLen] and the page is valid. header_inclusive indicates that
+ * reqLen is calculated including page header length.
*
- * Returns -1 if the required page cannot be read for some reason; errormsg_buf
- * is set in that case (unless the error occurs in the read_page callback).
+ * Returns false if the buffer already contains the requested data, or found
+ * error. state->page_verified is set to true for the former and false for the
+ * latter.
*
- * We fetch the page from a reader-local cache if we know we have the required
- * data and if there hasn't been any error since caching the data.
+ * Otherwise returns true and requests data loaded onto state->readBuf by
+ * state->readPagePtr and state->readLen. The caller shall call this function
+ * again after filling the buffer at least with that portion of data and set
+ * state->readLen to the length of actually loaded data.
+ *
+ * If header_inclusive is false, corrects reqLen internally by adding the
+ * actual page header length and may request caller for new data.
*/
-static int
-ReadPageInternal(XLogReaderState *state, XLogRecPtr pageptr, int reqLen)
+static bool
+XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
+ bool header_inclusive)
{
- int readLen;
uint32 targetPageOff;
XLogSegNo targetSegNo;
- XLogPageHeader hdr;
+ uint32 addLen = 0;
- Assert((pageptr % XLOG_BLCKSZ) == 0);
+ /* Some data is loaded, but page header is not verified yet. */
+ if (!state->page_verified &&
+ !XLogRecPtrIsInvalid(state->readPagePtr) && state->readLen >= 0)
+ {
+ uint32 pageHeaderSize;
+ /* just loaded new data so needs to verify page header */
+
+ /* The caller must have loaded at least page header */
+ Assert (state->readLen >= SizeOfXLogShortPHD);
+
+ /*
+ * We have enough data to check the header length. Recheck the loaded
+ * length against the actual header length.
+ */
+ pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+
+ /* Request more data if we don't have the full header. */
+ if (state->readLen < pageHeaderSize)
+ {
+ state->readLen = pageHeaderSize;
+ return true;
+ }
+
+ /* Now that we know we have the full header, validate it. */
+ if (!XLogReaderValidatePageHeader(state, state->readPagePtr,
+ (char *) state->readBuf))
+ {
+ /* That's bad. Force reading the page again. */
+ XLogReaderInvalReadState(state);
+
+ return false;
+ }
+
+ state->page_verified = true;
+
+ XLByteToSeg(state->readPagePtr, state->seg.ws_segno,
+ state->segcxt.ws_segsize);
+ }
+
+ /*
+ * The loaded page may not be the one caller is supposing to read when we
+ * are verifying the first page of new segment. In that case, skip further
+ * verification and immediately load the target page.
+ */
+ if (state->page_verified && pageptr == state->readPagePtr)
+ {
+
+ /*
+ * calculate additional length for page header keeping the total
+ * length within the block size.
+ */
+ if (!header_inclusive)
+ {
+ uint32 pageHeaderSize =
+ XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+
+ addLen = pageHeaderSize;
+ if (reqLen + pageHeaderSize <= XLOG_BLCKSZ)
+ addLen = pageHeaderSize;
+ else
+ addLen = XLOG_BLCKSZ - reqLen;
+
+ Assert(addLen >= 0);
+ }
+
+ /* Return if we already have it. */
+ if (reqLen + addLen <= state->readLen)
+ return false;
+ }
+
+ /* Data is not in our buffer, request the caller for it. */
XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize);
targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize);
+ Assert((pageptr % XLOG_BLCKSZ) == 0);
- /* check whether we have all the requested data already */
- if (targetSegNo == state->seg.ws_segno &&
- targetPageOff == state->segoff && reqLen <= state->readLen)
- return state->readLen;
+ /*
+ * Every time we request to load new data of a page to the caller, even if
+ * we looked at a part of it before, we need to do verification on the next
+ * invocation as the caller might now be rereading data from a different
+ * source.
+ */
+ state->page_verified = false;
/*
- * Data is not in our buffer.
- *
- * Every time we actually read the segment, even if we looked at parts of
- * it before, we need to do verification as the read_page callback might
- * now be rereading data from a different source.
- *
* Whenever switching to a new WAL segment, we read the first page of the
* file and validate its header, even if that's not where the target
* record is. This is so that we can check the additional identification
* info that is present in the first page's "long" header.
+ * Don't do this if the caller requested the first page in the segment.
*/
if (targetSegNo != state->seg.ws_segno && targetPageOff != 0)
{
- XLogRecPtr targetSegmentPtr = pageptr - targetPageOff;
-
- readLen = state->read_page(state, targetSegmentPtr, XLOG_BLCKSZ,
- state->currRecPtr,
- state->readBuf);
- if (readLen < 0)
- goto err;
-
- /* we can be sure to have enough WAL available, we scrolled back */
- Assert(readLen == XLOG_BLCKSZ);
-
- if (!XLogReaderValidatePageHeader(state, targetSegmentPtr,
- state->readBuf))
- goto err;
+ /*
+ * Then we'll see that the targetSegNo now matches the ws_segno, and
+ * will not come back here, but will request the actual target page.
+ */
+ state->readPagePtr = pageptr - targetPageOff;
+ state->readLen = XLOG_BLCKSZ;
+ return true;
}
/*
- * First, read the requested data length, but at least a short page header
- * so that we can validate it.
+ * Request the caller to load the page. We need at least a short page
+ * header so that we can validate it.
*/
- readLen = state->read_page(state, pageptr, Max(reqLen, SizeOfXLogShortPHD),
- state->currRecPtr,
- state->readBuf);
- if (readLen < 0)
- goto err;
-
- Assert(readLen <= XLOG_BLCKSZ);
-
- /* Do we have enough data to check the header length? */
- if (readLen <= SizeOfXLogShortPHD)
- goto err;
-
- Assert(readLen >= reqLen);
-
- hdr = (XLogPageHeader) state->readBuf;
-
- /* still not enough */
- if (readLen < XLogPageHeaderSize(hdr))
- {
- readLen = state->read_page(state, pageptr, XLogPageHeaderSize(hdr),
- state->currRecPtr,
- state->readBuf);
- if (readLen < 0)
- goto err;
- }
-
- /*
- * Now that we know we have the full header, validate it.
- */
- if (!XLogReaderValidatePageHeader(state, pageptr, (char *) hdr))
- goto err;
-
- /* update read state information */
- state->seg.ws_segno = targetSegNo;
- state->segoff = targetPageOff;
- state->readLen = readLen;
-
- return readLen;
-
-err:
- XLogReaderInvalReadState(state);
- return -1;
+ state->readPagePtr = pageptr;
+ state->readLen = Max(reqLen + addLen, SizeOfXLogShortPHD);
+ return true;
}
/*
@@ -669,9 +710,7 @@ err:
static void
XLogReaderInvalReadState(XLogReaderState *state)
{
- state->seg.ws_segno = 0;
- state->segoff = 0;
- state->readLen = 0;
+ state->readPagePtr = InvalidXLogRecPtr;
}
/*
@@ -952,7 +991,6 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
XLogRecPtr targetPagePtr;
int targetRecOff;
uint32 pageHeaderSize;
- int readLen;
/*
* Compute targetRecOff. It should typically be equal or greater than
@@ -960,7 +998,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
* that, except when caller has explicitly specified the offset that
* falls somewhere there or when we are skipping multi-page
* continuation record. It doesn't matter though because
- * ReadPageInternal() is prepared to handle that and will read at
+ * XLogNeedData() is prepared to handle that and will read at
* least short page-header worth of data
*/
targetRecOff = tmpRecPtr % XLOG_BLCKSZ;
@@ -968,19 +1006,23 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
/* scroll back to page boundary */
targetPagePtr = tmpRecPtr - targetRecOff;
- /* Read the page containing the record */
- readLen = ReadPageInternal(state, targetPagePtr, targetRecOff);
- if (readLen < 0)
+ while(XLogNeedData(state, targetPagePtr, targetRecOff,
+ targetRecOff != 0))
+ {
+ if (!state->read_page(state, state->readPagePtr, state->readLen,
+ state->ReadRecPtr, state->readBuf))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
header = (XLogPageHeader) state->readBuf;
pageHeaderSize = XLogPageHeaderSize(header);
- /* make sure we have enough data for the page header */
- readLen = ReadPageInternal(state, targetPagePtr, pageHeaderSize);
- if (readLen < 0)
- goto err;
+ /* we should have read the page header */
+ Assert (state->readLen >= pageHeaderSize);
/* skip over potential continuation data */
if (header->xlp_info & XLP_FIRST_IS_CONTRECORD)
diff --git a/src/backend/access/transam/xlogutils.c b/src/backend/access/transam/xlogutils.c
index 6cb143e161..2464a0d092 100644
--- a/src/backend/access/transam/xlogutils.c
+++ b/src/backend/access/transam/xlogutils.c
@@ -689,8 +689,8 @@ XLogTruncateRelation(RelFileNode rnode, ForkNumber forkNum,
void
XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wantLength)
{
- const XLogRecPtr lastReadPage = (state->seg.ws_segno *
- state->segcxt.ws_segsize + state->segoff);
+ const XLogRecPtr lastReadPage = state->seg.ws_segno *
+ state->segcxt.ws_segsize + state->readLen;
Assert(wantPage != InvalidXLogRecPtr && wantPage % XLOG_BLCKSZ == 0);
Assert(wantLength <= XLOG_BLCKSZ);
@@ -822,7 +822,7 @@ wal_segment_open(XLogSegNo nextSegNo, WALSegmentContext * segcxt,
* exists for normal backends, so we have to do a check/sleep/repeat style of
* loop for now.
*/
-int
+bool
read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *cur_page)
{
@@ -924,7 +924,8 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
else if (targetPagePtr + reqLen > read_upto)
{
/* not enough data there */
- return -1;
+ state->readLen = -1;
+ return false;
}
else
{
@@ -942,7 +943,8 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
WALReadRaiseError(&errinfo);
/* number of valid bytes in the buffer */
- return count;
+ state->readLen = count;
+ return true;
}
/*
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index 0e933228fc..e1c1de22c5 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -798,7 +798,7 @@ StartReplication(StartReplicationCmd *cmd)
* which has to do a plain sleep/busy loop, because the walsender's latch gets
* set every time WAL is flushed.
*/
-static int
+static bool
logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *cur_page)
{
@@ -818,7 +818,10 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
/* fail if not (implies we are going to shut down) */
if (flushptr < targetPagePtr + reqLen)
- return -1;
+ {
+ state->readLen = -1;
+ return false;
+ }
if (targetPagePtr + XLOG_BLCKSZ <= flushptr)
count = XLOG_BLCKSZ; /* more than one block available */
@@ -848,7 +851,8 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
XLByteToSeg(targetPagePtr, segno, sendCxt->ws_segsize);
CheckXLogRemoved(segno, sendSeg->ws_tli);
- return count;
+ state->readLen = count;
+ return true;
}
/*
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index 14a5db5433..1b0199f52f 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -46,7 +46,7 @@ typedef struct XLogPageReadPrivate
int tliIndex;
} XLogPageReadPrivate;
-static int SimpleXLogPageRead(XLogReaderState *xlogreader,
+static bool SimpleXLogPageRead(XLogReaderState *xlogreader,
XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
@@ -231,7 +231,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
}
/* XLogReader callback function, to read a WAL page */
-static int
+static bool
SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf)
{
@@ -291,7 +291,8 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
if (private->restoreCommand == NULL)
{
pg_log_error("could not open file \"%s\": %m", xlogfpath);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
/*
@@ -304,7 +305,10 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
private->restoreCommand);
if (xlogreadfd < 0)
- return -1;
+ {
+ xlogreader->readLen = -1;
+ return false;
+ }
else
pg_log_debug("using file \"%s\" restored from archive",
xlogfpath);
@@ -320,7 +324,8 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
if (lseek(xlogreadfd, (off_t) targetPageOff, SEEK_SET) < 0)
{
pg_log_error("could not seek in file \"%s\": %m", xlogfpath);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
@@ -333,13 +338,15 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
pg_log_error("could not read file \"%s\": read %d of %zu",
xlogfpath, r, (Size) XLOG_BLCKSZ);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
Assert(targetSegNo == xlogreadsegno);
xlogreader->seg.ws_tli = targetHistory[private->tliIndex].tli;
- return XLOG_BLCKSZ;
+ xlogreader->readLen = XLOG_BLCKSZ;
+ return true;
}
/*
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index d7bd9ccac2..33fde23e7b 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -323,7 +323,7 @@ WALDumpOpenSegment(XLogSegNo nextSegNo, WALSegmentContext *segcxt,
/*
* XLogReader read_page callback
*/
-static int
+static bool
WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetPtr, char *readBuff)
{
@@ -340,7 +340,8 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
else
{
private->endptr_reached = true;
- return -1;
+ state->readLen = -1;
+ return false;
}
}
@@ -365,7 +366,8 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
(Size) errinfo.wre_req);
}
- return count;
+ state->readLen = count;
+ return true;
}
/*
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index 4582196e18..6ad953eea3 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -51,7 +51,7 @@ typedef struct WALSegmentContext
typedef struct XLogReaderState XLogReaderState;
/* Function type definition for the read_page callback */
-typedef int (*XLogPageReadCB) (XLogReaderState *xlogreader,
+typedef bool (*XLogPageReadCB) (XLogReaderState *xlogreader,
XLogRecPtr targetPagePtr,
int reqLen,
XLogRecPtr targetRecPtr,
@@ -134,6 +134,20 @@ struct XLogReaderState
XLogRecPtr ReadRecPtr; /* start of last record read */
XLogRecPtr EndRecPtr; /* end+1 of last record read */
+ /* ----------------------------------------
+ * Communication with page reader
+ * readBuf is XLOG_BLCKSZ bytes, valid up to at least readLen bytes.
+ * ----------------------------------------
+ */
+ /* variables to communicate with page reader */
+ XLogRecPtr readPagePtr; /* page pointer to read */
+ int32 readLen; /* bytes requested to reader, or actual bytes
+ * read by reader, which must be larger than
+ * the request, or -1 on error */
+ TimeLineID readPageTLI; /* TLI for data currently in readBuf */
+ char *readBuf; /* buffer to store data */
+ bool page_verified; /* is the page on the buffer verified? */
+
/* ----------------------------------------
* Decoded representation of current record
@@ -160,13 +174,6 @@ struct XLogReaderState
* ----------------------------------------
*/
- /*
- * Buffer for currently read page (XLOG_BLCKSZ bytes, valid up to at least
- * readLen bytes)
- */
- char *readBuf;
- uint32 readLen;
-
/* last read XLOG position for data currently in readBuf */
WALSegmentContext segcxt;
WALOpenSegment seg;
diff --git a/src/include/access/xlogutils.h b/src/include/access/xlogutils.h
index 5181a077d9..dc7d894e5d 100644
--- a/src/include/access/xlogutils.h
+++ b/src/include/access/xlogutils.h
@@ -47,7 +47,7 @@ extern Buffer XLogReadBufferExtended(RelFileNode rnode, ForkNumber forknum,
extern Relation CreateFakeRelcacheEntry(RelFileNode rnode);
extern void FreeFakeRelcacheEntry(Relation fakerel);
-extern int read_local_xlog_page(XLogReaderState *state,
+extern bool read_local_xlog_page(XLogReaderState *state,
XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *cur_page);
--
2.18.2
----Next_Part(Tue_Apr_21_17_04_27_2020_275)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
filename="v8-0002-Move-page-reader-out-of-XLogReadRecord.patch"
^ permalink raw reply [nested|flat] 69+ messages in thread
* [PATCH v9 1/4] Move callback-call from ReadPageInternal to XLogReadRecord.
@ 2019-09-05 11:21 Kyotaro Horiguchi <[email protected]>
0 siblings, 0 replies; 69+ messages in thread
From: Kyotaro Horiguchi @ 2019-09-05 11:21 UTC (permalink / raw)
The current WAL record reader reads page data using a call back
function. Although it is not so problematic alone, it would be a
problem if we are going to do add tasks like encryption which is
performed on page data before WAL reader reads them. To avoid that the
record reader facility has to have a new code path corresponds to
every new callback, this patch separates page reader from WAL record
reading facility by modifying the current WAL record reader to a state
machine.
As the first step of that change, this patch moves the page reader
function out of ReadPageInternal, then the remaining tasks of the
function are taken over by the new function XLogNeedData. As the
result XLogPageRead directly calls the page reader callback function
according to the feedback from XLogNeedData.
---
src/backend/access/transam/xlog.c | 16 +-
src/backend/access/transam/xlogreader.c | 278 ++++++++++++++----------
src/backend/access/transam/xlogutils.c | 12 +-
src/backend/replication/walsender.c | 10 +-
src/bin/pg_rewind/parsexlog.c | 21 +-
src/bin/pg_waldump/pg_waldump.c | 8 +-
src/include/access/xlogreader.h | 23 +-
src/include/access/xlogutils.h | 2 +-
8 files changed, 218 insertions(+), 152 deletions(-)
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 11e32733c4..b0ad9376d6 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -908,7 +908,7 @@ static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath,
static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli,
XLogSource source, bool notfoundOk);
static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, XLogSource source);
-static int XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
+static bool XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
bool fetching_ckpt, XLogRecPtr tliRecPtr);
@@ -4328,7 +4328,6 @@ ReadRecord(XLogReaderState *xlogreader, int emode,
XLogRecord *record;
XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
- /* Pass through parameters to XLogPageRead */
private->fetching_ckpt = fetching_ckpt;
private->emode = emode;
private->randAccess = (xlogreader->ReadRecPtr == InvalidXLogRecPtr);
@@ -11810,7 +11809,7 @@ CancelBackup(void)
* XLogPageRead() to try fetching the record from another source, or to
* sleep and retry.
*/
-static int
+static bool
XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *readBuf)
{
@@ -11869,7 +11868,8 @@ retry:
readLen = 0;
readSource = XLOG_FROM_ANY;
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
}
@@ -11964,7 +11964,8 @@ retry:
goto next_record_is_invalid;
}
- return readLen;
+ xlogreader->readLen = readLen;
+ return true;
next_record_is_invalid:
lastSourceFailed = true;
@@ -11978,8 +11979,9 @@ next_record_is_invalid:
/* In standby-mode, keep trying */
if (StandbyMode)
goto retry;
- else
- return -1;
+
+ xlogreader->readLen = -1;
+ return false;
}
/*
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index 79ff976474..6250093dd9 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -36,8 +36,8 @@
static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
pg_attribute_printf(2, 3);
static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
-static int ReadPageInternal(XLogReaderState *state, XLogRecPtr pageptr,
- int reqLen);
+static bool XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr,
+ int reqLen, bool header_inclusive);
static void XLogReaderInvalReadState(XLogReaderState *state);
static bool ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
XLogRecPtr PrevRecPtr, XLogRecord *record, bool randAccess);
@@ -272,7 +272,6 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
uint32 targetRecOff;
uint32 pageHeaderSize;
bool gotheader;
- int readOff;
/*
* randAccess indicates whether to verify the previous-record pointer of
@@ -322,14 +321,20 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
* byte to cover the whole record header, or at least the part of it that
* fits on the same page.
*/
- readOff = ReadPageInternal(state, targetPagePtr,
- Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ));
- if (readOff < 0)
+ while (XLogNeedData(state, targetPagePtr,
+ Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+ targetRecOff != 0))
+ {
+ if (!state->read_page(state, state->readPagePtr, state->readLen,
+ RecPtr, state->readBuf))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
/*
- * ReadPageInternal always returns at least the page header, so we can
- * examine it now.
+ * We have at least the page header, so we can examine it now.
*/
pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
if (targetRecOff == 0)
@@ -355,8 +360,8 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
goto err;
}
- /* ReadPageInternal has verified the page header */
- Assert(pageHeaderSize <= readOff);
+ /* XLogNeedData has verified the page header */
+ Assert(pageHeaderSize <= state->readLen);
/*
* Read the record length.
@@ -429,18 +434,25 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
do
{
+ int rest_len = total_len - gotlen;
+
/* Calculate pointer to beginning of next page */
targetPagePtr += XLOG_BLCKSZ;
/* Wait for the next page to become available */
- readOff = ReadPageInternal(state, targetPagePtr,
- Min(total_len - gotlen + SizeOfXLogShortPHD,
- XLOG_BLCKSZ));
+ while (XLogNeedData(state, targetPagePtr,
+ Min(rest_len, XLOG_BLCKSZ),
+ false))
+ {
+ if (!state->read_page(state, state->readPagePtr, state->readLen,
+ state->ReadRecPtr, state->readBuf))
+ break;
+ }
- if (readOff < 0)
+ if (!state->page_verified)
goto err;
- Assert(SizeOfXLogShortPHD <= readOff);
+ Assert(SizeOfXLogShortPHD <= state->readLen);
/* Check that the continuation on next page looks valid */
pageHeader = (XLogPageHeader) state->readBuf;
@@ -469,21 +481,14 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
/* Append the continuation from this page to the buffer */
pageHeaderSize = XLogPageHeaderSize(pageHeader);
- if (readOff < pageHeaderSize)
- readOff = ReadPageInternal(state, targetPagePtr,
- pageHeaderSize);
-
- Assert(pageHeaderSize <= readOff);
+ Assert(pageHeaderSize <= state->readLen);
contdata = (char *) state->readBuf + pageHeaderSize;
len = XLOG_BLCKSZ - pageHeaderSize;
if (pageHeader->xlp_rem_len < len)
len = pageHeader->xlp_rem_len;
- if (readOff < pageHeaderSize + len)
- readOff = ReadPageInternal(state, targetPagePtr,
- pageHeaderSize + len);
-
+ Assert (pageHeaderSize + len <= state->readLen);
memcpy(buffer, (char *) contdata, len);
buffer += len;
gotlen += len;
@@ -513,9 +518,15 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
else
{
/* Wait for the record data to become available */
- readOff = ReadPageInternal(state, targetPagePtr,
- Min(targetRecOff + total_len, XLOG_BLCKSZ));
- if (readOff < 0)
+ while (XLogNeedData(state, targetPagePtr,
+ Min(targetRecOff + total_len, XLOG_BLCKSZ), true))
+ {
+ if (!state->read_page(state, state->readPagePtr, state->readLen,
+ state->ReadRecPtr, state->readBuf))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
/* Record does not cross a page boundary */
@@ -558,109 +569,139 @@ err:
}
/*
- * Read a single xlog page including at least [pageptr, reqLen] of valid data
- * via the read_page() callback.
+ * Checks that an xlog page loaded in state->readBuf is including at least
+ * [pageptr, reqLen] and the page is valid. header_inclusive indicates that
+ * reqLen is calculated including page header length.
*
- * Returns -1 if the required page cannot be read for some reason; errormsg_buf
- * is set in that case (unless the error occurs in the read_page callback).
+ * Returns false if the buffer already contains the requested data, or found
+ * error. state->page_verified is set to true for the former and false for the
+ * latter.
*
- * We fetch the page from a reader-local cache if we know we have the required
- * data and if there hasn't been any error since caching the data.
+ * Otherwise returns true and requests data loaded onto state->readBuf by
+ * state->readPagePtr and state->readLen. The caller shall call this function
+ * again after filling the buffer at least with that portion of data and set
+ * state->readLen to the length of actually loaded data.
+ *
+ * If header_inclusive is false, corrects reqLen internally by adding the
+ * actual page header length and may request caller for new data.
*/
-static int
-ReadPageInternal(XLogReaderState *state, XLogRecPtr pageptr, int reqLen)
+static bool
+XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
+ bool header_inclusive)
{
- int readLen;
uint32 targetPageOff;
XLogSegNo targetSegNo;
- XLogPageHeader hdr;
+ uint32 addLen = 0;
- Assert((pageptr % XLOG_BLCKSZ) == 0);
+ /* Some data is loaded, but page header is not verified yet. */
+ if (!state->page_verified &&
+ !XLogRecPtrIsInvalid(state->readPagePtr) && state->readLen >= 0)
+ {
+ uint32 pageHeaderSize;
+ /* just loaded new data so needs to verify page header */
+
+ /* The caller must have loaded at least page header */
+ Assert (state->readLen >= SizeOfXLogShortPHD);
+
+ /*
+ * We have enough data to check the header length. Recheck the loaded
+ * length against the actual header length.
+ */
+ pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+
+ /* Request more data if we don't have the full header. */
+ if (state->readLen < pageHeaderSize)
+ {
+ state->readLen = pageHeaderSize;
+ return true;
+ }
+
+ /* Now that we know we have the full header, validate it. */
+ if (!XLogReaderValidatePageHeader(state, state->readPagePtr,
+ (char *) state->readBuf))
+ {
+ /* That's bad. Force reading the page again. */
+ XLogReaderInvalReadState(state);
+
+ return false;
+ }
+
+ state->page_verified = true;
+
+ XLByteToSeg(state->readPagePtr, state->seg.ws_segno,
+ state->segcxt.ws_segsize);
+ }
+
+ /*
+ * The loaded page may not be the one caller is supposing to read when we
+ * are verifying the first page of new segment. In that case, skip further
+ * verification and immediately load the target page.
+ */
+ if (state->page_verified && pageptr == state->readPagePtr)
+ {
+
+ /*
+ * calculate additional length for page header keeping the total
+ * length within the block size.
+ */
+ if (!header_inclusive)
+ {
+ uint32 pageHeaderSize =
+ XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+
+ addLen = pageHeaderSize;
+ if (reqLen + pageHeaderSize <= XLOG_BLCKSZ)
+ addLen = pageHeaderSize;
+ else
+ addLen = XLOG_BLCKSZ - reqLen;
+
+ Assert(addLen >= 0);
+ }
+
+ /* Return if we already have it. */
+ if (reqLen + addLen <= state->readLen)
+ return false;
+ }
+
+ /* Data is not in our buffer, request the caller for it. */
XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize);
targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize);
+ Assert((pageptr % XLOG_BLCKSZ) == 0);
- /* check whether we have all the requested data already */
- if (targetSegNo == state->seg.ws_segno &&
- targetPageOff == state->segoff && reqLen <= state->readLen)
- return state->readLen;
+ /*
+ * Every time we request to load new data of a page to the caller, even if
+ * we looked at a part of it before, we need to do verification on the next
+ * invocation as the caller might now be rereading data from a different
+ * source.
+ */
+ state->page_verified = false;
/*
- * Data is not in our buffer.
- *
- * Every time we actually read the segment, even if we looked at parts of
- * it before, we need to do verification as the read_page callback might
- * now be rereading data from a different source.
- *
* Whenever switching to a new WAL segment, we read the first page of the
* file and validate its header, even if that's not where the target
* record is. This is so that we can check the additional identification
* info that is present in the first page's "long" header.
+ * Don't do this if the caller requested the first page in the segment.
*/
if (targetSegNo != state->seg.ws_segno && targetPageOff != 0)
{
- XLogRecPtr targetSegmentPtr = pageptr - targetPageOff;
-
- readLen = state->read_page(state, targetSegmentPtr, XLOG_BLCKSZ,
- state->currRecPtr,
- state->readBuf);
- if (readLen < 0)
- goto err;
-
- /* we can be sure to have enough WAL available, we scrolled back */
- Assert(readLen == XLOG_BLCKSZ);
-
- if (!XLogReaderValidatePageHeader(state, targetSegmentPtr,
- state->readBuf))
- goto err;
+ /*
+ * Then we'll see that the targetSegNo now matches the ws_segno, and
+ * will not come back here, but will request the actual target page.
+ */
+ state->readPagePtr = pageptr - targetPageOff;
+ state->readLen = XLOG_BLCKSZ;
+ return true;
}
/*
- * First, read the requested data length, but at least a short page header
- * so that we can validate it.
+ * Request the caller to load the page. We need at least a short page
+ * header so that we can validate it.
*/
- readLen = state->read_page(state, pageptr, Max(reqLen, SizeOfXLogShortPHD),
- state->currRecPtr,
- state->readBuf);
- if (readLen < 0)
- goto err;
-
- Assert(readLen <= XLOG_BLCKSZ);
-
- /* Do we have enough data to check the header length? */
- if (readLen <= SizeOfXLogShortPHD)
- goto err;
-
- Assert(readLen >= reqLen);
-
- hdr = (XLogPageHeader) state->readBuf;
-
- /* still not enough */
- if (readLen < XLogPageHeaderSize(hdr))
- {
- readLen = state->read_page(state, pageptr, XLogPageHeaderSize(hdr),
- state->currRecPtr,
- state->readBuf);
- if (readLen < 0)
- goto err;
- }
-
- /*
- * Now that we know we have the full header, validate it.
- */
- if (!XLogReaderValidatePageHeader(state, pageptr, (char *) hdr))
- goto err;
-
- /* update read state information */
- state->seg.ws_segno = targetSegNo;
- state->segoff = targetPageOff;
- state->readLen = readLen;
-
- return readLen;
-
-err:
- XLogReaderInvalReadState(state);
- return -1;
+ state->readPagePtr = pageptr;
+ state->readLen = Max(reqLen + addLen, SizeOfXLogShortPHD);
+ return true;
}
/*
@@ -669,9 +710,7 @@ err:
static void
XLogReaderInvalReadState(XLogReaderState *state)
{
- state->seg.ws_segno = 0;
- state->segoff = 0;
- state->readLen = 0;
+ state->readPagePtr = InvalidXLogRecPtr;
}
/*
@@ -952,7 +991,6 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
XLogRecPtr targetPagePtr;
int targetRecOff;
uint32 pageHeaderSize;
- int readLen;
/*
* Compute targetRecOff. It should typically be equal or greater than
@@ -960,7 +998,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
* that, except when caller has explicitly specified the offset that
* falls somewhere there or when we are skipping multi-page
* continuation record. It doesn't matter though because
- * ReadPageInternal() is prepared to handle that and will read at
+ * XLogNeedData() is prepared to handle that and will read at
* least short page-header worth of data
*/
targetRecOff = tmpRecPtr % XLOG_BLCKSZ;
@@ -968,19 +1006,23 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
/* scroll back to page boundary */
targetPagePtr = tmpRecPtr - targetRecOff;
- /* Read the page containing the record */
- readLen = ReadPageInternal(state, targetPagePtr, targetRecOff);
- if (readLen < 0)
+ while(XLogNeedData(state, targetPagePtr, targetRecOff,
+ targetRecOff != 0))
+ {
+ if (!state->read_page(state, state->readPagePtr, state->readLen,
+ state->ReadRecPtr, state->readBuf))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
header = (XLogPageHeader) state->readBuf;
pageHeaderSize = XLogPageHeaderSize(header);
- /* make sure we have enough data for the page header */
- readLen = ReadPageInternal(state, targetPagePtr, pageHeaderSize);
- if (readLen < 0)
- goto err;
+ /* we should have read the page header */
+ Assert (state->readLen >= pageHeaderSize);
/* skip over potential continuation data */
if (header->xlp_info & XLP_FIRST_IS_CONTRECORD)
diff --git a/src/backend/access/transam/xlogutils.c b/src/backend/access/transam/xlogutils.c
index 6cb143e161..2464a0d092 100644
--- a/src/backend/access/transam/xlogutils.c
+++ b/src/backend/access/transam/xlogutils.c
@@ -689,8 +689,8 @@ XLogTruncateRelation(RelFileNode rnode, ForkNumber forkNum,
void
XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wantLength)
{
- const XLogRecPtr lastReadPage = (state->seg.ws_segno *
- state->segcxt.ws_segsize + state->segoff);
+ const XLogRecPtr lastReadPage = state->seg.ws_segno *
+ state->segcxt.ws_segsize + state->readLen;
Assert(wantPage != InvalidXLogRecPtr && wantPage % XLOG_BLCKSZ == 0);
Assert(wantLength <= XLOG_BLCKSZ);
@@ -822,7 +822,7 @@ wal_segment_open(XLogSegNo nextSegNo, WALSegmentContext * segcxt,
* exists for normal backends, so we have to do a check/sleep/repeat style of
* loop for now.
*/
-int
+bool
read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *cur_page)
{
@@ -924,7 +924,8 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
else if (targetPagePtr + reqLen > read_upto)
{
/* not enough data there */
- return -1;
+ state->readLen = -1;
+ return false;
}
else
{
@@ -942,7 +943,8 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
WALReadRaiseError(&errinfo);
/* number of valid bytes in the buffer */
- return count;
+ state->readLen = count;
+ return true;
}
/*
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index 0e933228fc..e1c1de22c5 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -798,7 +798,7 @@ StartReplication(StartReplicationCmd *cmd)
* which has to do a plain sleep/busy loop, because the walsender's latch gets
* set every time WAL is flushed.
*/
-static int
+static bool
logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *cur_page)
{
@@ -818,7 +818,10 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
/* fail if not (implies we are going to shut down) */
if (flushptr < targetPagePtr + reqLen)
- return -1;
+ {
+ state->readLen = -1;
+ return false;
+ }
if (targetPagePtr + XLOG_BLCKSZ <= flushptr)
count = XLOG_BLCKSZ; /* more than one block available */
@@ -848,7 +851,8 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
XLByteToSeg(targetPagePtr, segno, sendCxt->ws_segsize);
CheckXLogRemoved(segno, sendSeg->ws_tli);
- return count;
+ state->readLen = count;
+ return true;
}
/*
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index c51b5db315..6cbe108129 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -46,7 +46,7 @@ typedef struct XLogPageReadPrivate
int tliIndex;
} XLogPageReadPrivate;
-static int SimpleXLogPageRead(XLogReaderState *xlogreader,
+static bool SimpleXLogPageRead(XLogReaderState *xlogreader,
XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
@@ -233,7 +233,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
}
/* XLogReader callback function, to read a WAL page */
-static int
+static bool
SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf)
{
@@ -293,7 +293,8 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
if (private->restoreCommand == NULL)
{
pg_log_error("could not open file \"%s\": %m", xlogfpath);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
/*
@@ -306,7 +307,10 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
private->restoreCommand);
if (xlogreadfd < 0)
- return -1;
+ {
+ xlogreader->readLen = -1;
+ return false;
+ }
else
pg_log_debug("using file \"%s\" restored from archive",
xlogfpath);
@@ -322,7 +326,8 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
if (lseek(xlogreadfd, (off_t) targetPageOff, SEEK_SET) < 0)
{
pg_log_error("could not seek in file \"%s\": %m", xlogfpath);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
@@ -335,13 +340,15 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
pg_log_error("could not read file \"%s\": read %d of %zu",
xlogfpath, r, (Size) XLOG_BLCKSZ);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
Assert(targetSegNo == xlogreadsegno);
xlogreader->seg.ws_tli = targetHistory[private->tliIndex].tli;
- return XLOG_BLCKSZ;
+ xlogreader->readLen = XLOG_BLCKSZ;
+ return true;
}
/*
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index d7bd9ccac2..33fde23e7b 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -323,7 +323,7 @@ WALDumpOpenSegment(XLogSegNo nextSegNo, WALSegmentContext *segcxt,
/*
* XLogReader read_page callback
*/
-static int
+static bool
WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetPtr, char *readBuff)
{
@@ -340,7 +340,8 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
else
{
private->endptr_reached = true;
- return -1;
+ state->readLen = -1;
+ return false;
}
}
@@ -365,7 +366,8 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
(Size) errinfo.wre_req);
}
- return count;
+ state->readLen = count;
+ return true;
}
/*
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index 4582196e18..6ad953eea3 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -51,7 +51,7 @@ typedef struct WALSegmentContext
typedef struct XLogReaderState XLogReaderState;
/* Function type definition for the read_page callback */
-typedef int (*XLogPageReadCB) (XLogReaderState *xlogreader,
+typedef bool (*XLogPageReadCB) (XLogReaderState *xlogreader,
XLogRecPtr targetPagePtr,
int reqLen,
XLogRecPtr targetRecPtr,
@@ -134,6 +134,20 @@ struct XLogReaderState
XLogRecPtr ReadRecPtr; /* start of last record read */
XLogRecPtr EndRecPtr; /* end+1 of last record read */
+ /* ----------------------------------------
+ * Communication with page reader
+ * readBuf is XLOG_BLCKSZ bytes, valid up to at least readLen bytes.
+ * ----------------------------------------
+ */
+ /* variables to communicate with page reader */
+ XLogRecPtr readPagePtr; /* page pointer to read */
+ int32 readLen; /* bytes requested to reader, or actual bytes
+ * read by reader, which must be larger than
+ * the request, or -1 on error */
+ TimeLineID readPageTLI; /* TLI for data currently in readBuf */
+ char *readBuf; /* buffer to store data */
+ bool page_verified; /* is the page on the buffer verified? */
+
/* ----------------------------------------
* Decoded representation of current record
@@ -160,13 +174,6 @@ struct XLogReaderState
* ----------------------------------------
*/
- /*
- * Buffer for currently read page (XLOG_BLCKSZ bytes, valid up to at least
- * readLen bytes)
- */
- char *readBuf;
- uint32 readLen;
-
/* last read XLOG position for data currently in readBuf */
WALSegmentContext segcxt;
WALOpenSegment seg;
diff --git a/src/include/access/xlogutils.h b/src/include/access/xlogutils.h
index 5181a077d9..dc7d894e5d 100644
--- a/src/include/access/xlogutils.h
+++ b/src/include/access/xlogutils.h
@@ -47,7 +47,7 @@ extern Buffer XLogReadBufferExtended(RelFileNode rnode, ForkNumber forknum,
extern Relation CreateFakeRelcacheEntry(RelFileNode rnode);
extern void FreeFakeRelcacheEntry(Relation fakerel);
-extern int read_local_xlog_page(XLogReaderState *state,
+extern bool read_local_xlog_page(XLogReaderState *state,
XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *cur_page);
--
2.18.2
----Next_Part(Wed_Apr_22_10_12_46_2020_740)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
filename="v9-0002-Move-page-reader-out-of-XLogReadRecord.patch"
^ permalink raw reply [nested|flat] 69+ messages in thread
* [PATCH v15 1/4] Move callback-call from ReadPageInternal to XLogReadRecord.
@ 2019-09-05 11:21 Kyotaro Horiguchi <[email protected]>
0 siblings, 0 replies; 69+ messages in thread
From: Kyotaro Horiguchi @ 2019-09-05 11:21 UTC (permalink / raw)
The current WAL record reader reads page data using a call back
function. Although it is not so problematic alone, it would be a
problem if we are going to do add tasks like encryption which is
performed on page data before WAL reader reads them. To avoid that the
record reader facility has to have a new code path corresponds to
every new callback, this patch separates page reader from WAL record
reading facility by modifying the current WAL record reader to a state
machine.
As the first step of that change, this patch moves the page reader
function out of ReadPageInternal, then the remaining tasks of the
function are taken over by the new function XLogNeedData. As the
result XLogPageRead directly calls the page reader callback function
according to the feedback from XLogNeedData.
---
src/backend/access/transam/xlog.c | 16 +-
src/backend/access/transam/xlogreader.c | 281 ++++++++++++++----------
src/backend/access/transam/xlogutils.c | 12 +-
src/backend/replication/walsender.c | 10 +-
src/bin/pg_rewind/parsexlog.c | 21 +-
src/bin/pg_waldump/pg_waldump.c | 8 +-
src/include/access/xlogreader.h | 25 ++-
src/include/access/xlogutils.h | 2 +-
8 files changed, 222 insertions(+), 153 deletions(-)
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index fd93bcfaeb..e570e56a24 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -912,7 +912,7 @@ static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath,
static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli,
XLogSource source, bool notfoundOk);
static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, XLogSource source);
-static int XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
+static bool XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
bool fetching_ckpt, XLogRecPtr tliRecPtr);
@@ -4332,7 +4332,6 @@ ReadRecord(XLogReaderState *xlogreader, int emode,
XLogRecord *record;
XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
- /* Pass through parameters to XLogPageRead */
private->fetching_ckpt = fetching_ckpt;
private->emode = emode;
private->randAccess = (xlogreader->ReadRecPtr == InvalidXLogRecPtr);
@@ -11860,7 +11859,7 @@ CancelBackup(void)
* XLogPageRead() to try fetching the record from another source, or to
* sleep and retry.
*/
-static int
+static bool
XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *readBuf)
{
@@ -11919,7 +11918,8 @@ retry:
readLen = 0;
readSource = XLOG_FROM_ANY;
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
}
@@ -12014,7 +12014,8 @@ retry:
goto next_record_is_invalid;
}
- return readLen;
+ xlogreader->readLen = readLen;
+ return true;
next_record_is_invalid:
lastSourceFailed = true;
@@ -12028,8 +12029,9 @@ next_record_is_invalid:
/* In standby-mode, keep trying */
if (StandbyMode)
goto retry;
- else
- return -1;
+
+ xlogreader->readLen = -1;
+ return false;
}
/*
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index cb76be4f46..3d599325ee 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -36,8 +36,8 @@
static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
pg_attribute_printf(2, 3);
static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
-static int ReadPageInternal(XLogReaderState *state, XLogRecPtr pageptr,
- int reqLen);
+static bool XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr,
+ int reqLen, bool header_inclusive);
static void XLogReaderInvalReadState(XLogReaderState *state);
static bool ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
XLogRecPtr PrevRecPtr, XLogRecord *record, bool randAccess);
@@ -276,7 +276,6 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
uint32 targetRecOff;
uint32 pageHeaderSize;
bool gotheader;
- int readOff;
/*
* randAccess indicates whether to verify the previous-record pointer of
@@ -326,14 +325,20 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
* byte to cover the whole record header, or at least the part of it that
* fits on the same page.
*/
- readOff = ReadPageInternal(state, targetPagePtr,
- Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ));
- if (readOff < 0)
+ while (XLogNeedData(state, targetPagePtr,
+ Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+ targetRecOff != 0))
+ {
+ if (!state->routine.page_read(state, state->readPagePtr, state->readLen,
+ RecPtr, state->readBuf))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
/*
- * ReadPageInternal always returns at least the page header, so we can
- * examine it now.
+ * We have at least the page header, so we can examine it now.
*/
pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
if (targetRecOff == 0)
@@ -359,8 +364,8 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
goto err;
}
- /* ReadPageInternal has verified the page header */
- Assert(pageHeaderSize <= readOff);
+ /* XLogNeedData has verified the page header */
+ Assert(pageHeaderSize <= state->readLen);
/*
* Read the record length.
@@ -433,18 +438,27 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
do
{
+ int rest_len = total_len - gotlen;
+
/* Calculate pointer to beginning of next page */
targetPagePtr += XLOG_BLCKSZ;
/* Wait for the next page to become available */
- readOff = ReadPageInternal(state, targetPagePtr,
- Min(total_len - gotlen + SizeOfXLogShortPHD,
- XLOG_BLCKSZ));
+ while (XLogNeedData(state, targetPagePtr,
+ Min(rest_len, XLOG_BLCKSZ),
+ false))
+ {
+ if (!state->routine.page_read(state, state->readPagePtr,
+ state->readLen,
+ state->ReadRecPtr,
+ state->readBuf))
+ break;
+ }
- if (readOff < 0)
+ if (!state->page_verified)
goto err;
- Assert(SizeOfXLogShortPHD <= readOff);
+ Assert(SizeOfXLogShortPHD <= state->readLen);
/* Check that the continuation on next page looks valid */
pageHeader = (XLogPageHeader) state->readBuf;
@@ -473,21 +487,14 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
/* Append the continuation from this page to the buffer */
pageHeaderSize = XLogPageHeaderSize(pageHeader);
- if (readOff < pageHeaderSize)
- readOff = ReadPageInternal(state, targetPagePtr,
- pageHeaderSize);
-
- Assert(pageHeaderSize <= readOff);
+ Assert(pageHeaderSize <= state->readLen);
contdata = (char *) state->readBuf + pageHeaderSize;
len = XLOG_BLCKSZ - pageHeaderSize;
if (pageHeader->xlp_rem_len < len)
len = pageHeader->xlp_rem_len;
- if (readOff < pageHeaderSize + len)
- readOff = ReadPageInternal(state, targetPagePtr,
- pageHeaderSize + len);
-
+ Assert (pageHeaderSize + len <= state->readLen);
memcpy(buffer, (char *) contdata, len);
buffer += len;
gotlen += len;
@@ -517,9 +524,16 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
else
{
/* Wait for the record data to become available */
- readOff = ReadPageInternal(state, targetPagePtr,
- Min(targetRecOff + total_len, XLOG_BLCKSZ));
- if (readOff < 0)
+ while (XLogNeedData(state, targetPagePtr,
+ Min(targetRecOff + total_len, XLOG_BLCKSZ), true))
+ {
+ if (!state->routine.page_read(state, state->readPagePtr,
+ state->readLen,
+ state->ReadRecPtr, state->readBuf))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
/* Record does not cross a page boundary */
@@ -562,109 +576,138 @@ err:
}
/*
- * Read a single xlog page including at least [pageptr, reqLen] of valid data
- * via the page_read() callback.
+ * Checks that an xlog page loaded in state->readBuf is including at least
+ * [pageptr, reqLen] and the page is valid. header_inclusive indicates that
+ * reqLen is calculated including page header length.
*
- * Returns -1 if the required page cannot be read for some reason; errormsg_buf
- * is set in that case (unless the error occurs in the page_read callback).
+ * Returns false if the buffer already contains the requested data, or found
+ * error. state->page_verified is set to true for the former and false for the
+ * latter.
*
- * We fetch the page from a reader-local cache if we know we have the required
- * data and if there hasn't been any error since caching the data.
+ * Otherwise returns true and requests data loaded onto state->readBuf by
+ * state->readPagePtr and state->readLen. The caller shall call this function
+ * again after filling the buffer at least with that portion of data and set
+ * state->readLen to the length of actually loaded data.
+ *
+ * If header_inclusive is false, corrects reqLen internally by adding the
+ * actual page header length and may request caller for new data.
*/
-static int
-ReadPageInternal(XLogReaderState *state, XLogRecPtr pageptr, int reqLen)
+static bool
+XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
+ bool header_inclusive)
{
- int readLen;
uint32 targetPageOff;
XLogSegNo targetSegNo;
- XLogPageHeader hdr;
+ uint32 addLen = 0;
- Assert((pageptr % XLOG_BLCKSZ) == 0);
+ /* Some data is loaded, but page header is not verified yet. */
+ if (!state->page_verified &&
+ !XLogRecPtrIsInvalid(state->readPagePtr) && state->readLen >= 0)
+ {
+ uint32 pageHeaderSize;
+ /* just loaded new data so needs to verify page header */
+
+ /* The caller must have loaded at least page header */
+ Assert (state->readLen >= SizeOfXLogShortPHD);
+
+ /*
+ * We have enough data to check the header length. Recheck the loaded
+ * length against the actual header length.
+ */
+ pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+
+ /* Request more data if we don't have the full header. */
+ if (state->readLen < pageHeaderSize)
+ {
+ state->readLen = pageHeaderSize;
+ return true;
+ }
+
+ /* Now that we know we have the full header, validate it. */
+ if (!XLogReaderValidatePageHeader(state, state->readPagePtr,
+ (char *) state->readBuf))
+ {
+ /* That's bad. Force reading the page again. */
+ XLogReaderInvalReadState(state);
+
+ return false;
+ }
+
+ state->page_verified = true;
+
+ XLByteToSeg(state->readPagePtr, state->seg.ws_segno,
+ state->segcxt.ws_segsize);
+ }
+
+ /*
+ * The loaded page may not be the one caller is supposing to read when we
+ * are verifying the first page of new segment. In that case, skip further
+ * verification and immediately load the target page.
+ */
+ if (state->page_verified && pageptr == state->readPagePtr)
+ {
+ /*
+ * calculate additional length for page header keeping the total
+ * length within the block size.
+ */
+ if (!header_inclusive)
+ {
+ uint32 pageHeaderSize =
+ XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+
+ addLen = pageHeaderSize;
+ if (reqLen + pageHeaderSize <= XLOG_BLCKSZ)
+ addLen = pageHeaderSize;
+ else
+ addLen = XLOG_BLCKSZ - reqLen;
+
+ Assert(addLen >= 0);
+ }
+
+ /* Return if we already have it. */
+ if (reqLen + addLen <= state->readLen)
+ return false;
+ }
+
+ /* Data is not in our buffer, request the caller for it. */
XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize);
targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize);
+ Assert((pageptr % XLOG_BLCKSZ) == 0);
- /* check whether we have all the requested data already */
- if (targetSegNo == state->seg.ws_segno &&
- targetPageOff == state->segoff && reqLen <= state->readLen)
- return state->readLen;
+ /*
+ * Every time we request to load new data of a page to the caller, even if
+ * we looked at a part of it before, we need to do verification on the next
+ * invocation as the caller might now be rereading data from a different
+ * source.
+ */
+ state->page_verified = false;
/*
- * Data is not in our buffer.
- *
- * Every time we actually read the segment, even if we looked at parts of
- * it before, we need to do verification as the page_read callback might
- * now be rereading data from a different source.
- *
* Whenever switching to a new WAL segment, we read the first page of the
* file and validate its header, even if that's not where the target
* record is. This is so that we can check the additional identification
* info that is present in the first page's "long" header.
+ * Don't do this if the caller requested the first page in the segment.
*/
if (targetSegNo != state->seg.ws_segno && targetPageOff != 0)
{
- XLogRecPtr targetSegmentPtr = pageptr - targetPageOff;
-
- readLen = state->routine.page_read(state, targetSegmentPtr, XLOG_BLCKSZ,
- state->currRecPtr,
- state->readBuf);
- if (readLen < 0)
- goto err;
-
- /* we can be sure to have enough WAL available, we scrolled back */
- Assert(readLen == XLOG_BLCKSZ);
-
- if (!XLogReaderValidatePageHeader(state, targetSegmentPtr,
- state->readBuf))
- goto err;
+ /*
+ * Then we'll see that the targetSegNo now matches the ws_segno, and
+ * will not come back here, but will request the actual target page.
+ */
+ state->readPagePtr = pageptr - targetPageOff;
+ state->readLen = XLOG_BLCKSZ;
+ return true;
}
/*
- * First, read the requested data length, but at least a short page header
- * so that we can validate it.
+ * Request the caller to load the page. We need at least a short page
+ * header so that we can validate it.
*/
- readLen = state->routine.page_read(state, pageptr, Max(reqLen, SizeOfXLogShortPHD),
- state->currRecPtr,
- state->readBuf);
- if (readLen < 0)
- goto err;
-
- Assert(readLen <= XLOG_BLCKSZ);
-
- /* Do we have enough data to check the header length? */
- if (readLen <= SizeOfXLogShortPHD)
- goto err;
-
- Assert(readLen >= reqLen);
-
- hdr = (XLogPageHeader) state->readBuf;
-
- /* still not enough */
- if (readLen < XLogPageHeaderSize(hdr))
- {
- readLen = state->routine.page_read(state, pageptr, XLogPageHeaderSize(hdr),
- state->currRecPtr,
- state->readBuf);
- if (readLen < 0)
- goto err;
- }
-
- /*
- * Now that we know we have the full header, validate it.
- */
- if (!XLogReaderValidatePageHeader(state, pageptr, (char *) hdr))
- goto err;
-
- /* update read state information */
- state->seg.ws_segno = targetSegNo;
- state->segoff = targetPageOff;
- state->readLen = readLen;
-
- return readLen;
-
-err:
- XLogReaderInvalReadState(state);
- return -1;
+ state->readPagePtr = pageptr;
+ state->readLen = Max(reqLen + addLen, SizeOfXLogShortPHD);
+ return true;
}
/*
@@ -673,9 +716,7 @@ err:
static void
XLogReaderInvalReadState(XLogReaderState *state)
{
- state->seg.ws_segno = 0;
- state->segoff = 0;
- state->readLen = 0;
+ state->readPagePtr = InvalidXLogRecPtr;
}
/*
@@ -956,7 +997,6 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
XLogRecPtr targetPagePtr;
int targetRecOff;
uint32 pageHeaderSize;
- int readLen;
/*
* Compute targetRecOff. It should typically be equal or greater than
@@ -964,7 +1004,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
* that, except when caller has explicitly specified the offset that
* falls somewhere there or when we are skipping multi-page
* continuation record. It doesn't matter though because
- * ReadPageInternal() is prepared to handle that and will read at
+ * XLogNeedData() is prepared to handle that and will read at
* least short page-header worth of data
*/
targetRecOff = tmpRecPtr % XLOG_BLCKSZ;
@@ -972,19 +1012,24 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
/* scroll back to page boundary */
targetPagePtr = tmpRecPtr - targetRecOff;
- /* Read the page containing the record */
- readLen = ReadPageInternal(state, targetPagePtr, targetRecOff);
- if (readLen < 0)
+ while(XLogNeedData(state, targetPagePtr, targetRecOff,
+ targetRecOff != 0))
+ {
+ if (!state->routine.page_read(state, state->readPagePtr,
+ state->readLen,
+ state->ReadRecPtr, state->readBuf))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
header = (XLogPageHeader) state->readBuf;
pageHeaderSize = XLogPageHeaderSize(header);
- /* make sure we have enough data for the page header */
- readLen = ReadPageInternal(state, targetPagePtr, pageHeaderSize);
- if (readLen < 0)
- goto err;
+ /* we should have read the page header */
+ Assert (state->readLen >= pageHeaderSize);
/* skip over potential continuation data */
if (header->xlp_info & XLP_FIRST_IS_CONTRECORD)
diff --git a/src/backend/access/transam/xlogutils.c b/src/backend/access/transam/xlogutils.c
index 322b0e8ff5..1b10b6df20 100644
--- a/src/backend/access/transam/xlogutils.c
+++ b/src/backend/access/transam/xlogutils.c
@@ -689,8 +689,8 @@ XLogTruncateRelation(RelFileNode rnode, ForkNumber forkNum,
void
XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wantLength)
{
- const XLogRecPtr lastReadPage = (state->seg.ws_segno *
- state->segcxt.ws_segsize + state->segoff);
+ const XLogRecPtr lastReadPage = state->seg.ws_segno *
+ state->segcxt.ws_segsize + state->readLen;
Assert(wantPage != InvalidXLogRecPtr && wantPage % XLOG_BLCKSZ == 0);
Assert(wantLength <= XLOG_BLCKSZ);
@@ -828,7 +828,7 @@ wal_segment_close(XLogReaderState *state)
* exists for normal backends, so we have to do a check/sleep/repeat style of
* loop for now.
*/
-int
+bool
read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *cur_page)
{
@@ -930,7 +930,8 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
else if (targetPagePtr + reqLen > read_upto)
{
/* not enough data there */
- return -1;
+ state->readLen = -1;
+ return false;
}
else
{
@@ -948,7 +949,8 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
WALReadRaiseError(&errinfo);
/* number of valid bytes in the buffer */
- return count;
+ state->readLen = count;
+ return true;
}
/*
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index e2477c47e0..bf2711bddd 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -811,7 +811,7 @@ StartReplication(StartReplicationCmd *cmd)
* which has to do a plain sleep/busy loop, because the walsender's latch gets
* set every time WAL is flushed.
*/
-static int
+static bool
logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *cur_page)
{
@@ -831,7 +831,10 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
/* fail if not (implies we are going to shut down) */
if (flushptr < targetPagePtr + reqLen)
- return -1;
+ {
+ state->readLen = -1;
+ return false;
+ }
if (targetPagePtr + XLOG_BLCKSZ <= flushptr)
count = XLOG_BLCKSZ; /* more than one block available */
@@ -859,7 +862,8 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
XLByteToSeg(targetPagePtr, segno, state->segcxt.ws_segsize);
CheckXLogRemoved(segno, state->seg.ws_tli);
- return count;
+ state->readLen = count;
+ return true;
}
/*
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index bc6f976994..0b7e73ae79 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -46,7 +46,7 @@ typedef struct XLogPageReadPrivate
int tliIndex;
} XLogPageReadPrivate;
-static int SimpleXLogPageRead(XLogReaderState *xlogreader,
+static bool SimpleXLogPageRead(XLogReaderState *xlogreader,
XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
@@ -236,7 +236,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
}
/* XLogReader callback function, to read a WAL page */
-static int
+static bool
SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf)
{
@@ -296,7 +296,8 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
if (private->restoreCommand == NULL)
{
pg_log_error("could not open file \"%s\": %m", xlogfpath);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
/*
@@ -309,7 +310,10 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
private->restoreCommand);
if (xlogreadfd < 0)
- return -1;
+ {
+ xlogreader->readLen = -1;
+ return false;
+ }
else
pg_log_debug("using file \"%s\" restored from archive",
xlogfpath);
@@ -325,7 +329,8 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
if (lseek(xlogreadfd, (off_t) targetPageOff, SEEK_SET) < 0)
{
pg_log_error("could not seek in file \"%s\": %m", xlogfpath);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
@@ -338,13 +343,15 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
pg_log_error("could not read file \"%s\": read %d of %zu",
xlogfpath, r, (Size) XLOG_BLCKSZ);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
Assert(targetSegNo == xlogreadsegno);
xlogreader->seg.ws_tli = targetHistory[private->tliIndex].tli;
- return XLOG_BLCKSZ;
+ xlogreader->readLen = XLOG_BLCKSZ;
+ return true;
}
/*
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index d1a0678935..ab0c28d9b7 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -331,7 +331,7 @@ WALDumpCloseSegment(XLogReaderState *state)
}
/* pg_waldump's XLogReaderRoutine->page_read callback */
-static int
+static bool
WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetPtr, char *readBuff)
{
@@ -348,7 +348,8 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
else
{
private->endptr_reached = true;
- return -1;
+ state->readLen = -1;
+ return false;
}
}
@@ -373,7 +374,8 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
(Size) errinfo.wre_req);
}
- return count;
+ state->readLen = count;
+ return true;
}
/*
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index b0f2a6ed43..6c8848e14f 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -57,8 +57,8 @@ typedef struct WALSegmentContext
typedef struct XLogReaderState XLogReaderState;
-/* Function type definitions for various xlogreader interactions */
-typedef int (*XLogPageReadCB) (XLogReaderState *xlogreader,
+/* Function type definition for the read_page callback */
+typedef bool (*XLogPageReadCB) (XLogReaderState *xlogreader,
XLogRecPtr targetPagePtr,
int reqLen,
XLogRecPtr targetRecPtr,
@@ -175,6 +175,20 @@ struct XLogReaderState
XLogRecPtr ReadRecPtr; /* start of last record read */
XLogRecPtr EndRecPtr; /* end+1 of last record read */
+ /* ----------------------------------------
+ * Communication with page reader
+ * readBuf is XLOG_BLCKSZ bytes, valid up to at least readLen bytes.
+ * ----------------------------------------
+ */
+ /* variables to communicate with page reader */
+ XLogRecPtr readPagePtr; /* page pointer to read */
+ int32 readLen; /* bytes requested to reader, or actual bytes
+ * read by reader, which must be larger than
+ * the request, or -1 on error */
+ TimeLineID readPageTLI; /* TLI for data currently in readBuf */
+ char *readBuf; /* buffer to store data */
+ bool page_verified; /* is the page on the buffer verified? */
+
/* ----------------------------------------
* Decoded representation of current record
@@ -201,13 +215,6 @@ struct XLogReaderState
* ----------------------------------------
*/
- /*
- * Buffer for currently read page (XLOG_BLCKSZ bytes, valid up to at least
- * readLen bytes)
- */
- char *readBuf;
- uint32 readLen;
-
/* last read XLOG position for data currently in readBuf */
WALSegmentContext segcxt;
WALOpenSegment seg;
diff --git a/src/include/access/xlogutils.h b/src/include/access/xlogutils.h
index e59b6cf3a9..c287edf206 100644
--- a/src/include/access/xlogutils.h
+++ b/src/include/access/xlogutils.h
@@ -47,7 +47,7 @@ extern Buffer XLogReadBufferExtended(RelFileNode rnode, ForkNumber forknum,
extern Relation CreateFakeRelcacheEntry(RelFileNode rnode);
extern void FreeFakeRelcacheEntry(Relation fakerel);
-extern int read_local_xlog_page(XLogReaderState *state,
+extern bool read_local_xlog_page(XLogReaderState *state,
XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *cur_page);
extern void wal_segment_open(XLogReaderState *state,
--
2.18.4
----Next_Part(Thu_Jul__2_13_53_30_2020_072)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
filename="v15-0002-Move-page-reader-out-of-XLogReadRecord.patch"
^ permalink raw reply [nested|flat] 69+ messages in thread
* [PATCH v17 1/4] Move callback-call from ReadPageInternal to XLogReadRecord.
@ 2019-09-05 11:21 Kyotaro Horiguchi <[email protected]>
0 siblings, 0 replies; 69+ messages in thread
From: Kyotaro Horiguchi @ 2019-09-05 11:21 UTC (permalink / raw)
The current WAL record reader reads page data using a call back
function. Although it is not so problematic alone, it would be a
problem if we are going to do add tasks like encryption which is
performed on page data before WAL reader reads them. To avoid that the
record reader facility has to have a new code path corresponds to
every new callback, this patch separates page reader from WAL record
reading facility by modifying the current WAL record reader to a state
machine.
As the first step of that change, this patch moves the page reader
function out of ReadPageInternal, then the remaining tasks of the
function are taken over by the new function XLogNeedData. As the
result XLogPageRead directly calls the page reader callback function
according to the feedback from XLogNeedData.
---
src/backend/access/transam/xlog.c | 16 +-
src/backend/access/transam/xlogreader.c | 281 ++++++++++++++----------
src/backend/access/transam/xlogutils.c | 12 +-
src/backend/replication/walsender.c | 10 +-
src/bin/pg_rewind/parsexlog.c | 21 +-
src/bin/pg_waldump/pg_waldump.c | 8 +-
src/include/access/xlogreader.h | 25 ++-
src/include/access/xlogutils.h | 2 +-
8 files changed, 222 insertions(+), 153 deletions(-)
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 377afb8732..606328a65a 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -917,7 +917,7 @@ static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath,
static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli,
XLogSource source, bool notfoundOk);
static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, XLogSource source);
-static int XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
+static bool XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
bool fetching_ckpt, XLogRecPtr tliRecPtr);
@@ -4350,7 +4350,6 @@ ReadRecord(XLogReaderState *xlogreader, int emode,
XLogRecord *record;
XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
- /* Pass through parameters to XLogPageRead */
private->fetching_ckpt = fetching_ckpt;
private->emode = emode;
private->randAccess = (xlogreader->ReadRecPtr == InvalidXLogRecPtr);
@@ -11987,7 +11986,7 @@ CancelBackup(void)
* XLogPageRead() to try fetching the record from another source, or to
* sleep and retry.
*/
-static int
+static bool
XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *readBuf)
{
@@ -12046,7 +12045,8 @@ retry:
readLen = 0;
readSource = XLOG_FROM_ANY;
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
}
@@ -12141,7 +12141,8 @@ retry:
goto next_record_is_invalid;
}
- return readLen;
+ xlogreader->readLen = readLen;
+ return true;
next_record_is_invalid:
lastSourceFailed = true;
@@ -12155,8 +12156,9 @@ next_record_is_invalid:
/* In standby-mode, keep trying */
if (StandbyMode)
goto retry;
- else
- return -1;
+
+ xlogreader->readLen = -1;
+ return false;
}
/*
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index 42738eb940..7d04995b7b 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -36,8 +36,8 @@
static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
pg_attribute_printf(2, 3);
static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
-static int ReadPageInternal(XLogReaderState *state, XLogRecPtr pageptr,
- int reqLen);
+static bool XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr,
+ int reqLen, bool header_inclusive);
static void XLogReaderInvalReadState(XLogReaderState *state);
static bool ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
XLogRecPtr PrevRecPtr, XLogRecord *record, bool randAccess);
@@ -276,7 +276,6 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
uint32 targetRecOff;
uint32 pageHeaderSize;
bool gotheader;
- int readOff;
/*
* randAccess indicates whether to verify the previous-record pointer of
@@ -326,14 +325,20 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
* byte to cover the whole record header, or at least the part of it that
* fits on the same page.
*/
- readOff = ReadPageInternal(state, targetPagePtr,
- Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ));
- if (readOff < 0)
+ while (XLogNeedData(state, targetPagePtr,
+ Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+ targetRecOff != 0))
+ {
+ if (!state->routine.page_read(state, state->readPagePtr, state->readLen,
+ RecPtr, state->readBuf))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
/*
- * ReadPageInternal always returns at least the page header, so we can
- * examine it now.
+ * We have at least the page header, so we can examine it now.
*/
pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
if (targetRecOff == 0)
@@ -359,8 +364,8 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
goto err;
}
- /* ReadPageInternal has verified the page header */
- Assert(pageHeaderSize <= readOff);
+ /* XLogNeedData has verified the page header */
+ Assert(pageHeaderSize <= state->readLen);
/*
* Read the record length.
@@ -432,18 +437,27 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
do
{
+ int rest_len = total_len - gotlen;
+
/* Calculate pointer to beginning of next page */
targetPagePtr += XLOG_BLCKSZ;
/* Wait for the next page to become available */
- readOff = ReadPageInternal(state, targetPagePtr,
- Min(total_len - gotlen + SizeOfXLogShortPHD,
- XLOG_BLCKSZ));
+ while (XLogNeedData(state, targetPagePtr,
+ Min(rest_len, XLOG_BLCKSZ),
+ false))
+ {
+ if (!state->routine.page_read(state, state->readPagePtr,
+ state->readLen,
+ state->ReadRecPtr,
+ state->readBuf))
+ break;
+ }
- if (readOff < 0)
+ if (!state->page_verified)
goto err;
- Assert(SizeOfXLogShortPHD <= readOff);
+ Assert(SizeOfXLogShortPHD <= state->readLen);
/* Check that the continuation on next page looks valid */
pageHeader = (XLogPageHeader) state->readBuf;
@@ -473,21 +487,14 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
/* Append the continuation from this page to the buffer */
pageHeaderSize = XLogPageHeaderSize(pageHeader);
- if (readOff < pageHeaderSize)
- readOff = ReadPageInternal(state, targetPagePtr,
- pageHeaderSize);
-
- Assert(pageHeaderSize <= readOff);
+ Assert(pageHeaderSize <= state->readLen);
contdata = (char *) state->readBuf + pageHeaderSize;
len = XLOG_BLCKSZ - pageHeaderSize;
if (pageHeader->xlp_rem_len < len)
len = pageHeader->xlp_rem_len;
- if (readOff < pageHeaderSize + len)
- readOff = ReadPageInternal(state, targetPagePtr,
- pageHeaderSize + len);
-
+ Assert (pageHeaderSize + len <= state->readLen);
memcpy(buffer, (char *) contdata, len);
buffer += len;
gotlen += len;
@@ -517,9 +524,16 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
else
{
/* Wait for the record data to become available */
- readOff = ReadPageInternal(state, targetPagePtr,
- Min(targetRecOff + total_len, XLOG_BLCKSZ));
- if (readOff < 0)
+ while (XLogNeedData(state, targetPagePtr,
+ Min(targetRecOff + total_len, XLOG_BLCKSZ), true))
+ {
+ if (!state->routine.page_read(state, state->readPagePtr,
+ state->readLen,
+ state->ReadRecPtr, state->readBuf))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
/* Record does not cross a page boundary */
@@ -562,109 +576,138 @@ err:
}
/*
- * Read a single xlog page including at least [pageptr, reqLen] of valid data
- * via the page_read() callback.
+ * Checks that an xlog page loaded in state->readBuf is including at least
+ * [pageptr, reqLen] and the page is valid. header_inclusive indicates that
+ * reqLen is calculated including page header length.
*
- * Returns -1 if the required page cannot be read for some reason; errormsg_buf
- * is set in that case (unless the error occurs in the page_read callback).
+ * Returns false if the buffer already contains the requested data, or found
+ * error. state->page_verified is set to true for the former and false for the
+ * latter.
*
- * We fetch the page from a reader-local cache if we know we have the required
- * data and if there hasn't been any error since caching the data.
+ * Otherwise returns true and requests data loaded onto state->readBuf by
+ * state->readPagePtr and state->readLen. The caller shall call this function
+ * again after filling the buffer at least with that portion of data and set
+ * state->readLen to the length of actually loaded data.
+ *
+ * If header_inclusive is false, corrects reqLen internally by adding the
+ * actual page header length and may request caller for new data.
*/
-static int
-ReadPageInternal(XLogReaderState *state, XLogRecPtr pageptr, int reqLen)
+static bool
+XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
+ bool header_inclusive)
{
- int readLen;
uint32 targetPageOff;
XLogSegNo targetSegNo;
- XLogPageHeader hdr;
+ uint32 addLen = 0;
- Assert((pageptr % XLOG_BLCKSZ) == 0);
+ /* Some data is loaded, but page header is not verified yet. */
+ if (!state->page_verified &&
+ !XLogRecPtrIsInvalid(state->readPagePtr) && state->readLen >= 0)
+ {
+ uint32 pageHeaderSize;
+ /* just loaded new data so needs to verify page header */
+
+ /* The caller must have loaded at least page header */
+ Assert (state->readLen >= SizeOfXLogShortPHD);
+
+ /*
+ * We have enough data to check the header length. Recheck the loaded
+ * length against the actual header length.
+ */
+ pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+
+ /* Request more data if we don't have the full header. */
+ if (state->readLen < pageHeaderSize)
+ {
+ state->readLen = pageHeaderSize;
+ return true;
+ }
+
+ /* Now that we know we have the full header, validate it. */
+ if (!XLogReaderValidatePageHeader(state, state->readPagePtr,
+ (char *) state->readBuf))
+ {
+ /* That's bad. Force reading the page again. */
+ XLogReaderInvalReadState(state);
+
+ return false;
+ }
+
+ state->page_verified = true;
+
+ XLByteToSeg(state->readPagePtr, state->seg.ws_segno,
+ state->segcxt.ws_segsize);
+ }
+
+ /*
+ * The loaded page may not be the one caller is supposing to read when we
+ * are verifying the first page of new segment. In that case, skip further
+ * verification and immediately load the target page.
+ */
+ if (state->page_verified && pageptr == state->readPagePtr)
+ {
+ /*
+ * calculate additional length for page header keeping the total
+ * length within the block size.
+ */
+ if (!header_inclusive)
+ {
+ uint32 pageHeaderSize =
+ XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+
+ addLen = pageHeaderSize;
+ if (reqLen + pageHeaderSize <= XLOG_BLCKSZ)
+ addLen = pageHeaderSize;
+ else
+ addLen = XLOG_BLCKSZ - reqLen;
+
+ Assert(addLen >= 0);
+ }
+
+ /* Return if we already have it. */
+ if (reqLen + addLen <= state->readLen)
+ return false;
+ }
+
+ /* Data is not in our buffer, request the caller for it. */
XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize);
targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize);
+ Assert((pageptr % XLOG_BLCKSZ) == 0);
- /* check whether we have all the requested data already */
- if (targetSegNo == state->seg.ws_segno &&
- targetPageOff == state->segoff && reqLen <= state->readLen)
- return state->readLen;
+ /*
+ * Every time we request to load new data of a page to the caller, even if
+ * we looked at a part of it before, we need to do verification on the next
+ * invocation as the caller might now be rereading data from a different
+ * source.
+ */
+ state->page_verified = false;
/*
- * Data is not in our buffer.
- *
- * Every time we actually read the segment, even if we looked at parts of
- * it before, we need to do verification as the page_read callback might
- * now be rereading data from a different source.
- *
* Whenever switching to a new WAL segment, we read the first page of the
* file and validate its header, even if that's not where the target
* record is. This is so that we can check the additional identification
* info that is present in the first page's "long" header.
+ * Don't do this if the caller requested the first page in the segment.
*/
if (targetSegNo != state->seg.ws_segno && targetPageOff != 0)
{
- XLogRecPtr targetSegmentPtr = pageptr - targetPageOff;
-
- readLen = state->routine.page_read(state, targetSegmentPtr, XLOG_BLCKSZ,
- state->currRecPtr,
- state->readBuf);
- if (readLen < 0)
- goto err;
-
- /* we can be sure to have enough WAL available, we scrolled back */
- Assert(readLen == XLOG_BLCKSZ);
-
- if (!XLogReaderValidatePageHeader(state, targetSegmentPtr,
- state->readBuf))
- goto err;
+ /*
+ * Then we'll see that the targetSegNo now matches the ws_segno, and
+ * will not come back here, but will request the actual target page.
+ */
+ state->readPagePtr = pageptr - targetPageOff;
+ state->readLen = XLOG_BLCKSZ;
+ return true;
}
/*
- * First, read the requested data length, but at least a short page header
- * so that we can validate it.
+ * Request the caller to load the page. We need at least a short page
+ * header so that we can validate it.
*/
- readLen = state->routine.page_read(state, pageptr, Max(reqLen, SizeOfXLogShortPHD),
- state->currRecPtr,
- state->readBuf);
- if (readLen < 0)
- goto err;
-
- Assert(readLen <= XLOG_BLCKSZ);
-
- /* Do we have enough data to check the header length? */
- if (readLen <= SizeOfXLogShortPHD)
- goto err;
-
- Assert(readLen >= reqLen);
-
- hdr = (XLogPageHeader) state->readBuf;
-
- /* still not enough */
- if (readLen < XLogPageHeaderSize(hdr))
- {
- readLen = state->routine.page_read(state, pageptr, XLogPageHeaderSize(hdr),
- state->currRecPtr,
- state->readBuf);
- if (readLen < 0)
- goto err;
- }
-
- /*
- * Now that we know we have the full header, validate it.
- */
- if (!XLogReaderValidatePageHeader(state, pageptr, (char *) hdr))
- goto err;
-
- /* update read state information */
- state->seg.ws_segno = targetSegNo;
- state->segoff = targetPageOff;
- state->readLen = readLen;
-
- return readLen;
-
-err:
- XLogReaderInvalReadState(state);
- return -1;
+ state->readPagePtr = pageptr;
+ state->readLen = Max(reqLen + addLen, SizeOfXLogShortPHD);
+ return true;
}
/*
@@ -673,9 +716,7 @@ err:
static void
XLogReaderInvalReadState(XLogReaderState *state)
{
- state->seg.ws_segno = 0;
- state->segoff = 0;
- state->readLen = 0;
+ state->readPagePtr = InvalidXLogRecPtr;
}
/*
@@ -953,7 +994,6 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
XLogRecPtr targetPagePtr;
int targetRecOff;
uint32 pageHeaderSize;
- int readLen;
/*
* Compute targetRecOff. It should typically be equal or greater than
@@ -961,7 +1001,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
* that, except when caller has explicitly specified the offset that
* falls somewhere there or when we are skipping multi-page
* continuation record. It doesn't matter though because
- * ReadPageInternal() is prepared to handle that and will read at
+ * XLogNeedData() is prepared to handle that and will read at
* least short page-header worth of data
*/
targetRecOff = tmpRecPtr % XLOG_BLCKSZ;
@@ -969,19 +1009,24 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
/* scroll back to page boundary */
targetPagePtr = tmpRecPtr - targetRecOff;
- /* Read the page containing the record */
- readLen = ReadPageInternal(state, targetPagePtr, targetRecOff);
- if (readLen < 0)
+ while(XLogNeedData(state, targetPagePtr, targetRecOff,
+ targetRecOff != 0))
+ {
+ if (!state->routine.page_read(state, state->readPagePtr,
+ state->readLen,
+ state->ReadRecPtr, state->readBuf))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
header = (XLogPageHeader) state->readBuf;
pageHeaderSize = XLogPageHeaderSize(header);
- /* make sure we have enough data for the page header */
- readLen = ReadPageInternal(state, targetPagePtr, pageHeaderSize);
- if (readLen < 0)
- goto err;
+ /* we should have read the page header */
+ Assert (state->readLen >= pageHeaderSize);
/* skip over potential continuation data */
if (header->xlp_info & XLP_FIRST_IS_CONTRECORD)
diff --git a/src/backend/access/transam/xlogutils.c b/src/backend/access/transam/xlogutils.c
index d17d660f46..46eda33f25 100644
--- a/src/backend/access/transam/xlogutils.c
+++ b/src/backend/access/transam/xlogutils.c
@@ -686,8 +686,8 @@ XLogTruncateRelation(RelFileNode rnode, ForkNumber forkNum,
void
XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wantLength)
{
- const XLogRecPtr lastReadPage = (state->seg.ws_segno *
- state->segcxt.ws_segsize + state->segoff);
+ const XLogRecPtr lastReadPage = state->seg.ws_segno *
+ state->segcxt.ws_segsize + state->readLen;
Assert(wantPage != InvalidXLogRecPtr && wantPage % XLOG_BLCKSZ == 0);
Assert(wantLength <= XLOG_BLCKSZ);
@@ -824,7 +824,7 @@ wal_segment_close(XLogReaderState *state)
* exists for normal backends, so we have to do a check/sleep/repeat style of
* loop for now.
*/
-int
+bool
read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *cur_page)
{
@@ -926,7 +926,8 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
else if (targetPagePtr + reqLen > read_upto)
{
/* not enough data there */
- return -1;
+ state->readLen = -1;
+ return false;
}
else
{
@@ -944,7 +945,8 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
WALReadRaiseError(&errinfo);
/* number of valid bytes in the buffer */
- return count;
+ state->readLen = count;
+ return true;
}
/*
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index 23baa4498a..c36b91ae5e 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -806,7 +806,7 @@ StartReplication(StartReplicationCmd *cmd)
* which has to do a plain sleep/busy loop, because the walsender's latch gets
* set every time WAL is flushed.
*/
-static int
+static bool
logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *cur_page)
{
@@ -826,7 +826,10 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
/* fail if not (implies we are going to shut down) */
if (flushptr < targetPagePtr + reqLen)
- return -1;
+ {
+ state->readLen = -1;
+ return false;
+ }
if (targetPagePtr + XLOG_BLCKSZ <= flushptr)
count = XLOG_BLCKSZ; /* more than one block available */
@@ -854,7 +857,8 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
XLByteToSeg(targetPagePtr, segno, state->segcxt.ws_segsize);
CheckXLogRemoved(segno, state->seg.ws_tli);
- return count;
+ state->readLen = count;
+ return true;
}
/*
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index 59ebac7d6a..cf119848b0 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -47,7 +47,7 @@ typedef struct XLogPageReadPrivate
int tliIndex;
} XLogPageReadPrivate;
-static int SimpleXLogPageRead(XLogReaderState *xlogreader,
+static bool SimpleXLogPageRead(XLogReaderState *xlogreader,
XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
@@ -246,7 +246,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
}
/* XLogReader callback function, to read a WAL page */
-static int
+static bool
SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf)
{
@@ -306,7 +306,8 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
if (private->restoreCommand == NULL)
{
pg_log_error("could not open file \"%s\": %m", xlogfpath);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
/*
@@ -319,7 +320,10 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
private->restoreCommand);
if (xlogreadfd < 0)
- return -1;
+ {
+ xlogreader->readLen = -1;
+ return false;
+ }
else
pg_log_debug("using file \"%s\" restored from archive",
xlogfpath);
@@ -335,7 +339,8 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
if (lseek(xlogreadfd, (off_t) targetPageOff, SEEK_SET) < 0)
{
pg_log_error("could not seek in file \"%s\": %m", xlogfpath);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
@@ -348,13 +353,15 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
pg_log_error("could not read file \"%s\": read %d of %zu",
xlogfpath, r, (Size) XLOG_BLCKSZ);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
Assert(targetSegNo == xlogreadsegno);
xlogreader->seg.ws_tli = targetHistory[private->tliIndex].tli;
- return XLOG_BLCKSZ;
+ xlogreader->readLen = XLOG_BLCKSZ;
+ return true;
}
/*
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index 610f65e471..0f0e0723a3 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -331,7 +331,7 @@ WALDumpCloseSegment(XLogReaderState *state)
}
/* pg_waldump's XLogReaderRoutine->page_read callback */
-static int
+static bool
WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetPtr, char *readBuff)
{
@@ -348,7 +348,8 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
else
{
private->endptr_reached = true;
- return -1;
+ state->readLen = -1;
+ return false;
}
}
@@ -373,7 +374,8 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
(Size) errinfo.wre_req);
}
- return count;
+ state->readLen = count;
+ return true;
}
/*
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index 21d200d3df..23d9e70a59 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -57,8 +57,8 @@ typedef struct WALSegmentContext
typedef struct XLogReaderState XLogReaderState;
-/* Function type definitions for various xlogreader interactions */
-typedef int (*XLogPageReadCB) (XLogReaderState *xlogreader,
+/* Function type definition for the read_page callback */
+typedef bool (*XLogPageReadCB) (XLogReaderState *xlogreader,
XLogRecPtr targetPagePtr,
int reqLen,
XLogRecPtr targetRecPtr,
@@ -175,6 +175,20 @@ struct XLogReaderState
XLogRecPtr ReadRecPtr; /* start of last record read */
XLogRecPtr EndRecPtr; /* end+1 of last record read */
+ /* ----------------------------------------
+ * Communication with page reader
+ * readBuf is XLOG_BLCKSZ bytes, valid up to at least readLen bytes.
+ * ----------------------------------------
+ */
+ /* variables to communicate with page reader */
+ XLogRecPtr readPagePtr; /* page pointer to read */
+ int32 readLen; /* bytes requested to reader, or actual bytes
+ * read by reader, which must be larger than
+ * the request, or -1 on error */
+ TimeLineID readPageTLI; /* TLI for data currently in readBuf */
+ char *readBuf; /* buffer to store data */
+ bool page_verified; /* is the page on the buffer verified? */
+
/* ----------------------------------------
* Decoded representation of current record
@@ -203,13 +217,6 @@ struct XLogReaderState
* ----------------------------------------
*/
- /*
- * Buffer for currently read page (XLOG_BLCKSZ bytes, valid up to at least
- * readLen bytes)
- */
- char *readBuf;
- uint32 readLen;
-
/* last read XLOG position for data currently in readBuf */
WALSegmentContext segcxt;
WALOpenSegment seg;
diff --git a/src/include/access/xlogutils.h b/src/include/access/xlogutils.h
index 9ac602b674..364a21c4ea 100644
--- a/src/include/access/xlogutils.h
+++ b/src/include/access/xlogutils.h
@@ -47,7 +47,7 @@ extern Buffer XLogReadBufferExtended(RelFileNode rnode, ForkNumber forknum,
extern Relation CreateFakeRelcacheEntry(RelFileNode rnode);
extern void FreeFakeRelcacheEntry(Relation fakerel);
-extern int read_local_xlog_page(XLogReaderState *state,
+extern bool read_local_xlog_page(XLogReaderState *state,
XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *cur_page);
extern void wal_segment_open(XLogReaderState *state,
--
2.27.0
----Next_Part(Thu_Mar__4_11_28_53_2021_014)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
filename="v17-0002-Move-page-reader-out-of-XLogReadRecord.patch"
^ permalink raw reply [nested|flat] 69+ messages in thread
* [PATCH v15 1/4] Move callback-call from ReadPageInternal to XLogReadRecord.
@ 2019-09-05 11:21 Kyotaro Horiguchi <[email protected]>
0 siblings, 0 replies; 69+ messages in thread
From: Kyotaro Horiguchi @ 2019-09-05 11:21 UTC (permalink / raw)
The current WAL record reader reads page data using a call back
function. Although it is not so problematic alone, it would be a
problem if we are going to do add tasks like encryption which is
performed on page data before WAL reader reads them. To avoid that the
record reader facility has to have a new code path corresponds to
every new callback, this patch separates page reader from WAL record
reading facility by modifying the current WAL record reader to a state
machine.
As the first step of that change, this patch moves the page reader
function out of ReadPageInternal, then the remaining tasks of the
function are taken over by the new function XLogNeedData. As the
result XLogPageRead directly calls the page reader callback function
according to the feedback from XLogNeedData.
---
src/backend/access/transam/xlog.c | 16 +-
src/backend/access/transam/xlogreader.c | 281 ++++++++++++++----------
src/backend/access/transam/xlogutils.c | 12 +-
src/backend/replication/walsender.c | 10 +-
src/bin/pg_rewind/parsexlog.c | 21 +-
src/bin/pg_waldump/pg_waldump.c | 8 +-
src/include/access/xlogreader.h | 25 ++-
src/include/access/xlogutils.h | 2 +-
8 files changed, 222 insertions(+), 153 deletions(-)
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index fd93bcfaeb..e570e56a24 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -912,7 +912,7 @@ static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath,
static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli,
XLogSource source, bool notfoundOk);
static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, XLogSource source);
-static int XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
+static bool XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
bool fetching_ckpt, XLogRecPtr tliRecPtr);
@@ -4332,7 +4332,6 @@ ReadRecord(XLogReaderState *xlogreader, int emode,
XLogRecord *record;
XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
- /* Pass through parameters to XLogPageRead */
private->fetching_ckpt = fetching_ckpt;
private->emode = emode;
private->randAccess = (xlogreader->ReadRecPtr == InvalidXLogRecPtr);
@@ -11860,7 +11859,7 @@ CancelBackup(void)
* XLogPageRead() to try fetching the record from another source, or to
* sleep and retry.
*/
-static int
+static bool
XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *readBuf)
{
@@ -11919,7 +11918,8 @@ retry:
readLen = 0;
readSource = XLOG_FROM_ANY;
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
}
@@ -12014,7 +12014,8 @@ retry:
goto next_record_is_invalid;
}
- return readLen;
+ xlogreader->readLen = readLen;
+ return true;
next_record_is_invalid:
lastSourceFailed = true;
@@ -12028,8 +12029,9 @@ next_record_is_invalid:
/* In standby-mode, keep trying */
if (StandbyMode)
goto retry;
- else
- return -1;
+
+ xlogreader->readLen = -1;
+ return false;
}
/*
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index cb76be4f46..3d599325ee 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -36,8 +36,8 @@
static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
pg_attribute_printf(2, 3);
static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
-static int ReadPageInternal(XLogReaderState *state, XLogRecPtr pageptr,
- int reqLen);
+static bool XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr,
+ int reqLen, bool header_inclusive);
static void XLogReaderInvalReadState(XLogReaderState *state);
static bool ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
XLogRecPtr PrevRecPtr, XLogRecord *record, bool randAccess);
@@ -276,7 +276,6 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
uint32 targetRecOff;
uint32 pageHeaderSize;
bool gotheader;
- int readOff;
/*
* randAccess indicates whether to verify the previous-record pointer of
@@ -326,14 +325,20 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
* byte to cover the whole record header, or at least the part of it that
* fits on the same page.
*/
- readOff = ReadPageInternal(state, targetPagePtr,
- Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ));
- if (readOff < 0)
+ while (XLogNeedData(state, targetPagePtr,
+ Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+ targetRecOff != 0))
+ {
+ if (!state->routine.page_read(state, state->readPagePtr, state->readLen,
+ RecPtr, state->readBuf))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
/*
- * ReadPageInternal always returns at least the page header, so we can
- * examine it now.
+ * We have at least the page header, so we can examine it now.
*/
pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
if (targetRecOff == 0)
@@ -359,8 +364,8 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
goto err;
}
- /* ReadPageInternal has verified the page header */
- Assert(pageHeaderSize <= readOff);
+ /* XLogNeedData has verified the page header */
+ Assert(pageHeaderSize <= state->readLen);
/*
* Read the record length.
@@ -433,18 +438,27 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
do
{
+ int rest_len = total_len - gotlen;
+
/* Calculate pointer to beginning of next page */
targetPagePtr += XLOG_BLCKSZ;
/* Wait for the next page to become available */
- readOff = ReadPageInternal(state, targetPagePtr,
- Min(total_len - gotlen + SizeOfXLogShortPHD,
- XLOG_BLCKSZ));
+ while (XLogNeedData(state, targetPagePtr,
+ Min(rest_len, XLOG_BLCKSZ),
+ false))
+ {
+ if (!state->routine.page_read(state, state->readPagePtr,
+ state->readLen,
+ state->ReadRecPtr,
+ state->readBuf))
+ break;
+ }
- if (readOff < 0)
+ if (!state->page_verified)
goto err;
- Assert(SizeOfXLogShortPHD <= readOff);
+ Assert(SizeOfXLogShortPHD <= state->readLen);
/* Check that the continuation on next page looks valid */
pageHeader = (XLogPageHeader) state->readBuf;
@@ -473,21 +487,14 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
/* Append the continuation from this page to the buffer */
pageHeaderSize = XLogPageHeaderSize(pageHeader);
- if (readOff < pageHeaderSize)
- readOff = ReadPageInternal(state, targetPagePtr,
- pageHeaderSize);
-
- Assert(pageHeaderSize <= readOff);
+ Assert(pageHeaderSize <= state->readLen);
contdata = (char *) state->readBuf + pageHeaderSize;
len = XLOG_BLCKSZ - pageHeaderSize;
if (pageHeader->xlp_rem_len < len)
len = pageHeader->xlp_rem_len;
- if (readOff < pageHeaderSize + len)
- readOff = ReadPageInternal(state, targetPagePtr,
- pageHeaderSize + len);
-
+ Assert (pageHeaderSize + len <= state->readLen);
memcpy(buffer, (char *) contdata, len);
buffer += len;
gotlen += len;
@@ -517,9 +524,16 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
else
{
/* Wait for the record data to become available */
- readOff = ReadPageInternal(state, targetPagePtr,
- Min(targetRecOff + total_len, XLOG_BLCKSZ));
- if (readOff < 0)
+ while (XLogNeedData(state, targetPagePtr,
+ Min(targetRecOff + total_len, XLOG_BLCKSZ), true))
+ {
+ if (!state->routine.page_read(state, state->readPagePtr,
+ state->readLen,
+ state->ReadRecPtr, state->readBuf))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
/* Record does not cross a page boundary */
@@ -562,109 +576,138 @@ err:
}
/*
- * Read a single xlog page including at least [pageptr, reqLen] of valid data
- * via the page_read() callback.
+ * Checks that an xlog page loaded in state->readBuf is including at least
+ * [pageptr, reqLen] and the page is valid. header_inclusive indicates that
+ * reqLen is calculated including page header length.
*
- * Returns -1 if the required page cannot be read for some reason; errormsg_buf
- * is set in that case (unless the error occurs in the page_read callback).
+ * Returns false if the buffer already contains the requested data, or found
+ * error. state->page_verified is set to true for the former and false for the
+ * latter.
*
- * We fetch the page from a reader-local cache if we know we have the required
- * data and if there hasn't been any error since caching the data.
+ * Otherwise returns true and requests data loaded onto state->readBuf by
+ * state->readPagePtr and state->readLen. The caller shall call this function
+ * again after filling the buffer at least with that portion of data and set
+ * state->readLen to the length of actually loaded data.
+ *
+ * If header_inclusive is false, corrects reqLen internally by adding the
+ * actual page header length and may request caller for new data.
*/
-static int
-ReadPageInternal(XLogReaderState *state, XLogRecPtr pageptr, int reqLen)
+static bool
+XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
+ bool header_inclusive)
{
- int readLen;
uint32 targetPageOff;
XLogSegNo targetSegNo;
- XLogPageHeader hdr;
+ uint32 addLen = 0;
- Assert((pageptr % XLOG_BLCKSZ) == 0);
+ /* Some data is loaded, but page header is not verified yet. */
+ if (!state->page_verified &&
+ !XLogRecPtrIsInvalid(state->readPagePtr) && state->readLen >= 0)
+ {
+ uint32 pageHeaderSize;
+ /* just loaded new data so needs to verify page header */
+
+ /* The caller must have loaded at least page header */
+ Assert (state->readLen >= SizeOfXLogShortPHD);
+
+ /*
+ * We have enough data to check the header length. Recheck the loaded
+ * length against the actual header length.
+ */
+ pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+
+ /* Request more data if we don't have the full header. */
+ if (state->readLen < pageHeaderSize)
+ {
+ state->readLen = pageHeaderSize;
+ return true;
+ }
+
+ /* Now that we know we have the full header, validate it. */
+ if (!XLogReaderValidatePageHeader(state, state->readPagePtr,
+ (char *) state->readBuf))
+ {
+ /* That's bad. Force reading the page again. */
+ XLogReaderInvalReadState(state);
+
+ return false;
+ }
+
+ state->page_verified = true;
+
+ XLByteToSeg(state->readPagePtr, state->seg.ws_segno,
+ state->segcxt.ws_segsize);
+ }
+
+ /*
+ * The loaded page may not be the one caller is supposing to read when we
+ * are verifying the first page of new segment. In that case, skip further
+ * verification and immediately load the target page.
+ */
+ if (state->page_verified && pageptr == state->readPagePtr)
+ {
+ /*
+ * calculate additional length for page header keeping the total
+ * length within the block size.
+ */
+ if (!header_inclusive)
+ {
+ uint32 pageHeaderSize =
+ XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+
+ addLen = pageHeaderSize;
+ if (reqLen + pageHeaderSize <= XLOG_BLCKSZ)
+ addLen = pageHeaderSize;
+ else
+ addLen = XLOG_BLCKSZ - reqLen;
+
+ Assert(addLen >= 0);
+ }
+
+ /* Return if we already have it. */
+ if (reqLen + addLen <= state->readLen)
+ return false;
+ }
+
+ /* Data is not in our buffer, request the caller for it. */
XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize);
targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize);
+ Assert((pageptr % XLOG_BLCKSZ) == 0);
- /* check whether we have all the requested data already */
- if (targetSegNo == state->seg.ws_segno &&
- targetPageOff == state->segoff && reqLen <= state->readLen)
- return state->readLen;
+ /*
+ * Every time we request to load new data of a page to the caller, even if
+ * we looked at a part of it before, we need to do verification on the next
+ * invocation as the caller might now be rereading data from a different
+ * source.
+ */
+ state->page_verified = false;
/*
- * Data is not in our buffer.
- *
- * Every time we actually read the segment, even if we looked at parts of
- * it before, we need to do verification as the page_read callback might
- * now be rereading data from a different source.
- *
* Whenever switching to a new WAL segment, we read the first page of the
* file and validate its header, even if that's not where the target
* record is. This is so that we can check the additional identification
* info that is present in the first page's "long" header.
+ * Don't do this if the caller requested the first page in the segment.
*/
if (targetSegNo != state->seg.ws_segno && targetPageOff != 0)
{
- XLogRecPtr targetSegmentPtr = pageptr - targetPageOff;
-
- readLen = state->routine.page_read(state, targetSegmentPtr, XLOG_BLCKSZ,
- state->currRecPtr,
- state->readBuf);
- if (readLen < 0)
- goto err;
-
- /* we can be sure to have enough WAL available, we scrolled back */
- Assert(readLen == XLOG_BLCKSZ);
-
- if (!XLogReaderValidatePageHeader(state, targetSegmentPtr,
- state->readBuf))
- goto err;
+ /*
+ * Then we'll see that the targetSegNo now matches the ws_segno, and
+ * will not come back here, but will request the actual target page.
+ */
+ state->readPagePtr = pageptr - targetPageOff;
+ state->readLen = XLOG_BLCKSZ;
+ return true;
}
/*
- * First, read the requested data length, but at least a short page header
- * so that we can validate it.
+ * Request the caller to load the page. We need at least a short page
+ * header so that we can validate it.
*/
- readLen = state->routine.page_read(state, pageptr, Max(reqLen, SizeOfXLogShortPHD),
- state->currRecPtr,
- state->readBuf);
- if (readLen < 0)
- goto err;
-
- Assert(readLen <= XLOG_BLCKSZ);
-
- /* Do we have enough data to check the header length? */
- if (readLen <= SizeOfXLogShortPHD)
- goto err;
-
- Assert(readLen >= reqLen);
-
- hdr = (XLogPageHeader) state->readBuf;
-
- /* still not enough */
- if (readLen < XLogPageHeaderSize(hdr))
- {
- readLen = state->routine.page_read(state, pageptr, XLogPageHeaderSize(hdr),
- state->currRecPtr,
- state->readBuf);
- if (readLen < 0)
- goto err;
- }
-
- /*
- * Now that we know we have the full header, validate it.
- */
- if (!XLogReaderValidatePageHeader(state, pageptr, (char *) hdr))
- goto err;
-
- /* update read state information */
- state->seg.ws_segno = targetSegNo;
- state->segoff = targetPageOff;
- state->readLen = readLen;
-
- return readLen;
-
-err:
- XLogReaderInvalReadState(state);
- return -1;
+ state->readPagePtr = pageptr;
+ state->readLen = Max(reqLen + addLen, SizeOfXLogShortPHD);
+ return true;
}
/*
@@ -673,9 +716,7 @@ err:
static void
XLogReaderInvalReadState(XLogReaderState *state)
{
- state->seg.ws_segno = 0;
- state->segoff = 0;
- state->readLen = 0;
+ state->readPagePtr = InvalidXLogRecPtr;
}
/*
@@ -956,7 +997,6 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
XLogRecPtr targetPagePtr;
int targetRecOff;
uint32 pageHeaderSize;
- int readLen;
/*
* Compute targetRecOff. It should typically be equal or greater than
@@ -964,7 +1004,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
* that, except when caller has explicitly specified the offset that
* falls somewhere there or when we are skipping multi-page
* continuation record. It doesn't matter though because
- * ReadPageInternal() is prepared to handle that and will read at
+ * XLogNeedData() is prepared to handle that and will read at
* least short page-header worth of data
*/
targetRecOff = tmpRecPtr % XLOG_BLCKSZ;
@@ -972,19 +1012,24 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
/* scroll back to page boundary */
targetPagePtr = tmpRecPtr - targetRecOff;
- /* Read the page containing the record */
- readLen = ReadPageInternal(state, targetPagePtr, targetRecOff);
- if (readLen < 0)
+ while(XLogNeedData(state, targetPagePtr, targetRecOff,
+ targetRecOff != 0))
+ {
+ if (!state->routine.page_read(state, state->readPagePtr,
+ state->readLen,
+ state->ReadRecPtr, state->readBuf))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
header = (XLogPageHeader) state->readBuf;
pageHeaderSize = XLogPageHeaderSize(header);
- /* make sure we have enough data for the page header */
- readLen = ReadPageInternal(state, targetPagePtr, pageHeaderSize);
- if (readLen < 0)
- goto err;
+ /* we should have read the page header */
+ Assert (state->readLen >= pageHeaderSize);
/* skip over potential continuation data */
if (header->xlp_info & XLP_FIRST_IS_CONTRECORD)
diff --git a/src/backend/access/transam/xlogutils.c b/src/backend/access/transam/xlogutils.c
index 322b0e8ff5..1b10b6df20 100644
--- a/src/backend/access/transam/xlogutils.c
+++ b/src/backend/access/transam/xlogutils.c
@@ -689,8 +689,8 @@ XLogTruncateRelation(RelFileNode rnode, ForkNumber forkNum,
void
XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wantLength)
{
- const XLogRecPtr lastReadPage = (state->seg.ws_segno *
- state->segcxt.ws_segsize + state->segoff);
+ const XLogRecPtr lastReadPage = state->seg.ws_segno *
+ state->segcxt.ws_segsize + state->readLen;
Assert(wantPage != InvalidXLogRecPtr && wantPage % XLOG_BLCKSZ == 0);
Assert(wantLength <= XLOG_BLCKSZ);
@@ -828,7 +828,7 @@ wal_segment_close(XLogReaderState *state)
* exists for normal backends, so we have to do a check/sleep/repeat style of
* loop for now.
*/
-int
+bool
read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *cur_page)
{
@@ -930,7 +930,8 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
else if (targetPagePtr + reqLen > read_upto)
{
/* not enough data there */
- return -1;
+ state->readLen = -1;
+ return false;
}
else
{
@@ -948,7 +949,8 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
WALReadRaiseError(&errinfo);
/* number of valid bytes in the buffer */
- return count;
+ state->readLen = count;
+ return true;
}
/*
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index e2477c47e0..bf2711bddd 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -811,7 +811,7 @@ StartReplication(StartReplicationCmd *cmd)
* which has to do a plain sleep/busy loop, because the walsender's latch gets
* set every time WAL is flushed.
*/
-static int
+static bool
logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *cur_page)
{
@@ -831,7 +831,10 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
/* fail if not (implies we are going to shut down) */
if (flushptr < targetPagePtr + reqLen)
- return -1;
+ {
+ state->readLen = -1;
+ return false;
+ }
if (targetPagePtr + XLOG_BLCKSZ <= flushptr)
count = XLOG_BLCKSZ; /* more than one block available */
@@ -859,7 +862,8 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
XLByteToSeg(targetPagePtr, segno, state->segcxt.ws_segsize);
CheckXLogRemoved(segno, state->seg.ws_tli);
- return count;
+ state->readLen = count;
+ return true;
}
/*
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index bc6f976994..0b7e73ae79 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -46,7 +46,7 @@ typedef struct XLogPageReadPrivate
int tliIndex;
} XLogPageReadPrivate;
-static int SimpleXLogPageRead(XLogReaderState *xlogreader,
+static bool SimpleXLogPageRead(XLogReaderState *xlogreader,
XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
@@ -236,7 +236,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
}
/* XLogReader callback function, to read a WAL page */
-static int
+static bool
SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf)
{
@@ -296,7 +296,8 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
if (private->restoreCommand == NULL)
{
pg_log_error("could not open file \"%s\": %m", xlogfpath);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
/*
@@ -309,7 +310,10 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
private->restoreCommand);
if (xlogreadfd < 0)
- return -1;
+ {
+ xlogreader->readLen = -1;
+ return false;
+ }
else
pg_log_debug("using file \"%s\" restored from archive",
xlogfpath);
@@ -325,7 +329,8 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
if (lseek(xlogreadfd, (off_t) targetPageOff, SEEK_SET) < 0)
{
pg_log_error("could not seek in file \"%s\": %m", xlogfpath);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
@@ -338,13 +343,15 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
pg_log_error("could not read file \"%s\": read %d of %zu",
xlogfpath, r, (Size) XLOG_BLCKSZ);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
Assert(targetSegNo == xlogreadsegno);
xlogreader->seg.ws_tli = targetHistory[private->tliIndex].tli;
- return XLOG_BLCKSZ;
+ xlogreader->readLen = XLOG_BLCKSZ;
+ return true;
}
/*
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index d1a0678935..ab0c28d9b7 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -331,7 +331,7 @@ WALDumpCloseSegment(XLogReaderState *state)
}
/* pg_waldump's XLogReaderRoutine->page_read callback */
-static int
+static bool
WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetPtr, char *readBuff)
{
@@ -348,7 +348,8 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
else
{
private->endptr_reached = true;
- return -1;
+ state->readLen = -1;
+ return false;
}
}
@@ -373,7 +374,8 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
(Size) errinfo.wre_req);
}
- return count;
+ state->readLen = count;
+ return true;
}
/*
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index b0f2a6ed43..6c8848e14f 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -57,8 +57,8 @@ typedef struct WALSegmentContext
typedef struct XLogReaderState XLogReaderState;
-/* Function type definitions for various xlogreader interactions */
-typedef int (*XLogPageReadCB) (XLogReaderState *xlogreader,
+/* Function type definition for the read_page callback */
+typedef bool (*XLogPageReadCB) (XLogReaderState *xlogreader,
XLogRecPtr targetPagePtr,
int reqLen,
XLogRecPtr targetRecPtr,
@@ -175,6 +175,20 @@ struct XLogReaderState
XLogRecPtr ReadRecPtr; /* start of last record read */
XLogRecPtr EndRecPtr; /* end+1 of last record read */
+ /* ----------------------------------------
+ * Communication with page reader
+ * readBuf is XLOG_BLCKSZ bytes, valid up to at least readLen bytes.
+ * ----------------------------------------
+ */
+ /* variables to communicate with page reader */
+ XLogRecPtr readPagePtr; /* page pointer to read */
+ int32 readLen; /* bytes requested to reader, or actual bytes
+ * read by reader, which must be larger than
+ * the request, or -1 on error */
+ TimeLineID readPageTLI; /* TLI for data currently in readBuf */
+ char *readBuf; /* buffer to store data */
+ bool page_verified; /* is the page on the buffer verified? */
+
/* ----------------------------------------
* Decoded representation of current record
@@ -201,13 +215,6 @@ struct XLogReaderState
* ----------------------------------------
*/
- /*
- * Buffer for currently read page (XLOG_BLCKSZ bytes, valid up to at least
- * readLen bytes)
- */
- char *readBuf;
- uint32 readLen;
-
/* last read XLOG position for data currently in readBuf */
WALSegmentContext segcxt;
WALOpenSegment seg;
diff --git a/src/include/access/xlogutils.h b/src/include/access/xlogutils.h
index e59b6cf3a9..c287edf206 100644
--- a/src/include/access/xlogutils.h
+++ b/src/include/access/xlogutils.h
@@ -47,7 +47,7 @@ extern Buffer XLogReadBufferExtended(RelFileNode rnode, ForkNumber forknum,
extern Relation CreateFakeRelcacheEntry(RelFileNode rnode);
extern void FreeFakeRelcacheEntry(Relation fakerel);
-extern int read_local_xlog_page(XLogReaderState *state,
+extern bool read_local_xlog_page(XLogReaderState *state,
XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *cur_page);
extern void wal_segment_open(XLogReaderState *state,
--
2.18.4
----Next_Part(Thu_Jul__2_13_53_30_2020_072)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
filename="v15-0002-Move-page-reader-out-of-XLogReadRecord.patch"
^ permalink raw reply [nested|flat] 69+ messages in thread
* [PATCH v17 1/4] Move callback-call from ReadPageInternal to XLogReadRecord.
@ 2019-09-05 11:21 Kyotaro Horiguchi <[email protected]>
0 siblings, 0 replies; 69+ messages in thread
From: Kyotaro Horiguchi @ 2019-09-05 11:21 UTC (permalink / raw)
The current WAL record reader reads page data using a call back
function. Although it is not so problematic alone, it would be a
problem if we are going to do add tasks like encryption which is
performed on page data before WAL reader reads them. To avoid that the
record reader facility has to have a new code path corresponds to
every new callback, this patch separates page reader from WAL record
reading facility by modifying the current WAL record reader to a state
machine.
As the first step of that change, this patch moves the page reader
function out of ReadPageInternal, then the remaining tasks of the
function are taken over by the new function XLogNeedData. As the
result XLogPageRead directly calls the page reader callback function
according to the feedback from XLogNeedData.
---
src/backend/access/transam/xlog.c | 16 +-
src/backend/access/transam/xlogreader.c | 281 ++++++++++++++----------
src/backend/access/transam/xlogutils.c | 12 +-
src/backend/replication/walsender.c | 10 +-
src/bin/pg_rewind/parsexlog.c | 21 +-
src/bin/pg_waldump/pg_waldump.c | 8 +-
src/include/access/xlogreader.h | 25 ++-
src/include/access/xlogutils.h | 2 +-
8 files changed, 222 insertions(+), 153 deletions(-)
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 377afb8732..606328a65a 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -917,7 +917,7 @@ static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath,
static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli,
XLogSource source, bool notfoundOk);
static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, XLogSource source);
-static int XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
+static bool XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
bool fetching_ckpt, XLogRecPtr tliRecPtr);
@@ -4350,7 +4350,6 @@ ReadRecord(XLogReaderState *xlogreader, int emode,
XLogRecord *record;
XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
- /* Pass through parameters to XLogPageRead */
private->fetching_ckpt = fetching_ckpt;
private->emode = emode;
private->randAccess = (xlogreader->ReadRecPtr == InvalidXLogRecPtr);
@@ -11987,7 +11986,7 @@ CancelBackup(void)
* XLogPageRead() to try fetching the record from another source, or to
* sleep and retry.
*/
-static int
+static bool
XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *readBuf)
{
@@ -12046,7 +12045,8 @@ retry:
readLen = 0;
readSource = XLOG_FROM_ANY;
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
}
@@ -12141,7 +12141,8 @@ retry:
goto next_record_is_invalid;
}
- return readLen;
+ xlogreader->readLen = readLen;
+ return true;
next_record_is_invalid:
lastSourceFailed = true;
@@ -12155,8 +12156,9 @@ next_record_is_invalid:
/* In standby-mode, keep trying */
if (StandbyMode)
goto retry;
- else
- return -1;
+
+ xlogreader->readLen = -1;
+ return false;
}
/*
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index 42738eb940..7d04995b7b 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -36,8 +36,8 @@
static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
pg_attribute_printf(2, 3);
static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
-static int ReadPageInternal(XLogReaderState *state, XLogRecPtr pageptr,
- int reqLen);
+static bool XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr,
+ int reqLen, bool header_inclusive);
static void XLogReaderInvalReadState(XLogReaderState *state);
static bool ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
XLogRecPtr PrevRecPtr, XLogRecord *record, bool randAccess);
@@ -276,7 +276,6 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
uint32 targetRecOff;
uint32 pageHeaderSize;
bool gotheader;
- int readOff;
/*
* randAccess indicates whether to verify the previous-record pointer of
@@ -326,14 +325,20 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
* byte to cover the whole record header, or at least the part of it that
* fits on the same page.
*/
- readOff = ReadPageInternal(state, targetPagePtr,
- Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ));
- if (readOff < 0)
+ while (XLogNeedData(state, targetPagePtr,
+ Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+ targetRecOff != 0))
+ {
+ if (!state->routine.page_read(state, state->readPagePtr, state->readLen,
+ RecPtr, state->readBuf))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
/*
- * ReadPageInternal always returns at least the page header, so we can
- * examine it now.
+ * We have at least the page header, so we can examine it now.
*/
pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
if (targetRecOff == 0)
@@ -359,8 +364,8 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
goto err;
}
- /* ReadPageInternal has verified the page header */
- Assert(pageHeaderSize <= readOff);
+ /* XLogNeedData has verified the page header */
+ Assert(pageHeaderSize <= state->readLen);
/*
* Read the record length.
@@ -432,18 +437,27 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
do
{
+ int rest_len = total_len - gotlen;
+
/* Calculate pointer to beginning of next page */
targetPagePtr += XLOG_BLCKSZ;
/* Wait for the next page to become available */
- readOff = ReadPageInternal(state, targetPagePtr,
- Min(total_len - gotlen + SizeOfXLogShortPHD,
- XLOG_BLCKSZ));
+ while (XLogNeedData(state, targetPagePtr,
+ Min(rest_len, XLOG_BLCKSZ),
+ false))
+ {
+ if (!state->routine.page_read(state, state->readPagePtr,
+ state->readLen,
+ state->ReadRecPtr,
+ state->readBuf))
+ break;
+ }
- if (readOff < 0)
+ if (!state->page_verified)
goto err;
- Assert(SizeOfXLogShortPHD <= readOff);
+ Assert(SizeOfXLogShortPHD <= state->readLen);
/* Check that the continuation on next page looks valid */
pageHeader = (XLogPageHeader) state->readBuf;
@@ -473,21 +487,14 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
/* Append the continuation from this page to the buffer */
pageHeaderSize = XLogPageHeaderSize(pageHeader);
- if (readOff < pageHeaderSize)
- readOff = ReadPageInternal(state, targetPagePtr,
- pageHeaderSize);
-
- Assert(pageHeaderSize <= readOff);
+ Assert(pageHeaderSize <= state->readLen);
contdata = (char *) state->readBuf + pageHeaderSize;
len = XLOG_BLCKSZ - pageHeaderSize;
if (pageHeader->xlp_rem_len < len)
len = pageHeader->xlp_rem_len;
- if (readOff < pageHeaderSize + len)
- readOff = ReadPageInternal(state, targetPagePtr,
- pageHeaderSize + len);
-
+ Assert (pageHeaderSize + len <= state->readLen);
memcpy(buffer, (char *) contdata, len);
buffer += len;
gotlen += len;
@@ -517,9 +524,16 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
else
{
/* Wait for the record data to become available */
- readOff = ReadPageInternal(state, targetPagePtr,
- Min(targetRecOff + total_len, XLOG_BLCKSZ));
- if (readOff < 0)
+ while (XLogNeedData(state, targetPagePtr,
+ Min(targetRecOff + total_len, XLOG_BLCKSZ), true))
+ {
+ if (!state->routine.page_read(state, state->readPagePtr,
+ state->readLen,
+ state->ReadRecPtr, state->readBuf))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
/* Record does not cross a page boundary */
@@ -562,109 +576,138 @@ err:
}
/*
- * Read a single xlog page including at least [pageptr, reqLen] of valid data
- * via the page_read() callback.
+ * Checks that an xlog page loaded in state->readBuf is including at least
+ * [pageptr, reqLen] and the page is valid. header_inclusive indicates that
+ * reqLen is calculated including page header length.
*
- * Returns -1 if the required page cannot be read for some reason; errormsg_buf
- * is set in that case (unless the error occurs in the page_read callback).
+ * Returns false if the buffer already contains the requested data, or found
+ * error. state->page_verified is set to true for the former and false for the
+ * latter.
*
- * We fetch the page from a reader-local cache if we know we have the required
- * data and if there hasn't been any error since caching the data.
+ * Otherwise returns true and requests data loaded onto state->readBuf by
+ * state->readPagePtr and state->readLen. The caller shall call this function
+ * again after filling the buffer at least with that portion of data and set
+ * state->readLen to the length of actually loaded data.
+ *
+ * If header_inclusive is false, corrects reqLen internally by adding the
+ * actual page header length and may request caller for new data.
*/
-static int
-ReadPageInternal(XLogReaderState *state, XLogRecPtr pageptr, int reqLen)
+static bool
+XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
+ bool header_inclusive)
{
- int readLen;
uint32 targetPageOff;
XLogSegNo targetSegNo;
- XLogPageHeader hdr;
+ uint32 addLen = 0;
- Assert((pageptr % XLOG_BLCKSZ) == 0);
+ /* Some data is loaded, but page header is not verified yet. */
+ if (!state->page_verified &&
+ !XLogRecPtrIsInvalid(state->readPagePtr) && state->readLen >= 0)
+ {
+ uint32 pageHeaderSize;
+ /* just loaded new data so needs to verify page header */
+
+ /* The caller must have loaded at least page header */
+ Assert (state->readLen >= SizeOfXLogShortPHD);
+
+ /*
+ * We have enough data to check the header length. Recheck the loaded
+ * length against the actual header length.
+ */
+ pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+
+ /* Request more data if we don't have the full header. */
+ if (state->readLen < pageHeaderSize)
+ {
+ state->readLen = pageHeaderSize;
+ return true;
+ }
+
+ /* Now that we know we have the full header, validate it. */
+ if (!XLogReaderValidatePageHeader(state, state->readPagePtr,
+ (char *) state->readBuf))
+ {
+ /* That's bad. Force reading the page again. */
+ XLogReaderInvalReadState(state);
+
+ return false;
+ }
+
+ state->page_verified = true;
+
+ XLByteToSeg(state->readPagePtr, state->seg.ws_segno,
+ state->segcxt.ws_segsize);
+ }
+
+ /*
+ * The loaded page may not be the one caller is supposing to read when we
+ * are verifying the first page of new segment. In that case, skip further
+ * verification and immediately load the target page.
+ */
+ if (state->page_verified && pageptr == state->readPagePtr)
+ {
+ /*
+ * calculate additional length for page header keeping the total
+ * length within the block size.
+ */
+ if (!header_inclusive)
+ {
+ uint32 pageHeaderSize =
+ XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+
+ addLen = pageHeaderSize;
+ if (reqLen + pageHeaderSize <= XLOG_BLCKSZ)
+ addLen = pageHeaderSize;
+ else
+ addLen = XLOG_BLCKSZ - reqLen;
+
+ Assert(addLen >= 0);
+ }
+
+ /* Return if we already have it. */
+ if (reqLen + addLen <= state->readLen)
+ return false;
+ }
+
+ /* Data is not in our buffer, request the caller for it. */
XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize);
targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize);
+ Assert((pageptr % XLOG_BLCKSZ) == 0);
- /* check whether we have all the requested data already */
- if (targetSegNo == state->seg.ws_segno &&
- targetPageOff == state->segoff && reqLen <= state->readLen)
- return state->readLen;
+ /*
+ * Every time we request to load new data of a page to the caller, even if
+ * we looked at a part of it before, we need to do verification on the next
+ * invocation as the caller might now be rereading data from a different
+ * source.
+ */
+ state->page_verified = false;
/*
- * Data is not in our buffer.
- *
- * Every time we actually read the segment, even if we looked at parts of
- * it before, we need to do verification as the page_read callback might
- * now be rereading data from a different source.
- *
* Whenever switching to a new WAL segment, we read the first page of the
* file and validate its header, even if that's not where the target
* record is. This is so that we can check the additional identification
* info that is present in the first page's "long" header.
+ * Don't do this if the caller requested the first page in the segment.
*/
if (targetSegNo != state->seg.ws_segno && targetPageOff != 0)
{
- XLogRecPtr targetSegmentPtr = pageptr - targetPageOff;
-
- readLen = state->routine.page_read(state, targetSegmentPtr, XLOG_BLCKSZ,
- state->currRecPtr,
- state->readBuf);
- if (readLen < 0)
- goto err;
-
- /* we can be sure to have enough WAL available, we scrolled back */
- Assert(readLen == XLOG_BLCKSZ);
-
- if (!XLogReaderValidatePageHeader(state, targetSegmentPtr,
- state->readBuf))
- goto err;
+ /*
+ * Then we'll see that the targetSegNo now matches the ws_segno, and
+ * will not come back here, but will request the actual target page.
+ */
+ state->readPagePtr = pageptr - targetPageOff;
+ state->readLen = XLOG_BLCKSZ;
+ return true;
}
/*
- * First, read the requested data length, but at least a short page header
- * so that we can validate it.
+ * Request the caller to load the page. We need at least a short page
+ * header so that we can validate it.
*/
- readLen = state->routine.page_read(state, pageptr, Max(reqLen, SizeOfXLogShortPHD),
- state->currRecPtr,
- state->readBuf);
- if (readLen < 0)
- goto err;
-
- Assert(readLen <= XLOG_BLCKSZ);
-
- /* Do we have enough data to check the header length? */
- if (readLen <= SizeOfXLogShortPHD)
- goto err;
-
- Assert(readLen >= reqLen);
-
- hdr = (XLogPageHeader) state->readBuf;
-
- /* still not enough */
- if (readLen < XLogPageHeaderSize(hdr))
- {
- readLen = state->routine.page_read(state, pageptr, XLogPageHeaderSize(hdr),
- state->currRecPtr,
- state->readBuf);
- if (readLen < 0)
- goto err;
- }
-
- /*
- * Now that we know we have the full header, validate it.
- */
- if (!XLogReaderValidatePageHeader(state, pageptr, (char *) hdr))
- goto err;
-
- /* update read state information */
- state->seg.ws_segno = targetSegNo;
- state->segoff = targetPageOff;
- state->readLen = readLen;
-
- return readLen;
-
-err:
- XLogReaderInvalReadState(state);
- return -1;
+ state->readPagePtr = pageptr;
+ state->readLen = Max(reqLen + addLen, SizeOfXLogShortPHD);
+ return true;
}
/*
@@ -673,9 +716,7 @@ err:
static void
XLogReaderInvalReadState(XLogReaderState *state)
{
- state->seg.ws_segno = 0;
- state->segoff = 0;
- state->readLen = 0;
+ state->readPagePtr = InvalidXLogRecPtr;
}
/*
@@ -953,7 +994,6 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
XLogRecPtr targetPagePtr;
int targetRecOff;
uint32 pageHeaderSize;
- int readLen;
/*
* Compute targetRecOff. It should typically be equal or greater than
@@ -961,7 +1001,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
* that, except when caller has explicitly specified the offset that
* falls somewhere there or when we are skipping multi-page
* continuation record. It doesn't matter though because
- * ReadPageInternal() is prepared to handle that and will read at
+ * XLogNeedData() is prepared to handle that and will read at
* least short page-header worth of data
*/
targetRecOff = tmpRecPtr % XLOG_BLCKSZ;
@@ -969,19 +1009,24 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
/* scroll back to page boundary */
targetPagePtr = tmpRecPtr - targetRecOff;
- /* Read the page containing the record */
- readLen = ReadPageInternal(state, targetPagePtr, targetRecOff);
- if (readLen < 0)
+ while(XLogNeedData(state, targetPagePtr, targetRecOff,
+ targetRecOff != 0))
+ {
+ if (!state->routine.page_read(state, state->readPagePtr,
+ state->readLen,
+ state->ReadRecPtr, state->readBuf))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
header = (XLogPageHeader) state->readBuf;
pageHeaderSize = XLogPageHeaderSize(header);
- /* make sure we have enough data for the page header */
- readLen = ReadPageInternal(state, targetPagePtr, pageHeaderSize);
- if (readLen < 0)
- goto err;
+ /* we should have read the page header */
+ Assert (state->readLen >= pageHeaderSize);
/* skip over potential continuation data */
if (header->xlp_info & XLP_FIRST_IS_CONTRECORD)
diff --git a/src/backend/access/transam/xlogutils.c b/src/backend/access/transam/xlogutils.c
index d17d660f46..46eda33f25 100644
--- a/src/backend/access/transam/xlogutils.c
+++ b/src/backend/access/transam/xlogutils.c
@@ -686,8 +686,8 @@ XLogTruncateRelation(RelFileNode rnode, ForkNumber forkNum,
void
XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wantLength)
{
- const XLogRecPtr lastReadPage = (state->seg.ws_segno *
- state->segcxt.ws_segsize + state->segoff);
+ const XLogRecPtr lastReadPage = state->seg.ws_segno *
+ state->segcxt.ws_segsize + state->readLen;
Assert(wantPage != InvalidXLogRecPtr && wantPage % XLOG_BLCKSZ == 0);
Assert(wantLength <= XLOG_BLCKSZ);
@@ -824,7 +824,7 @@ wal_segment_close(XLogReaderState *state)
* exists for normal backends, so we have to do a check/sleep/repeat style of
* loop for now.
*/
-int
+bool
read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *cur_page)
{
@@ -926,7 +926,8 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
else if (targetPagePtr + reqLen > read_upto)
{
/* not enough data there */
- return -1;
+ state->readLen = -1;
+ return false;
}
else
{
@@ -944,7 +945,8 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
WALReadRaiseError(&errinfo);
/* number of valid bytes in the buffer */
- return count;
+ state->readLen = count;
+ return true;
}
/*
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index 23baa4498a..c36b91ae5e 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -806,7 +806,7 @@ StartReplication(StartReplicationCmd *cmd)
* which has to do a plain sleep/busy loop, because the walsender's latch gets
* set every time WAL is flushed.
*/
-static int
+static bool
logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *cur_page)
{
@@ -826,7 +826,10 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
/* fail if not (implies we are going to shut down) */
if (flushptr < targetPagePtr + reqLen)
- return -1;
+ {
+ state->readLen = -1;
+ return false;
+ }
if (targetPagePtr + XLOG_BLCKSZ <= flushptr)
count = XLOG_BLCKSZ; /* more than one block available */
@@ -854,7 +857,8 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
XLByteToSeg(targetPagePtr, segno, state->segcxt.ws_segsize);
CheckXLogRemoved(segno, state->seg.ws_tli);
- return count;
+ state->readLen = count;
+ return true;
}
/*
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index 59ebac7d6a..cf119848b0 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -47,7 +47,7 @@ typedef struct XLogPageReadPrivate
int tliIndex;
} XLogPageReadPrivate;
-static int SimpleXLogPageRead(XLogReaderState *xlogreader,
+static bool SimpleXLogPageRead(XLogReaderState *xlogreader,
XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
@@ -246,7 +246,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
}
/* XLogReader callback function, to read a WAL page */
-static int
+static bool
SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf)
{
@@ -306,7 +306,8 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
if (private->restoreCommand == NULL)
{
pg_log_error("could not open file \"%s\": %m", xlogfpath);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
/*
@@ -319,7 +320,10 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
private->restoreCommand);
if (xlogreadfd < 0)
- return -1;
+ {
+ xlogreader->readLen = -1;
+ return false;
+ }
else
pg_log_debug("using file \"%s\" restored from archive",
xlogfpath);
@@ -335,7 +339,8 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
if (lseek(xlogreadfd, (off_t) targetPageOff, SEEK_SET) < 0)
{
pg_log_error("could not seek in file \"%s\": %m", xlogfpath);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
@@ -348,13 +353,15 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
pg_log_error("could not read file \"%s\": read %d of %zu",
xlogfpath, r, (Size) XLOG_BLCKSZ);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
Assert(targetSegNo == xlogreadsegno);
xlogreader->seg.ws_tli = targetHistory[private->tliIndex].tli;
- return XLOG_BLCKSZ;
+ xlogreader->readLen = XLOG_BLCKSZ;
+ return true;
}
/*
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index 610f65e471..0f0e0723a3 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -331,7 +331,7 @@ WALDumpCloseSegment(XLogReaderState *state)
}
/* pg_waldump's XLogReaderRoutine->page_read callback */
-static int
+static bool
WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetPtr, char *readBuff)
{
@@ -348,7 +348,8 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
else
{
private->endptr_reached = true;
- return -1;
+ state->readLen = -1;
+ return false;
}
}
@@ -373,7 +374,8 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
(Size) errinfo.wre_req);
}
- return count;
+ state->readLen = count;
+ return true;
}
/*
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index 21d200d3df..23d9e70a59 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -57,8 +57,8 @@ typedef struct WALSegmentContext
typedef struct XLogReaderState XLogReaderState;
-/* Function type definitions for various xlogreader interactions */
-typedef int (*XLogPageReadCB) (XLogReaderState *xlogreader,
+/* Function type definition for the read_page callback */
+typedef bool (*XLogPageReadCB) (XLogReaderState *xlogreader,
XLogRecPtr targetPagePtr,
int reqLen,
XLogRecPtr targetRecPtr,
@@ -175,6 +175,20 @@ struct XLogReaderState
XLogRecPtr ReadRecPtr; /* start of last record read */
XLogRecPtr EndRecPtr; /* end+1 of last record read */
+ /* ----------------------------------------
+ * Communication with page reader
+ * readBuf is XLOG_BLCKSZ bytes, valid up to at least readLen bytes.
+ * ----------------------------------------
+ */
+ /* variables to communicate with page reader */
+ XLogRecPtr readPagePtr; /* page pointer to read */
+ int32 readLen; /* bytes requested to reader, or actual bytes
+ * read by reader, which must be larger than
+ * the request, or -1 on error */
+ TimeLineID readPageTLI; /* TLI for data currently in readBuf */
+ char *readBuf; /* buffer to store data */
+ bool page_verified; /* is the page on the buffer verified? */
+
/* ----------------------------------------
* Decoded representation of current record
@@ -203,13 +217,6 @@ struct XLogReaderState
* ----------------------------------------
*/
- /*
- * Buffer for currently read page (XLOG_BLCKSZ bytes, valid up to at least
- * readLen bytes)
- */
- char *readBuf;
- uint32 readLen;
-
/* last read XLOG position for data currently in readBuf */
WALSegmentContext segcxt;
WALOpenSegment seg;
diff --git a/src/include/access/xlogutils.h b/src/include/access/xlogutils.h
index 9ac602b674..364a21c4ea 100644
--- a/src/include/access/xlogutils.h
+++ b/src/include/access/xlogutils.h
@@ -47,7 +47,7 @@ extern Buffer XLogReadBufferExtended(RelFileNode rnode, ForkNumber forknum,
extern Relation CreateFakeRelcacheEntry(RelFileNode rnode);
extern void FreeFakeRelcacheEntry(Relation fakerel);
-extern int read_local_xlog_page(XLogReaderState *state,
+extern bool read_local_xlog_page(XLogReaderState *state,
XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *cur_page);
extern void wal_segment_open(XLogReaderState *state,
--
2.27.0
----Next_Part(Thu_Mar__4_11_28_53_2021_014)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
filename="v17-0002-Move-page-reader-out-of-XLogReadRecord.patch"
^ permalink raw reply [nested|flat] 69+ messages in thread
* [PATCH v15 1/4] Move callback-call from ReadPageInternal to XLogReadRecord.
@ 2019-09-05 11:21 Kyotaro Horiguchi <[email protected]>
0 siblings, 0 replies; 69+ messages in thread
From: Kyotaro Horiguchi @ 2019-09-05 11:21 UTC (permalink / raw)
The current WAL record reader reads page data using a call back
function. Although it is not so problematic alone, it would be a
problem if we are going to do add tasks like encryption which is
performed on page data before WAL reader reads them. To avoid that the
record reader facility has to have a new code path corresponds to
every new callback, this patch separates page reader from WAL record
reading facility by modifying the current WAL record reader to a state
machine.
As the first step of that change, this patch moves the page reader
function out of ReadPageInternal, then the remaining tasks of the
function are taken over by the new function XLogNeedData. As the
result XLogPageRead directly calls the page reader callback function
according to the feedback from XLogNeedData.
---
src/backend/access/transam/xlog.c | 16 +-
src/backend/access/transam/xlogreader.c | 281 ++++++++++++++----------
src/backend/access/transam/xlogutils.c | 12 +-
src/backend/replication/walsender.c | 10 +-
src/bin/pg_rewind/parsexlog.c | 21 +-
src/bin/pg_waldump/pg_waldump.c | 8 +-
src/include/access/xlogreader.h | 25 ++-
src/include/access/xlogutils.h | 2 +-
8 files changed, 222 insertions(+), 153 deletions(-)
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index fd93bcfaeb..e570e56a24 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -912,7 +912,7 @@ static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath,
static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli,
XLogSource source, bool notfoundOk);
static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, XLogSource source);
-static int XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
+static bool XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
bool fetching_ckpt, XLogRecPtr tliRecPtr);
@@ -4332,7 +4332,6 @@ ReadRecord(XLogReaderState *xlogreader, int emode,
XLogRecord *record;
XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
- /* Pass through parameters to XLogPageRead */
private->fetching_ckpt = fetching_ckpt;
private->emode = emode;
private->randAccess = (xlogreader->ReadRecPtr == InvalidXLogRecPtr);
@@ -11860,7 +11859,7 @@ CancelBackup(void)
* XLogPageRead() to try fetching the record from another source, or to
* sleep and retry.
*/
-static int
+static bool
XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *readBuf)
{
@@ -11919,7 +11918,8 @@ retry:
readLen = 0;
readSource = XLOG_FROM_ANY;
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
}
@@ -12014,7 +12014,8 @@ retry:
goto next_record_is_invalid;
}
- return readLen;
+ xlogreader->readLen = readLen;
+ return true;
next_record_is_invalid:
lastSourceFailed = true;
@@ -12028,8 +12029,9 @@ next_record_is_invalid:
/* In standby-mode, keep trying */
if (StandbyMode)
goto retry;
- else
- return -1;
+
+ xlogreader->readLen = -1;
+ return false;
}
/*
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index cb76be4f46..3d599325ee 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -36,8 +36,8 @@
static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
pg_attribute_printf(2, 3);
static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
-static int ReadPageInternal(XLogReaderState *state, XLogRecPtr pageptr,
- int reqLen);
+static bool XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr,
+ int reqLen, bool header_inclusive);
static void XLogReaderInvalReadState(XLogReaderState *state);
static bool ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
XLogRecPtr PrevRecPtr, XLogRecord *record, bool randAccess);
@@ -276,7 +276,6 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
uint32 targetRecOff;
uint32 pageHeaderSize;
bool gotheader;
- int readOff;
/*
* randAccess indicates whether to verify the previous-record pointer of
@@ -326,14 +325,20 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
* byte to cover the whole record header, or at least the part of it that
* fits on the same page.
*/
- readOff = ReadPageInternal(state, targetPagePtr,
- Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ));
- if (readOff < 0)
+ while (XLogNeedData(state, targetPagePtr,
+ Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+ targetRecOff != 0))
+ {
+ if (!state->routine.page_read(state, state->readPagePtr, state->readLen,
+ RecPtr, state->readBuf))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
/*
- * ReadPageInternal always returns at least the page header, so we can
- * examine it now.
+ * We have at least the page header, so we can examine it now.
*/
pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
if (targetRecOff == 0)
@@ -359,8 +364,8 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
goto err;
}
- /* ReadPageInternal has verified the page header */
- Assert(pageHeaderSize <= readOff);
+ /* XLogNeedData has verified the page header */
+ Assert(pageHeaderSize <= state->readLen);
/*
* Read the record length.
@@ -433,18 +438,27 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
do
{
+ int rest_len = total_len - gotlen;
+
/* Calculate pointer to beginning of next page */
targetPagePtr += XLOG_BLCKSZ;
/* Wait for the next page to become available */
- readOff = ReadPageInternal(state, targetPagePtr,
- Min(total_len - gotlen + SizeOfXLogShortPHD,
- XLOG_BLCKSZ));
+ while (XLogNeedData(state, targetPagePtr,
+ Min(rest_len, XLOG_BLCKSZ),
+ false))
+ {
+ if (!state->routine.page_read(state, state->readPagePtr,
+ state->readLen,
+ state->ReadRecPtr,
+ state->readBuf))
+ break;
+ }
- if (readOff < 0)
+ if (!state->page_verified)
goto err;
- Assert(SizeOfXLogShortPHD <= readOff);
+ Assert(SizeOfXLogShortPHD <= state->readLen);
/* Check that the continuation on next page looks valid */
pageHeader = (XLogPageHeader) state->readBuf;
@@ -473,21 +487,14 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
/* Append the continuation from this page to the buffer */
pageHeaderSize = XLogPageHeaderSize(pageHeader);
- if (readOff < pageHeaderSize)
- readOff = ReadPageInternal(state, targetPagePtr,
- pageHeaderSize);
-
- Assert(pageHeaderSize <= readOff);
+ Assert(pageHeaderSize <= state->readLen);
contdata = (char *) state->readBuf + pageHeaderSize;
len = XLOG_BLCKSZ - pageHeaderSize;
if (pageHeader->xlp_rem_len < len)
len = pageHeader->xlp_rem_len;
- if (readOff < pageHeaderSize + len)
- readOff = ReadPageInternal(state, targetPagePtr,
- pageHeaderSize + len);
-
+ Assert (pageHeaderSize + len <= state->readLen);
memcpy(buffer, (char *) contdata, len);
buffer += len;
gotlen += len;
@@ -517,9 +524,16 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
else
{
/* Wait for the record data to become available */
- readOff = ReadPageInternal(state, targetPagePtr,
- Min(targetRecOff + total_len, XLOG_BLCKSZ));
- if (readOff < 0)
+ while (XLogNeedData(state, targetPagePtr,
+ Min(targetRecOff + total_len, XLOG_BLCKSZ), true))
+ {
+ if (!state->routine.page_read(state, state->readPagePtr,
+ state->readLen,
+ state->ReadRecPtr, state->readBuf))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
/* Record does not cross a page boundary */
@@ -562,109 +576,138 @@ err:
}
/*
- * Read a single xlog page including at least [pageptr, reqLen] of valid data
- * via the page_read() callback.
+ * Checks that an xlog page loaded in state->readBuf is including at least
+ * [pageptr, reqLen] and the page is valid. header_inclusive indicates that
+ * reqLen is calculated including page header length.
*
- * Returns -1 if the required page cannot be read for some reason; errormsg_buf
- * is set in that case (unless the error occurs in the page_read callback).
+ * Returns false if the buffer already contains the requested data, or found
+ * error. state->page_verified is set to true for the former and false for the
+ * latter.
*
- * We fetch the page from a reader-local cache if we know we have the required
- * data and if there hasn't been any error since caching the data.
+ * Otherwise returns true and requests data loaded onto state->readBuf by
+ * state->readPagePtr and state->readLen. The caller shall call this function
+ * again after filling the buffer at least with that portion of data and set
+ * state->readLen to the length of actually loaded data.
+ *
+ * If header_inclusive is false, corrects reqLen internally by adding the
+ * actual page header length and may request caller for new data.
*/
-static int
-ReadPageInternal(XLogReaderState *state, XLogRecPtr pageptr, int reqLen)
+static bool
+XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
+ bool header_inclusive)
{
- int readLen;
uint32 targetPageOff;
XLogSegNo targetSegNo;
- XLogPageHeader hdr;
+ uint32 addLen = 0;
- Assert((pageptr % XLOG_BLCKSZ) == 0);
+ /* Some data is loaded, but page header is not verified yet. */
+ if (!state->page_verified &&
+ !XLogRecPtrIsInvalid(state->readPagePtr) && state->readLen >= 0)
+ {
+ uint32 pageHeaderSize;
+ /* just loaded new data so needs to verify page header */
+
+ /* The caller must have loaded at least page header */
+ Assert (state->readLen >= SizeOfXLogShortPHD);
+
+ /*
+ * We have enough data to check the header length. Recheck the loaded
+ * length against the actual header length.
+ */
+ pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+
+ /* Request more data if we don't have the full header. */
+ if (state->readLen < pageHeaderSize)
+ {
+ state->readLen = pageHeaderSize;
+ return true;
+ }
+
+ /* Now that we know we have the full header, validate it. */
+ if (!XLogReaderValidatePageHeader(state, state->readPagePtr,
+ (char *) state->readBuf))
+ {
+ /* That's bad. Force reading the page again. */
+ XLogReaderInvalReadState(state);
+
+ return false;
+ }
+
+ state->page_verified = true;
+
+ XLByteToSeg(state->readPagePtr, state->seg.ws_segno,
+ state->segcxt.ws_segsize);
+ }
+
+ /*
+ * The loaded page may not be the one caller is supposing to read when we
+ * are verifying the first page of new segment. In that case, skip further
+ * verification and immediately load the target page.
+ */
+ if (state->page_verified && pageptr == state->readPagePtr)
+ {
+ /*
+ * calculate additional length for page header keeping the total
+ * length within the block size.
+ */
+ if (!header_inclusive)
+ {
+ uint32 pageHeaderSize =
+ XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+
+ addLen = pageHeaderSize;
+ if (reqLen + pageHeaderSize <= XLOG_BLCKSZ)
+ addLen = pageHeaderSize;
+ else
+ addLen = XLOG_BLCKSZ - reqLen;
+
+ Assert(addLen >= 0);
+ }
+
+ /* Return if we already have it. */
+ if (reqLen + addLen <= state->readLen)
+ return false;
+ }
+
+ /* Data is not in our buffer, request the caller for it. */
XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize);
targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize);
+ Assert((pageptr % XLOG_BLCKSZ) == 0);
- /* check whether we have all the requested data already */
- if (targetSegNo == state->seg.ws_segno &&
- targetPageOff == state->segoff && reqLen <= state->readLen)
- return state->readLen;
+ /*
+ * Every time we request to load new data of a page to the caller, even if
+ * we looked at a part of it before, we need to do verification on the next
+ * invocation as the caller might now be rereading data from a different
+ * source.
+ */
+ state->page_verified = false;
/*
- * Data is not in our buffer.
- *
- * Every time we actually read the segment, even if we looked at parts of
- * it before, we need to do verification as the page_read callback might
- * now be rereading data from a different source.
- *
* Whenever switching to a new WAL segment, we read the first page of the
* file and validate its header, even if that's not where the target
* record is. This is so that we can check the additional identification
* info that is present in the first page's "long" header.
+ * Don't do this if the caller requested the first page in the segment.
*/
if (targetSegNo != state->seg.ws_segno && targetPageOff != 0)
{
- XLogRecPtr targetSegmentPtr = pageptr - targetPageOff;
-
- readLen = state->routine.page_read(state, targetSegmentPtr, XLOG_BLCKSZ,
- state->currRecPtr,
- state->readBuf);
- if (readLen < 0)
- goto err;
-
- /* we can be sure to have enough WAL available, we scrolled back */
- Assert(readLen == XLOG_BLCKSZ);
-
- if (!XLogReaderValidatePageHeader(state, targetSegmentPtr,
- state->readBuf))
- goto err;
+ /*
+ * Then we'll see that the targetSegNo now matches the ws_segno, and
+ * will not come back here, but will request the actual target page.
+ */
+ state->readPagePtr = pageptr - targetPageOff;
+ state->readLen = XLOG_BLCKSZ;
+ return true;
}
/*
- * First, read the requested data length, but at least a short page header
- * so that we can validate it.
+ * Request the caller to load the page. We need at least a short page
+ * header so that we can validate it.
*/
- readLen = state->routine.page_read(state, pageptr, Max(reqLen, SizeOfXLogShortPHD),
- state->currRecPtr,
- state->readBuf);
- if (readLen < 0)
- goto err;
-
- Assert(readLen <= XLOG_BLCKSZ);
-
- /* Do we have enough data to check the header length? */
- if (readLen <= SizeOfXLogShortPHD)
- goto err;
-
- Assert(readLen >= reqLen);
-
- hdr = (XLogPageHeader) state->readBuf;
-
- /* still not enough */
- if (readLen < XLogPageHeaderSize(hdr))
- {
- readLen = state->routine.page_read(state, pageptr, XLogPageHeaderSize(hdr),
- state->currRecPtr,
- state->readBuf);
- if (readLen < 0)
- goto err;
- }
-
- /*
- * Now that we know we have the full header, validate it.
- */
- if (!XLogReaderValidatePageHeader(state, pageptr, (char *) hdr))
- goto err;
-
- /* update read state information */
- state->seg.ws_segno = targetSegNo;
- state->segoff = targetPageOff;
- state->readLen = readLen;
-
- return readLen;
-
-err:
- XLogReaderInvalReadState(state);
- return -1;
+ state->readPagePtr = pageptr;
+ state->readLen = Max(reqLen + addLen, SizeOfXLogShortPHD);
+ return true;
}
/*
@@ -673,9 +716,7 @@ err:
static void
XLogReaderInvalReadState(XLogReaderState *state)
{
- state->seg.ws_segno = 0;
- state->segoff = 0;
- state->readLen = 0;
+ state->readPagePtr = InvalidXLogRecPtr;
}
/*
@@ -956,7 +997,6 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
XLogRecPtr targetPagePtr;
int targetRecOff;
uint32 pageHeaderSize;
- int readLen;
/*
* Compute targetRecOff. It should typically be equal or greater than
@@ -964,7 +1004,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
* that, except when caller has explicitly specified the offset that
* falls somewhere there or when we are skipping multi-page
* continuation record. It doesn't matter though because
- * ReadPageInternal() is prepared to handle that and will read at
+ * XLogNeedData() is prepared to handle that and will read at
* least short page-header worth of data
*/
targetRecOff = tmpRecPtr % XLOG_BLCKSZ;
@@ -972,19 +1012,24 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
/* scroll back to page boundary */
targetPagePtr = tmpRecPtr - targetRecOff;
- /* Read the page containing the record */
- readLen = ReadPageInternal(state, targetPagePtr, targetRecOff);
- if (readLen < 0)
+ while(XLogNeedData(state, targetPagePtr, targetRecOff,
+ targetRecOff != 0))
+ {
+ if (!state->routine.page_read(state, state->readPagePtr,
+ state->readLen,
+ state->ReadRecPtr, state->readBuf))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
header = (XLogPageHeader) state->readBuf;
pageHeaderSize = XLogPageHeaderSize(header);
- /* make sure we have enough data for the page header */
- readLen = ReadPageInternal(state, targetPagePtr, pageHeaderSize);
- if (readLen < 0)
- goto err;
+ /* we should have read the page header */
+ Assert (state->readLen >= pageHeaderSize);
/* skip over potential continuation data */
if (header->xlp_info & XLP_FIRST_IS_CONTRECORD)
diff --git a/src/backend/access/transam/xlogutils.c b/src/backend/access/transam/xlogutils.c
index 322b0e8ff5..1b10b6df20 100644
--- a/src/backend/access/transam/xlogutils.c
+++ b/src/backend/access/transam/xlogutils.c
@@ -689,8 +689,8 @@ XLogTruncateRelation(RelFileNode rnode, ForkNumber forkNum,
void
XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wantLength)
{
- const XLogRecPtr lastReadPage = (state->seg.ws_segno *
- state->segcxt.ws_segsize + state->segoff);
+ const XLogRecPtr lastReadPage = state->seg.ws_segno *
+ state->segcxt.ws_segsize + state->readLen;
Assert(wantPage != InvalidXLogRecPtr && wantPage % XLOG_BLCKSZ == 0);
Assert(wantLength <= XLOG_BLCKSZ);
@@ -828,7 +828,7 @@ wal_segment_close(XLogReaderState *state)
* exists for normal backends, so we have to do a check/sleep/repeat style of
* loop for now.
*/
-int
+bool
read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *cur_page)
{
@@ -930,7 +930,8 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
else if (targetPagePtr + reqLen > read_upto)
{
/* not enough data there */
- return -1;
+ state->readLen = -1;
+ return false;
}
else
{
@@ -948,7 +949,8 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
WALReadRaiseError(&errinfo);
/* number of valid bytes in the buffer */
- return count;
+ state->readLen = count;
+ return true;
}
/*
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index e2477c47e0..bf2711bddd 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -811,7 +811,7 @@ StartReplication(StartReplicationCmd *cmd)
* which has to do a plain sleep/busy loop, because the walsender's latch gets
* set every time WAL is flushed.
*/
-static int
+static bool
logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *cur_page)
{
@@ -831,7 +831,10 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
/* fail if not (implies we are going to shut down) */
if (flushptr < targetPagePtr + reqLen)
- return -1;
+ {
+ state->readLen = -1;
+ return false;
+ }
if (targetPagePtr + XLOG_BLCKSZ <= flushptr)
count = XLOG_BLCKSZ; /* more than one block available */
@@ -859,7 +862,8 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
XLByteToSeg(targetPagePtr, segno, state->segcxt.ws_segsize);
CheckXLogRemoved(segno, state->seg.ws_tli);
- return count;
+ state->readLen = count;
+ return true;
}
/*
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index bc6f976994..0b7e73ae79 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -46,7 +46,7 @@ typedef struct XLogPageReadPrivate
int tliIndex;
} XLogPageReadPrivate;
-static int SimpleXLogPageRead(XLogReaderState *xlogreader,
+static bool SimpleXLogPageRead(XLogReaderState *xlogreader,
XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
@@ -236,7 +236,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
}
/* XLogReader callback function, to read a WAL page */
-static int
+static bool
SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf)
{
@@ -296,7 +296,8 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
if (private->restoreCommand == NULL)
{
pg_log_error("could not open file \"%s\": %m", xlogfpath);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
/*
@@ -309,7 +310,10 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
private->restoreCommand);
if (xlogreadfd < 0)
- return -1;
+ {
+ xlogreader->readLen = -1;
+ return false;
+ }
else
pg_log_debug("using file \"%s\" restored from archive",
xlogfpath);
@@ -325,7 +329,8 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
if (lseek(xlogreadfd, (off_t) targetPageOff, SEEK_SET) < 0)
{
pg_log_error("could not seek in file \"%s\": %m", xlogfpath);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
@@ -338,13 +343,15 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
pg_log_error("could not read file \"%s\": read %d of %zu",
xlogfpath, r, (Size) XLOG_BLCKSZ);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
Assert(targetSegNo == xlogreadsegno);
xlogreader->seg.ws_tli = targetHistory[private->tliIndex].tli;
- return XLOG_BLCKSZ;
+ xlogreader->readLen = XLOG_BLCKSZ;
+ return true;
}
/*
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index d1a0678935..ab0c28d9b7 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -331,7 +331,7 @@ WALDumpCloseSegment(XLogReaderState *state)
}
/* pg_waldump's XLogReaderRoutine->page_read callback */
-static int
+static bool
WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetPtr, char *readBuff)
{
@@ -348,7 +348,8 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
else
{
private->endptr_reached = true;
- return -1;
+ state->readLen = -1;
+ return false;
}
}
@@ -373,7 +374,8 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
(Size) errinfo.wre_req);
}
- return count;
+ state->readLen = count;
+ return true;
}
/*
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index b0f2a6ed43..6c8848e14f 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -57,8 +57,8 @@ typedef struct WALSegmentContext
typedef struct XLogReaderState XLogReaderState;
-/* Function type definitions for various xlogreader interactions */
-typedef int (*XLogPageReadCB) (XLogReaderState *xlogreader,
+/* Function type definition for the read_page callback */
+typedef bool (*XLogPageReadCB) (XLogReaderState *xlogreader,
XLogRecPtr targetPagePtr,
int reqLen,
XLogRecPtr targetRecPtr,
@@ -175,6 +175,20 @@ struct XLogReaderState
XLogRecPtr ReadRecPtr; /* start of last record read */
XLogRecPtr EndRecPtr; /* end+1 of last record read */
+ /* ----------------------------------------
+ * Communication with page reader
+ * readBuf is XLOG_BLCKSZ bytes, valid up to at least readLen bytes.
+ * ----------------------------------------
+ */
+ /* variables to communicate with page reader */
+ XLogRecPtr readPagePtr; /* page pointer to read */
+ int32 readLen; /* bytes requested to reader, or actual bytes
+ * read by reader, which must be larger than
+ * the request, or -1 on error */
+ TimeLineID readPageTLI; /* TLI for data currently in readBuf */
+ char *readBuf; /* buffer to store data */
+ bool page_verified; /* is the page on the buffer verified? */
+
/* ----------------------------------------
* Decoded representation of current record
@@ -201,13 +215,6 @@ struct XLogReaderState
* ----------------------------------------
*/
- /*
- * Buffer for currently read page (XLOG_BLCKSZ bytes, valid up to at least
- * readLen bytes)
- */
- char *readBuf;
- uint32 readLen;
-
/* last read XLOG position for data currently in readBuf */
WALSegmentContext segcxt;
WALOpenSegment seg;
diff --git a/src/include/access/xlogutils.h b/src/include/access/xlogutils.h
index e59b6cf3a9..c287edf206 100644
--- a/src/include/access/xlogutils.h
+++ b/src/include/access/xlogutils.h
@@ -47,7 +47,7 @@ extern Buffer XLogReadBufferExtended(RelFileNode rnode, ForkNumber forknum,
extern Relation CreateFakeRelcacheEntry(RelFileNode rnode);
extern void FreeFakeRelcacheEntry(Relation fakerel);
-extern int read_local_xlog_page(XLogReaderState *state,
+extern bool read_local_xlog_page(XLogReaderState *state,
XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *cur_page);
extern void wal_segment_open(XLogReaderState *state,
--
2.18.4
----Next_Part(Thu_Jul__2_13_53_30_2020_072)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
filename="v15-0002-Move-page-reader-out-of-XLogReadRecord.patch"
^ permalink raw reply [nested|flat] 69+ messages in thread
* [PATCH v17 1/4] Move callback-call from ReadPageInternal to XLogReadRecord.
@ 2019-09-05 11:21 Kyotaro Horiguchi <[email protected]>
0 siblings, 0 replies; 69+ messages in thread
From: Kyotaro Horiguchi @ 2019-09-05 11:21 UTC (permalink / raw)
The current WAL record reader reads page data using a call back
function. Although it is not so problematic alone, it would be a
problem if we are going to do add tasks like encryption which is
performed on page data before WAL reader reads them. To avoid that the
record reader facility has to have a new code path corresponds to
every new callback, this patch separates page reader from WAL record
reading facility by modifying the current WAL record reader to a state
machine.
As the first step of that change, this patch moves the page reader
function out of ReadPageInternal, then the remaining tasks of the
function are taken over by the new function XLogNeedData. As the
result XLogPageRead directly calls the page reader callback function
according to the feedback from XLogNeedData.
---
src/backend/access/transam/xlog.c | 16 +-
src/backend/access/transam/xlogreader.c | 281 ++++++++++++++----------
src/backend/access/transam/xlogutils.c | 12 +-
src/backend/replication/walsender.c | 10 +-
src/bin/pg_rewind/parsexlog.c | 21 +-
src/bin/pg_waldump/pg_waldump.c | 8 +-
src/include/access/xlogreader.h | 25 ++-
src/include/access/xlogutils.h | 2 +-
8 files changed, 222 insertions(+), 153 deletions(-)
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 377afb8732..606328a65a 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -917,7 +917,7 @@ static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath,
static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli,
XLogSource source, bool notfoundOk);
static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, XLogSource source);
-static int XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
+static bool XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
bool fetching_ckpt, XLogRecPtr tliRecPtr);
@@ -4350,7 +4350,6 @@ ReadRecord(XLogReaderState *xlogreader, int emode,
XLogRecord *record;
XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
- /* Pass through parameters to XLogPageRead */
private->fetching_ckpt = fetching_ckpt;
private->emode = emode;
private->randAccess = (xlogreader->ReadRecPtr == InvalidXLogRecPtr);
@@ -11987,7 +11986,7 @@ CancelBackup(void)
* XLogPageRead() to try fetching the record from another source, or to
* sleep and retry.
*/
-static int
+static bool
XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *readBuf)
{
@@ -12046,7 +12045,8 @@ retry:
readLen = 0;
readSource = XLOG_FROM_ANY;
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
}
@@ -12141,7 +12141,8 @@ retry:
goto next_record_is_invalid;
}
- return readLen;
+ xlogreader->readLen = readLen;
+ return true;
next_record_is_invalid:
lastSourceFailed = true;
@@ -12155,8 +12156,9 @@ next_record_is_invalid:
/* In standby-mode, keep trying */
if (StandbyMode)
goto retry;
- else
- return -1;
+
+ xlogreader->readLen = -1;
+ return false;
}
/*
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index 42738eb940..7d04995b7b 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -36,8 +36,8 @@
static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
pg_attribute_printf(2, 3);
static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
-static int ReadPageInternal(XLogReaderState *state, XLogRecPtr pageptr,
- int reqLen);
+static bool XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr,
+ int reqLen, bool header_inclusive);
static void XLogReaderInvalReadState(XLogReaderState *state);
static bool ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
XLogRecPtr PrevRecPtr, XLogRecord *record, bool randAccess);
@@ -276,7 +276,6 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
uint32 targetRecOff;
uint32 pageHeaderSize;
bool gotheader;
- int readOff;
/*
* randAccess indicates whether to verify the previous-record pointer of
@@ -326,14 +325,20 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
* byte to cover the whole record header, or at least the part of it that
* fits on the same page.
*/
- readOff = ReadPageInternal(state, targetPagePtr,
- Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ));
- if (readOff < 0)
+ while (XLogNeedData(state, targetPagePtr,
+ Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+ targetRecOff != 0))
+ {
+ if (!state->routine.page_read(state, state->readPagePtr, state->readLen,
+ RecPtr, state->readBuf))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
/*
- * ReadPageInternal always returns at least the page header, so we can
- * examine it now.
+ * We have at least the page header, so we can examine it now.
*/
pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
if (targetRecOff == 0)
@@ -359,8 +364,8 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
goto err;
}
- /* ReadPageInternal has verified the page header */
- Assert(pageHeaderSize <= readOff);
+ /* XLogNeedData has verified the page header */
+ Assert(pageHeaderSize <= state->readLen);
/*
* Read the record length.
@@ -432,18 +437,27 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
do
{
+ int rest_len = total_len - gotlen;
+
/* Calculate pointer to beginning of next page */
targetPagePtr += XLOG_BLCKSZ;
/* Wait for the next page to become available */
- readOff = ReadPageInternal(state, targetPagePtr,
- Min(total_len - gotlen + SizeOfXLogShortPHD,
- XLOG_BLCKSZ));
+ while (XLogNeedData(state, targetPagePtr,
+ Min(rest_len, XLOG_BLCKSZ),
+ false))
+ {
+ if (!state->routine.page_read(state, state->readPagePtr,
+ state->readLen,
+ state->ReadRecPtr,
+ state->readBuf))
+ break;
+ }
- if (readOff < 0)
+ if (!state->page_verified)
goto err;
- Assert(SizeOfXLogShortPHD <= readOff);
+ Assert(SizeOfXLogShortPHD <= state->readLen);
/* Check that the continuation on next page looks valid */
pageHeader = (XLogPageHeader) state->readBuf;
@@ -473,21 +487,14 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
/* Append the continuation from this page to the buffer */
pageHeaderSize = XLogPageHeaderSize(pageHeader);
- if (readOff < pageHeaderSize)
- readOff = ReadPageInternal(state, targetPagePtr,
- pageHeaderSize);
-
- Assert(pageHeaderSize <= readOff);
+ Assert(pageHeaderSize <= state->readLen);
contdata = (char *) state->readBuf + pageHeaderSize;
len = XLOG_BLCKSZ - pageHeaderSize;
if (pageHeader->xlp_rem_len < len)
len = pageHeader->xlp_rem_len;
- if (readOff < pageHeaderSize + len)
- readOff = ReadPageInternal(state, targetPagePtr,
- pageHeaderSize + len);
-
+ Assert (pageHeaderSize + len <= state->readLen);
memcpy(buffer, (char *) contdata, len);
buffer += len;
gotlen += len;
@@ -517,9 +524,16 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
else
{
/* Wait for the record data to become available */
- readOff = ReadPageInternal(state, targetPagePtr,
- Min(targetRecOff + total_len, XLOG_BLCKSZ));
- if (readOff < 0)
+ while (XLogNeedData(state, targetPagePtr,
+ Min(targetRecOff + total_len, XLOG_BLCKSZ), true))
+ {
+ if (!state->routine.page_read(state, state->readPagePtr,
+ state->readLen,
+ state->ReadRecPtr, state->readBuf))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
/* Record does not cross a page boundary */
@@ -562,109 +576,138 @@ err:
}
/*
- * Read a single xlog page including at least [pageptr, reqLen] of valid data
- * via the page_read() callback.
+ * Checks that an xlog page loaded in state->readBuf is including at least
+ * [pageptr, reqLen] and the page is valid. header_inclusive indicates that
+ * reqLen is calculated including page header length.
*
- * Returns -1 if the required page cannot be read for some reason; errormsg_buf
- * is set in that case (unless the error occurs in the page_read callback).
+ * Returns false if the buffer already contains the requested data, or found
+ * error. state->page_verified is set to true for the former and false for the
+ * latter.
*
- * We fetch the page from a reader-local cache if we know we have the required
- * data and if there hasn't been any error since caching the data.
+ * Otherwise returns true and requests data loaded onto state->readBuf by
+ * state->readPagePtr and state->readLen. The caller shall call this function
+ * again after filling the buffer at least with that portion of data and set
+ * state->readLen to the length of actually loaded data.
+ *
+ * If header_inclusive is false, corrects reqLen internally by adding the
+ * actual page header length and may request caller for new data.
*/
-static int
-ReadPageInternal(XLogReaderState *state, XLogRecPtr pageptr, int reqLen)
+static bool
+XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
+ bool header_inclusive)
{
- int readLen;
uint32 targetPageOff;
XLogSegNo targetSegNo;
- XLogPageHeader hdr;
+ uint32 addLen = 0;
- Assert((pageptr % XLOG_BLCKSZ) == 0);
+ /* Some data is loaded, but page header is not verified yet. */
+ if (!state->page_verified &&
+ !XLogRecPtrIsInvalid(state->readPagePtr) && state->readLen >= 0)
+ {
+ uint32 pageHeaderSize;
+ /* just loaded new data so needs to verify page header */
+
+ /* The caller must have loaded at least page header */
+ Assert (state->readLen >= SizeOfXLogShortPHD);
+
+ /*
+ * We have enough data to check the header length. Recheck the loaded
+ * length against the actual header length.
+ */
+ pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+
+ /* Request more data if we don't have the full header. */
+ if (state->readLen < pageHeaderSize)
+ {
+ state->readLen = pageHeaderSize;
+ return true;
+ }
+
+ /* Now that we know we have the full header, validate it. */
+ if (!XLogReaderValidatePageHeader(state, state->readPagePtr,
+ (char *) state->readBuf))
+ {
+ /* That's bad. Force reading the page again. */
+ XLogReaderInvalReadState(state);
+
+ return false;
+ }
+
+ state->page_verified = true;
+
+ XLByteToSeg(state->readPagePtr, state->seg.ws_segno,
+ state->segcxt.ws_segsize);
+ }
+
+ /*
+ * The loaded page may not be the one caller is supposing to read when we
+ * are verifying the first page of new segment. In that case, skip further
+ * verification and immediately load the target page.
+ */
+ if (state->page_verified && pageptr == state->readPagePtr)
+ {
+ /*
+ * calculate additional length for page header keeping the total
+ * length within the block size.
+ */
+ if (!header_inclusive)
+ {
+ uint32 pageHeaderSize =
+ XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+
+ addLen = pageHeaderSize;
+ if (reqLen + pageHeaderSize <= XLOG_BLCKSZ)
+ addLen = pageHeaderSize;
+ else
+ addLen = XLOG_BLCKSZ - reqLen;
+
+ Assert(addLen >= 0);
+ }
+
+ /* Return if we already have it. */
+ if (reqLen + addLen <= state->readLen)
+ return false;
+ }
+
+ /* Data is not in our buffer, request the caller for it. */
XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize);
targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize);
+ Assert((pageptr % XLOG_BLCKSZ) == 0);
- /* check whether we have all the requested data already */
- if (targetSegNo == state->seg.ws_segno &&
- targetPageOff == state->segoff && reqLen <= state->readLen)
- return state->readLen;
+ /*
+ * Every time we request to load new data of a page to the caller, even if
+ * we looked at a part of it before, we need to do verification on the next
+ * invocation as the caller might now be rereading data from a different
+ * source.
+ */
+ state->page_verified = false;
/*
- * Data is not in our buffer.
- *
- * Every time we actually read the segment, even if we looked at parts of
- * it before, we need to do verification as the page_read callback might
- * now be rereading data from a different source.
- *
* Whenever switching to a new WAL segment, we read the first page of the
* file and validate its header, even if that's not where the target
* record is. This is so that we can check the additional identification
* info that is present in the first page's "long" header.
+ * Don't do this if the caller requested the first page in the segment.
*/
if (targetSegNo != state->seg.ws_segno && targetPageOff != 0)
{
- XLogRecPtr targetSegmentPtr = pageptr - targetPageOff;
-
- readLen = state->routine.page_read(state, targetSegmentPtr, XLOG_BLCKSZ,
- state->currRecPtr,
- state->readBuf);
- if (readLen < 0)
- goto err;
-
- /* we can be sure to have enough WAL available, we scrolled back */
- Assert(readLen == XLOG_BLCKSZ);
-
- if (!XLogReaderValidatePageHeader(state, targetSegmentPtr,
- state->readBuf))
- goto err;
+ /*
+ * Then we'll see that the targetSegNo now matches the ws_segno, and
+ * will not come back here, but will request the actual target page.
+ */
+ state->readPagePtr = pageptr - targetPageOff;
+ state->readLen = XLOG_BLCKSZ;
+ return true;
}
/*
- * First, read the requested data length, but at least a short page header
- * so that we can validate it.
+ * Request the caller to load the page. We need at least a short page
+ * header so that we can validate it.
*/
- readLen = state->routine.page_read(state, pageptr, Max(reqLen, SizeOfXLogShortPHD),
- state->currRecPtr,
- state->readBuf);
- if (readLen < 0)
- goto err;
-
- Assert(readLen <= XLOG_BLCKSZ);
-
- /* Do we have enough data to check the header length? */
- if (readLen <= SizeOfXLogShortPHD)
- goto err;
-
- Assert(readLen >= reqLen);
-
- hdr = (XLogPageHeader) state->readBuf;
-
- /* still not enough */
- if (readLen < XLogPageHeaderSize(hdr))
- {
- readLen = state->routine.page_read(state, pageptr, XLogPageHeaderSize(hdr),
- state->currRecPtr,
- state->readBuf);
- if (readLen < 0)
- goto err;
- }
-
- /*
- * Now that we know we have the full header, validate it.
- */
- if (!XLogReaderValidatePageHeader(state, pageptr, (char *) hdr))
- goto err;
-
- /* update read state information */
- state->seg.ws_segno = targetSegNo;
- state->segoff = targetPageOff;
- state->readLen = readLen;
-
- return readLen;
-
-err:
- XLogReaderInvalReadState(state);
- return -1;
+ state->readPagePtr = pageptr;
+ state->readLen = Max(reqLen + addLen, SizeOfXLogShortPHD);
+ return true;
}
/*
@@ -673,9 +716,7 @@ err:
static void
XLogReaderInvalReadState(XLogReaderState *state)
{
- state->seg.ws_segno = 0;
- state->segoff = 0;
- state->readLen = 0;
+ state->readPagePtr = InvalidXLogRecPtr;
}
/*
@@ -953,7 +994,6 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
XLogRecPtr targetPagePtr;
int targetRecOff;
uint32 pageHeaderSize;
- int readLen;
/*
* Compute targetRecOff. It should typically be equal or greater than
@@ -961,7 +1001,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
* that, except when caller has explicitly specified the offset that
* falls somewhere there or when we are skipping multi-page
* continuation record. It doesn't matter though because
- * ReadPageInternal() is prepared to handle that and will read at
+ * XLogNeedData() is prepared to handle that and will read at
* least short page-header worth of data
*/
targetRecOff = tmpRecPtr % XLOG_BLCKSZ;
@@ -969,19 +1009,24 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
/* scroll back to page boundary */
targetPagePtr = tmpRecPtr - targetRecOff;
- /* Read the page containing the record */
- readLen = ReadPageInternal(state, targetPagePtr, targetRecOff);
- if (readLen < 0)
+ while(XLogNeedData(state, targetPagePtr, targetRecOff,
+ targetRecOff != 0))
+ {
+ if (!state->routine.page_read(state, state->readPagePtr,
+ state->readLen,
+ state->ReadRecPtr, state->readBuf))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
header = (XLogPageHeader) state->readBuf;
pageHeaderSize = XLogPageHeaderSize(header);
- /* make sure we have enough data for the page header */
- readLen = ReadPageInternal(state, targetPagePtr, pageHeaderSize);
- if (readLen < 0)
- goto err;
+ /* we should have read the page header */
+ Assert (state->readLen >= pageHeaderSize);
/* skip over potential continuation data */
if (header->xlp_info & XLP_FIRST_IS_CONTRECORD)
diff --git a/src/backend/access/transam/xlogutils.c b/src/backend/access/transam/xlogutils.c
index d17d660f46..46eda33f25 100644
--- a/src/backend/access/transam/xlogutils.c
+++ b/src/backend/access/transam/xlogutils.c
@@ -686,8 +686,8 @@ XLogTruncateRelation(RelFileNode rnode, ForkNumber forkNum,
void
XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wantLength)
{
- const XLogRecPtr lastReadPage = (state->seg.ws_segno *
- state->segcxt.ws_segsize + state->segoff);
+ const XLogRecPtr lastReadPage = state->seg.ws_segno *
+ state->segcxt.ws_segsize + state->readLen;
Assert(wantPage != InvalidXLogRecPtr && wantPage % XLOG_BLCKSZ == 0);
Assert(wantLength <= XLOG_BLCKSZ);
@@ -824,7 +824,7 @@ wal_segment_close(XLogReaderState *state)
* exists for normal backends, so we have to do a check/sleep/repeat style of
* loop for now.
*/
-int
+bool
read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *cur_page)
{
@@ -926,7 +926,8 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
else if (targetPagePtr + reqLen > read_upto)
{
/* not enough data there */
- return -1;
+ state->readLen = -1;
+ return false;
}
else
{
@@ -944,7 +945,8 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
WALReadRaiseError(&errinfo);
/* number of valid bytes in the buffer */
- return count;
+ state->readLen = count;
+ return true;
}
/*
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index 23baa4498a..c36b91ae5e 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -806,7 +806,7 @@ StartReplication(StartReplicationCmd *cmd)
* which has to do a plain sleep/busy loop, because the walsender's latch gets
* set every time WAL is flushed.
*/
-static int
+static bool
logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *cur_page)
{
@@ -826,7 +826,10 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
/* fail if not (implies we are going to shut down) */
if (flushptr < targetPagePtr + reqLen)
- return -1;
+ {
+ state->readLen = -1;
+ return false;
+ }
if (targetPagePtr + XLOG_BLCKSZ <= flushptr)
count = XLOG_BLCKSZ; /* more than one block available */
@@ -854,7 +857,8 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
XLByteToSeg(targetPagePtr, segno, state->segcxt.ws_segsize);
CheckXLogRemoved(segno, state->seg.ws_tli);
- return count;
+ state->readLen = count;
+ return true;
}
/*
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index 59ebac7d6a..cf119848b0 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -47,7 +47,7 @@ typedef struct XLogPageReadPrivate
int tliIndex;
} XLogPageReadPrivate;
-static int SimpleXLogPageRead(XLogReaderState *xlogreader,
+static bool SimpleXLogPageRead(XLogReaderState *xlogreader,
XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
@@ -246,7 +246,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
}
/* XLogReader callback function, to read a WAL page */
-static int
+static bool
SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf)
{
@@ -306,7 +306,8 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
if (private->restoreCommand == NULL)
{
pg_log_error("could not open file \"%s\": %m", xlogfpath);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
/*
@@ -319,7 +320,10 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
private->restoreCommand);
if (xlogreadfd < 0)
- return -1;
+ {
+ xlogreader->readLen = -1;
+ return false;
+ }
else
pg_log_debug("using file \"%s\" restored from archive",
xlogfpath);
@@ -335,7 +339,8 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
if (lseek(xlogreadfd, (off_t) targetPageOff, SEEK_SET) < 0)
{
pg_log_error("could not seek in file \"%s\": %m", xlogfpath);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
@@ -348,13 +353,15 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
pg_log_error("could not read file \"%s\": read %d of %zu",
xlogfpath, r, (Size) XLOG_BLCKSZ);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
Assert(targetSegNo == xlogreadsegno);
xlogreader->seg.ws_tli = targetHistory[private->tliIndex].tli;
- return XLOG_BLCKSZ;
+ xlogreader->readLen = XLOG_BLCKSZ;
+ return true;
}
/*
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index 610f65e471..0f0e0723a3 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -331,7 +331,7 @@ WALDumpCloseSegment(XLogReaderState *state)
}
/* pg_waldump's XLogReaderRoutine->page_read callback */
-static int
+static bool
WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetPtr, char *readBuff)
{
@@ -348,7 +348,8 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
else
{
private->endptr_reached = true;
- return -1;
+ state->readLen = -1;
+ return false;
}
}
@@ -373,7 +374,8 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
(Size) errinfo.wre_req);
}
- return count;
+ state->readLen = count;
+ return true;
}
/*
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index 21d200d3df..23d9e70a59 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -57,8 +57,8 @@ typedef struct WALSegmentContext
typedef struct XLogReaderState XLogReaderState;
-/* Function type definitions for various xlogreader interactions */
-typedef int (*XLogPageReadCB) (XLogReaderState *xlogreader,
+/* Function type definition for the read_page callback */
+typedef bool (*XLogPageReadCB) (XLogReaderState *xlogreader,
XLogRecPtr targetPagePtr,
int reqLen,
XLogRecPtr targetRecPtr,
@@ -175,6 +175,20 @@ struct XLogReaderState
XLogRecPtr ReadRecPtr; /* start of last record read */
XLogRecPtr EndRecPtr; /* end+1 of last record read */
+ /* ----------------------------------------
+ * Communication with page reader
+ * readBuf is XLOG_BLCKSZ bytes, valid up to at least readLen bytes.
+ * ----------------------------------------
+ */
+ /* variables to communicate with page reader */
+ XLogRecPtr readPagePtr; /* page pointer to read */
+ int32 readLen; /* bytes requested to reader, or actual bytes
+ * read by reader, which must be larger than
+ * the request, or -1 on error */
+ TimeLineID readPageTLI; /* TLI for data currently in readBuf */
+ char *readBuf; /* buffer to store data */
+ bool page_verified; /* is the page on the buffer verified? */
+
/* ----------------------------------------
* Decoded representation of current record
@@ -203,13 +217,6 @@ struct XLogReaderState
* ----------------------------------------
*/
- /*
- * Buffer for currently read page (XLOG_BLCKSZ bytes, valid up to at least
- * readLen bytes)
- */
- char *readBuf;
- uint32 readLen;
-
/* last read XLOG position for data currently in readBuf */
WALSegmentContext segcxt;
WALOpenSegment seg;
diff --git a/src/include/access/xlogutils.h b/src/include/access/xlogutils.h
index 9ac602b674..364a21c4ea 100644
--- a/src/include/access/xlogutils.h
+++ b/src/include/access/xlogutils.h
@@ -47,7 +47,7 @@ extern Buffer XLogReadBufferExtended(RelFileNode rnode, ForkNumber forknum,
extern Relation CreateFakeRelcacheEntry(RelFileNode rnode);
extern void FreeFakeRelcacheEntry(Relation fakerel);
-extern int read_local_xlog_page(XLogReaderState *state,
+extern bool read_local_xlog_page(XLogReaderState *state,
XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *cur_page);
extern void wal_segment_open(XLogReaderState *state,
--
2.27.0
----Next_Part(Thu_Mar__4_11_28_53_2021_014)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
filename="v17-0002-Move-page-reader-out-of-XLogReadRecord.patch"
^ permalink raw reply [nested|flat] 69+ messages in thread
* [PATCH v15 1/4] Move callback-call from ReadPageInternal to XLogReadRecord.
@ 2019-09-05 11:21 Kyotaro Horiguchi <[email protected]>
0 siblings, 0 replies; 69+ messages in thread
From: Kyotaro Horiguchi @ 2019-09-05 11:21 UTC (permalink / raw)
The current WAL record reader reads page data using a call back
function. Although it is not so problematic alone, it would be a
problem if we are going to do add tasks like encryption which is
performed on page data before WAL reader reads them. To avoid that the
record reader facility has to have a new code path corresponds to
every new callback, this patch separates page reader from WAL record
reading facility by modifying the current WAL record reader to a state
machine.
As the first step of that change, this patch moves the page reader
function out of ReadPageInternal, then the remaining tasks of the
function are taken over by the new function XLogNeedData. As the
result XLogPageRead directly calls the page reader callback function
according to the feedback from XLogNeedData.
---
src/backend/access/transam/xlog.c | 16 +-
src/backend/access/transam/xlogreader.c | 281 ++++++++++++++----------
src/backend/access/transam/xlogutils.c | 12 +-
src/backend/replication/walsender.c | 10 +-
src/bin/pg_rewind/parsexlog.c | 21 +-
src/bin/pg_waldump/pg_waldump.c | 8 +-
src/include/access/xlogreader.h | 25 ++-
src/include/access/xlogutils.h | 2 +-
8 files changed, 222 insertions(+), 153 deletions(-)
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index fd93bcfaeb..e570e56a24 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -912,7 +912,7 @@ static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath,
static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli,
XLogSource source, bool notfoundOk);
static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, XLogSource source);
-static int XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
+static bool XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
bool fetching_ckpt, XLogRecPtr tliRecPtr);
@@ -4332,7 +4332,6 @@ ReadRecord(XLogReaderState *xlogreader, int emode,
XLogRecord *record;
XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
- /* Pass through parameters to XLogPageRead */
private->fetching_ckpt = fetching_ckpt;
private->emode = emode;
private->randAccess = (xlogreader->ReadRecPtr == InvalidXLogRecPtr);
@@ -11860,7 +11859,7 @@ CancelBackup(void)
* XLogPageRead() to try fetching the record from another source, or to
* sleep and retry.
*/
-static int
+static bool
XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *readBuf)
{
@@ -11919,7 +11918,8 @@ retry:
readLen = 0;
readSource = XLOG_FROM_ANY;
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
}
@@ -12014,7 +12014,8 @@ retry:
goto next_record_is_invalid;
}
- return readLen;
+ xlogreader->readLen = readLen;
+ return true;
next_record_is_invalid:
lastSourceFailed = true;
@@ -12028,8 +12029,9 @@ next_record_is_invalid:
/* In standby-mode, keep trying */
if (StandbyMode)
goto retry;
- else
- return -1;
+
+ xlogreader->readLen = -1;
+ return false;
}
/*
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index cb76be4f46..3d599325ee 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -36,8 +36,8 @@
static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
pg_attribute_printf(2, 3);
static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
-static int ReadPageInternal(XLogReaderState *state, XLogRecPtr pageptr,
- int reqLen);
+static bool XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr,
+ int reqLen, bool header_inclusive);
static void XLogReaderInvalReadState(XLogReaderState *state);
static bool ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
XLogRecPtr PrevRecPtr, XLogRecord *record, bool randAccess);
@@ -276,7 +276,6 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
uint32 targetRecOff;
uint32 pageHeaderSize;
bool gotheader;
- int readOff;
/*
* randAccess indicates whether to verify the previous-record pointer of
@@ -326,14 +325,20 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
* byte to cover the whole record header, or at least the part of it that
* fits on the same page.
*/
- readOff = ReadPageInternal(state, targetPagePtr,
- Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ));
- if (readOff < 0)
+ while (XLogNeedData(state, targetPagePtr,
+ Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+ targetRecOff != 0))
+ {
+ if (!state->routine.page_read(state, state->readPagePtr, state->readLen,
+ RecPtr, state->readBuf))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
/*
- * ReadPageInternal always returns at least the page header, so we can
- * examine it now.
+ * We have at least the page header, so we can examine it now.
*/
pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
if (targetRecOff == 0)
@@ -359,8 +364,8 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
goto err;
}
- /* ReadPageInternal has verified the page header */
- Assert(pageHeaderSize <= readOff);
+ /* XLogNeedData has verified the page header */
+ Assert(pageHeaderSize <= state->readLen);
/*
* Read the record length.
@@ -433,18 +438,27 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
do
{
+ int rest_len = total_len - gotlen;
+
/* Calculate pointer to beginning of next page */
targetPagePtr += XLOG_BLCKSZ;
/* Wait for the next page to become available */
- readOff = ReadPageInternal(state, targetPagePtr,
- Min(total_len - gotlen + SizeOfXLogShortPHD,
- XLOG_BLCKSZ));
+ while (XLogNeedData(state, targetPagePtr,
+ Min(rest_len, XLOG_BLCKSZ),
+ false))
+ {
+ if (!state->routine.page_read(state, state->readPagePtr,
+ state->readLen,
+ state->ReadRecPtr,
+ state->readBuf))
+ break;
+ }
- if (readOff < 0)
+ if (!state->page_verified)
goto err;
- Assert(SizeOfXLogShortPHD <= readOff);
+ Assert(SizeOfXLogShortPHD <= state->readLen);
/* Check that the continuation on next page looks valid */
pageHeader = (XLogPageHeader) state->readBuf;
@@ -473,21 +487,14 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
/* Append the continuation from this page to the buffer */
pageHeaderSize = XLogPageHeaderSize(pageHeader);
- if (readOff < pageHeaderSize)
- readOff = ReadPageInternal(state, targetPagePtr,
- pageHeaderSize);
-
- Assert(pageHeaderSize <= readOff);
+ Assert(pageHeaderSize <= state->readLen);
contdata = (char *) state->readBuf + pageHeaderSize;
len = XLOG_BLCKSZ - pageHeaderSize;
if (pageHeader->xlp_rem_len < len)
len = pageHeader->xlp_rem_len;
- if (readOff < pageHeaderSize + len)
- readOff = ReadPageInternal(state, targetPagePtr,
- pageHeaderSize + len);
-
+ Assert (pageHeaderSize + len <= state->readLen);
memcpy(buffer, (char *) contdata, len);
buffer += len;
gotlen += len;
@@ -517,9 +524,16 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
else
{
/* Wait for the record data to become available */
- readOff = ReadPageInternal(state, targetPagePtr,
- Min(targetRecOff + total_len, XLOG_BLCKSZ));
- if (readOff < 0)
+ while (XLogNeedData(state, targetPagePtr,
+ Min(targetRecOff + total_len, XLOG_BLCKSZ), true))
+ {
+ if (!state->routine.page_read(state, state->readPagePtr,
+ state->readLen,
+ state->ReadRecPtr, state->readBuf))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
/* Record does not cross a page boundary */
@@ -562,109 +576,138 @@ err:
}
/*
- * Read a single xlog page including at least [pageptr, reqLen] of valid data
- * via the page_read() callback.
+ * Checks that an xlog page loaded in state->readBuf is including at least
+ * [pageptr, reqLen] and the page is valid. header_inclusive indicates that
+ * reqLen is calculated including page header length.
*
- * Returns -1 if the required page cannot be read for some reason; errormsg_buf
- * is set in that case (unless the error occurs in the page_read callback).
+ * Returns false if the buffer already contains the requested data, or found
+ * error. state->page_verified is set to true for the former and false for the
+ * latter.
*
- * We fetch the page from a reader-local cache if we know we have the required
- * data and if there hasn't been any error since caching the data.
+ * Otherwise returns true and requests data loaded onto state->readBuf by
+ * state->readPagePtr and state->readLen. The caller shall call this function
+ * again after filling the buffer at least with that portion of data and set
+ * state->readLen to the length of actually loaded data.
+ *
+ * If header_inclusive is false, corrects reqLen internally by adding the
+ * actual page header length and may request caller for new data.
*/
-static int
-ReadPageInternal(XLogReaderState *state, XLogRecPtr pageptr, int reqLen)
+static bool
+XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
+ bool header_inclusive)
{
- int readLen;
uint32 targetPageOff;
XLogSegNo targetSegNo;
- XLogPageHeader hdr;
+ uint32 addLen = 0;
- Assert((pageptr % XLOG_BLCKSZ) == 0);
+ /* Some data is loaded, but page header is not verified yet. */
+ if (!state->page_verified &&
+ !XLogRecPtrIsInvalid(state->readPagePtr) && state->readLen >= 0)
+ {
+ uint32 pageHeaderSize;
+ /* just loaded new data so needs to verify page header */
+
+ /* The caller must have loaded at least page header */
+ Assert (state->readLen >= SizeOfXLogShortPHD);
+
+ /*
+ * We have enough data to check the header length. Recheck the loaded
+ * length against the actual header length.
+ */
+ pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+
+ /* Request more data if we don't have the full header. */
+ if (state->readLen < pageHeaderSize)
+ {
+ state->readLen = pageHeaderSize;
+ return true;
+ }
+
+ /* Now that we know we have the full header, validate it. */
+ if (!XLogReaderValidatePageHeader(state, state->readPagePtr,
+ (char *) state->readBuf))
+ {
+ /* That's bad. Force reading the page again. */
+ XLogReaderInvalReadState(state);
+
+ return false;
+ }
+
+ state->page_verified = true;
+
+ XLByteToSeg(state->readPagePtr, state->seg.ws_segno,
+ state->segcxt.ws_segsize);
+ }
+
+ /*
+ * The loaded page may not be the one caller is supposing to read when we
+ * are verifying the first page of new segment. In that case, skip further
+ * verification and immediately load the target page.
+ */
+ if (state->page_verified && pageptr == state->readPagePtr)
+ {
+ /*
+ * calculate additional length for page header keeping the total
+ * length within the block size.
+ */
+ if (!header_inclusive)
+ {
+ uint32 pageHeaderSize =
+ XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+
+ addLen = pageHeaderSize;
+ if (reqLen + pageHeaderSize <= XLOG_BLCKSZ)
+ addLen = pageHeaderSize;
+ else
+ addLen = XLOG_BLCKSZ - reqLen;
+
+ Assert(addLen >= 0);
+ }
+
+ /* Return if we already have it. */
+ if (reqLen + addLen <= state->readLen)
+ return false;
+ }
+
+ /* Data is not in our buffer, request the caller for it. */
XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize);
targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize);
+ Assert((pageptr % XLOG_BLCKSZ) == 0);
- /* check whether we have all the requested data already */
- if (targetSegNo == state->seg.ws_segno &&
- targetPageOff == state->segoff && reqLen <= state->readLen)
- return state->readLen;
+ /*
+ * Every time we request to load new data of a page to the caller, even if
+ * we looked at a part of it before, we need to do verification on the next
+ * invocation as the caller might now be rereading data from a different
+ * source.
+ */
+ state->page_verified = false;
/*
- * Data is not in our buffer.
- *
- * Every time we actually read the segment, even if we looked at parts of
- * it before, we need to do verification as the page_read callback might
- * now be rereading data from a different source.
- *
* Whenever switching to a new WAL segment, we read the first page of the
* file and validate its header, even if that's not where the target
* record is. This is so that we can check the additional identification
* info that is present in the first page's "long" header.
+ * Don't do this if the caller requested the first page in the segment.
*/
if (targetSegNo != state->seg.ws_segno && targetPageOff != 0)
{
- XLogRecPtr targetSegmentPtr = pageptr - targetPageOff;
-
- readLen = state->routine.page_read(state, targetSegmentPtr, XLOG_BLCKSZ,
- state->currRecPtr,
- state->readBuf);
- if (readLen < 0)
- goto err;
-
- /* we can be sure to have enough WAL available, we scrolled back */
- Assert(readLen == XLOG_BLCKSZ);
-
- if (!XLogReaderValidatePageHeader(state, targetSegmentPtr,
- state->readBuf))
- goto err;
+ /*
+ * Then we'll see that the targetSegNo now matches the ws_segno, and
+ * will not come back here, but will request the actual target page.
+ */
+ state->readPagePtr = pageptr - targetPageOff;
+ state->readLen = XLOG_BLCKSZ;
+ return true;
}
/*
- * First, read the requested data length, but at least a short page header
- * so that we can validate it.
+ * Request the caller to load the page. We need at least a short page
+ * header so that we can validate it.
*/
- readLen = state->routine.page_read(state, pageptr, Max(reqLen, SizeOfXLogShortPHD),
- state->currRecPtr,
- state->readBuf);
- if (readLen < 0)
- goto err;
-
- Assert(readLen <= XLOG_BLCKSZ);
-
- /* Do we have enough data to check the header length? */
- if (readLen <= SizeOfXLogShortPHD)
- goto err;
-
- Assert(readLen >= reqLen);
-
- hdr = (XLogPageHeader) state->readBuf;
-
- /* still not enough */
- if (readLen < XLogPageHeaderSize(hdr))
- {
- readLen = state->routine.page_read(state, pageptr, XLogPageHeaderSize(hdr),
- state->currRecPtr,
- state->readBuf);
- if (readLen < 0)
- goto err;
- }
-
- /*
- * Now that we know we have the full header, validate it.
- */
- if (!XLogReaderValidatePageHeader(state, pageptr, (char *) hdr))
- goto err;
-
- /* update read state information */
- state->seg.ws_segno = targetSegNo;
- state->segoff = targetPageOff;
- state->readLen = readLen;
-
- return readLen;
-
-err:
- XLogReaderInvalReadState(state);
- return -1;
+ state->readPagePtr = pageptr;
+ state->readLen = Max(reqLen + addLen, SizeOfXLogShortPHD);
+ return true;
}
/*
@@ -673,9 +716,7 @@ err:
static void
XLogReaderInvalReadState(XLogReaderState *state)
{
- state->seg.ws_segno = 0;
- state->segoff = 0;
- state->readLen = 0;
+ state->readPagePtr = InvalidXLogRecPtr;
}
/*
@@ -956,7 +997,6 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
XLogRecPtr targetPagePtr;
int targetRecOff;
uint32 pageHeaderSize;
- int readLen;
/*
* Compute targetRecOff. It should typically be equal or greater than
@@ -964,7 +1004,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
* that, except when caller has explicitly specified the offset that
* falls somewhere there or when we are skipping multi-page
* continuation record. It doesn't matter though because
- * ReadPageInternal() is prepared to handle that and will read at
+ * XLogNeedData() is prepared to handle that and will read at
* least short page-header worth of data
*/
targetRecOff = tmpRecPtr % XLOG_BLCKSZ;
@@ -972,19 +1012,24 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
/* scroll back to page boundary */
targetPagePtr = tmpRecPtr - targetRecOff;
- /* Read the page containing the record */
- readLen = ReadPageInternal(state, targetPagePtr, targetRecOff);
- if (readLen < 0)
+ while(XLogNeedData(state, targetPagePtr, targetRecOff,
+ targetRecOff != 0))
+ {
+ if (!state->routine.page_read(state, state->readPagePtr,
+ state->readLen,
+ state->ReadRecPtr, state->readBuf))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
header = (XLogPageHeader) state->readBuf;
pageHeaderSize = XLogPageHeaderSize(header);
- /* make sure we have enough data for the page header */
- readLen = ReadPageInternal(state, targetPagePtr, pageHeaderSize);
- if (readLen < 0)
- goto err;
+ /* we should have read the page header */
+ Assert (state->readLen >= pageHeaderSize);
/* skip over potential continuation data */
if (header->xlp_info & XLP_FIRST_IS_CONTRECORD)
diff --git a/src/backend/access/transam/xlogutils.c b/src/backend/access/transam/xlogutils.c
index 322b0e8ff5..1b10b6df20 100644
--- a/src/backend/access/transam/xlogutils.c
+++ b/src/backend/access/transam/xlogutils.c
@@ -689,8 +689,8 @@ XLogTruncateRelation(RelFileNode rnode, ForkNumber forkNum,
void
XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wantLength)
{
- const XLogRecPtr lastReadPage = (state->seg.ws_segno *
- state->segcxt.ws_segsize + state->segoff);
+ const XLogRecPtr lastReadPage = state->seg.ws_segno *
+ state->segcxt.ws_segsize + state->readLen;
Assert(wantPage != InvalidXLogRecPtr && wantPage % XLOG_BLCKSZ == 0);
Assert(wantLength <= XLOG_BLCKSZ);
@@ -828,7 +828,7 @@ wal_segment_close(XLogReaderState *state)
* exists for normal backends, so we have to do a check/sleep/repeat style of
* loop for now.
*/
-int
+bool
read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *cur_page)
{
@@ -930,7 +930,8 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
else if (targetPagePtr + reqLen > read_upto)
{
/* not enough data there */
- return -1;
+ state->readLen = -1;
+ return false;
}
else
{
@@ -948,7 +949,8 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
WALReadRaiseError(&errinfo);
/* number of valid bytes in the buffer */
- return count;
+ state->readLen = count;
+ return true;
}
/*
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index e2477c47e0..bf2711bddd 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -811,7 +811,7 @@ StartReplication(StartReplicationCmd *cmd)
* which has to do a plain sleep/busy loop, because the walsender's latch gets
* set every time WAL is flushed.
*/
-static int
+static bool
logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *cur_page)
{
@@ -831,7 +831,10 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
/* fail if not (implies we are going to shut down) */
if (flushptr < targetPagePtr + reqLen)
- return -1;
+ {
+ state->readLen = -1;
+ return false;
+ }
if (targetPagePtr + XLOG_BLCKSZ <= flushptr)
count = XLOG_BLCKSZ; /* more than one block available */
@@ -859,7 +862,8 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
XLByteToSeg(targetPagePtr, segno, state->segcxt.ws_segsize);
CheckXLogRemoved(segno, state->seg.ws_tli);
- return count;
+ state->readLen = count;
+ return true;
}
/*
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index bc6f976994..0b7e73ae79 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -46,7 +46,7 @@ typedef struct XLogPageReadPrivate
int tliIndex;
} XLogPageReadPrivate;
-static int SimpleXLogPageRead(XLogReaderState *xlogreader,
+static bool SimpleXLogPageRead(XLogReaderState *xlogreader,
XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
@@ -236,7 +236,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
}
/* XLogReader callback function, to read a WAL page */
-static int
+static bool
SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf)
{
@@ -296,7 +296,8 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
if (private->restoreCommand == NULL)
{
pg_log_error("could not open file \"%s\": %m", xlogfpath);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
/*
@@ -309,7 +310,10 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
private->restoreCommand);
if (xlogreadfd < 0)
- return -1;
+ {
+ xlogreader->readLen = -1;
+ return false;
+ }
else
pg_log_debug("using file \"%s\" restored from archive",
xlogfpath);
@@ -325,7 +329,8 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
if (lseek(xlogreadfd, (off_t) targetPageOff, SEEK_SET) < 0)
{
pg_log_error("could not seek in file \"%s\": %m", xlogfpath);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
@@ -338,13 +343,15 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
pg_log_error("could not read file \"%s\": read %d of %zu",
xlogfpath, r, (Size) XLOG_BLCKSZ);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
Assert(targetSegNo == xlogreadsegno);
xlogreader->seg.ws_tli = targetHistory[private->tliIndex].tli;
- return XLOG_BLCKSZ;
+ xlogreader->readLen = XLOG_BLCKSZ;
+ return true;
}
/*
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index d1a0678935..ab0c28d9b7 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -331,7 +331,7 @@ WALDumpCloseSegment(XLogReaderState *state)
}
/* pg_waldump's XLogReaderRoutine->page_read callback */
-static int
+static bool
WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetPtr, char *readBuff)
{
@@ -348,7 +348,8 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
else
{
private->endptr_reached = true;
- return -1;
+ state->readLen = -1;
+ return false;
}
}
@@ -373,7 +374,8 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
(Size) errinfo.wre_req);
}
- return count;
+ state->readLen = count;
+ return true;
}
/*
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index b0f2a6ed43..6c8848e14f 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -57,8 +57,8 @@ typedef struct WALSegmentContext
typedef struct XLogReaderState XLogReaderState;
-/* Function type definitions for various xlogreader interactions */
-typedef int (*XLogPageReadCB) (XLogReaderState *xlogreader,
+/* Function type definition for the read_page callback */
+typedef bool (*XLogPageReadCB) (XLogReaderState *xlogreader,
XLogRecPtr targetPagePtr,
int reqLen,
XLogRecPtr targetRecPtr,
@@ -175,6 +175,20 @@ struct XLogReaderState
XLogRecPtr ReadRecPtr; /* start of last record read */
XLogRecPtr EndRecPtr; /* end+1 of last record read */
+ /* ----------------------------------------
+ * Communication with page reader
+ * readBuf is XLOG_BLCKSZ bytes, valid up to at least readLen bytes.
+ * ----------------------------------------
+ */
+ /* variables to communicate with page reader */
+ XLogRecPtr readPagePtr; /* page pointer to read */
+ int32 readLen; /* bytes requested to reader, or actual bytes
+ * read by reader, which must be larger than
+ * the request, or -1 on error */
+ TimeLineID readPageTLI; /* TLI for data currently in readBuf */
+ char *readBuf; /* buffer to store data */
+ bool page_verified; /* is the page on the buffer verified? */
+
/* ----------------------------------------
* Decoded representation of current record
@@ -201,13 +215,6 @@ struct XLogReaderState
* ----------------------------------------
*/
- /*
- * Buffer for currently read page (XLOG_BLCKSZ bytes, valid up to at least
- * readLen bytes)
- */
- char *readBuf;
- uint32 readLen;
-
/* last read XLOG position for data currently in readBuf */
WALSegmentContext segcxt;
WALOpenSegment seg;
diff --git a/src/include/access/xlogutils.h b/src/include/access/xlogutils.h
index e59b6cf3a9..c287edf206 100644
--- a/src/include/access/xlogutils.h
+++ b/src/include/access/xlogutils.h
@@ -47,7 +47,7 @@ extern Buffer XLogReadBufferExtended(RelFileNode rnode, ForkNumber forknum,
extern Relation CreateFakeRelcacheEntry(RelFileNode rnode);
extern void FreeFakeRelcacheEntry(Relation fakerel);
-extern int read_local_xlog_page(XLogReaderState *state,
+extern bool read_local_xlog_page(XLogReaderState *state,
XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *cur_page);
extern void wal_segment_open(XLogReaderState *state,
--
2.18.4
----Next_Part(Thu_Jul__2_13_53_30_2020_072)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
filename="v15-0002-Move-page-reader-out-of-XLogReadRecord.patch"
^ permalink raw reply [nested|flat] 69+ messages in thread
* [PATCH v17 1/4] Move callback-call from ReadPageInternal to XLogReadRecord.
@ 2019-09-05 11:21 Kyotaro Horiguchi <[email protected]>
0 siblings, 0 replies; 69+ messages in thread
From: Kyotaro Horiguchi @ 2019-09-05 11:21 UTC (permalink / raw)
The current WAL record reader reads page data using a call back
function. Although it is not so problematic alone, it would be a
problem if we are going to do add tasks like encryption which is
performed on page data before WAL reader reads them. To avoid that the
record reader facility has to have a new code path corresponds to
every new callback, this patch separates page reader from WAL record
reading facility by modifying the current WAL record reader to a state
machine.
As the first step of that change, this patch moves the page reader
function out of ReadPageInternal, then the remaining tasks of the
function are taken over by the new function XLogNeedData. As the
result XLogPageRead directly calls the page reader callback function
according to the feedback from XLogNeedData.
---
src/backend/access/transam/xlog.c | 16 +-
src/backend/access/transam/xlogreader.c | 281 ++++++++++++++----------
src/backend/access/transam/xlogutils.c | 12 +-
src/backend/replication/walsender.c | 10 +-
src/bin/pg_rewind/parsexlog.c | 21 +-
src/bin/pg_waldump/pg_waldump.c | 8 +-
src/include/access/xlogreader.h | 25 ++-
src/include/access/xlogutils.h | 2 +-
8 files changed, 222 insertions(+), 153 deletions(-)
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 377afb8732..606328a65a 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -917,7 +917,7 @@ static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath,
static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli,
XLogSource source, bool notfoundOk);
static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, XLogSource source);
-static int XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
+static bool XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
bool fetching_ckpt, XLogRecPtr tliRecPtr);
@@ -4350,7 +4350,6 @@ ReadRecord(XLogReaderState *xlogreader, int emode,
XLogRecord *record;
XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
- /* Pass through parameters to XLogPageRead */
private->fetching_ckpt = fetching_ckpt;
private->emode = emode;
private->randAccess = (xlogreader->ReadRecPtr == InvalidXLogRecPtr);
@@ -11987,7 +11986,7 @@ CancelBackup(void)
* XLogPageRead() to try fetching the record from another source, or to
* sleep and retry.
*/
-static int
+static bool
XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *readBuf)
{
@@ -12046,7 +12045,8 @@ retry:
readLen = 0;
readSource = XLOG_FROM_ANY;
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
}
@@ -12141,7 +12141,8 @@ retry:
goto next_record_is_invalid;
}
- return readLen;
+ xlogreader->readLen = readLen;
+ return true;
next_record_is_invalid:
lastSourceFailed = true;
@@ -12155,8 +12156,9 @@ next_record_is_invalid:
/* In standby-mode, keep trying */
if (StandbyMode)
goto retry;
- else
- return -1;
+
+ xlogreader->readLen = -1;
+ return false;
}
/*
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index 42738eb940..7d04995b7b 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -36,8 +36,8 @@
static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
pg_attribute_printf(2, 3);
static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
-static int ReadPageInternal(XLogReaderState *state, XLogRecPtr pageptr,
- int reqLen);
+static bool XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr,
+ int reqLen, bool header_inclusive);
static void XLogReaderInvalReadState(XLogReaderState *state);
static bool ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
XLogRecPtr PrevRecPtr, XLogRecord *record, bool randAccess);
@@ -276,7 +276,6 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
uint32 targetRecOff;
uint32 pageHeaderSize;
bool gotheader;
- int readOff;
/*
* randAccess indicates whether to verify the previous-record pointer of
@@ -326,14 +325,20 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
* byte to cover the whole record header, or at least the part of it that
* fits on the same page.
*/
- readOff = ReadPageInternal(state, targetPagePtr,
- Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ));
- if (readOff < 0)
+ while (XLogNeedData(state, targetPagePtr,
+ Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+ targetRecOff != 0))
+ {
+ if (!state->routine.page_read(state, state->readPagePtr, state->readLen,
+ RecPtr, state->readBuf))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
/*
- * ReadPageInternal always returns at least the page header, so we can
- * examine it now.
+ * We have at least the page header, so we can examine it now.
*/
pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
if (targetRecOff == 0)
@@ -359,8 +364,8 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
goto err;
}
- /* ReadPageInternal has verified the page header */
- Assert(pageHeaderSize <= readOff);
+ /* XLogNeedData has verified the page header */
+ Assert(pageHeaderSize <= state->readLen);
/*
* Read the record length.
@@ -432,18 +437,27 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
do
{
+ int rest_len = total_len - gotlen;
+
/* Calculate pointer to beginning of next page */
targetPagePtr += XLOG_BLCKSZ;
/* Wait for the next page to become available */
- readOff = ReadPageInternal(state, targetPagePtr,
- Min(total_len - gotlen + SizeOfXLogShortPHD,
- XLOG_BLCKSZ));
+ while (XLogNeedData(state, targetPagePtr,
+ Min(rest_len, XLOG_BLCKSZ),
+ false))
+ {
+ if (!state->routine.page_read(state, state->readPagePtr,
+ state->readLen,
+ state->ReadRecPtr,
+ state->readBuf))
+ break;
+ }
- if (readOff < 0)
+ if (!state->page_verified)
goto err;
- Assert(SizeOfXLogShortPHD <= readOff);
+ Assert(SizeOfXLogShortPHD <= state->readLen);
/* Check that the continuation on next page looks valid */
pageHeader = (XLogPageHeader) state->readBuf;
@@ -473,21 +487,14 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
/* Append the continuation from this page to the buffer */
pageHeaderSize = XLogPageHeaderSize(pageHeader);
- if (readOff < pageHeaderSize)
- readOff = ReadPageInternal(state, targetPagePtr,
- pageHeaderSize);
-
- Assert(pageHeaderSize <= readOff);
+ Assert(pageHeaderSize <= state->readLen);
contdata = (char *) state->readBuf + pageHeaderSize;
len = XLOG_BLCKSZ - pageHeaderSize;
if (pageHeader->xlp_rem_len < len)
len = pageHeader->xlp_rem_len;
- if (readOff < pageHeaderSize + len)
- readOff = ReadPageInternal(state, targetPagePtr,
- pageHeaderSize + len);
-
+ Assert (pageHeaderSize + len <= state->readLen);
memcpy(buffer, (char *) contdata, len);
buffer += len;
gotlen += len;
@@ -517,9 +524,16 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
else
{
/* Wait for the record data to become available */
- readOff = ReadPageInternal(state, targetPagePtr,
- Min(targetRecOff + total_len, XLOG_BLCKSZ));
- if (readOff < 0)
+ while (XLogNeedData(state, targetPagePtr,
+ Min(targetRecOff + total_len, XLOG_BLCKSZ), true))
+ {
+ if (!state->routine.page_read(state, state->readPagePtr,
+ state->readLen,
+ state->ReadRecPtr, state->readBuf))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
/* Record does not cross a page boundary */
@@ -562,109 +576,138 @@ err:
}
/*
- * Read a single xlog page including at least [pageptr, reqLen] of valid data
- * via the page_read() callback.
+ * Checks that an xlog page loaded in state->readBuf is including at least
+ * [pageptr, reqLen] and the page is valid. header_inclusive indicates that
+ * reqLen is calculated including page header length.
*
- * Returns -1 if the required page cannot be read for some reason; errormsg_buf
- * is set in that case (unless the error occurs in the page_read callback).
+ * Returns false if the buffer already contains the requested data, or found
+ * error. state->page_verified is set to true for the former and false for the
+ * latter.
*
- * We fetch the page from a reader-local cache if we know we have the required
- * data and if there hasn't been any error since caching the data.
+ * Otherwise returns true and requests data loaded onto state->readBuf by
+ * state->readPagePtr and state->readLen. The caller shall call this function
+ * again after filling the buffer at least with that portion of data and set
+ * state->readLen to the length of actually loaded data.
+ *
+ * If header_inclusive is false, corrects reqLen internally by adding the
+ * actual page header length and may request caller for new data.
*/
-static int
-ReadPageInternal(XLogReaderState *state, XLogRecPtr pageptr, int reqLen)
+static bool
+XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
+ bool header_inclusive)
{
- int readLen;
uint32 targetPageOff;
XLogSegNo targetSegNo;
- XLogPageHeader hdr;
+ uint32 addLen = 0;
- Assert((pageptr % XLOG_BLCKSZ) == 0);
+ /* Some data is loaded, but page header is not verified yet. */
+ if (!state->page_verified &&
+ !XLogRecPtrIsInvalid(state->readPagePtr) && state->readLen >= 0)
+ {
+ uint32 pageHeaderSize;
+ /* just loaded new data so needs to verify page header */
+
+ /* The caller must have loaded at least page header */
+ Assert (state->readLen >= SizeOfXLogShortPHD);
+
+ /*
+ * We have enough data to check the header length. Recheck the loaded
+ * length against the actual header length.
+ */
+ pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+
+ /* Request more data if we don't have the full header. */
+ if (state->readLen < pageHeaderSize)
+ {
+ state->readLen = pageHeaderSize;
+ return true;
+ }
+
+ /* Now that we know we have the full header, validate it. */
+ if (!XLogReaderValidatePageHeader(state, state->readPagePtr,
+ (char *) state->readBuf))
+ {
+ /* That's bad. Force reading the page again. */
+ XLogReaderInvalReadState(state);
+
+ return false;
+ }
+
+ state->page_verified = true;
+
+ XLByteToSeg(state->readPagePtr, state->seg.ws_segno,
+ state->segcxt.ws_segsize);
+ }
+
+ /*
+ * The loaded page may not be the one caller is supposing to read when we
+ * are verifying the first page of new segment. In that case, skip further
+ * verification and immediately load the target page.
+ */
+ if (state->page_verified && pageptr == state->readPagePtr)
+ {
+ /*
+ * calculate additional length for page header keeping the total
+ * length within the block size.
+ */
+ if (!header_inclusive)
+ {
+ uint32 pageHeaderSize =
+ XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+
+ addLen = pageHeaderSize;
+ if (reqLen + pageHeaderSize <= XLOG_BLCKSZ)
+ addLen = pageHeaderSize;
+ else
+ addLen = XLOG_BLCKSZ - reqLen;
+
+ Assert(addLen >= 0);
+ }
+
+ /* Return if we already have it. */
+ if (reqLen + addLen <= state->readLen)
+ return false;
+ }
+
+ /* Data is not in our buffer, request the caller for it. */
XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize);
targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize);
+ Assert((pageptr % XLOG_BLCKSZ) == 0);
- /* check whether we have all the requested data already */
- if (targetSegNo == state->seg.ws_segno &&
- targetPageOff == state->segoff && reqLen <= state->readLen)
- return state->readLen;
+ /*
+ * Every time we request to load new data of a page to the caller, even if
+ * we looked at a part of it before, we need to do verification on the next
+ * invocation as the caller might now be rereading data from a different
+ * source.
+ */
+ state->page_verified = false;
/*
- * Data is not in our buffer.
- *
- * Every time we actually read the segment, even if we looked at parts of
- * it before, we need to do verification as the page_read callback might
- * now be rereading data from a different source.
- *
* Whenever switching to a new WAL segment, we read the first page of the
* file and validate its header, even if that's not where the target
* record is. This is so that we can check the additional identification
* info that is present in the first page's "long" header.
+ * Don't do this if the caller requested the first page in the segment.
*/
if (targetSegNo != state->seg.ws_segno && targetPageOff != 0)
{
- XLogRecPtr targetSegmentPtr = pageptr - targetPageOff;
-
- readLen = state->routine.page_read(state, targetSegmentPtr, XLOG_BLCKSZ,
- state->currRecPtr,
- state->readBuf);
- if (readLen < 0)
- goto err;
-
- /* we can be sure to have enough WAL available, we scrolled back */
- Assert(readLen == XLOG_BLCKSZ);
-
- if (!XLogReaderValidatePageHeader(state, targetSegmentPtr,
- state->readBuf))
- goto err;
+ /*
+ * Then we'll see that the targetSegNo now matches the ws_segno, and
+ * will not come back here, but will request the actual target page.
+ */
+ state->readPagePtr = pageptr - targetPageOff;
+ state->readLen = XLOG_BLCKSZ;
+ return true;
}
/*
- * First, read the requested data length, but at least a short page header
- * so that we can validate it.
+ * Request the caller to load the page. We need at least a short page
+ * header so that we can validate it.
*/
- readLen = state->routine.page_read(state, pageptr, Max(reqLen, SizeOfXLogShortPHD),
- state->currRecPtr,
- state->readBuf);
- if (readLen < 0)
- goto err;
-
- Assert(readLen <= XLOG_BLCKSZ);
-
- /* Do we have enough data to check the header length? */
- if (readLen <= SizeOfXLogShortPHD)
- goto err;
-
- Assert(readLen >= reqLen);
-
- hdr = (XLogPageHeader) state->readBuf;
-
- /* still not enough */
- if (readLen < XLogPageHeaderSize(hdr))
- {
- readLen = state->routine.page_read(state, pageptr, XLogPageHeaderSize(hdr),
- state->currRecPtr,
- state->readBuf);
- if (readLen < 0)
- goto err;
- }
-
- /*
- * Now that we know we have the full header, validate it.
- */
- if (!XLogReaderValidatePageHeader(state, pageptr, (char *) hdr))
- goto err;
-
- /* update read state information */
- state->seg.ws_segno = targetSegNo;
- state->segoff = targetPageOff;
- state->readLen = readLen;
-
- return readLen;
-
-err:
- XLogReaderInvalReadState(state);
- return -1;
+ state->readPagePtr = pageptr;
+ state->readLen = Max(reqLen + addLen, SizeOfXLogShortPHD);
+ return true;
}
/*
@@ -673,9 +716,7 @@ err:
static void
XLogReaderInvalReadState(XLogReaderState *state)
{
- state->seg.ws_segno = 0;
- state->segoff = 0;
- state->readLen = 0;
+ state->readPagePtr = InvalidXLogRecPtr;
}
/*
@@ -953,7 +994,6 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
XLogRecPtr targetPagePtr;
int targetRecOff;
uint32 pageHeaderSize;
- int readLen;
/*
* Compute targetRecOff. It should typically be equal or greater than
@@ -961,7 +1001,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
* that, except when caller has explicitly specified the offset that
* falls somewhere there or when we are skipping multi-page
* continuation record. It doesn't matter though because
- * ReadPageInternal() is prepared to handle that and will read at
+ * XLogNeedData() is prepared to handle that and will read at
* least short page-header worth of data
*/
targetRecOff = tmpRecPtr % XLOG_BLCKSZ;
@@ -969,19 +1009,24 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
/* scroll back to page boundary */
targetPagePtr = tmpRecPtr - targetRecOff;
- /* Read the page containing the record */
- readLen = ReadPageInternal(state, targetPagePtr, targetRecOff);
- if (readLen < 0)
+ while(XLogNeedData(state, targetPagePtr, targetRecOff,
+ targetRecOff != 0))
+ {
+ if (!state->routine.page_read(state, state->readPagePtr,
+ state->readLen,
+ state->ReadRecPtr, state->readBuf))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
header = (XLogPageHeader) state->readBuf;
pageHeaderSize = XLogPageHeaderSize(header);
- /* make sure we have enough data for the page header */
- readLen = ReadPageInternal(state, targetPagePtr, pageHeaderSize);
- if (readLen < 0)
- goto err;
+ /* we should have read the page header */
+ Assert (state->readLen >= pageHeaderSize);
/* skip over potential continuation data */
if (header->xlp_info & XLP_FIRST_IS_CONTRECORD)
diff --git a/src/backend/access/transam/xlogutils.c b/src/backend/access/transam/xlogutils.c
index d17d660f46..46eda33f25 100644
--- a/src/backend/access/transam/xlogutils.c
+++ b/src/backend/access/transam/xlogutils.c
@@ -686,8 +686,8 @@ XLogTruncateRelation(RelFileNode rnode, ForkNumber forkNum,
void
XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wantLength)
{
- const XLogRecPtr lastReadPage = (state->seg.ws_segno *
- state->segcxt.ws_segsize + state->segoff);
+ const XLogRecPtr lastReadPage = state->seg.ws_segno *
+ state->segcxt.ws_segsize + state->readLen;
Assert(wantPage != InvalidXLogRecPtr && wantPage % XLOG_BLCKSZ == 0);
Assert(wantLength <= XLOG_BLCKSZ);
@@ -824,7 +824,7 @@ wal_segment_close(XLogReaderState *state)
* exists for normal backends, so we have to do a check/sleep/repeat style of
* loop for now.
*/
-int
+bool
read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *cur_page)
{
@@ -926,7 +926,8 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
else if (targetPagePtr + reqLen > read_upto)
{
/* not enough data there */
- return -1;
+ state->readLen = -1;
+ return false;
}
else
{
@@ -944,7 +945,8 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
WALReadRaiseError(&errinfo);
/* number of valid bytes in the buffer */
- return count;
+ state->readLen = count;
+ return true;
}
/*
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index 23baa4498a..c36b91ae5e 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -806,7 +806,7 @@ StartReplication(StartReplicationCmd *cmd)
* which has to do a plain sleep/busy loop, because the walsender's latch gets
* set every time WAL is flushed.
*/
-static int
+static bool
logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *cur_page)
{
@@ -826,7 +826,10 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
/* fail if not (implies we are going to shut down) */
if (flushptr < targetPagePtr + reqLen)
- return -1;
+ {
+ state->readLen = -1;
+ return false;
+ }
if (targetPagePtr + XLOG_BLCKSZ <= flushptr)
count = XLOG_BLCKSZ; /* more than one block available */
@@ -854,7 +857,8 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
XLByteToSeg(targetPagePtr, segno, state->segcxt.ws_segsize);
CheckXLogRemoved(segno, state->seg.ws_tli);
- return count;
+ state->readLen = count;
+ return true;
}
/*
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index 59ebac7d6a..cf119848b0 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -47,7 +47,7 @@ typedef struct XLogPageReadPrivate
int tliIndex;
} XLogPageReadPrivate;
-static int SimpleXLogPageRead(XLogReaderState *xlogreader,
+static bool SimpleXLogPageRead(XLogReaderState *xlogreader,
XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
@@ -246,7 +246,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
}
/* XLogReader callback function, to read a WAL page */
-static int
+static bool
SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf)
{
@@ -306,7 +306,8 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
if (private->restoreCommand == NULL)
{
pg_log_error("could not open file \"%s\": %m", xlogfpath);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
/*
@@ -319,7 +320,10 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
private->restoreCommand);
if (xlogreadfd < 0)
- return -1;
+ {
+ xlogreader->readLen = -1;
+ return false;
+ }
else
pg_log_debug("using file \"%s\" restored from archive",
xlogfpath);
@@ -335,7 +339,8 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
if (lseek(xlogreadfd, (off_t) targetPageOff, SEEK_SET) < 0)
{
pg_log_error("could not seek in file \"%s\": %m", xlogfpath);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
@@ -348,13 +353,15 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
pg_log_error("could not read file \"%s\": read %d of %zu",
xlogfpath, r, (Size) XLOG_BLCKSZ);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
Assert(targetSegNo == xlogreadsegno);
xlogreader->seg.ws_tli = targetHistory[private->tliIndex].tli;
- return XLOG_BLCKSZ;
+ xlogreader->readLen = XLOG_BLCKSZ;
+ return true;
}
/*
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index 610f65e471..0f0e0723a3 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -331,7 +331,7 @@ WALDumpCloseSegment(XLogReaderState *state)
}
/* pg_waldump's XLogReaderRoutine->page_read callback */
-static int
+static bool
WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetPtr, char *readBuff)
{
@@ -348,7 +348,8 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
else
{
private->endptr_reached = true;
- return -1;
+ state->readLen = -1;
+ return false;
}
}
@@ -373,7 +374,8 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
(Size) errinfo.wre_req);
}
- return count;
+ state->readLen = count;
+ return true;
}
/*
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index 21d200d3df..23d9e70a59 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -57,8 +57,8 @@ typedef struct WALSegmentContext
typedef struct XLogReaderState XLogReaderState;
-/* Function type definitions for various xlogreader interactions */
-typedef int (*XLogPageReadCB) (XLogReaderState *xlogreader,
+/* Function type definition for the read_page callback */
+typedef bool (*XLogPageReadCB) (XLogReaderState *xlogreader,
XLogRecPtr targetPagePtr,
int reqLen,
XLogRecPtr targetRecPtr,
@@ -175,6 +175,20 @@ struct XLogReaderState
XLogRecPtr ReadRecPtr; /* start of last record read */
XLogRecPtr EndRecPtr; /* end+1 of last record read */
+ /* ----------------------------------------
+ * Communication with page reader
+ * readBuf is XLOG_BLCKSZ bytes, valid up to at least readLen bytes.
+ * ----------------------------------------
+ */
+ /* variables to communicate with page reader */
+ XLogRecPtr readPagePtr; /* page pointer to read */
+ int32 readLen; /* bytes requested to reader, or actual bytes
+ * read by reader, which must be larger than
+ * the request, or -1 on error */
+ TimeLineID readPageTLI; /* TLI for data currently in readBuf */
+ char *readBuf; /* buffer to store data */
+ bool page_verified; /* is the page on the buffer verified? */
+
/* ----------------------------------------
* Decoded representation of current record
@@ -203,13 +217,6 @@ struct XLogReaderState
* ----------------------------------------
*/
- /*
- * Buffer for currently read page (XLOG_BLCKSZ bytes, valid up to at least
- * readLen bytes)
- */
- char *readBuf;
- uint32 readLen;
-
/* last read XLOG position for data currently in readBuf */
WALSegmentContext segcxt;
WALOpenSegment seg;
diff --git a/src/include/access/xlogutils.h b/src/include/access/xlogutils.h
index 9ac602b674..364a21c4ea 100644
--- a/src/include/access/xlogutils.h
+++ b/src/include/access/xlogutils.h
@@ -47,7 +47,7 @@ extern Buffer XLogReadBufferExtended(RelFileNode rnode, ForkNumber forknum,
extern Relation CreateFakeRelcacheEntry(RelFileNode rnode);
extern void FreeFakeRelcacheEntry(Relation fakerel);
-extern int read_local_xlog_page(XLogReaderState *state,
+extern bool read_local_xlog_page(XLogReaderState *state,
XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *cur_page);
extern void wal_segment_open(XLogReaderState *state,
--
2.27.0
----Next_Part(Thu_Mar__4_11_28_53_2021_014)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
filename="v17-0002-Move-page-reader-out-of-XLogReadRecord.patch"
^ permalink raw reply [nested|flat] 69+ messages in thread
* [PATCH v15 1/4] Move callback-call from ReadPageInternal to XLogReadRecord.
@ 2019-09-05 11:21 Kyotaro Horiguchi <[email protected]>
0 siblings, 0 replies; 69+ messages in thread
From: Kyotaro Horiguchi @ 2019-09-05 11:21 UTC (permalink / raw)
The current WAL record reader reads page data using a call back
function. Although it is not so problematic alone, it would be a
problem if we are going to do add tasks like encryption which is
performed on page data before WAL reader reads them. To avoid that the
record reader facility has to have a new code path corresponds to
every new callback, this patch separates page reader from WAL record
reading facility by modifying the current WAL record reader to a state
machine.
As the first step of that change, this patch moves the page reader
function out of ReadPageInternal, then the remaining tasks of the
function are taken over by the new function XLogNeedData. As the
result XLogPageRead directly calls the page reader callback function
according to the feedback from XLogNeedData.
---
src/backend/access/transam/xlog.c | 16 +-
src/backend/access/transam/xlogreader.c | 281 ++++++++++++++----------
src/backend/access/transam/xlogutils.c | 12 +-
src/backend/replication/walsender.c | 10 +-
src/bin/pg_rewind/parsexlog.c | 21 +-
src/bin/pg_waldump/pg_waldump.c | 8 +-
src/include/access/xlogreader.h | 25 ++-
src/include/access/xlogutils.h | 2 +-
8 files changed, 222 insertions(+), 153 deletions(-)
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index fd93bcfaeb..e570e56a24 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -912,7 +912,7 @@ static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath,
static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli,
XLogSource source, bool notfoundOk);
static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, XLogSource source);
-static int XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
+static bool XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
bool fetching_ckpt, XLogRecPtr tliRecPtr);
@@ -4332,7 +4332,6 @@ ReadRecord(XLogReaderState *xlogreader, int emode,
XLogRecord *record;
XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
- /* Pass through parameters to XLogPageRead */
private->fetching_ckpt = fetching_ckpt;
private->emode = emode;
private->randAccess = (xlogreader->ReadRecPtr == InvalidXLogRecPtr);
@@ -11860,7 +11859,7 @@ CancelBackup(void)
* XLogPageRead() to try fetching the record from another source, or to
* sleep and retry.
*/
-static int
+static bool
XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *readBuf)
{
@@ -11919,7 +11918,8 @@ retry:
readLen = 0;
readSource = XLOG_FROM_ANY;
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
}
@@ -12014,7 +12014,8 @@ retry:
goto next_record_is_invalid;
}
- return readLen;
+ xlogreader->readLen = readLen;
+ return true;
next_record_is_invalid:
lastSourceFailed = true;
@@ -12028,8 +12029,9 @@ next_record_is_invalid:
/* In standby-mode, keep trying */
if (StandbyMode)
goto retry;
- else
- return -1;
+
+ xlogreader->readLen = -1;
+ return false;
}
/*
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index cb76be4f46..3d599325ee 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -36,8 +36,8 @@
static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
pg_attribute_printf(2, 3);
static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
-static int ReadPageInternal(XLogReaderState *state, XLogRecPtr pageptr,
- int reqLen);
+static bool XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr,
+ int reqLen, bool header_inclusive);
static void XLogReaderInvalReadState(XLogReaderState *state);
static bool ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
XLogRecPtr PrevRecPtr, XLogRecord *record, bool randAccess);
@@ -276,7 +276,6 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
uint32 targetRecOff;
uint32 pageHeaderSize;
bool gotheader;
- int readOff;
/*
* randAccess indicates whether to verify the previous-record pointer of
@@ -326,14 +325,20 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
* byte to cover the whole record header, or at least the part of it that
* fits on the same page.
*/
- readOff = ReadPageInternal(state, targetPagePtr,
- Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ));
- if (readOff < 0)
+ while (XLogNeedData(state, targetPagePtr,
+ Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+ targetRecOff != 0))
+ {
+ if (!state->routine.page_read(state, state->readPagePtr, state->readLen,
+ RecPtr, state->readBuf))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
/*
- * ReadPageInternal always returns at least the page header, so we can
- * examine it now.
+ * We have at least the page header, so we can examine it now.
*/
pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
if (targetRecOff == 0)
@@ -359,8 +364,8 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
goto err;
}
- /* ReadPageInternal has verified the page header */
- Assert(pageHeaderSize <= readOff);
+ /* XLogNeedData has verified the page header */
+ Assert(pageHeaderSize <= state->readLen);
/*
* Read the record length.
@@ -433,18 +438,27 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
do
{
+ int rest_len = total_len - gotlen;
+
/* Calculate pointer to beginning of next page */
targetPagePtr += XLOG_BLCKSZ;
/* Wait for the next page to become available */
- readOff = ReadPageInternal(state, targetPagePtr,
- Min(total_len - gotlen + SizeOfXLogShortPHD,
- XLOG_BLCKSZ));
+ while (XLogNeedData(state, targetPagePtr,
+ Min(rest_len, XLOG_BLCKSZ),
+ false))
+ {
+ if (!state->routine.page_read(state, state->readPagePtr,
+ state->readLen,
+ state->ReadRecPtr,
+ state->readBuf))
+ break;
+ }
- if (readOff < 0)
+ if (!state->page_verified)
goto err;
- Assert(SizeOfXLogShortPHD <= readOff);
+ Assert(SizeOfXLogShortPHD <= state->readLen);
/* Check that the continuation on next page looks valid */
pageHeader = (XLogPageHeader) state->readBuf;
@@ -473,21 +487,14 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
/* Append the continuation from this page to the buffer */
pageHeaderSize = XLogPageHeaderSize(pageHeader);
- if (readOff < pageHeaderSize)
- readOff = ReadPageInternal(state, targetPagePtr,
- pageHeaderSize);
-
- Assert(pageHeaderSize <= readOff);
+ Assert(pageHeaderSize <= state->readLen);
contdata = (char *) state->readBuf + pageHeaderSize;
len = XLOG_BLCKSZ - pageHeaderSize;
if (pageHeader->xlp_rem_len < len)
len = pageHeader->xlp_rem_len;
- if (readOff < pageHeaderSize + len)
- readOff = ReadPageInternal(state, targetPagePtr,
- pageHeaderSize + len);
-
+ Assert (pageHeaderSize + len <= state->readLen);
memcpy(buffer, (char *) contdata, len);
buffer += len;
gotlen += len;
@@ -517,9 +524,16 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
else
{
/* Wait for the record data to become available */
- readOff = ReadPageInternal(state, targetPagePtr,
- Min(targetRecOff + total_len, XLOG_BLCKSZ));
- if (readOff < 0)
+ while (XLogNeedData(state, targetPagePtr,
+ Min(targetRecOff + total_len, XLOG_BLCKSZ), true))
+ {
+ if (!state->routine.page_read(state, state->readPagePtr,
+ state->readLen,
+ state->ReadRecPtr, state->readBuf))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
/* Record does not cross a page boundary */
@@ -562,109 +576,138 @@ err:
}
/*
- * Read a single xlog page including at least [pageptr, reqLen] of valid data
- * via the page_read() callback.
+ * Checks that an xlog page loaded in state->readBuf is including at least
+ * [pageptr, reqLen] and the page is valid. header_inclusive indicates that
+ * reqLen is calculated including page header length.
*
- * Returns -1 if the required page cannot be read for some reason; errormsg_buf
- * is set in that case (unless the error occurs in the page_read callback).
+ * Returns false if the buffer already contains the requested data, or found
+ * error. state->page_verified is set to true for the former and false for the
+ * latter.
*
- * We fetch the page from a reader-local cache if we know we have the required
- * data and if there hasn't been any error since caching the data.
+ * Otherwise returns true and requests data loaded onto state->readBuf by
+ * state->readPagePtr and state->readLen. The caller shall call this function
+ * again after filling the buffer at least with that portion of data and set
+ * state->readLen to the length of actually loaded data.
+ *
+ * If header_inclusive is false, corrects reqLen internally by adding the
+ * actual page header length and may request caller for new data.
*/
-static int
-ReadPageInternal(XLogReaderState *state, XLogRecPtr pageptr, int reqLen)
+static bool
+XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
+ bool header_inclusive)
{
- int readLen;
uint32 targetPageOff;
XLogSegNo targetSegNo;
- XLogPageHeader hdr;
+ uint32 addLen = 0;
- Assert((pageptr % XLOG_BLCKSZ) == 0);
+ /* Some data is loaded, but page header is not verified yet. */
+ if (!state->page_verified &&
+ !XLogRecPtrIsInvalid(state->readPagePtr) && state->readLen >= 0)
+ {
+ uint32 pageHeaderSize;
+ /* just loaded new data so needs to verify page header */
+
+ /* The caller must have loaded at least page header */
+ Assert (state->readLen >= SizeOfXLogShortPHD);
+
+ /*
+ * We have enough data to check the header length. Recheck the loaded
+ * length against the actual header length.
+ */
+ pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+
+ /* Request more data if we don't have the full header. */
+ if (state->readLen < pageHeaderSize)
+ {
+ state->readLen = pageHeaderSize;
+ return true;
+ }
+
+ /* Now that we know we have the full header, validate it. */
+ if (!XLogReaderValidatePageHeader(state, state->readPagePtr,
+ (char *) state->readBuf))
+ {
+ /* That's bad. Force reading the page again. */
+ XLogReaderInvalReadState(state);
+
+ return false;
+ }
+
+ state->page_verified = true;
+
+ XLByteToSeg(state->readPagePtr, state->seg.ws_segno,
+ state->segcxt.ws_segsize);
+ }
+
+ /*
+ * The loaded page may not be the one caller is supposing to read when we
+ * are verifying the first page of new segment. In that case, skip further
+ * verification and immediately load the target page.
+ */
+ if (state->page_verified && pageptr == state->readPagePtr)
+ {
+ /*
+ * calculate additional length for page header keeping the total
+ * length within the block size.
+ */
+ if (!header_inclusive)
+ {
+ uint32 pageHeaderSize =
+ XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+
+ addLen = pageHeaderSize;
+ if (reqLen + pageHeaderSize <= XLOG_BLCKSZ)
+ addLen = pageHeaderSize;
+ else
+ addLen = XLOG_BLCKSZ - reqLen;
+
+ Assert(addLen >= 0);
+ }
+
+ /* Return if we already have it. */
+ if (reqLen + addLen <= state->readLen)
+ return false;
+ }
+
+ /* Data is not in our buffer, request the caller for it. */
XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize);
targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize);
+ Assert((pageptr % XLOG_BLCKSZ) == 0);
- /* check whether we have all the requested data already */
- if (targetSegNo == state->seg.ws_segno &&
- targetPageOff == state->segoff && reqLen <= state->readLen)
- return state->readLen;
+ /*
+ * Every time we request to load new data of a page to the caller, even if
+ * we looked at a part of it before, we need to do verification on the next
+ * invocation as the caller might now be rereading data from a different
+ * source.
+ */
+ state->page_verified = false;
/*
- * Data is not in our buffer.
- *
- * Every time we actually read the segment, even if we looked at parts of
- * it before, we need to do verification as the page_read callback might
- * now be rereading data from a different source.
- *
* Whenever switching to a new WAL segment, we read the first page of the
* file and validate its header, even if that's not where the target
* record is. This is so that we can check the additional identification
* info that is present in the first page's "long" header.
+ * Don't do this if the caller requested the first page in the segment.
*/
if (targetSegNo != state->seg.ws_segno && targetPageOff != 0)
{
- XLogRecPtr targetSegmentPtr = pageptr - targetPageOff;
-
- readLen = state->routine.page_read(state, targetSegmentPtr, XLOG_BLCKSZ,
- state->currRecPtr,
- state->readBuf);
- if (readLen < 0)
- goto err;
-
- /* we can be sure to have enough WAL available, we scrolled back */
- Assert(readLen == XLOG_BLCKSZ);
-
- if (!XLogReaderValidatePageHeader(state, targetSegmentPtr,
- state->readBuf))
- goto err;
+ /*
+ * Then we'll see that the targetSegNo now matches the ws_segno, and
+ * will not come back here, but will request the actual target page.
+ */
+ state->readPagePtr = pageptr - targetPageOff;
+ state->readLen = XLOG_BLCKSZ;
+ return true;
}
/*
- * First, read the requested data length, but at least a short page header
- * so that we can validate it.
+ * Request the caller to load the page. We need at least a short page
+ * header so that we can validate it.
*/
- readLen = state->routine.page_read(state, pageptr, Max(reqLen, SizeOfXLogShortPHD),
- state->currRecPtr,
- state->readBuf);
- if (readLen < 0)
- goto err;
-
- Assert(readLen <= XLOG_BLCKSZ);
-
- /* Do we have enough data to check the header length? */
- if (readLen <= SizeOfXLogShortPHD)
- goto err;
-
- Assert(readLen >= reqLen);
-
- hdr = (XLogPageHeader) state->readBuf;
-
- /* still not enough */
- if (readLen < XLogPageHeaderSize(hdr))
- {
- readLen = state->routine.page_read(state, pageptr, XLogPageHeaderSize(hdr),
- state->currRecPtr,
- state->readBuf);
- if (readLen < 0)
- goto err;
- }
-
- /*
- * Now that we know we have the full header, validate it.
- */
- if (!XLogReaderValidatePageHeader(state, pageptr, (char *) hdr))
- goto err;
-
- /* update read state information */
- state->seg.ws_segno = targetSegNo;
- state->segoff = targetPageOff;
- state->readLen = readLen;
-
- return readLen;
-
-err:
- XLogReaderInvalReadState(state);
- return -1;
+ state->readPagePtr = pageptr;
+ state->readLen = Max(reqLen + addLen, SizeOfXLogShortPHD);
+ return true;
}
/*
@@ -673,9 +716,7 @@ err:
static void
XLogReaderInvalReadState(XLogReaderState *state)
{
- state->seg.ws_segno = 0;
- state->segoff = 0;
- state->readLen = 0;
+ state->readPagePtr = InvalidXLogRecPtr;
}
/*
@@ -956,7 +997,6 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
XLogRecPtr targetPagePtr;
int targetRecOff;
uint32 pageHeaderSize;
- int readLen;
/*
* Compute targetRecOff. It should typically be equal or greater than
@@ -964,7 +1004,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
* that, except when caller has explicitly specified the offset that
* falls somewhere there or when we are skipping multi-page
* continuation record. It doesn't matter though because
- * ReadPageInternal() is prepared to handle that and will read at
+ * XLogNeedData() is prepared to handle that and will read at
* least short page-header worth of data
*/
targetRecOff = tmpRecPtr % XLOG_BLCKSZ;
@@ -972,19 +1012,24 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
/* scroll back to page boundary */
targetPagePtr = tmpRecPtr - targetRecOff;
- /* Read the page containing the record */
- readLen = ReadPageInternal(state, targetPagePtr, targetRecOff);
- if (readLen < 0)
+ while(XLogNeedData(state, targetPagePtr, targetRecOff,
+ targetRecOff != 0))
+ {
+ if (!state->routine.page_read(state, state->readPagePtr,
+ state->readLen,
+ state->ReadRecPtr, state->readBuf))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
header = (XLogPageHeader) state->readBuf;
pageHeaderSize = XLogPageHeaderSize(header);
- /* make sure we have enough data for the page header */
- readLen = ReadPageInternal(state, targetPagePtr, pageHeaderSize);
- if (readLen < 0)
- goto err;
+ /* we should have read the page header */
+ Assert (state->readLen >= pageHeaderSize);
/* skip over potential continuation data */
if (header->xlp_info & XLP_FIRST_IS_CONTRECORD)
diff --git a/src/backend/access/transam/xlogutils.c b/src/backend/access/transam/xlogutils.c
index 322b0e8ff5..1b10b6df20 100644
--- a/src/backend/access/transam/xlogutils.c
+++ b/src/backend/access/transam/xlogutils.c
@@ -689,8 +689,8 @@ XLogTruncateRelation(RelFileNode rnode, ForkNumber forkNum,
void
XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wantLength)
{
- const XLogRecPtr lastReadPage = (state->seg.ws_segno *
- state->segcxt.ws_segsize + state->segoff);
+ const XLogRecPtr lastReadPage = state->seg.ws_segno *
+ state->segcxt.ws_segsize + state->readLen;
Assert(wantPage != InvalidXLogRecPtr && wantPage % XLOG_BLCKSZ == 0);
Assert(wantLength <= XLOG_BLCKSZ);
@@ -828,7 +828,7 @@ wal_segment_close(XLogReaderState *state)
* exists for normal backends, so we have to do a check/sleep/repeat style of
* loop for now.
*/
-int
+bool
read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *cur_page)
{
@@ -930,7 +930,8 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
else if (targetPagePtr + reqLen > read_upto)
{
/* not enough data there */
- return -1;
+ state->readLen = -1;
+ return false;
}
else
{
@@ -948,7 +949,8 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
WALReadRaiseError(&errinfo);
/* number of valid bytes in the buffer */
- return count;
+ state->readLen = count;
+ return true;
}
/*
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index e2477c47e0..bf2711bddd 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -811,7 +811,7 @@ StartReplication(StartReplicationCmd *cmd)
* which has to do a plain sleep/busy loop, because the walsender's latch gets
* set every time WAL is flushed.
*/
-static int
+static bool
logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *cur_page)
{
@@ -831,7 +831,10 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
/* fail if not (implies we are going to shut down) */
if (flushptr < targetPagePtr + reqLen)
- return -1;
+ {
+ state->readLen = -1;
+ return false;
+ }
if (targetPagePtr + XLOG_BLCKSZ <= flushptr)
count = XLOG_BLCKSZ; /* more than one block available */
@@ -859,7 +862,8 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
XLByteToSeg(targetPagePtr, segno, state->segcxt.ws_segsize);
CheckXLogRemoved(segno, state->seg.ws_tli);
- return count;
+ state->readLen = count;
+ return true;
}
/*
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index bc6f976994..0b7e73ae79 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -46,7 +46,7 @@ typedef struct XLogPageReadPrivate
int tliIndex;
} XLogPageReadPrivate;
-static int SimpleXLogPageRead(XLogReaderState *xlogreader,
+static bool SimpleXLogPageRead(XLogReaderState *xlogreader,
XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
@@ -236,7 +236,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
}
/* XLogReader callback function, to read a WAL page */
-static int
+static bool
SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf)
{
@@ -296,7 +296,8 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
if (private->restoreCommand == NULL)
{
pg_log_error("could not open file \"%s\": %m", xlogfpath);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
/*
@@ -309,7 +310,10 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
private->restoreCommand);
if (xlogreadfd < 0)
- return -1;
+ {
+ xlogreader->readLen = -1;
+ return false;
+ }
else
pg_log_debug("using file \"%s\" restored from archive",
xlogfpath);
@@ -325,7 +329,8 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
if (lseek(xlogreadfd, (off_t) targetPageOff, SEEK_SET) < 0)
{
pg_log_error("could not seek in file \"%s\": %m", xlogfpath);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
@@ -338,13 +343,15 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
pg_log_error("could not read file \"%s\": read %d of %zu",
xlogfpath, r, (Size) XLOG_BLCKSZ);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
Assert(targetSegNo == xlogreadsegno);
xlogreader->seg.ws_tli = targetHistory[private->tliIndex].tli;
- return XLOG_BLCKSZ;
+ xlogreader->readLen = XLOG_BLCKSZ;
+ return true;
}
/*
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index d1a0678935..ab0c28d9b7 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -331,7 +331,7 @@ WALDumpCloseSegment(XLogReaderState *state)
}
/* pg_waldump's XLogReaderRoutine->page_read callback */
-static int
+static bool
WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetPtr, char *readBuff)
{
@@ -348,7 +348,8 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
else
{
private->endptr_reached = true;
- return -1;
+ state->readLen = -1;
+ return false;
}
}
@@ -373,7 +374,8 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
(Size) errinfo.wre_req);
}
- return count;
+ state->readLen = count;
+ return true;
}
/*
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index b0f2a6ed43..6c8848e14f 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -57,8 +57,8 @@ typedef struct WALSegmentContext
typedef struct XLogReaderState XLogReaderState;
-/* Function type definitions for various xlogreader interactions */
-typedef int (*XLogPageReadCB) (XLogReaderState *xlogreader,
+/* Function type definition for the read_page callback */
+typedef bool (*XLogPageReadCB) (XLogReaderState *xlogreader,
XLogRecPtr targetPagePtr,
int reqLen,
XLogRecPtr targetRecPtr,
@@ -175,6 +175,20 @@ struct XLogReaderState
XLogRecPtr ReadRecPtr; /* start of last record read */
XLogRecPtr EndRecPtr; /* end+1 of last record read */
+ /* ----------------------------------------
+ * Communication with page reader
+ * readBuf is XLOG_BLCKSZ bytes, valid up to at least readLen bytes.
+ * ----------------------------------------
+ */
+ /* variables to communicate with page reader */
+ XLogRecPtr readPagePtr; /* page pointer to read */
+ int32 readLen; /* bytes requested to reader, or actual bytes
+ * read by reader, which must be larger than
+ * the request, or -1 on error */
+ TimeLineID readPageTLI; /* TLI for data currently in readBuf */
+ char *readBuf; /* buffer to store data */
+ bool page_verified; /* is the page on the buffer verified? */
+
/* ----------------------------------------
* Decoded representation of current record
@@ -201,13 +215,6 @@ struct XLogReaderState
* ----------------------------------------
*/
- /*
- * Buffer for currently read page (XLOG_BLCKSZ bytes, valid up to at least
- * readLen bytes)
- */
- char *readBuf;
- uint32 readLen;
-
/* last read XLOG position for data currently in readBuf */
WALSegmentContext segcxt;
WALOpenSegment seg;
diff --git a/src/include/access/xlogutils.h b/src/include/access/xlogutils.h
index e59b6cf3a9..c287edf206 100644
--- a/src/include/access/xlogutils.h
+++ b/src/include/access/xlogutils.h
@@ -47,7 +47,7 @@ extern Buffer XLogReadBufferExtended(RelFileNode rnode, ForkNumber forknum,
extern Relation CreateFakeRelcacheEntry(RelFileNode rnode);
extern void FreeFakeRelcacheEntry(Relation fakerel);
-extern int read_local_xlog_page(XLogReaderState *state,
+extern bool read_local_xlog_page(XLogReaderState *state,
XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *cur_page);
extern void wal_segment_open(XLogReaderState *state,
--
2.18.4
----Next_Part(Thu_Jul__2_13_53_30_2020_072)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
filename="v15-0002-Move-page-reader-out-of-XLogReadRecord.patch"
^ permalink raw reply [nested|flat] 69+ messages in thread
* [PATCH v17 1/4] Move callback-call from ReadPageInternal to XLogReadRecord.
@ 2019-09-05 11:21 Kyotaro Horiguchi <[email protected]>
0 siblings, 0 replies; 69+ messages in thread
From: Kyotaro Horiguchi @ 2019-09-05 11:21 UTC (permalink / raw)
The current WAL record reader reads page data using a call back
function. Although it is not so problematic alone, it would be a
problem if we are going to do add tasks like encryption which is
performed on page data before WAL reader reads them. To avoid that the
record reader facility has to have a new code path corresponds to
every new callback, this patch separates page reader from WAL record
reading facility by modifying the current WAL record reader to a state
machine.
As the first step of that change, this patch moves the page reader
function out of ReadPageInternal, then the remaining tasks of the
function are taken over by the new function XLogNeedData. As the
result XLogPageRead directly calls the page reader callback function
according to the feedback from XLogNeedData.
---
src/backend/access/transam/xlog.c | 16 +-
src/backend/access/transam/xlogreader.c | 281 ++++++++++++++----------
src/backend/access/transam/xlogutils.c | 12 +-
src/backend/replication/walsender.c | 10 +-
src/bin/pg_rewind/parsexlog.c | 21 +-
src/bin/pg_waldump/pg_waldump.c | 8 +-
src/include/access/xlogreader.h | 25 ++-
src/include/access/xlogutils.h | 2 +-
8 files changed, 222 insertions(+), 153 deletions(-)
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 377afb8732..606328a65a 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -917,7 +917,7 @@ static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath,
static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli,
XLogSource source, bool notfoundOk);
static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, XLogSource source);
-static int XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
+static bool XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
bool fetching_ckpt, XLogRecPtr tliRecPtr);
@@ -4350,7 +4350,6 @@ ReadRecord(XLogReaderState *xlogreader, int emode,
XLogRecord *record;
XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
- /* Pass through parameters to XLogPageRead */
private->fetching_ckpt = fetching_ckpt;
private->emode = emode;
private->randAccess = (xlogreader->ReadRecPtr == InvalidXLogRecPtr);
@@ -11987,7 +11986,7 @@ CancelBackup(void)
* XLogPageRead() to try fetching the record from another source, or to
* sleep and retry.
*/
-static int
+static bool
XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *readBuf)
{
@@ -12046,7 +12045,8 @@ retry:
readLen = 0;
readSource = XLOG_FROM_ANY;
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
}
@@ -12141,7 +12141,8 @@ retry:
goto next_record_is_invalid;
}
- return readLen;
+ xlogreader->readLen = readLen;
+ return true;
next_record_is_invalid:
lastSourceFailed = true;
@@ -12155,8 +12156,9 @@ next_record_is_invalid:
/* In standby-mode, keep trying */
if (StandbyMode)
goto retry;
- else
- return -1;
+
+ xlogreader->readLen = -1;
+ return false;
}
/*
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index 42738eb940..7d04995b7b 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -36,8 +36,8 @@
static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
pg_attribute_printf(2, 3);
static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
-static int ReadPageInternal(XLogReaderState *state, XLogRecPtr pageptr,
- int reqLen);
+static bool XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr,
+ int reqLen, bool header_inclusive);
static void XLogReaderInvalReadState(XLogReaderState *state);
static bool ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
XLogRecPtr PrevRecPtr, XLogRecord *record, bool randAccess);
@@ -276,7 +276,6 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
uint32 targetRecOff;
uint32 pageHeaderSize;
bool gotheader;
- int readOff;
/*
* randAccess indicates whether to verify the previous-record pointer of
@@ -326,14 +325,20 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
* byte to cover the whole record header, or at least the part of it that
* fits on the same page.
*/
- readOff = ReadPageInternal(state, targetPagePtr,
- Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ));
- if (readOff < 0)
+ while (XLogNeedData(state, targetPagePtr,
+ Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+ targetRecOff != 0))
+ {
+ if (!state->routine.page_read(state, state->readPagePtr, state->readLen,
+ RecPtr, state->readBuf))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
/*
- * ReadPageInternal always returns at least the page header, so we can
- * examine it now.
+ * We have at least the page header, so we can examine it now.
*/
pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
if (targetRecOff == 0)
@@ -359,8 +364,8 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
goto err;
}
- /* ReadPageInternal has verified the page header */
- Assert(pageHeaderSize <= readOff);
+ /* XLogNeedData has verified the page header */
+ Assert(pageHeaderSize <= state->readLen);
/*
* Read the record length.
@@ -432,18 +437,27 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
do
{
+ int rest_len = total_len - gotlen;
+
/* Calculate pointer to beginning of next page */
targetPagePtr += XLOG_BLCKSZ;
/* Wait for the next page to become available */
- readOff = ReadPageInternal(state, targetPagePtr,
- Min(total_len - gotlen + SizeOfXLogShortPHD,
- XLOG_BLCKSZ));
+ while (XLogNeedData(state, targetPagePtr,
+ Min(rest_len, XLOG_BLCKSZ),
+ false))
+ {
+ if (!state->routine.page_read(state, state->readPagePtr,
+ state->readLen,
+ state->ReadRecPtr,
+ state->readBuf))
+ break;
+ }
- if (readOff < 0)
+ if (!state->page_verified)
goto err;
- Assert(SizeOfXLogShortPHD <= readOff);
+ Assert(SizeOfXLogShortPHD <= state->readLen);
/* Check that the continuation on next page looks valid */
pageHeader = (XLogPageHeader) state->readBuf;
@@ -473,21 +487,14 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
/* Append the continuation from this page to the buffer */
pageHeaderSize = XLogPageHeaderSize(pageHeader);
- if (readOff < pageHeaderSize)
- readOff = ReadPageInternal(state, targetPagePtr,
- pageHeaderSize);
-
- Assert(pageHeaderSize <= readOff);
+ Assert(pageHeaderSize <= state->readLen);
contdata = (char *) state->readBuf + pageHeaderSize;
len = XLOG_BLCKSZ - pageHeaderSize;
if (pageHeader->xlp_rem_len < len)
len = pageHeader->xlp_rem_len;
- if (readOff < pageHeaderSize + len)
- readOff = ReadPageInternal(state, targetPagePtr,
- pageHeaderSize + len);
-
+ Assert (pageHeaderSize + len <= state->readLen);
memcpy(buffer, (char *) contdata, len);
buffer += len;
gotlen += len;
@@ -517,9 +524,16 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
else
{
/* Wait for the record data to become available */
- readOff = ReadPageInternal(state, targetPagePtr,
- Min(targetRecOff + total_len, XLOG_BLCKSZ));
- if (readOff < 0)
+ while (XLogNeedData(state, targetPagePtr,
+ Min(targetRecOff + total_len, XLOG_BLCKSZ), true))
+ {
+ if (!state->routine.page_read(state, state->readPagePtr,
+ state->readLen,
+ state->ReadRecPtr, state->readBuf))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
/* Record does not cross a page boundary */
@@ -562,109 +576,138 @@ err:
}
/*
- * Read a single xlog page including at least [pageptr, reqLen] of valid data
- * via the page_read() callback.
+ * Checks that an xlog page loaded in state->readBuf is including at least
+ * [pageptr, reqLen] and the page is valid. header_inclusive indicates that
+ * reqLen is calculated including page header length.
*
- * Returns -1 if the required page cannot be read for some reason; errormsg_buf
- * is set in that case (unless the error occurs in the page_read callback).
+ * Returns false if the buffer already contains the requested data, or found
+ * error. state->page_verified is set to true for the former and false for the
+ * latter.
*
- * We fetch the page from a reader-local cache if we know we have the required
- * data and if there hasn't been any error since caching the data.
+ * Otherwise returns true and requests data loaded onto state->readBuf by
+ * state->readPagePtr and state->readLen. The caller shall call this function
+ * again after filling the buffer at least with that portion of data and set
+ * state->readLen to the length of actually loaded data.
+ *
+ * If header_inclusive is false, corrects reqLen internally by adding the
+ * actual page header length and may request caller for new data.
*/
-static int
-ReadPageInternal(XLogReaderState *state, XLogRecPtr pageptr, int reqLen)
+static bool
+XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
+ bool header_inclusive)
{
- int readLen;
uint32 targetPageOff;
XLogSegNo targetSegNo;
- XLogPageHeader hdr;
+ uint32 addLen = 0;
- Assert((pageptr % XLOG_BLCKSZ) == 0);
+ /* Some data is loaded, but page header is not verified yet. */
+ if (!state->page_verified &&
+ !XLogRecPtrIsInvalid(state->readPagePtr) && state->readLen >= 0)
+ {
+ uint32 pageHeaderSize;
+ /* just loaded new data so needs to verify page header */
+
+ /* The caller must have loaded at least page header */
+ Assert (state->readLen >= SizeOfXLogShortPHD);
+
+ /*
+ * We have enough data to check the header length. Recheck the loaded
+ * length against the actual header length.
+ */
+ pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+
+ /* Request more data if we don't have the full header. */
+ if (state->readLen < pageHeaderSize)
+ {
+ state->readLen = pageHeaderSize;
+ return true;
+ }
+
+ /* Now that we know we have the full header, validate it. */
+ if (!XLogReaderValidatePageHeader(state, state->readPagePtr,
+ (char *) state->readBuf))
+ {
+ /* That's bad. Force reading the page again. */
+ XLogReaderInvalReadState(state);
+
+ return false;
+ }
+
+ state->page_verified = true;
+
+ XLByteToSeg(state->readPagePtr, state->seg.ws_segno,
+ state->segcxt.ws_segsize);
+ }
+
+ /*
+ * The loaded page may not be the one caller is supposing to read when we
+ * are verifying the first page of new segment. In that case, skip further
+ * verification and immediately load the target page.
+ */
+ if (state->page_verified && pageptr == state->readPagePtr)
+ {
+ /*
+ * calculate additional length for page header keeping the total
+ * length within the block size.
+ */
+ if (!header_inclusive)
+ {
+ uint32 pageHeaderSize =
+ XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+
+ addLen = pageHeaderSize;
+ if (reqLen + pageHeaderSize <= XLOG_BLCKSZ)
+ addLen = pageHeaderSize;
+ else
+ addLen = XLOG_BLCKSZ - reqLen;
+
+ Assert(addLen >= 0);
+ }
+
+ /* Return if we already have it. */
+ if (reqLen + addLen <= state->readLen)
+ return false;
+ }
+
+ /* Data is not in our buffer, request the caller for it. */
XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize);
targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize);
+ Assert((pageptr % XLOG_BLCKSZ) == 0);
- /* check whether we have all the requested data already */
- if (targetSegNo == state->seg.ws_segno &&
- targetPageOff == state->segoff && reqLen <= state->readLen)
- return state->readLen;
+ /*
+ * Every time we request to load new data of a page to the caller, even if
+ * we looked at a part of it before, we need to do verification on the next
+ * invocation as the caller might now be rereading data from a different
+ * source.
+ */
+ state->page_verified = false;
/*
- * Data is not in our buffer.
- *
- * Every time we actually read the segment, even if we looked at parts of
- * it before, we need to do verification as the page_read callback might
- * now be rereading data from a different source.
- *
* Whenever switching to a new WAL segment, we read the first page of the
* file and validate its header, even if that's not where the target
* record is. This is so that we can check the additional identification
* info that is present in the first page's "long" header.
+ * Don't do this if the caller requested the first page in the segment.
*/
if (targetSegNo != state->seg.ws_segno && targetPageOff != 0)
{
- XLogRecPtr targetSegmentPtr = pageptr - targetPageOff;
-
- readLen = state->routine.page_read(state, targetSegmentPtr, XLOG_BLCKSZ,
- state->currRecPtr,
- state->readBuf);
- if (readLen < 0)
- goto err;
-
- /* we can be sure to have enough WAL available, we scrolled back */
- Assert(readLen == XLOG_BLCKSZ);
-
- if (!XLogReaderValidatePageHeader(state, targetSegmentPtr,
- state->readBuf))
- goto err;
+ /*
+ * Then we'll see that the targetSegNo now matches the ws_segno, and
+ * will not come back here, but will request the actual target page.
+ */
+ state->readPagePtr = pageptr - targetPageOff;
+ state->readLen = XLOG_BLCKSZ;
+ return true;
}
/*
- * First, read the requested data length, but at least a short page header
- * so that we can validate it.
+ * Request the caller to load the page. We need at least a short page
+ * header so that we can validate it.
*/
- readLen = state->routine.page_read(state, pageptr, Max(reqLen, SizeOfXLogShortPHD),
- state->currRecPtr,
- state->readBuf);
- if (readLen < 0)
- goto err;
-
- Assert(readLen <= XLOG_BLCKSZ);
-
- /* Do we have enough data to check the header length? */
- if (readLen <= SizeOfXLogShortPHD)
- goto err;
-
- Assert(readLen >= reqLen);
-
- hdr = (XLogPageHeader) state->readBuf;
-
- /* still not enough */
- if (readLen < XLogPageHeaderSize(hdr))
- {
- readLen = state->routine.page_read(state, pageptr, XLogPageHeaderSize(hdr),
- state->currRecPtr,
- state->readBuf);
- if (readLen < 0)
- goto err;
- }
-
- /*
- * Now that we know we have the full header, validate it.
- */
- if (!XLogReaderValidatePageHeader(state, pageptr, (char *) hdr))
- goto err;
-
- /* update read state information */
- state->seg.ws_segno = targetSegNo;
- state->segoff = targetPageOff;
- state->readLen = readLen;
-
- return readLen;
-
-err:
- XLogReaderInvalReadState(state);
- return -1;
+ state->readPagePtr = pageptr;
+ state->readLen = Max(reqLen + addLen, SizeOfXLogShortPHD);
+ return true;
}
/*
@@ -673,9 +716,7 @@ err:
static void
XLogReaderInvalReadState(XLogReaderState *state)
{
- state->seg.ws_segno = 0;
- state->segoff = 0;
- state->readLen = 0;
+ state->readPagePtr = InvalidXLogRecPtr;
}
/*
@@ -953,7 +994,6 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
XLogRecPtr targetPagePtr;
int targetRecOff;
uint32 pageHeaderSize;
- int readLen;
/*
* Compute targetRecOff. It should typically be equal or greater than
@@ -961,7 +1001,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
* that, except when caller has explicitly specified the offset that
* falls somewhere there or when we are skipping multi-page
* continuation record. It doesn't matter though because
- * ReadPageInternal() is prepared to handle that and will read at
+ * XLogNeedData() is prepared to handle that and will read at
* least short page-header worth of data
*/
targetRecOff = tmpRecPtr % XLOG_BLCKSZ;
@@ -969,19 +1009,24 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
/* scroll back to page boundary */
targetPagePtr = tmpRecPtr - targetRecOff;
- /* Read the page containing the record */
- readLen = ReadPageInternal(state, targetPagePtr, targetRecOff);
- if (readLen < 0)
+ while(XLogNeedData(state, targetPagePtr, targetRecOff,
+ targetRecOff != 0))
+ {
+ if (!state->routine.page_read(state, state->readPagePtr,
+ state->readLen,
+ state->ReadRecPtr, state->readBuf))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
header = (XLogPageHeader) state->readBuf;
pageHeaderSize = XLogPageHeaderSize(header);
- /* make sure we have enough data for the page header */
- readLen = ReadPageInternal(state, targetPagePtr, pageHeaderSize);
- if (readLen < 0)
- goto err;
+ /* we should have read the page header */
+ Assert (state->readLen >= pageHeaderSize);
/* skip over potential continuation data */
if (header->xlp_info & XLP_FIRST_IS_CONTRECORD)
diff --git a/src/backend/access/transam/xlogutils.c b/src/backend/access/transam/xlogutils.c
index d17d660f46..46eda33f25 100644
--- a/src/backend/access/transam/xlogutils.c
+++ b/src/backend/access/transam/xlogutils.c
@@ -686,8 +686,8 @@ XLogTruncateRelation(RelFileNode rnode, ForkNumber forkNum,
void
XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wantLength)
{
- const XLogRecPtr lastReadPage = (state->seg.ws_segno *
- state->segcxt.ws_segsize + state->segoff);
+ const XLogRecPtr lastReadPage = state->seg.ws_segno *
+ state->segcxt.ws_segsize + state->readLen;
Assert(wantPage != InvalidXLogRecPtr && wantPage % XLOG_BLCKSZ == 0);
Assert(wantLength <= XLOG_BLCKSZ);
@@ -824,7 +824,7 @@ wal_segment_close(XLogReaderState *state)
* exists for normal backends, so we have to do a check/sleep/repeat style of
* loop for now.
*/
-int
+bool
read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *cur_page)
{
@@ -926,7 +926,8 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
else if (targetPagePtr + reqLen > read_upto)
{
/* not enough data there */
- return -1;
+ state->readLen = -1;
+ return false;
}
else
{
@@ -944,7 +945,8 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
WALReadRaiseError(&errinfo);
/* number of valid bytes in the buffer */
- return count;
+ state->readLen = count;
+ return true;
}
/*
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index 23baa4498a..c36b91ae5e 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -806,7 +806,7 @@ StartReplication(StartReplicationCmd *cmd)
* which has to do a plain sleep/busy loop, because the walsender's latch gets
* set every time WAL is flushed.
*/
-static int
+static bool
logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *cur_page)
{
@@ -826,7 +826,10 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
/* fail if not (implies we are going to shut down) */
if (flushptr < targetPagePtr + reqLen)
- return -1;
+ {
+ state->readLen = -1;
+ return false;
+ }
if (targetPagePtr + XLOG_BLCKSZ <= flushptr)
count = XLOG_BLCKSZ; /* more than one block available */
@@ -854,7 +857,8 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
XLByteToSeg(targetPagePtr, segno, state->segcxt.ws_segsize);
CheckXLogRemoved(segno, state->seg.ws_tli);
- return count;
+ state->readLen = count;
+ return true;
}
/*
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index 59ebac7d6a..cf119848b0 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -47,7 +47,7 @@ typedef struct XLogPageReadPrivate
int tliIndex;
} XLogPageReadPrivate;
-static int SimpleXLogPageRead(XLogReaderState *xlogreader,
+static bool SimpleXLogPageRead(XLogReaderState *xlogreader,
XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
@@ -246,7 +246,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
}
/* XLogReader callback function, to read a WAL page */
-static int
+static bool
SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf)
{
@@ -306,7 +306,8 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
if (private->restoreCommand == NULL)
{
pg_log_error("could not open file \"%s\": %m", xlogfpath);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
/*
@@ -319,7 +320,10 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
private->restoreCommand);
if (xlogreadfd < 0)
- return -1;
+ {
+ xlogreader->readLen = -1;
+ return false;
+ }
else
pg_log_debug("using file \"%s\" restored from archive",
xlogfpath);
@@ -335,7 +339,8 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
if (lseek(xlogreadfd, (off_t) targetPageOff, SEEK_SET) < 0)
{
pg_log_error("could not seek in file \"%s\": %m", xlogfpath);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
@@ -348,13 +353,15 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
pg_log_error("could not read file \"%s\": read %d of %zu",
xlogfpath, r, (Size) XLOG_BLCKSZ);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
Assert(targetSegNo == xlogreadsegno);
xlogreader->seg.ws_tli = targetHistory[private->tliIndex].tli;
- return XLOG_BLCKSZ;
+ xlogreader->readLen = XLOG_BLCKSZ;
+ return true;
}
/*
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index 610f65e471..0f0e0723a3 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -331,7 +331,7 @@ WALDumpCloseSegment(XLogReaderState *state)
}
/* pg_waldump's XLogReaderRoutine->page_read callback */
-static int
+static bool
WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetPtr, char *readBuff)
{
@@ -348,7 +348,8 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
else
{
private->endptr_reached = true;
- return -1;
+ state->readLen = -1;
+ return false;
}
}
@@ -373,7 +374,8 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
(Size) errinfo.wre_req);
}
- return count;
+ state->readLen = count;
+ return true;
}
/*
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index 21d200d3df..23d9e70a59 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -57,8 +57,8 @@ typedef struct WALSegmentContext
typedef struct XLogReaderState XLogReaderState;
-/* Function type definitions for various xlogreader interactions */
-typedef int (*XLogPageReadCB) (XLogReaderState *xlogreader,
+/* Function type definition for the read_page callback */
+typedef bool (*XLogPageReadCB) (XLogReaderState *xlogreader,
XLogRecPtr targetPagePtr,
int reqLen,
XLogRecPtr targetRecPtr,
@@ -175,6 +175,20 @@ struct XLogReaderState
XLogRecPtr ReadRecPtr; /* start of last record read */
XLogRecPtr EndRecPtr; /* end+1 of last record read */
+ /* ----------------------------------------
+ * Communication with page reader
+ * readBuf is XLOG_BLCKSZ bytes, valid up to at least readLen bytes.
+ * ----------------------------------------
+ */
+ /* variables to communicate with page reader */
+ XLogRecPtr readPagePtr; /* page pointer to read */
+ int32 readLen; /* bytes requested to reader, or actual bytes
+ * read by reader, which must be larger than
+ * the request, or -1 on error */
+ TimeLineID readPageTLI; /* TLI for data currently in readBuf */
+ char *readBuf; /* buffer to store data */
+ bool page_verified; /* is the page on the buffer verified? */
+
/* ----------------------------------------
* Decoded representation of current record
@@ -203,13 +217,6 @@ struct XLogReaderState
* ----------------------------------------
*/
- /*
- * Buffer for currently read page (XLOG_BLCKSZ bytes, valid up to at least
- * readLen bytes)
- */
- char *readBuf;
- uint32 readLen;
-
/* last read XLOG position for data currently in readBuf */
WALSegmentContext segcxt;
WALOpenSegment seg;
diff --git a/src/include/access/xlogutils.h b/src/include/access/xlogutils.h
index 9ac602b674..364a21c4ea 100644
--- a/src/include/access/xlogutils.h
+++ b/src/include/access/xlogutils.h
@@ -47,7 +47,7 @@ extern Buffer XLogReadBufferExtended(RelFileNode rnode, ForkNumber forknum,
extern Relation CreateFakeRelcacheEntry(RelFileNode rnode);
extern void FreeFakeRelcacheEntry(Relation fakerel);
-extern int read_local_xlog_page(XLogReaderState *state,
+extern bool read_local_xlog_page(XLogReaderState *state,
XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *cur_page);
extern void wal_segment_open(XLogReaderState *state,
--
2.27.0
----Next_Part(Thu_Mar__4_11_28_53_2021_014)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
filename="v17-0002-Move-page-reader-out-of-XLogReadRecord.patch"
^ permalink raw reply [nested|flat] 69+ messages in thread
* [PATCH v15 1/4] Move callback-call from ReadPageInternal to XLogReadRecord.
@ 2019-09-05 11:21 Kyotaro Horiguchi <[email protected]>
0 siblings, 0 replies; 69+ messages in thread
From: Kyotaro Horiguchi @ 2019-09-05 11:21 UTC (permalink / raw)
The current WAL record reader reads page data using a call back
function. Although it is not so problematic alone, it would be a
problem if we are going to do add tasks like encryption which is
performed on page data before WAL reader reads them. To avoid that the
record reader facility has to have a new code path corresponds to
every new callback, this patch separates page reader from WAL record
reading facility by modifying the current WAL record reader to a state
machine.
As the first step of that change, this patch moves the page reader
function out of ReadPageInternal, then the remaining tasks of the
function are taken over by the new function XLogNeedData. As the
result XLogPageRead directly calls the page reader callback function
according to the feedback from XLogNeedData.
---
src/backend/access/transam/xlog.c | 16 +-
src/backend/access/transam/xlogreader.c | 281 ++++++++++++++----------
src/backend/access/transam/xlogutils.c | 12 +-
src/backend/replication/walsender.c | 10 +-
src/bin/pg_rewind/parsexlog.c | 21 +-
src/bin/pg_waldump/pg_waldump.c | 8 +-
src/include/access/xlogreader.h | 25 ++-
src/include/access/xlogutils.h | 2 +-
8 files changed, 222 insertions(+), 153 deletions(-)
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index fd93bcfaeb..e570e56a24 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -912,7 +912,7 @@ static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath,
static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli,
XLogSource source, bool notfoundOk);
static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, XLogSource source);
-static int XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
+static bool XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
bool fetching_ckpt, XLogRecPtr tliRecPtr);
@@ -4332,7 +4332,6 @@ ReadRecord(XLogReaderState *xlogreader, int emode,
XLogRecord *record;
XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
- /* Pass through parameters to XLogPageRead */
private->fetching_ckpt = fetching_ckpt;
private->emode = emode;
private->randAccess = (xlogreader->ReadRecPtr == InvalidXLogRecPtr);
@@ -11860,7 +11859,7 @@ CancelBackup(void)
* XLogPageRead() to try fetching the record from another source, or to
* sleep and retry.
*/
-static int
+static bool
XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *readBuf)
{
@@ -11919,7 +11918,8 @@ retry:
readLen = 0;
readSource = XLOG_FROM_ANY;
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
}
@@ -12014,7 +12014,8 @@ retry:
goto next_record_is_invalid;
}
- return readLen;
+ xlogreader->readLen = readLen;
+ return true;
next_record_is_invalid:
lastSourceFailed = true;
@@ -12028,8 +12029,9 @@ next_record_is_invalid:
/* In standby-mode, keep trying */
if (StandbyMode)
goto retry;
- else
- return -1;
+
+ xlogreader->readLen = -1;
+ return false;
}
/*
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index cb76be4f46..3d599325ee 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -36,8 +36,8 @@
static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
pg_attribute_printf(2, 3);
static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
-static int ReadPageInternal(XLogReaderState *state, XLogRecPtr pageptr,
- int reqLen);
+static bool XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr,
+ int reqLen, bool header_inclusive);
static void XLogReaderInvalReadState(XLogReaderState *state);
static bool ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
XLogRecPtr PrevRecPtr, XLogRecord *record, bool randAccess);
@@ -276,7 +276,6 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
uint32 targetRecOff;
uint32 pageHeaderSize;
bool gotheader;
- int readOff;
/*
* randAccess indicates whether to verify the previous-record pointer of
@@ -326,14 +325,20 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
* byte to cover the whole record header, or at least the part of it that
* fits on the same page.
*/
- readOff = ReadPageInternal(state, targetPagePtr,
- Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ));
- if (readOff < 0)
+ while (XLogNeedData(state, targetPagePtr,
+ Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+ targetRecOff != 0))
+ {
+ if (!state->routine.page_read(state, state->readPagePtr, state->readLen,
+ RecPtr, state->readBuf))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
/*
- * ReadPageInternal always returns at least the page header, so we can
- * examine it now.
+ * We have at least the page header, so we can examine it now.
*/
pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
if (targetRecOff == 0)
@@ -359,8 +364,8 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
goto err;
}
- /* ReadPageInternal has verified the page header */
- Assert(pageHeaderSize <= readOff);
+ /* XLogNeedData has verified the page header */
+ Assert(pageHeaderSize <= state->readLen);
/*
* Read the record length.
@@ -433,18 +438,27 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
do
{
+ int rest_len = total_len - gotlen;
+
/* Calculate pointer to beginning of next page */
targetPagePtr += XLOG_BLCKSZ;
/* Wait for the next page to become available */
- readOff = ReadPageInternal(state, targetPagePtr,
- Min(total_len - gotlen + SizeOfXLogShortPHD,
- XLOG_BLCKSZ));
+ while (XLogNeedData(state, targetPagePtr,
+ Min(rest_len, XLOG_BLCKSZ),
+ false))
+ {
+ if (!state->routine.page_read(state, state->readPagePtr,
+ state->readLen,
+ state->ReadRecPtr,
+ state->readBuf))
+ break;
+ }
- if (readOff < 0)
+ if (!state->page_verified)
goto err;
- Assert(SizeOfXLogShortPHD <= readOff);
+ Assert(SizeOfXLogShortPHD <= state->readLen);
/* Check that the continuation on next page looks valid */
pageHeader = (XLogPageHeader) state->readBuf;
@@ -473,21 +487,14 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
/* Append the continuation from this page to the buffer */
pageHeaderSize = XLogPageHeaderSize(pageHeader);
- if (readOff < pageHeaderSize)
- readOff = ReadPageInternal(state, targetPagePtr,
- pageHeaderSize);
-
- Assert(pageHeaderSize <= readOff);
+ Assert(pageHeaderSize <= state->readLen);
contdata = (char *) state->readBuf + pageHeaderSize;
len = XLOG_BLCKSZ - pageHeaderSize;
if (pageHeader->xlp_rem_len < len)
len = pageHeader->xlp_rem_len;
- if (readOff < pageHeaderSize + len)
- readOff = ReadPageInternal(state, targetPagePtr,
- pageHeaderSize + len);
-
+ Assert (pageHeaderSize + len <= state->readLen);
memcpy(buffer, (char *) contdata, len);
buffer += len;
gotlen += len;
@@ -517,9 +524,16 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
else
{
/* Wait for the record data to become available */
- readOff = ReadPageInternal(state, targetPagePtr,
- Min(targetRecOff + total_len, XLOG_BLCKSZ));
- if (readOff < 0)
+ while (XLogNeedData(state, targetPagePtr,
+ Min(targetRecOff + total_len, XLOG_BLCKSZ), true))
+ {
+ if (!state->routine.page_read(state, state->readPagePtr,
+ state->readLen,
+ state->ReadRecPtr, state->readBuf))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
/* Record does not cross a page boundary */
@@ -562,109 +576,138 @@ err:
}
/*
- * Read a single xlog page including at least [pageptr, reqLen] of valid data
- * via the page_read() callback.
+ * Checks that an xlog page loaded in state->readBuf is including at least
+ * [pageptr, reqLen] and the page is valid. header_inclusive indicates that
+ * reqLen is calculated including page header length.
*
- * Returns -1 if the required page cannot be read for some reason; errormsg_buf
- * is set in that case (unless the error occurs in the page_read callback).
+ * Returns false if the buffer already contains the requested data, or found
+ * error. state->page_verified is set to true for the former and false for the
+ * latter.
*
- * We fetch the page from a reader-local cache if we know we have the required
- * data and if there hasn't been any error since caching the data.
+ * Otherwise returns true and requests data loaded onto state->readBuf by
+ * state->readPagePtr and state->readLen. The caller shall call this function
+ * again after filling the buffer at least with that portion of data and set
+ * state->readLen to the length of actually loaded data.
+ *
+ * If header_inclusive is false, corrects reqLen internally by adding the
+ * actual page header length and may request caller for new data.
*/
-static int
-ReadPageInternal(XLogReaderState *state, XLogRecPtr pageptr, int reqLen)
+static bool
+XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
+ bool header_inclusive)
{
- int readLen;
uint32 targetPageOff;
XLogSegNo targetSegNo;
- XLogPageHeader hdr;
+ uint32 addLen = 0;
- Assert((pageptr % XLOG_BLCKSZ) == 0);
+ /* Some data is loaded, but page header is not verified yet. */
+ if (!state->page_verified &&
+ !XLogRecPtrIsInvalid(state->readPagePtr) && state->readLen >= 0)
+ {
+ uint32 pageHeaderSize;
+ /* just loaded new data so needs to verify page header */
+
+ /* The caller must have loaded at least page header */
+ Assert (state->readLen >= SizeOfXLogShortPHD);
+
+ /*
+ * We have enough data to check the header length. Recheck the loaded
+ * length against the actual header length.
+ */
+ pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+
+ /* Request more data if we don't have the full header. */
+ if (state->readLen < pageHeaderSize)
+ {
+ state->readLen = pageHeaderSize;
+ return true;
+ }
+
+ /* Now that we know we have the full header, validate it. */
+ if (!XLogReaderValidatePageHeader(state, state->readPagePtr,
+ (char *) state->readBuf))
+ {
+ /* That's bad. Force reading the page again. */
+ XLogReaderInvalReadState(state);
+
+ return false;
+ }
+
+ state->page_verified = true;
+
+ XLByteToSeg(state->readPagePtr, state->seg.ws_segno,
+ state->segcxt.ws_segsize);
+ }
+
+ /*
+ * The loaded page may not be the one caller is supposing to read when we
+ * are verifying the first page of new segment. In that case, skip further
+ * verification and immediately load the target page.
+ */
+ if (state->page_verified && pageptr == state->readPagePtr)
+ {
+ /*
+ * calculate additional length for page header keeping the total
+ * length within the block size.
+ */
+ if (!header_inclusive)
+ {
+ uint32 pageHeaderSize =
+ XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+
+ addLen = pageHeaderSize;
+ if (reqLen + pageHeaderSize <= XLOG_BLCKSZ)
+ addLen = pageHeaderSize;
+ else
+ addLen = XLOG_BLCKSZ - reqLen;
+
+ Assert(addLen >= 0);
+ }
+
+ /* Return if we already have it. */
+ if (reqLen + addLen <= state->readLen)
+ return false;
+ }
+
+ /* Data is not in our buffer, request the caller for it. */
XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize);
targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize);
+ Assert((pageptr % XLOG_BLCKSZ) == 0);
- /* check whether we have all the requested data already */
- if (targetSegNo == state->seg.ws_segno &&
- targetPageOff == state->segoff && reqLen <= state->readLen)
- return state->readLen;
+ /*
+ * Every time we request to load new data of a page to the caller, even if
+ * we looked at a part of it before, we need to do verification on the next
+ * invocation as the caller might now be rereading data from a different
+ * source.
+ */
+ state->page_verified = false;
/*
- * Data is not in our buffer.
- *
- * Every time we actually read the segment, even if we looked at parts of
- * it before, we need to do verification as the page_read callback might
- * now be rereading data from a different source.
- *
* Whenever switching to a new WAL segment, we read the first page of the
* file and validate its header, even if that's not where the target
* record is. This is so that we can check the additional identification
* info that is present in the first page's "long" header.
+ * Don't do this if the caller requested the first page in the segment.
*/
if (targetSegNo != state->seg.ws_segno && targetPageOff != 0)
{
- XLogRecPtr targetSegmentPtr = pageptr - targetPageOff;
-
- readLen = state->routine.page_read(state, targetSegmentPtr, XLOG_BLCKSZ,
- state->currRecPtr,
- state->readBuf);
- if (readLen < 0)
- goto err;
-
- /* we can be sure to have enough WAL available, we scrolled back */
- Assert(readLen == XLOG_BLCKSZ);
-
- if (!XLogReaderValidatePageHeader(state, targetSegmentPtr,
- state->readBuf))
- goto err;
+ /*
+ * Then we'll see that the targetSegNo now matches the ws_segno, and
+ * will not come back here, but will request the actual target page.
+ */
+ state->readPagePtr = pageptr - targetPageOff;
+ state->readLen = XLOG_BLCKSZ;
+ return true;
}
/*
- * First, read the requested data length, but at least a short page header
- * so that we can validate it.
+ * Request the caller to load the page. We need at least a short page
+ * header so that we can validate it.
*/
- readLen = state->routine.page_read(state, pageptr, Max(reqLen, SizeOfXLogShortPHD),
- state->currRecPtr,
- state->readBuf);
- if (readLen < 0)
- goto err;
-
- Assert(readLen <= XLOG_BLCKSZ);
-
- /* Do we have enough data to check the header length? */
- if (readLen <= SizeOfXLogShortPHD)
- goto err;
-
- Assert(readLen >= reqLen);
-
- hdr = (XLogPageHeader) state->readBuf;
-
- /* still not enough */
- if (readLen < XLogPageHeaderSize(hdr))
- {
- readLen = state->routine.page_read(state, pageptr, XLogPageHeaderSize(hdr),
- state->currRecPtr,
- state->readBuf);
- if (readLen < 0)
- goto err;
- }
-
- /*
- * Now that we know we have the full header, validate it.
- */
- if (!XLogReaderValidatePageHeader(state, pageptr, (char *) hdr))
- goto err;
-
- /* update read state information */
- state->seg.ws_segno = targetSegNo;
- state->segoff = targetPageOff;
- state->readLen = readLen;
-
- return readLen;
-
-err:
- XLogReaderInvalReadState(state);
- return -1;
+ state->readPagePtr = pageptr;
+ state->readLen = Max(reqLen + addLen, SizeOfXLogShortPHD);
+ return true;
}
/*
@@ -673,9 +716,7 @@ err:
static void
XLogReaderInvalReadState(XLogReaderState *state)
{
- state->seg.ws_segno = 0;
- state->segoff = 0;
- state->readLen = 0;
+ state->readPagePtr = InvalidXLogRecPtr;
}
/*
@@ -956,7 +997,6 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
XLogRecPtr targetPagePtr;
int targetRecOff;
uint32 pageHeaderSize;
- int readLen;
/*
* Compute targetRecOff. It should typically be equal or greater than
@@ -964,7 +1004,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
* that, except when caller has explicitly specified the offset that
* falls somewhere there or when we are skipping multi-page
* continuation record. It doesn't matter though because
- * ReadPageInternal() is prepared to handle that and will read at
+ * XLogNeedData() is prepared to handle that and will read at
* least short page-header worth of data
*/
targetRecOff = tmpRecPtr % XLOG_BLCKSZ;
@@ -972,19 +1012,24 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
/* scroll back to page boundary */
targetPagePtr = tmpRecPtr - targetRecOff;
- /* Read the page containing the record */
- readLen = ReadPageInternal(state, targetPagePtr, targetRecOff);
- if (readLen < 0)
+ while(XLogNeedData(state, targetPagePtr, targetRecOff,
+ targetRecOff != 0))
+ {
+ if (!state->routine.page_read(state, state->readPagePtr,
+ state->readLen,
+ state->ReadRecPtr, state->readBuf))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
header = (XLogPageHeader) state->readBuf;
pageHeaderSize = XLogPageHeaderSize(header);
- /* make sure we have enough data for the page header */
- readLen = ReadPageInternal(state, targetPagePtr, pageHeaderSize);
- if (readLen < 0)
- goto err;
+ /* we should have read the page header */
+ Assert (state->readLen >= pageHeaderSize);
/* skip over potential continuation data */
if (header->xlp_info & XLP_FIRST_IS_CONTRECORD)
diff --git a/src/backend/access/transam/xlogutils.c b/src/backend/access/transam/xlogutils.c
index 322b0e8ff5..1b10b6df20 100644
--- a/src/backend/access/transam/xlogutils.c
+++ b/src/backend/access/transam/xlogutils.c
@@ -689,8 +689,8 @@ XLogTruncateRelation(RelFileNode rnode, ForkNumber forkNum,
void
XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wantLength)
{
- const XLogRecPtr lastReadPage = (state->seg.ws_segno *
- state->segcxt.ws_segsize + state->segoff);
+ const XLogRecPtr lastReadPage = state->seg.ws_segno *
+ state->segcxt.ws_segsize + state->readLen;
Assert(wantPage != InvalidXLogRecPtr && wantPage % XLOG_BLCKSZ == 0);
Assert(wantLength <= XLOG_BLCKSZ);
@@ -828,7 +828,7 @@ wal_segment_close(XLogReaderState *state)
* exists for normal backends, so we have to do a check/sleep/repeat style of
* loop for now.
*/
-int
+bool
read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *cur_page)
{
@@ -930,7 +930,8 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
else if (targetPagePtr + reqLen > read_upto)
{
/* not enough data there */
- return -1;
+ state->readLen = -1;
+ return false;
}
else
{
@@ -948,7 +949,8 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
WALReadRaiseError(&errinfo);
/* number of valid bytes in the buffer */
- return count;
+ state->readLen = count;
+ return true;
}
/*
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index e2477c47e0..bf2711bddd 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -811,7 +811,7 @@ StartReplication(StartReplicationCmd *cmd)
* which has to do a plain sleep/busy loop, because the walsender's latch gets
* set every time WAL is flushed.
*/
-static int
+static bool
logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *cur_page)
{
@@ -831,7 +831,10 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
/* fail if not (implies we are going to shut down) */
if (flushptr < targetPagePtr + reqLen)
- return -1;
+ {
+ state->readLen = -1;
+ return false;
+ }
if (targetPagePtr + XLOG_BLCKSZ <= flushptr)
count = XLOG_BLCKSZ; /* more than one block available */
@@ -859,7 +862,8 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
XLByteToSeg(targetPagePtr, segno, state->segcxt.ws_segsize);
CheckXLogRemoved(segno, state->seg.ws_tli);
- return count;
+ state->readLen = count;
+ return true;
}
/*
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index bc6f976994..0b7e73ae79 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -46,7 +46,7 @@ typedef struct XLogPageReadPrivate
int tliIndex;
} XLogPageReadPrivate;
-static int SimpleXLogPageRead(XLogReaderState *xlogreader,
+static bool SimpleXLogPageRead(XLogReaderState *xlogreader,
XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
@@ -236,7 +236,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
}
/* XLogReader callback function, to read a WAL page */
-static int
+static bool
SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf)
{
@@ -296,7 +296,8 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
if (private->restoreCommand == NULL)
{
pg_log_error("could not open file \"%s\": %m", xlogfpath);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
/*
@@ -309,7 +310,10 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
private->restoreCommand);
if (xlogreadfd < 0)
- return -1;
+ {
+ xlogreader->readLen = -1;
+ return false;
+ }
else
pg_log_debug("using file \"%s\" restored from archive",
xlogfpath);
@@ -325,7 +329,8 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
if (lseek(xlogreadfd, (off_t) targetPageOff, SEEK_SET) < 0)
{
pg_log_error("could not seek in file \"%s\": %m", xlogfpath);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
@@ -338,13 +343,15 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
pg_log_error("could not read file \"%s\": read %d of %zu",
xlogfpath, r, (Size) XLOG_BLCKSZ);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
Assert(targetSegNo == xlogreadsegno);
xlogreader->seg.ws_tli = targetHistory[private->tliIndex].tli;
- return XLOG_BLCKSZ;
+ xlogreader->readLen = XLOG_BLCKSZ;
+ return true;
}
/*
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index d1a0678935..ab0c28d9b7 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -331,7 +331,7 @@ WALDumpCloseSegment(XLogReaderState *state)
}
/* pg_waldump's XLogReaderRoutine->page_read callback */
-static int
+static bool
WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetPtr, char *readBuff)
{
@@ -348,7 +348,8 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
else
{
private->endptr_reached = true;
- return -1;
+ state->readLen = -1;
+ return false;
}
}
@@ -373,7 +374,8 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
(Size) errinfo.wre_req);
}
- return count;
+ state->readLen = count;
+ return true;
}
/*
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index b0f2a6ed43..6c8848e14f 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -57,8 +57,8 @@ typedef struct WALSegmentContext
typedef struct XLogReaderState XLogReaderState;
-/* Function type definitions for various xlogreader interactions */
-typedef int (*XLogPageReadCB) (XLogReaderState *xlogreader,
+/* Function type definition for the read_page callback */
+typedef bool (*XLogPageReadCB) (XLogReaderState *xlogreader,
XLogRecPtr targetPagePtr,
int reqLen,
XLogRecPtr targetRecPtr,
@@ -175,6 +175,20 @@ struct XLogReaderState
XLogRecPtr ReadRecPtr; /* start of last record read */
XLogRecPtr EndRecPtr; /* end+1 of last record read */
+ /* ----------------------------------------
+ * Communication with page reader
+ * readBuf is XLOG_BLCKSZ bytes, valid up to at least readLen bytes.
+ * ----------------------------------------
+ */
+ /* variables to communicate with page reader */
+ XLogRecPtr readPagePtr; /* page pointer to read */
+ int32 readLen; /* bytes requested to reader, or actual bytes
+ * read by reader, which must be larger than
+ * the request, or -1 on error */
+ TimeLineID readPageTLI; /* TLI for data currently in readBuf */
+ char *readBuf; /* buffer to store data */
+ bool page_verified; /* is the page on the buffer verified? */
+
/* ----------------------------------------
* Decoded representation of current record
@@ -201,13 +215,6 @@ struct XLogReaderState
* ----------------------------------------
*/
- /*
- * Buffer for currently read page (XLOG_BLCKSZ bytes, valid up to at least
- * readLen bytes)
- */
- char *readBuf;
- uint32 readLen;
-
/* last read XLOG position for data currently in readBuf */
WALSegmentContext segcxt;
WALOpenSegment seg;
diff --git a/src/include/access/xlogutils.h b/src/include/access/xlogutils.h
index e59b6cf3a9..c287edf206 100644
--- a/src/include/access/xlogutils.h
+++ b/src/include/access/xlogutils.h
@@ -47,7 +47,7 @@ extern Buffer XLogReadBufferExtended(RelFileNode rnode, ForkNumber forknum,
extern Relation CreateFakeRelcacheEntry(RelFileNode rnode);
extern void FreeFakeRelcacheEntry(Relation fakerel);
-extern int read_local_xlog_page(XLogReaderState *state,
+extern bool read_local_xlog_page(XLogReaderState *state,
XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *cur_page);
extern void wal_segment_open(XLogReaderState *state,
--
2.18.4
----Next_Part(Thu_Jul__2_13_53_30_2020_072)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
filename="v15-0002-Move-page-reader-out-of-XLogReadRecord.patch"
^ permalink raw reply [nested|flat] 69+ messages in thread
* [PATCH v17 1/4] Move callback-call from ReadPageInternal to XLogReadRecord.
@ 2019-09-05 11:21 Kyotaro Horiguchi <[email protected]>
0 siblings, 0 replies; 69+ messages in thread
From: Kyotaro Horiguchi @ 2019-09-05 11:21 UTC (permalink / raw)
The current WAL record reader reads page data using a call back
function. Although it is not so problematic alone, it would be a
problem if we are going to do add tasks like encryption which is
performed on page data before WAL reader reads them. To avoid that the
record reader facility has to have a new code path corresponds to
every new callback, this patch separates page reader from WAL record
reading facility by modifying the current WAL record reader to a state
machine.
As the first step of that change, this patch moves the page reader
function out of ReadPageInternal, then the remaining tasks of the
function are taken over by the new function XLogNeedData. As the
result XLogPageRead directly calls the page reader callback function
according to the feedback from XLogNeedData.
---
src/backend/access/transam/xlog.c | 16 +-
src/backend/access/transam/xlogreader.c | 281 ++++++++++++++----------
src/backend/access/transam/xlogutils.c | 12 +-
src/backend/replication/walsender.c | 10 +-
src/bin/pg_rewind/parsexlog.c | 21 +-
src/bin/pg_waldump/pg_waldump.c | 8 +-
src/include/access/xlogreader.h | 25 ++-
src/include/access/xlogutils.h | 2 +-
8 files changed, 222 insertions(+), 153 deletions(-)
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 377afb8732..606328a65a 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -917,7 +917,7 @@ static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath,
static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli,
XLogSource source, bool notfoundOk);
static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, XLogSource source);
-static int XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
+static bool XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
bool fetching_ckpt, XLogRecPtr tliRecPtr);
@@ -4350,7 +4350,6 @@ ReadRecord(XLogReaderState *xlogreader, int emode,
XLogRecord *record;
XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
- /* Pass through parameters to XLogPageRead */
private->fetching_ckpt = fetching_ckpt;
private->emode = emode;
private->randAccess = (xlogreader->ReadRecPtr == InvalidXLogRecPtr);
@@ -11987,7 +11986,7 @@ CancelBackup(void)
* XLogPageRead() to try fetching the record from another source, or to
* sleep and retry.
*/
-static int
+static bool
XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *readBuf)
{
@@ -12046,7 +12045,8 @@ retry:
readLen = 0;
readSource = XLOG_FROM_ANY;
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
}
@@ -12141,7 +12141,8 @@ retry:
goto next_record_is_invalid;
}
- return readLen;
+ xlogreader->readLen = readLen;
+ return true;
next_record_is_invalid:
lastSourceFailed = true;
@@ -12155,8 +12156,9 @@ next_record_is_invalid:
/* In standby-mode, keep trying */
if (StandbyMode)
goto retry;
- else
- return -1;
+
+ xlogreader->readLen = -1;
+ return false;
}
/*
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index 42738eb940..7d04995b7b 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -36,8 +36,8 @@
static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
pg_attribute_printf(2, 3);
static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
-static int ReadPageInternal(XLogReaderState *state, XLogRecPtr pageptr,
- int reqLen);
+static bool XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr,
+ int reqLen, bool header_inclusive);
static void XLogReaderInvalReadState(XLogReaderState *state);
static bool ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
XLogRecPtr PrevRecPtr, XLogRecord *record, bool randAccess);
@@ -276,7 +276,6 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
uint32 targetRecOff;
uint32 pageHeaderSize;
bool gotheader;
- int readOff;
/*
* randAccess indicates whether to verify the previous-record pointer of
@@ -326,14 +325,20 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
* byte to cover the whole record header, or at least the part of it that
* fits on the same page.
*/
- readOff = ReadPageInternal(state, targetPagePtr,
- Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ));
- if (readOff < 0)
+ while (XLogNeedData(state, targetPagePtr,
+ Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+ targetRecOff != 0))
+ {
+ if (!state->routine.page_read(state, state->readPagePtr, state->readLen,
+ RecPtr, state->readBuf))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
/*
- * ReadPageInternal always returns at least the page header, so we can
- * examine it now.
+ * We have at least the page header, so we can examine it now.
*/
pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
if (targetRecOff == 0)
@@ -359,8 +364,8 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
goto err;
}
- /* ReadPageInternal has verified the page header */
- Assert(pageHeaderSize <= readOff);
+ /* XLogNeedData has verified the page header */
+ Assert(pageHeaderSize <= state->readLen);
/*
* Read the record length.
@@ -432,18 +437,27 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
do
{
+ int rest_len = total_len - gotlen;
+
/* Calculate pointer to beginning of next page */
targetPagePtr += XLOG_BLCKSZ;
/* Wait for the next page to become available */
- readOff = ReadPageInternal(state, targetPagePtr,
- Min(total_len - gotlen + SizeOfXLogShortPHD,
- XLOG_BLCKSZ));
+ while (XLogNeedData(state, targetPagePtr,
+ Min(rest_len, XLOG_BLCKSZ),
+ false))
+ {
+ if (!state->routine.page_read(state, state->readPagePtr,
+ state->readLen,
+ state->ReadRecPtr,
+ state->readBuf))
+ break;
+ }
- if (readOff < 0)
+ if (!state->page_verified)
goto err;
- Assert(SizeOfXLogShortPHD <= readOff);
+ Assert(SizeOfXLogShortPHD <= state->readLen);
/* Check that the continuation on next page looks valid */
pageHeader = (XLogPageHeader) state->readBuf;
@@ -473,21 +487,14 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
/* Append the continuation from this page to the buffer */
pageHeaderSize = XLogPageHeaderSize(pageHeader);
- if (readOff < pageHeaderSize)
- readOff = ReadPageInternal(state, targetPagePtr,
- pageHeaderSize);
-
- Assert(pageHeaderSize <= readOff);
+ Assert(pageHeaderSize <= state->readLen);
contdata = (char *) state->readBuf + pageHeaderSize;
len = XLOG_BLCKSZ - pageHeaderSize;
if (pageHeader->xlp_rem_len < len)
len = pageHeader->xlp_rem_len;
- if (readOff < pageHeaderSize + len)
- readOff = ReadPageInternal(state, targetPagePtr,
- pageHeaderSize + len);
-
+ Assert (pageHeaderSize + len <= state->readLen);
memcpy(buffer, (char *) contdata, len);
buffer += len;
gotlen += len;
@@ -517,9 +524,16 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
else
{
/* Wait for the record data to become available */
- readOff = ReadPageInternal(state, targetPagePtr,
- Min(targetRecOff + total_len, XLOG_BLCKSZ));
- if (readOff < 0)
+ while (XLogNeedData(state, targetPagePtr,
+ Min(targetRecOff + total_len, XLOG_BLCKSZ), true))
+ {
+ if (!state->routine.page_read(state, state->readPagePtr,
+ state->readLen,
+ state->ReadRecPtr, state->readBuf))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
/* Record does not cross a page boundary */
@@ -562,109 +576,138 @@ err:
}
/*
- * Read a single xlog page including at least [pageptr, reqLen] of valid data
- * via the page_read() callback.
+ * Checks that an xlog page loaded in state->readBuf is including at least
+ * [pageptr, reqLen] and the page is valid. header_inclusive indicates that
+ * reqLen is calculated including page header length.
*
- * Returns -1 if the required page cannot be read for some reason; errormsg_buf
- * is set in that case (unless the error occurs in the page_read callback).
+ * Returns false if the buffer already contains the requested data, or found
+ * error. state->page_verified is set to true for the former and false for the
+ * latter.
*
- * We fetch the page from a reader-local cache if we know we have the required
- * data and if there hasn't been any error since caching the data.
+ * Otherwise returns true and requests data loaded onto state->readBuf by
+ * state->readPagePtr and state->readLen. The caller shall call this function
+ * again after filling the buffer at least with that portion of data and set
+ * state->readLen to the length of actually loaded data.
+ *
+ * If header_inclusive is false, corrects reqLen internally by adding the
+ * actual page header length and may request caller for new data.
*/
-static int
-ReadPageInternal(XLogReaderState *state, XLogRecPtr pageptr, int reqLen)
+static bool
+XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
+ bool header_inclusive)
{
- int readLen;
uint32 targetPageOff;
XLogSegNo targetSegNo;
- XLogPageHeader hdr;
+ uint32 addLen = 0;
- Assert((pageptr % XLOG_BLCKSZ) == 0);
+ /* Some data is loaded, but page header is not verified yet. */
+ if (!state->page_verified &&
+ !XLogRecPtrIsInvalid(state->readPagePtr) && state->readLen >= 0)
+ {
+ uint32 pageHeaderSize;
+ /* just loaded new data so needs to verify page header */
+
+ /* The caller must have loaded at least page header */
+ Assert (state->readLen >= SizeOfXLogShortPHD);
+
+ /*
+ * We have enough data to check the header length. Recheck the loaded
+ * length against the actual header length.
+ */
+ pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+
+ /* Request more data if we don't have the full header. */
+ if (state->readLen < pageHeaderSize)
+ {
+ state->readLen = pageHeaderSize;
+ return true;
+ }
+
+ /* Now that we know we have the full header, validate it. */
+ if (!XLogReaderValidatePageHeader(state, state->readPagePtr,
+ (char *) state->readBuf))
+ {
+ /* That's bad. Force reading the page again. */
+ XLogReaderInvalReadState(state);
+
+ return false;
+ }
+
+ state->page_verified = true;
+
+ XLByteToSeg(state->readPagePtr, state->seg.ws_segno,
+ state->segcxt.ws_segsize);
+ }
+
+ /*
+ * The loaded page may not be the one caller is supposing to read when we
+ * are verifying the first page of new segment. In that case, skip further
+ * verification and immediately load the target page.
+ */
+ if (state->page_verified && pageptr == state->readPagePtr)
+ {
+ /*
+ * calculate additional length for page header keeping the total
+ * length within the block size.
+ */
+ if (!header_inclusive)
+ {
+ uint32 pageHeaderSize =
+ XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+
+ addLen = pageHeaderSize;
+ if (reqLen + pageHeaderSize <= XLOG_BLCKSZ)
+ addLen = pageHeaderSize;
+ else
+ addLen = XLOG_BLCKSZ - reqLen;
+
+ Assert(addLen >= 0);
+ }
+
+ /* Return if we already have it. */
+ if (reqLen + addLen <= state->readLen)
+ return false;
+ }
+
+ /* Data is not in our buffer, request the caller for it. */
XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize);
targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize);
+ Assert((pageptr % XLOG_BLCKSZ) == 0);
- /* check whether we have all the requested data already */
- if (targetSegNo == state->seg.ws_segno &&
- targetPageOff == state->segoff && reqLen <= state->readLen)
- return state->readLen;
+ /*
+ * Every time we request to load new data of a page to the caller, even if
+ * we looked at a part of it before, we need to do verification on the next
+ * invocation as the caller might now be rereading data from a different
+ * source.
+ */
+ state->page_verified = false;
/*
- * Data is not in our buffer.
- *
- * Every time we actually read the segment, even if we looked at parts of
- * it before, we need to do verification as the page_read callback might
- * now be rereading data from a different source.
- *
* Whenever switching to a new WAL segment, we read the first page of the
* file and validate its header, even if that's not where the target
* record is. This is so that we can check the additional identification
* info that is present in the first page's "long" header.
+ * Don't do this if the caller requested the first page in the segment.
*/
if (targetSegNo != state->seg.ws_segno && targetPageOff != 0)
{
- XLogRecPtr targetSegmentPtr = pageptr - targetPageOff;
-
- readLen = state->routine.page_read(state, targetSegmentPtr, XLOG_BLCKSZ,
- state->currRecPtr,
- state->readBuf);
- if (readLen < 0)
- goto err;
-
- /* we can be sure to have enough WAL available, we scrolled back */
- Assert(readLen == XLOG_BLCKSZ);
-
- if (!XLogReaderValidatePageHeader(state, targetSegmentPtr,
- state->readBuf))
- goto err;
+ /*
+ * Then we'll see that the targetSegNo now matches the ws_segno, and
+ * will not come back here, but will request the actual target page.
+ */
+ state->readPagePtr = pageptr - targetPageOff;
+ state->readLen = XLOG_BLCKSZ;
+ return true;
}
/*
- * First, read the requested data length, but at least a short page header
- * so that we can validate it.
+ * Request the caller to load the page. We need at least a short page
+ * header so that we can validate it.
*/
- readLen = state->routine.page_read(state, pageptr, Max(reqLen, SizeOfXLogShortPHD),
- state->currRecPtr,
- state->readBuf);
- if (readLen < 0)
- goto err;
-
- Assert(readLen <= XLOG_BLCKSZ);
-
- /* Do we have enough data to check the header length? */
- if (readLen <= SizeOfXLogShortPHD)
- goto err;
-
- Assert(readLen >= reqLen);
-
- hdr = (XLogPageHeader) state->readBuf;
-
- /* still not enough */
- if (readLen < XLogPageHeaderSize(hdr))
- {
- readLen = state->routine.page_read(state, pageptr, XLogPageHeaderSize(hdr),
- state->currRecPtr,
- state->readBuf);
- if (readLen < 0)
- goto err;
- }
-
- /*
- * Now that we know we have the full header, validate it.
- */
- if (!XLogReaderValidatePageHeader(state, pageptr, (char *) hdr))
- goto err;
-
- /* update read state information */
- state->seg.ws_segno = targetSegNo;
- state->segoff = targetPageOff;
- state->readLen = readLen;
-
- return readLen;
-
-err:
- XLogReaderInvalReadState(state);
- return -1;
+ state->readPagePtr = pageptr;
+ state->readLen = Max(reqLen + addLen, SizeOfXLogShortPHD);
+ return true;
}
/*
@@ -673,9 +716,7 @@ err:
static void
XLogReaderInvalReadState(XLogReaderState *state)
{
- state->seg.ws_segno = 0;
- state->segoff = 0;
- state->readLen = 0;
+ state->readPagePtr = InvalidXLogRecPtr;
}
/*
@@ -953,7 +994,6 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
XLogRecPtr targetPagePtr;
int targetRecOff;
uint32 pageHeaderSize;
- int readLen;
/*
* Compute targetRecOff. It should typically be equal or greater than
@@ -961,7 +1001,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
* that, except when caller has explicitly specified the offset that
* falls somewhere there or when we are skipping multi-page
* continuation record. It doesn't matter though because
- * ReadPageInternal() is prepared to handle that and will read at
+ * XLogNeedData() is prepared to handle that and will read at
* least short page-header worth of data
*/
targetRecOff = tmpRecPtr % XLOG_BLCKSZ;
@@ -969,19 +1009,24 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
/* scroll back to page boundary */
targetPagePtr = tmpRecPtr - targetRecOff;
- /* Read the page containing the record */
- readLen = ReadPageInternal(state, targetPagePtr, targetRecOff);
- if (readLen < 0)
+ while(XLogNeedData(state, targetPagePtr, targetRecOff,
+ targetRecOff != 0))
+ {
+ if (!state->routine.page_read(state, state->readPagePtr,
+ state->readLen,
+ state->ReadRecPtr, state->readBuf))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
header = (XLogPageHeader) state->readBuf;
pageHeaderSize = XLogPageHeaderSize(header);
- /* make sure we have enough data for the page header */
- readLen = ReadPageInternal(state, targetPagePtr, pageHeaderSize);
- if (readLen < 0)
- goto err;
+ /* we should have read the page header */
+ Assert (state->readLen >= pageHeaderSize);
/* skip over potential continuation data */
if (header->xlp_info & XLP_FIRST_IS_CONTRECORD)
diff --git a/src/backend/access/transam/xlogutils.c b/src/backend/access/transam/xlogutils.c
index d17d660f46..46eda33f25 100644
--- a/src/backend/access/transam/xlogutils.c
+++ b/src/backend/access/transam/xlogutils.c
@@ -686,8 +686,8 @@ XLogTruncateRelation(RelFileNode rnode, ForkNumber forkNum,
void
XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wantLength)
{
- const XLogRecPtr lastReadPage = (state->seg.ws_segno *
- state->segcxt.ws_segsize + state->segoff);
+ const XLogRecPtr lastReadPage = state->seg.ws_segno *
+ state->segcxt.ws_segsize + state->readLen;
Assert(wantPage != InvalidXLogRecPtr && wantPage % XLOG_BLCKSZ == 0);
Assert(wantLength <= XLOG_BLCKSZ);
@@ -824,7 +824,7 @@ wal_segment_close(XLogReaderState *state)
* exists for normal backends, so we have to do a check/sleep/repeat style of
* loop for now.
*/
-int
+bool
read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *cur_page)
{
@@ -926,7 +926,8 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
else if (targetPagePtr + reqLen > read_upto)
{
/* not enough data there */
- return -1;
+ state->readLen = -1;
+ return false;
}
else
{
@@ -944,7 +945,8 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
WALReadRaiseError(&errinfo);
/* number of valid bytes in the buffer */
- return count;
+ state->readLen = count;
+ return true;
}
/*
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index 23baa4498a..c36b91ae5e 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -806,7 +806,7 @@ StartReplication(StartReplicationCmd *cmd)
* which has to do a plain sleep/busy loop, because the walsender's latch gets
* set every time WAL is flushed.
*/
-static int
+static bool
logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *cur_page)
{
@@ -826,7 +826,10 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
/* fail if not (implies we are going to shut down) */
if (flushptr < targetPagePtr + reqLen)
- return -1;
+ {
+ state->readLen = -1;
+ return false;
+ }
if (targetPagePtr + XLOG_BLCKSZ <= flushptr)
count = XLOG_BLCKSZ; /* more than one block available */
@@ -854,7 +857,8 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
XLByteToSeg(targetPagePtr, segno, state->segcxt.ws_segsize);
CheckXLogRemoved(segno, state->seg.ws_tli);
- return count;
+ state->readLen = count;
+ return true;
}
/*
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index 59ebac7d6a..cf119848b0 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -47,7 +47,7 @@ typedef struct XLogPageReadPrivate
int tliIndex;
} XLogPageReadPrivate;
-static int SimpleXLogPageRead(XLogReaderState *xlogreader,
+static bool SimpleXLogPageRead(XLogReaderState *xlogreader,
XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
@@ -246,7 +246,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
}
/* XLogReader callback function, to read a WAL page */
-static int
+static bool
SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf)
{
@@ -306,7 +306,8 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
if (private->restoreCommand == NULL)
{
pg_log_error("could not open file \"%s\": %m", xlogfpath);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
/*
@@ -319,7 +320,10 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
private->restoreCommand);
if (xlogreadfd < 0)
- return -1;
+ {
+ xlogreader->readLen = -1;
+ return false;
+ }
else
pg_log_debug("using file \"%s\" restored from archive",
xlogfpath);
@@ -335,7 +339,8 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
if (lseek(xlogreadfd, (off_t) targetPageOff, SEEK_SET) < 0)
{
pg_log_error("could not seek in file \"%s\": %m", xlogfpath);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
@@ -348,13 +353,15 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
pg_log_error("could not read file \"%s\": read %d of %zu",
xlogfpath, r, (Size) XLOG_BLCKSZ);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
Assert(targetSegNo == xlogreadsegno);
xlogreader->seg.ws_tli = targetHistory[private->tliIndex].tli;
- return XLOG_BLCKSZ;
+ xlogreader->readLen = XLOG_BLCKSZ;
+ return true;
}
/*
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index 610f65e471..0f0e0723a3 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -331,7 +331,7 @@ WALDumpCloseSegment(XLogReaderState *state)
}
/* pg_waldump's XLogReaderRoutine->page_read callback */
-static int
+static bool
WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetPtr, char *readBuff)
{
@@ -348,7 +348,8 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
else
{
private->endptr_reached = true;
- return -1;
+ state->readLen = -1;
+ return false;
}
}
@@ -373,7 +374,8 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
(Size) errinfo.wre_req);
}
- return count;
+ state->readLen = count;
+ return true;
}
/*
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index 21d200d3df..23d9e70a59 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -57,8 +57,8 @@ typedef struct WALSegmentContext
typedef struct XLogReaderState XLogReaderState;
-/* Function type definitions for various xlogreader interactions */
-typedef int (*XLogPageReadCB) (XLogReaderState *xlogreader,
+/* Function type definition for the read_page callback */
+typedef bool (*XLogPageReadCB) (XLogReaderState *xlogreader,
XLogRecPtr targetPagePtr,
int reqLen,
XLogRecPtr targetRecPtr,
@@ -175,6 +175,20 @@ struct XLogReaderState
XLogRecPtr ReadRecPtr; /* start of last record read */
XLogRecPtr EndRecPtr; /* end+1 of last record read */
+ /* ----------------------------------------
+ * Communication with page reader
+ * readBuf is XLOG_BLCKSZ bytes, valid up to at least readLen bytes.
+ * ----------------------------------------
+ */
+ /* variables to communicate with page reader */
+ XLogRecPtr readPagePtr; /* page pointer to read */
+ int32 readLen; /* bytes requested to reader, or actual bytes
+ * read by reader, which must be larger than
+ * the request, or -1 on error */
+ TimeLineID readPageTLI; /* TLI for data currently in readBuf */
+ char *readBuf; /* buffer to store data */
+ bool page_verified; /* is the page on the buffer verified? */
+
/* ----------------------------------------
* Decoded representation of current record
@@ -203,13 +217,6 @@ struct XLogReaderState
* ----------------------------------------
*/
- /*
- * Buffer for currently read page (XLOG_BLCKSZ bytes, valid up to at least
- * readLen bytes)
- */
- char *readBuf;
- uint32 readLen;
-
/* last read XLOG position for data currently in readBuf */
WALSegmentContext segcxt;
WALOpenSegment seg;
diff --git a/src/include/access/xlogutils.h b/src/include/access/xlogutils.h
index 9ac602b674..364a21c4ea 100644
--- a/src/include/access/xlogutils.h
+++ b/src/include/access/xlogutils.h
@@ -47,7 +47,7 @@ extern Buffer XLogReadBufferExtended(RelFileNode rnode, ForkNumber forknum,
extern Relation CreateFakeRelcacheEntry(RelFileNode rnode);
extern void FreeFakeRelcacheEntry(Relation fakerel);
-extern int read_local_xlog_page(XLogReaderState *state,
+extern bool read_local_xlog_page(XLogReaderState *state,
XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *cur_page);
extern void wal_segment_open(XLogReaderState *state,
--
2.27.0
----Next_Part(Thu_Mar__4_11_28_53_2021_014)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
filename="v17-0002-Move-page-reader-out-of-XLogReadRecord.patch"
^ permalink raw reply [nested|flat] 69+ messages in thread
* [PATCH v15 1/4] Move callback-call from ReadPageInternal to XLogReadRecord.
@ 2019-09-05 11:21 Kyotaro Horiguchi <[email protected]>
0 siblings, 0 replies; 69+ messages in thread
From: Kyotaro Horiguchi @ 2019-09-05 11:21 UTC (permalink / raw)
The current WAL record reader reads page data using a call back
function. Although it is not so problematic alone, it would be a
problem if we are going to do add tasks like encryption which is
performed on page data before WAL reader reads them. To avoid that the
record reader facility has to have a new code path corresponds to
every new callback, this patch separates page reader from WAL record
reading facility by modifying the current WAL record reader to a state
machine.
As the first step of that change, this patch moves the page reader
function out of ReadPageInternal, then the remaining tasks of the
function are taken over by the new function XLogNeedData. As the
result XLogPageRead directly calls the page reader callback function
according to the feedback from XLogNeedData.
---
src/backend/access/transam/xlog.c | 16 +-
src/backend/access/transam/xlogreader.c | 281 ++++++++++++++----------
src/backend/access/transam/xlogutils.c | 12 +-
src/backend/replication/walsender.c | 10 +-
src/bin/pg_rewind/parsexlog.c | 21 +-
src/bin/pg_waldump/pg_waldump.c | 8 +-
src/include/access/xlogreader.h | 25 ++-
src/include/access/xlogutils.h | 2 +-
8 files changed, 222 insertions(+), 153 deletions(-)
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index fd93bcfaeb..e570e56a24 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -912,7 +912,7 @@ static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath,
static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli,
XLogSource source, bool notfoundOk);
static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, XLogSource source);
-static int XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
+static bool XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
bool fetching_ckpt, XLogRecPtr tliRecPtr);
@@ -4332,7 +4332,6 @@ ReadRecord(XLogReaderState *xlogreader, int emode,
XLogRecord *record;
XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
- /* Pass through parameters to XLogPageRead */
private->fetching_ckpt = fetching_ckpt;
private->emode = emode;
private->randAccess = (xlogreader->ReadRecPtr == InvalidXLogRecPtr);
@@ -11860,7 +11859,7 @@ CancelBackup(void)
* XLogPageRead() to try fetching the record from another source, or to
* sleep and retry.
*/
-static int
+static bool
XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *readBuf)
{
@@ -11919,7 +11918,8 @@ retry:
readLen = 0;
readSource = XLOG_FROM_ANY;
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
}
@@ -12014,7 +12014,8 @@ retry:
goto next_record_is_invalid;
}
- return readLen;
+ xlogreader->readLen = readLen;
+ return true;
next_record_is_invalid:
lastSourceFailed = true;
@@ -12028,8 +12029,9 @@ next_record_is_invalid:
/* In standby-mode, keep trying */
if (StandbyMode)
goto retry;
- else
- return -1;
+
+ xlogreader->readLen = -1;
+ return false;
}
/*
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index cb76be4f46..3d599325ee 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -36,8 +36,8 @@
static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
pg_attribute_printf(2, 3);
static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
-static int ReadPageInternal(XLogReaderState *state, XLogRecPtr pageptr,
- int reqLen);
+static bool XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr,
+ int reqLen, bool header_inclusive);
static void XLogReaderInvalReadState(XLogReaderState *state);
static bool ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
XLogRecPtr PrevRecPtr, XLogRecord *record, bool randAccess);
@@ -276,7 +276,6 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
uint32 targetRecOff;
uint32 pageHeaderSize;
bool gotheader;
- int readOff;
/*
* randAccess indicates whether to verify the previous-record pointer of
@@ -326,14 +325,20 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
* byte to cover the whole record header, or at least the part of it that
* fits on the same page.
*/
- readOff = ReadPageInternal(state, targetPagePtr,
- Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ));
- if (readOff < 0)
+ while (XLogNeedData(state, targetPagePtr,
+ Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+ targetRecOff != 0))
+ {
+ if (!state->routine.page_read(state, state->readPagePtr, state->readLen,
+ RecPtr, state->readBuf))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
/*
- * ReadPageInternal always returns at least the page header, so we can
- * examine it now.
+ * We have at least the page header, so we can examine it now.
*/
pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
if (targetRecOff == 0)
@@ -359,8 +364,8 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
goto err;
}
- /* ReadPageInternal has verified the page header */
- Assert(pageHeaderSize <= readOff);
+ /* XLogNeedData has verified the page header */
+ Assert(pageHeaderSize <= state->readLen);
/*
* Read the record length.
@@ -433,18 +438,27 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
do
{
+ int rest_len = total_len - gotlen;
+
/* Calculate pointer to beginning of next page */
targetPagePtr += XLOG_BLCKSZ;
/* Wait for the next page to become available */
- readOff = ReadPageInternal(state, targetPagePtr,
- Min(total_len - gotlen + SizeOfXLogShortPHD,
- XLOG_BLCKSZ));
+ while (XLogNeedData(state, targetPagePtr,
+ Min(rest_len, XLOG_BLCKSZ),
+ false))
+ {
+ if (!state->routine.page_read(state, state->readPagePtr,
+ state->readLen,
+ state->ReadRecPtr,
+ state->readBuf))
+ break;
+ }
- if (readOff < 0)
+ if (!state->page_verified)
goto err;
- Assert(SizeOfXLogShortPHD <= readOff);
+ Assert(SizeOfXLogShortPHD <= state->readLen);
/* Check that the continuation on next page looks valid */
pageHeader = (XLogPageHeader) state->readBuf;
@@ -473,21 +487,14 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
/* Append the continuation from this page to the buffer */
pageHeaderSize = XLogPageHeaderSize(pageHeader);
- if (readOff < pageHeaderSize)
- readOff = ReadPageInternal(state, targetPagePtr,
- pageHeaderSize);
-
- Assert(pageHeaderSize <= readOff);
+ Assert(pageHeaderSize <= state->readLen);
contdata = (char *) state->readBuf + pageHeaderSize;
len = XLOG_BLCKSZ - pageHeaderSize;
if (pageHeader->xlp_rem_len < len)
len = pageHeader->xlp_rem_len;
- if (readOff < pageHeaderSize + len)
- readOff = ReadPageInternal(state, targetPagePtr,
- pageHeaderSize + len);
-
+ Assert (pageHeaderSize + len <= state->readLen);
memcpy(buffer, (char *) contdata, len);
buffer += len;
gotlen += len;
@@ -517,9 +524,16 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
else
{
/* Wait for the record data to become available */
- readOff = ReadPageInternal(state, targetPagePtr,
- Min(targetRecOff + total_len, XLOG_BLCKSZ));
- if (readOff < 0)
+ while (XLogNeedData(state, targetPagePtr,
+ Min(targetRecOff + total_len, XLOG_BLCKSZ), true))
+ {
+ if (!state->routine.page_read(state, state->readPagePtr,
+ state->readLen,
+ state->ReadRecPtr, state->readBuf))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
/* Record does not cross a page boundary */
@@ -562,109 +576,138 @@ err:
}
/*
- * Read a single xlog page including at least [pageptr, reqLen] of valid data
- * via the page_read() callback.
+ * Checks that an xlog page loaded in state->readBuf is including at least
+ * [pageptr, reqLen] and the page is valid. header_inclusive indicates that
+ * reqLen is calculated including page header length.
*
- * Returns -1 if the required page cannot be read for some reason; errormsg_buf
- * is set in that case (unless the error occurs in the page_read callback).
+ * Returns false if the buffer already contains the requested data, or found
+ * error. state->page_verified is set to true for the former and false for the
+ * latter.
*
- * We fetch the page from a reader-local cache if we know we have the required
- * data and if there hasn't been any error since caching the data.
+ * Otherwise returns true and requests data loaded onto state->readBuf by
+ * state->readPagePtr and state->readLen. The caller shall call this function
+ * again after filling the buffer at least with that portion of data and set
+ * state->readLen to the length of actually loaded data.
+ *
+ * If header_inclusive is false, corrects reqLen internally by adding the
+ * actual page header length and may request caller for new data.
*/
-static int
-ReadPageInternal(XLogReaderState *state, XLogRecPtr pageptr, int reqLen)
+static bool
+XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
+ bool header_inclusive)
{
- int readLen;
uint32 targetPageOff;
XLogSegNo targetSegNo;
- XLogPageHeader hdr;
+ uint32 addLen = 0;
- Assert((pageptr % XLOG_BLCKSZ) == 0);
+ /* Some data is loaded, but page header is not verified yet. */
+ if (!state->page_verified &&
+ !XLogRecPtrIsInvalid(state->readPagePtr) && state->readLen >= 0)
+ {
+ uint32 pageHeaderSize;
+ /* just loaded new data so needs to verify page header */
+
+ /* The caller must have loaded at least page header */
+ Assert (state->readLen >= SizeOfXLogShortPHD);
+
+ /*
+ * We have enough data to check the header length. Recheck the loaded
+ * length against the actual header length.
+ */
+ pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+
+ /* Request more data if we don't have the full header. */
+ if (state->readLen < pageHeaderSize)
+ {
+ state->readLen = pageHeaderSize;
+ return true;
+ }
+
+ /* Now that we know we have the full header, validate it. */
+ if (!XLogReaderValidatePageHeader(state, state->readPagePtr,
+ (char *) state->readBuf))
+ {
+ /* That's bad. Force reading the page again. */
+ XLogReaderInvalReadState(state);
+
+ return false;
+ }
+
+ state->page_verified = true;
+
+ XLByteToSeg(state->readPagePtr, state->seg.ws_segno,
+ state->segcxt.ws_segsize);
+ }
+
+ /*
+ * The loaded page may not be the one caller is supposing to read when we
+ * are verifying the first page of new segment. In that case, skip further
+ * verification and immediately load the target page.
+ */
+ if (state->page_verified && pageptr == state->readPagePtr)
+ {
+ /*
+ * calculate additional length for page header keeping the total
+ * length within the block size.
+ */
+ if (!header_inclusive)
+ {
+ uint32 pageHeaderSize =
+ XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+
+ addLen = pageHeaderSize;
+ if (reqLen + pageHeaderSize <= XLOG_BLCKSZ)
+ addLen = pageHeaderSize;
+ else
+ addLen = XLOG_BLCKSZ - reqLen;
+
+ Assert(addLen >= 0);
+ }
+
+ /* Return if we already have it. */
+ if (reqLen + addLen <= state->readLen)
+ return false;
+ }
+
+ /* Data is not in our buffer, request the caller for it. */
XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize);
targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize);
+ Assert((pageptr % XLOG_BLCKSZ) == 0);
- /* check whether we have all the requested data already */
- if (targetSegNo == state->seg.ws_segno &&
- targetPageOff == state->segoff && reqLen <= state->readLen)
- return state->readLen;
+ /*
+ * Every time we request to load new data of a page to the caller, even if
+ * we looked at a part of it before, we need to do verification on the next
+ * invocation as the caller might now be rereading data from a different
+ * source.
+ */
+ state->page_verified = false;
/*
- * Data is not in our buffer.
- *
- * Every time we actually read the segment, even if we looked at parts of
- * it before, we need to do verification as the page_read callback might
- * now be rereading data from a different source.
- *
* Whenever switching to a new WAL segment, we read the first page of the
* file and validate its header, even if that's not where the target
* record is. This is so that we can check the additional identification
* info that is present in the first page's "long" header.
+ * Don't do this if the caller requested the first page in the segment.
*/
if (targetSegNo != state->seg.ws_segno && targetPageOff != 0)
{
- XLogRecPtr targetSegmentPtr = pageptr - targetPageOff;
-
- readLen = state->routine.page_read(state, targetSegmentPtr, XLOG_BLCKSZ,
- state->currRecPtr,
- state->readBuf);
- if (readLen < 0)
- goto err;
-
- /* we can be sure to have enough WAL available, we scrolled back */
- Assert(readLen == XLOG_BLCKSZ);
-
- if (!XLogReaderValidatePageHeader(state, targetSegmentPtr,
- state->readBuf))
- goto err;
+ /*
+ * Then we'll see that the targetSegNo now matches the ws_segno, and
+ * will not come back here, but will request the actual target page.
+ */
+ state->readPagePtr = pageptr - targetPageOff;
+ state->readLen = XLOG_BLCKSZ;
+ return true;
}
/*
- * First, read the requested data length, but at least a short page header
- * so that we can validate it.
+ * Request the caller to load the page. We need at least a short page
+ * header so that we can validate it.
*/
- readLen = state->routine.page_read(state, pageptr, Max(reqLen, SizeOfXLogShortPHD),
- state->currRecPtr,
- state->readBuf);
- if (readLen < 0)
- goto err;
-
- Assert(readLen <= XLOG_BLCKSZ);
-
- /* Do we have enough data to check the header length? */
- if (readLen <= SizeOfXLogShortPHD)
- goto err;
-
- Assert(readLen >= reqLen);
-
- hdr = (XLogPageHeader) state->readBuf;
-
- /* still not enough */
- if (readLen < XLogPageHeaderSize(hdr))
- {
- readLen = state->routine.page_read(state, pageptr, XLogPageHeaderSize(hdr),
- state->currRecPtr,
- state->readBuf);
- if (readLen < 0)
- goto err;
- }
-
- /*
- * Now that we know we have the full header, validate it.
- */
- if (!XLogReaderValidatePageHeader(state, pageptr, (char *) hdr))
- goto err;
-
- /* update read state information */
- state->seg.ws_segno = targetSegNo;
- state->segoff = targetPageOff;
- state->readLen = readLen;
-
- return readLen;
-
-err:
- XLogReaderInvalReadState(state);
- return -1;
+ state->readPagePtr = pageptr;
+ state->readLen = Max(reqLen + addLen, SizeOfXLogShortPHD);
+ return true;
}
/*
@@ -673,9 +716,7 @@ err:
static void
XLogReaderInvalReadState(XLogReaderState *state)
{
- state->seg.ws_segno = 0;
- state->segoff = 0;
- state->readLen = 0;
+ state->readPagePtr = InvalidXLogRecPtr;
}
/*
@@ -956,7 +997,6 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
XLogRecPtr targetPagePtr;
int targetRecOff;
uint32 pageHeaderSize;
- int readLen;
/*
* Compute targetRecOff. It should typically be equal or greater than
@@ -964,7 +1004,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
* that, except when caller has explicitly specified the offset that
* falls somewhere there or when we are skipping multi-page
* continuation record. It doesn't matter though because
- * ReadPageInternal() is prepared to handle that and will read at
+ * XLogNeedData() is prepared to handle that and will read at
* least short page-header worth of data
*/
targetRecOff = tmpRecPtr % XLOG_BLCKSZ;
@@ -972,19 +1012,24 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
/* scroll back to page boundary */
targetPagePtr = tmpRecPtr - targetRecOff;
- /* Read the page containing the record */
- readLen = ReadPageInternal(state, targetPagePtr, targetRecOff);
- if (readLen < 0)
+ while(XLogNeedData(state, targetPagePtr, targetRecOff,
+ targetRecOff != 0))
+ {
+ if (!state->routine.page_read(state, state->readPagePtr,
+ state->readLen,
+ state->ReadRecPtr, state->readBuf))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
header = (XLogPageHeader) state->readBuf;
pageHeaderSize = XLogPageHeaderSize(header);
- /* make sure we have enough data for the page header */
- readLen = ReadPageInternal(state, targetPagePtr, pageHeaderSize);
- if (readLen < 0)
- goto err;
+ /* we should have read the page header */
+ Assert (state->readLen >= pageHeaderSize);
/* skip over potential continuation data */
if (header->xlp_info & XLP_FIRST_IS_CONTRECORD)
diff --git a/src/backend/access/transam/xlogutils.c b/src/backend/access/transam/xlogutils.c
index 322b0e8ff5..1b10b6df20 100644
--- a/src/backend/access/transam/xlogutils.c
+++ b/src/backend/access/transam/xlogutils.c
@@ -689,8 +689,8 @@ XLogTruncateRelation(RelFileNode rnode, ForkNumber forkNum,
void
XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wantLength)
{
- const XLogRecPtr lastReadPage = (state->seg.ws_segno *
- state->segcxt.ws_segsize + state->segoff);
+ const XLogRecPtr lastReadPage = state->seg.ws_segno *
+ state->segcxt.ws_segsize + state->readLen;
Assert(wantPage != InvalidXLogRecPtr && wantPage % XLOG_BLCKSZ == 0);
Assert(wantLength <= XLOG_BLCKSZ);
@@ -828,7 +828,7 @@ wal_segment_close(XLogReaderState *state)
* exists for normal backends, so we have to do a check/sleep/repeat style of
* loop for now.
*/
-int
+bool
read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *cur_page)
{
@@ -930,7 +930,8 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
else if (targetPagePtr + reqLen > read_upto)
{
/* not enough data there */
- return -1;
+ state->readLen = -1;
+ return false;
}
else
{
@@ -948,7 +949,8 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
WALReadRaiseError(&errinfo);
/* number of valid bytes in the buffer */
- return count;
+ state->readLen = count;
+ return true;
}
/*
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index e2477c47e0..bf2711bddd 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -811,7 +811,7 @@ StartReplication(StartReplicationCmd *cmd)
* which has to do a plain sleep/busy loop, because the walsender's latch gets
* set every time WAL is flushed.
*/
-static int
+static bool
logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *cur_page)
{
@@ -831,7 +831,10 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
/* fail if not (implies we are going to shut down) */
if (flushptr < targetPagePtr + reqLen)
- return -1;
+ {
+ state->readLen = -1;
+ return false;
+ }
if (targetPagePtr + XLOG_BLCKSZ <= flushptr)
count = XLOG_BLCKSZ; /* more than one block available */
@@ -859,7 +862,8 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
XLByteToSeg(targetPagePtr, segno, state->segcxt.ws_segsize);
CheckXLogRemoved(segno, state->seg.ws_tli);
- return count;
+ state->readLen = count;
+ return true;
}
/*
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index bc6f976994..0b7e73ae79 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -46,7 +46,7 @@ typedef struct XLogPageReadPrivate
int tliIndex;
} XLogPageReadPrivate;
-static int SimpleXLogPageRead(XLogReaderState *xlogreader,
+static bool SimpleXLogPageRead(XLogReaderState *xlogreader,
XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
@@ -236,7 +236,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
}
/* XLogReader callback function, to read a WAL page */
-static int
+static bool
SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf)
{
@@ -296,7 +296,8 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
if (private->restoreCommand == NULL)
{
pg_log_error("could not open file \"%s\": %m", xlogfpath);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
/*
@@ -309,7 +310,10 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
private->restoreCommand);
if (xlogreadfd < 0)
- return -1;
+ {
+ xlogreader->readLen = -1;
+ return false;
+ }
else
pg_log_debug("using file \"%s\" restored from archive",
xlogfpath);
@@ -325,7 +329,8 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
if (lseek(xlogreadfd, (off_t) targetPageOff, SEEK_SET) < 0)
{
pg_log_error("could not seek in file \"%s\": %m", xlogfpath);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
@@ -338,13 +343,15 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
pg_log_error("could not read file \"%s\": read %d of %zu",
xlogfpath, r, (Size) XLOG_BLCKSZ);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
Assert(targetSegNo == xlogreadsegno);
xlogreader->seg.ws_tli = targetHistory[private->tliIndex].tli;
- return XLOG_BLCKSZ;
+ xlogreader->readLen = XLOG_BLCKSZ;
+ return true;
}
/*
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index d1a0678935..ab0c28d9b7 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -331,7 +331,7 @@ WALDumpCloseSegment(XLogReaderState *state)
}
/* pg_waldump's XLogReaderRoutine->page_read callback */
-static int
+static bool
WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetPtr, char *readBuff)
{
@@ -348,7 +348,8 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
else
{
private->endptr_reached = true;
- return -1;
+ state->readLen = -1;
+ return false;
}
}
@@ -373,7 +374,8 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
(Size) errinfo.wre_req);
}
- return count;
+ state->readLen = count;
+ return true;
}
/*
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index b0f2a6ed43..6c8848e14f 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -57,8 +57,8 @@ typedef struct WALSegmentContext
typedef struct XLogReaderState XLogReaderState;
-/* Function type definitions for various xlogreader interactions */
-typedef int (*XLogPageReadCB) (XLogReaderState *xlogreader,
+/* Function type definition for the read_page callback */
+typedef bool (*XLogPageReadCB) (XLogReaderState *xlogreader,
XLogRecPtr targetPagePtr,
int reqLen,
XLogRecPtr targetRecPtr,
@@ -175,6 +175,20 @@ struct XLogReaderState
XLogRecPtr ReadRecPtr; /* start of last record read */
XLogRecPtr EndRecPtr; /* end+1 of last record read */
+ /* ----------------------------------------
+ * Communication with page reader
+ * readBuf is XLOG_BLCKSZ bytes, valid up to at least readLen bytes.
+ * ----------------------------------------
+ */
+ /* variables to communicate with page reader */
+ XLogRecPtr readPagePtr; /* page pointer to read */
+ int32 readLen; /* bytes requested to reader, or actual bytes
+ * read by reader, which must be larger than
+ * the request, or -1 on error */
+ TimeLineID readPageTLI; /* TLI for data currently in readBuf */
+ char *readBuf; /* buffer to store data */
+ bool page_verified; /* is the page on the buffer verified? */
+
/* ----------------------------------------
* Decoded representation of current record
@@ -201,13 +215,6 @@ struct XLogReaderState
* ----------------------------------------
*/
- /*
- * Buffer for currently read page (XLOG_BLCKSZ bytes, valid up to at least
- * readLen bytes)
- */
- char *readBuf;
- uint32 readLen;
-
/* last read XLOG position for data currently in readBuf */
WALSegmentContext segcxt;
WALOpenSegment seg;
diff --git a/src/include/access/xlogutils.h b/src/include/access/xlogutils.h
index e59b6cf3a9..c287edf206 100644
--- a/src/include/access/xlogutils.h
+++ b/src/include/access/xlogutils.h
@@ -47,7 +47,7 @@ extern Buffer XLogReadBufferExtended(RelFileNode rnode, ForkNumber forknum,
extern Relation CreateFakeRelcacheEntry(RelFileNode rnode);
extern void FreeFakeRelcacheEntry(Relation fakerel);
-extern int read_local_xlog_page(XLogReaderState *state,
+extern bool read_local_xlog_page(XLogReaderState *state,
XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *cur_page);
extern void wal_segment_open(XLogReaderState *state,
--
2.18.4
----Next_Part(Thu_Jul__2_13_53_30_2020_072)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
filename="v15-0002-Move-page-reader-out-of-XLogReadRecord.patch"
^ permalink raw reply [nested|flat] 69+ messages in thread
* [PATCH v17 1/4] Move callback-call from ReadPageInternal to XLogReadRecord.
@ 2019-09-05 11:21 Kyotaro Horiguchi <[email protected]>
0 siblings, 0 replies; 69+ messages in thread
From: Kyotaro Horiguchi @ 2019-09-05 11:21 UTC (permalink / raw)
The current WAL record reader reads page data using a call back
function. Although it is not so problematic alone, it would be a
problem if we are going to do add tasks like encryption which is
performed on page data before WAL reader reads them. To avoid that the
record reader facility has to have a new code path corresponds to
every new callback, this patch separates page reader from WAL record
reading facility by modifying the current WAL record reader to a state
machine.
As the first step of that change, this patch moves the page reader
function out of ReadPageInternal, then the remaining tasks of the
function are taken over by the new function XLogNeedData. As the
result XLogPageRead directly calls the page reader callback function
according to the feedback from XLogNeedData.
---
src/backend/access/transam/xlog.c | 16 +-
src/backend/access/transam/xlogreader.c | 281 ++++++++++++++----------
src/backend/access/transam/xlogutils.c | 12 +-
src/backend/replication/walsender.c | 10 +-
src/bin/pg_rewind/parsexlog.c | 21 +-
src/bin/pg_waldump/pg_waldump.c | 8 +-
src/include/access/xlogreader.h | 25 ++-
src/include/access/xlogutils.h | 2 +-
8 files changed, 222 insertions(+), 153 deletions(-)
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 377afb8732..606328a65a 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -917,7 +917,7 @@ static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath,
static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli,
XLogSource source, bool notfoundOk);
static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, XLogSource source);
-static int XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
+static bool XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
bool fetching_ckpt, XLogRecPtr tliRecPtr);
@@ -4350,7 +4350,6 @@ ReadRecord(XLogReaderState *xlogreader, int emode,
XLogRecord *record;
XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
- /* Pass through parameters to XLogPageRead */
private->fetching_ckpt = fetching_ckpt;
private->emode = emode;
private->randAccess = (xlogreader->ReadRecPtr == InvalidXLogRecPtr);
@@ -11987,7 +11986,7 @@ CancelBackup(void)
* XLogPageRead() to try fetching the record from another source, or to
* sleep and retry.
*/
-static int
+static bool
XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *readBuf)
{
@@ -12046,7 +12045,8 @@ retry:
readLen = 0;
readSource = XLOG_FROM_ANY;
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
}
@@ -12141,7 +12141,8 @@ retry:
goto next_record_is_invalid;
}
- return readLen;
+ xlogreader->readLen = readLen;
+ return true;
next_record_is_invalid:
lastSourceFailed = true;
@@ -12155,8 +12156,9 @@ next_record_is_invalid:
/* In standby-mode, keep trying */
if (StandbyMode)
goto retry;
- else
- return -1;
+
+ xlogreader->readLen = -1;
+ return false;
}
/*
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index 42738eb940..7d04995b7b 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -36,8 +36,8 @@
static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
pg_attribute_printf(2, 3);
static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
-static int ReadPageInternal(XLogReaderState *state, XLogRecPtr pageptr,
- int reqLen);
+static bool XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr,
+ int reqLen, bool header_inclusive);
static void XLogReaderInvalReadState(XLogReaderState *state);
static bool ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
XLogRecPtr PrevRecPtr, XLogRecord *record, bool randAccess);
@@ -276,7 +276,6 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
uint32 targetRecOff;
uint32 pageHeaderSize;
bool gotheader;
- int readOff;
/*
* randAccess indicates whether to verify the previous-record pointer of
@@ -326,14 +325,20 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
* byte to cover the whole record header, or at least the part of it that
* fits on the same page.
*/
- readOff = ReadPageInternal(state, targetPagePtr,
- Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ));
- if (readOff < 0)
+ while (XLogNeedData(state, targetPagePtr,
+ Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+ targetRecOff != 0))
+ {
+ if (!state->routine.page_read(state, state->readPagePtr, state->readLen,
+ RecPtr, state->readBuf))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
/*
- * ReadPageInternal always returns at least the page header, so we can
- * examine it now.
+ * We have at least the page header, so we can examine it now.
*/
pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
if (targetRecOff == 0)
@@ -359,8 +364,8 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
goto err;
}
- /* ReadPageInternal has verified the page header */
- Assert(pageHeaderSize <= readOff);
+ /* XLogNeedData has verified the page header */
+ Assert(pageHeaderSize <= state->readLen);
/*
* Read the record length.
@@ -432,18 +437,27 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
do
{
+ int rest_len = total_len - gotlen;
+
/* Calculate pointer to beginning of next page */
targetPagePtr += XLOG_BLCKSZ;
/* Wait for the next page to become available */
- readOff = ReadPageInternal(state, targetPagePtr,
- Min(total_len - gotlen + SizeOfXLogShortPHD,
- XLOG_BLCKSZ));
+ while (XLogNeedData(state, targetPagePtr,
+ Min(rest_len, XLOG_BLCKSZ),
+ false))
+ {
+ if (!state->routine.page_read(state, state->readPagePtr,
+ state->readLen,
+ state->ReadRecPtr,
+ state->readBuf))
+ break;
+ }
- if (readOff < 0)
+ if (!state->page_verified)
goto err;
- Assert(SizeOfXLogShortPHD <= readOff);
+ Assert(SizeOfXLogShortPHD <= state->readLen);
/* Check that the continuation on next page looks valid */
pageHeader = (XLogPageHeader) state->readBuf;
@@ -473,21 +487,14 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
/* Append the continuation from this page to the buffer */
pageHeaderSize = XLogPageHeaderSize(pageHeader);
- if (readOff < pageHeaderSize)
- readOff = ReadPageInternal(state, targetPagePtr,
- pageHeaderSize);
-
- Assert(pageHeaderSize <= readOff);
+ Assert(pageHeaderSize <= state->readLen);
contdata = (char *) state->readBuf + pageHeaderSize;
len = XLOG_BLCKSZ - pageHeaderSize;
if (pageHeader->xlp_rem_len < len)
len = pageHeader->xlp_rem_len;
- if (readOff < pageHeaderSize + len)
- readOff = ReadPageInternal(state, targetPagePtr,
- pageHeaderSize + len);
-
+ Assert (pageHeaderSize + len <= state->readLen);
memcpy(buffer, (char *) contdata, len);
buffer += len;
gotlen += len;
@@ -517,9 +524,16 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
else
{
/* Wait for the record data to become available */
- readOff = ReadPageInternal(state, targetPagePtr,
- Min(targetRecOff + total_len, XLOG_BLCKSZ));
- if (readOff < 0)
+ while (XLogNeedData(state, targetPagePtr,
+ Min(targetRecOff + total_len, XLOG_BLCKSZ), true))
+ {
+ if (!state->routine.page_read(state, state->readPagePtr,
+ state->readLen,
+ state->ReadRecPtr, state->readBuf))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
/* Record does not cross a page boundary */
@@ -562,109 +576,138 @@ err:
}
/*
- * Read a single xlog page including at least [pageptr, reqLen] of valid data
- * via the page_read() callback.
+ * Checks that an xlog page loaded in state->readBuf is including at least
+ * [pageptr, reqLen] and the page is valid. header_inclusive indicates that
+ * reqLen is calculated including page header length.
*
- * Returns -1 if the required page cannot be read for some reason; errormsg_buf
- * is set in that case (unless the error occurs in the page_read callback).
+ * Returns false if the buffer already contains the requested data, or found
+ * error. state->page_verified is set to true for the former and false for the
+ * latter.
*
- * We fetch the page from a reader-local cache if we know we have the required
- * data and if there hasn't been any error since caching the data.
+ * Otherwise returns true and requests data loaded onto state->readBuf by
+ * state->readPagePtr and state->readLen. The caller shall call this function
+ * again after filling the buffer at least with that portion of data and set
+ * state->readLen to the length of actually loaded data.
+ *
+ * If header_inclusive is false, corrects reqLen internally by adding the
+ * actual page header length and may request caller for new data.
*/
-static int
-ReadPageInternal(XLogReaderState *state, XLogRecPtr pageptr, int reqLen)
+static bool
+XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
+ bool header_inclusive)
{
- int readLen;
uint32 targetPageOff;
XLogSegNo targetSegNo;
- XLogPageHeader hdr;
+ uint32 addLen = 0;
- Assert((pageptr % XLOG_BLCKSZ) == 0);
+ /* Some data is loaded, but page header is not verified yet. */
+ if (!state->page_verified &&
+ !XLogRecPtrIsInvalid(state->readPagePtr) && state->readLen >= 0)
+ {
+ uint32 pageHeaderSize;
+ /* just loaded new data so needs to verify page header */
+
+ /* The caller must have loaded at least page header */
+ Assert (state->readLen >= SizeOfXLogShortPHD);
+
+ /*
+ * We have enough data to check the header length. Recheck the loaded
+ * length against the actual header length.
+ */
+ pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+
+ /* Request more data if we don't have the full header. */
+ if (state->readLen < pageHeaderSize)
+ {
+ state->readLen = pageHeaderSize;
+ return true;
+ }
+
+ /* Now that we know we have the full header, validate it. */
+ if (!XLogReaderValidatePageHeader(state, state->readPagePtr,
+ (char *) state->readBuf))
+ {
+ /* That's bad. Force reading the page again. */
+ XLogReaderInvalReadState(state);
+
+ return false;
+ }
+
+ state->page_verified = true;
+
+ XLByteToSeg(state->readPagePtr, state->seg.ws_segno,
+ state->segcxt.ws_segsize);
+ }
+
+ /*
+ * The loaded page may not be the one caller is supposing to read when we
+ * are verifying the first page of new segment. In that case, skip further
+ * verification and immediately load the target page.
+ */
+ if (state->page_verified && pageptr == state->readPagePtr)
+ {
+ /*
+ * calculate additional length for page header keeping the total
+ * length within the block size.
+ */
+ if (!header_inclusive)
+ {
+ uint32 pageHeaderSize =
+ XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+
+ addLen = pageHeaderSize;
+ if (reqLen + pageHeaderSize <= XLOG_BLCKSZ)
+ addLen = pageHeaderSize;
+ else
+ addLen = XLOG_BLCKSZ - reqLen;
+
+ Assert(addLen >= 0);
+ }
+
+ /* Return if we already have it. */
+ if (reqLen + addLen <= state->readLen)
+ return false;
+ }
+
+ /* Data is not in our buffer, request the caller for it. */
XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize);
targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize);
+ Assert((pageptr % XLOG_BLCKSZ) == 0);
- /* check whether we have all the requested data already */
- if (targetSegNo == state->seg.ws_segno &&
- targetPageOff == state->segoff && reqLen <= state->readLen)
- return state->readLen;
+ /*
+ * Every time we request to load new data of a page to the caller, even if
+ * we looked at a part of it before, we need to do verification on the next
+ * invocation as the caller might now be rereading data from a different
+ * source.
+ */
+ state->page_verified = false;
/*
- * Data is not in our buffer.
- *
- * Every time we actually read the segment, even if we looked at parts of
- * it before, we need to do verification as the page_read callback might
- * now be rereading data from a different source.
- *
* Whenever switching to a new WAL segment, we read the first page of the
* file and validate its header, even if that's not where the target
* record is. This is so that we can check the additional identification
* info that is present in the first page's "long" header.
+ * Don't do this if the caller requested the first page in the segment.
*/
if (targetSegNo != state->seg.ws_segno && targetPageOff != 0)
{
- XLogRecPtr targetSegmentPtr = pageptr - targetPageOff;
-
- readLen = state->routine.page_read(state, targetSegmentPtr, XLOG_BLCKSZ,
- state->currRecPtr,
- state->readBuf);
- if (readLen < 0)
- goto err;
-
- /* we can be sure to have enough WAL available, we scrolled back */
- Assert(readLen == XLOG_BLCKSZ);
-
- if (!XLogReaderValidatePageHeader(state, targetSegmentPtr,
- state->readBuf))
- goto err;
+ /*
+ * Then we'll see that the targetSegNo now matches the ws_segno, and
+ * will not come back here, but will request the actual target page.
+ */
+ state->readPagePtr = pageptr - targetPageOff;
+ state->readLen = XLOG_BLCKSZ;
+ return true;
}
/*
- * First, read the requested data length, but at least a short page header
- * so that we can validate it.
+ * Request the caller to load the page. We need at least a short page
+ * header so that we can validate it.
*/
- readLen = state->routine.page_read(state, pageptr, Max(reqLen, SizeOfXLogShortPHD),
- state->currRecPtr,
- state->readBuf);
- if (readLen < 0)
- goto err;
-
- Assert(readLen <= XLOG_BLCKSZ);
-
- /* Do we have enough data to check the header length? */
- if (readLen <= SizeOfXLogShortPHD)
- goto err;
-
- Assert(readLen >= reqLen);
-
- hdr = (XLogPageHeader) state->readBuf;
-
- /* still not enough */
- if (readLen < XLogPageHeaderSize(hdr))
- {
- readLen = state->routine.page_read(state, pageptr, XLogPageHeaderSize(hdr),
- state->currRecPtr,
- state->readBuf);
- if (readLen < 0)
- goto err;
- }
-
- /*
- * Now that we know we have the full header, validate it.
- */
- if (!XLogReaderValidatePageHeader(state, pageptr, (char *) hdr))
- goto err;
-
- /* update read state information */
- state->seg.ws_segno = targetSegNo;
- state->segoff = targetPageOff;
- state->readLen = readLen;
-
- return readLen;
-
-err:
- XLogReaderInvalReadState(state);
- return -1;
+ state->readPagePtr = pageptr;
+ state->readLen = Max(reqLen + addLen, SizeOfXLogShortPHD);
+ return true;
}
/*
@@ -673,9 +716,7 @@ err:
static void
XLogReaderInvalReadState(XLogReaderState *state)
{
- state->seg.ws_segno = 0;
- state->segoff = 0;
- state->readLen = 0;
+ state->readPagePtr = InvalidXLogRecPtr;
}
/*
@@ -953,7 +994,6 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
XLogRecPtr targetPagePtr;
int targetRecOff;
uint32 pageHeaderSize;
- int readLen;
/*
* Compute targetRecOff. It should typically be equal or greater than
@@ -961,7 +1001,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
* that, except when caller has explicitly specified the offset that
* falls somewhere there or when we are skipping multi-page
* continuation record. It doesn't matter though because
- * ReadPageInternal() is prepared to handle that and will read at
+ * XLogNeedData() is prepared to handle that and will read at
* least short page-header worth of data
*/
targetRecOff = tmpRecPtr % XLOG_BLCKSZ;
@@ -969,19 +1009,24 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
/* scroll back to page boundary */
targetPagePtr = tmpRecPtr - targetRecOff;
- /* Read the page containing the record */
- readLen = ReadPageInternal(state, targetPagePtr, targetRecOff);
- if (readLen < 0)
+ while(XLogNeedData(state, targetPagePtr, targetRecOff,
+ targetRecOff != 0))
+ {
+ if (!state->routine.page_read(state, state->readPagePtr,
+ state->readLen,
+ state->ReadRecPtr, state->readBuf))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
header = (XLogPageHeader) state->readBuf;
pageHeaderSize = XLogPageHeaderSize(header);
- /* make sure we have enough data for the page header */
- readLen = ReadPageInternal(state, targetPagePtr, pageHeaderSize);
- if (readLen < 0)
- goto err;
+ /* we should have read the page header */
+ Assert (state->readLen >= pageHeaderSize);
/* skip over potential continuation data */
if (header->xlp_info & XLP_FIRST_IS_CONTRECORD)
diff --git a/src/backend/access/transam/xlogutils.c b/src/backend/access/transam/xlogutils.c
index d17d660f46..46eda33f25 100644
--- a/src/backend/access/transam/xlogutils.c
+++ b/src/backend/access/transam/xlogutils.c
@@ -686,8 +686,8 @@ XLogTruncateRelation(RelFileNode rnode, ForkNumber forkNum,
void
XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wantLength)
{
- const XLogRecPtr lastReadPage = (state->seg.ws_segno *
- state->segcxt.ws_segsize + state->segoff);
+ const XLogRecPtr lastReadPage = state->seg.ws_segno *
+ state->segcxt.ws_segsize + state->readLen;
Assert(wantPage != InvalidXLogRecPtr && wantPage % XLOG_BLCKSZ == 0);
Assert(wantLength <= XLOG_BLCKSZ);
@@ -824,7 +824,7 @@ wal_segment_close(XLogReaderState *state)
* exists for normal backends, so we have to do a check/sleep/repeat style of
* loop for now.
*/
-int
+bool
read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *cur_page)
{
@@ -926,7 +926,8 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
else if (targetPagePtr + reqLen > read_upto)
{
/* not enough data there */
- return -1;
+ state->readLen = -1;
+ return false;
}
else
{
@@ -944,7 +945,8 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
WALReadRaiseError(&errinfo);
/* number of valid bytes in the buffer */
- return count;
+ state->readLen = count;
+ return true;
}
/*
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index 23baa4498a..c36b91ae5e 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -806,7 +806,7 @@ StartReplication(StartReplicationCmd *cmd)
* which has to do a plain sleep/busy loop, because the walsender's latch gets
* set every time WAL is flushed.
*/
-static int
+static bool
logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *cur_page)
{
@@ -826,7 +826,10 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
/* fail if not (implies we are going to shut down) */
if (flushptr < targetPagePtr + reqLen)
- return -1;
+ {
+ state->readLen = -1;
+ return false;
+ }
if (targetPagePtr + XLOG_BLCKSZ <= flushptr)
count = XLOG_BLCKSZ; /* more than one block available */
@@ -854,7 +857,8 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
XLByteToSeg(targetPagePtr, segno, state->segcxt.ws_segsize);
CheckXLogRemoved(segno, state->seg.ws_tli);
- return count;
+ state->readLen = count;
+ return true;
}
/*
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index 59ebac7d6a..cf119848b0 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -47,7 +47,7 @@ typedef struct XLogPageReadPrivate
int tliIndex;
} XLogPageReadPrivate;
-static int SimpleXLogPageRead(XLogReaderState *xlogreader,
+static bool SimpleXLogPageRead(XLogReaderState *xlogreader,
XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
@@ -246,7 +246,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
}
/* XLogReader callback function, to read a WAL page */
-static int
+static bool
SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf)
{
@@ -306,7 +306,8 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
if (private->restoreCommand == NULL)
{
pg_log_error("could not open file \"%s\": %m", xlogfpath);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
/*
@@ -319,7 +320,10 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
private->restoreCommand);
if (xlogreadfd < 0)
- return -1;
+ {
+ xlogreader->readLen = -1;
+ return false;
+ }
else
pg_log_debug("using file \"%s\" restored from archive",
xlogfpath);
@@ -335,7 +339,8 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
if (lseek(xlogreadfd, (off_t) targetPageOff, SEEK_SET) < 0)
{
pg_log_error("could not seek in file \"%s\": %m", xlogfpath);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
@@ -348,13 +353,15 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
pg_log_error("could not read file \"%s\": read %d of %zu",
xlogfpath, r, (Size) XLOG_BLCKSZ);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
Assert(targetSegNo == xlogreadsegno);
xlogreader->seg.ws_tli = targetHistory[private->tliIndex].tli;
- return XLOG_BLCKSZ;
+ xlogreader->readLen = XLOG_BLCKSZ;
+ return true;
}
/*
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index 610f65e471..0f0e0723a3 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -331,7 +331,7 @@ WALDumpCloseSegment(XLogReaderState *state)
}
/* pg_waldump's XLogReaderRoutine->page_read callback */
-static int
+static bool
WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetPtr, char *readBuff)
{
@@ -348,7 +348,8 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
else
{
private->endptr_reached = true;
- return -1;
+ state->readLen = -1;
+ return false;
}
}
@@ -373,7 +374,8 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
(Size) errinfo.wre_req);
}
- return count;
+ state->readLen = count;
+ return true;
}
/*
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index 21d200d3df..23d9e70a59 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -57,8 +57,8 @@ typedef struct WALSegmentContext
typedef struct XLogReaderState XLogReaderState;
-/* Function type definitions for various xlogreader interactions */
-typedef int (*XLogPageReadCB) (XLogReaderState *xlogreader,
+/* Function type definition for the read_page callback */
+typedef bool (*XLogPageReadCB) (XLogReaderState *xlogreader,
XLogRecPtr targetPagePtr,
int reqLen,
XLogRecPtr targetRecPtr,
@@ -175,6 +175,20 @@ struct XLogReaderState
XLogRecPtr ReadRecPtr; /* start of last record read */
XLogRecPtr EndRecPtr; /* end+1 of last record read */
+ /* ----------------------------------------
+ * Communication with page reader
+ * readBuf is XLOG_BLCKSZ bytes, valid up to at least readLen bytes.
+ * ----------------------------------------
+ */
+ /* variables to communicate with page reader */
+ XLogRecPtr readPagePtr; /* page pointer to read */
+ int32 readLen; /* bytes requested to reader, or actual bytes
+ * read by reader, which must be larger than
+ * the request, or -1 on error */
+ TimeLineID readPageTLI; /* TLI for data currently in readBuf */
+ char *readBuf; /* buffer to store data */
+ bool page_verified; /* is the page on the buffer verified? */
+
/* ----------------------------------------
* Decoded representation of current record
@@ -203,13 +217,6 @@ struct XLogReaderState
* ----------------------------------------
*/
- /*
- * Buffer for currently read page (XLOG_BLCKSZ bytes, valid up to at least
- * readLen bytes)
- */
- char *readBuf;
- uint32 readLen;
-
/* last read XLOG position for data currently in readBuf */
WALSegmentContext segcxt;
WALOpenSegment seg;
diff --git a/src/include/access/xlogutils.h b/src/include/access/xlogutils.h
index 9ac602b674..364a21c4ea 100644
--- a/src/include/access/xlogutils.h
+++ b/src/include/access/xlogutils.h
@@ -47,7 +47,7 @@ extern Buffer XLogReadBufferExtended(RelFileNode rnode, ForkNumber forknum,
extern Relation CreateFakeRelcacheEntry(RelFileNode rnode);
extern void FreeFakeRelcacheEntry(Relation fakerel);
-extern int read_local_xlog_page(XLogReaderState *state,
+extern bool read_local_xlog_page(XLogReaderState *state,
XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *cur_page);
extern void wal_segment_open(XLogReaderState *state,
--
2.27.0
----Next_Part(Thu_Mar__4_11_28_53_2021_014)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
filename="v17-0002-Move-page-reader-out-of-XLogReadRecord.patch"
^ permalink raw reply [nested|flat] 69+ messages in thread
* [PATCH v15 1/4] Move callback-call from ReadPageInternal to XLogReadRecord.
@ 2019-09-05 11:21 Kyotaro Horiguchi <[email protected]>
0 siblings, 0 replies; 69+ messages in thread
From: Kyotaro Horiguchi @ 2019-09-05 11:21 UTC (permalink / raw)
The current WAL record reader reads page data using a call back
function. Although it is not so problematic alone, it would be a
problem if we are going to do add tasks like encryption which is
performed on page data before WAL reader reads them. To avoid that the
record reader facility has to have a new code path corresponds to
every new callback, this patch separates page reader from WAL record
reading facility by modifying the current WAL record reader to a state
machine.
As the first step of that change, this patch moves the page reader
function out of ReadPageInternal, then the remaining tasks of the
function are taken over by the new function XLogNeedData. As the
result XLogPageRead directly calls the page reader callback function
according to the feedback from XLogNeedData.
---
src/backend/access/transam/xlog.c | 16 +-
src/backend/access/transam/xlogreader.c | 281 ++++++++++++++----------
src/backend/access/transam/xlogutils.c | 12 +-
src/backend/replication/walsender.c | 10 +-
src/bin/pg_rewind/parsexlog.c | 21 +-
src/bin/pg_waldump/pg_waldump.c | 8 +-
src/include/access/xlogreader.h | 25 ++-
src/include/access/xlogutils.h | 2 +-
8 files changed, 222 insertions(+), 153 deletions(-)
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index fd93bcfaeb..e570e56a24 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -912,7 +912,7 @@ static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath,
static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli,
XLogSource source, bool notfoundOk);
static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, XLogSource source);
-static int XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
+static bool XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
bool fetching_ckpt, XLogRecPtr tliRecPtr);
@@ -4332,7 +4332,6 @@ ReadRecord(XLogReaderState *xlogreader, int emode,
XLogRecord *record;
XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
- /* Pass through parameters to XLogPageRead */
private->fetching_ckpt = fetching_ckpt;
private->emode = emode;
private->randAccess = (xlogreader->ReadRecPtr == InvalidXLogRecPtr);
@@ -11860,7 +11859,7 @@ CancelBackup(void)
* XLogPageRead() to try fetching the record from another source, or to
* sleep and retry.
*/
-static int
+static bool
XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *readBuf)
{
@@ -11919,7 +11918,8 @@ retry:
readLen = 0;
readSource = XLOG_FROM_ANY;
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
}
@@ -12014,7 +12014,8 @@ retry:
goto next_record_is_invalid;
}
- return readLen;
+ xlogreader->readLen = readLen;
+ return true;
next_record_is_invalid:
lastSourceFailed = true;
@@ -12028,8 +12029,9 @@ next_record_is_invalid:
/* In standby-mode, keep trying */
if (StandbyMode)
goto retry;
- else
- return -1;
+
+ xlogreader->readLen = -1;
+ return false;
}
/*
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index cb76be4f46..3d599325ee 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -36,8 +36,8 @@
static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
pg_attribute_printf(2, 3);
static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
-static int ReadPageInternal(XLogReaderState *state, XLogRecPtr pageptr,
- int reqLen);
+static bool XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr,
+ int reqLen, bool header_inclusive);
static void XLogReaderInvalReadState(XLogReaderState *state);
static bool ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
XLogRecPtr PrevRecPtr, XLogRecord *record, bool randAccess);
@@ -276,7 +276,6 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
uint32 targetRecOff;
uint32 pageHeaderSize;
bool gotheader;
- int readOff;
/*
* randAccess indicates whether to verify the previous-record pointer of
@@ -326,14 +325,20 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
* byte to cover the whole record header, or at least the part of it that
* fits on the same page.
*/
- readOff = ReadPageInternal(state, targetPagePtr,
- Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ));
- if (readOff < 0)
+ while (XLogNeedData(state, targetPagePtr,
+ Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+ targetRecOff != 0))
+ {
+ if (!state->routine.page_read(state, state->readPagePtr, state->readLen,
+ RecPtr, state->readBuf))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
/*
- * ReadPageInternal always returns at least the page header, so we can
- * examine it now.
+ * We have at least the page header, so we can examine it now.
*/
pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
if (targetRecOff == 0)
@@ -359,8 +364,8 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
goto err;
}
- /* ReadPageInternal has verified the page header */
- Assert(pageHeaderSize <= readOff);
+ /* XLogNeedData has verified the page header */
+ Assert(pageHeaderSize <= state->readLen);
/*
* Read the record length.
@@ -433,18 +438,27 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
do
{
+ int rest_len = total_len - gotlen;
+
/* Calculate pointer to beginning of next page */
targetPagePtr += XLOG_BLCKSZ;
/* Wait for the next page to become available */
- readOff = ReadPageInternal(state, targetPagePtr,
- Min(total_len - gotlen + SizeOfXLogShortPHD,
- XLOG_BLCKSZ));
+ while (XLogNeedData(state, targetPagePtr,
+ Min(rest_len, XLOG_BLCKSZ),
+ false))
+ {
+ if (!state->routine.page_read(state, state->readPagePtr,
+ state->readLen,
+ state->ReadRecPtr,
+ state->readBuf))
+ break;
+ }
- if (readOff < 0)
+ if (!state->page_verified)
goto err;
- Assert(SizeOfXLogShortPHD <= readOff);
+ Assert(SizeOfXLogShortPHD <= state->readLen);
/* Check that the continuation on next page looks valid */
pageHeader = (XLogPageHeader) state->readBuf;
@@ -473,21 +487,14 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
/* Append the continuation from this page to the buffer */
pageHeaderSize = XLogPageHeaderSize(pageHeader);
- if (readOff < pageHeaderSize)
- readOff = ReadPageInternal(state, targetPagePtr,
- pageHeaderSize);
-
- Assert(pageHeaderSize <= readOff);
+ Assert(pageHeaderSize <= state->readLen);
contdata = (char *) state->readBuf + pageHeaderSize;
len = XLOG_BLCKSZ - pageHeaderSize;
if (pageHeader->xlp_rem_len < len)
len = pageHeader->xlp_rem_len;
- if (readOff < pageHeaderSize + len)
- readOff = ReadPageInternal(state, targetPagePtr,
- pageHeaderSize + len);
-
+ Assert (pageHeaderSize + len <= state->readLen);
memcpy(buffer, (char *) contdata, len);
buffer += len;
gotlen += len;
@@ -517,9 +524,16 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
else
{
/* Wait for the record data to become available */
- readOff = ReadPageInternal(state, targetPagePtr,
- Min(targetRecOff + total_len, XLOG_BLCKSZ));
- if (readOff < 0)
+ while (XLogNeedData(state, targetPagePtr,
+ Min(targetRecOff + total_len, XLOG_BLCKSZ), true))
+ {
+ if (!state->routine.page_read(state, state->readPagePtr,
+ state->readLen,
+ state->ReadRecPtr, state->readBuf))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
/* Record does not cross a page boundary */
@@ -562,109 +576,138 @@ err:
}
/*
- * Read a single xlog page including at least [pageptr, reqLen] of valid data
- * via the page_read() callback.
+ * Checks that an xlog page loaded in state->readBuf is including at least
+ * [pageptr, reqLen] and the page is valid. header_inclusive indicates that
+ * reqLen is calculated including page header length.
*
- * Returns -1 if the required page cannot be read for some reason; errormsg_buf
- * is set in that case (unless the error occurs in the page_read callback).
+ * Returns false if the buffer already contains the requested data, or found
+ * error. state->page_verified is set to true for the former and false for the
+ * latter.
*
- * We fetch the page from a reader-local cache if we know we have the required
- * data and if there hasn't been any error since caching the data.
+ * Otherwise returns true and requests data loaded onto state->readBuf by
+ * state->readPagePtr and state->readLen. The caller shall call this function
+ * again after filling the buffer at least with that portion of data and set
+ * state->readLen to the length of actually loaded data.
+ *
+ * If header_inclusive is false, corrects reqLen internally by adding the
+ * actual page header length and may request caller for new data.
*/
-static int
-ReadPageInternal(XLogReaderState *state, XLogRecPtr pageptr, int reqLen)
+static bool
+XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
+ bool header_inclusive)
{
- int readLen;
uint32 targetPageOff;
XLogSegNo targetSegNo;
- XLogPageHeader hdr;
+ uint32 addLen = 0;
- Assert((pageptr % XLOG_BLCKSZ) == 0);
+ /* Some data is loaded, but page header is not verified yet. */
+ if (!state->page_verified &&
+ !XLogRecPtrIsInvalid(state->readPagePtr) && state->readLen >= 0)
+ {
+ uint32 pageHeaderSize;
+ /* just loaded new data so needs to verify page header */
+
+ /* The caller must have loaded at least page header */
+ Assert (state->readLen >= SizeOfXLogShortPHD);
+
+ /*
+ * We have enough data to check the header length. Recheck the loaded
+ * length against the actual header length.
+ */
+ pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+
+ /* Request more data if we don't have the full header. */
+ if (state->readLen < pageHeaderSize)
+ {
+ state->readLen = pageHeaderSize;
+ return true;
+ }
+
+ /* Now that we know we have the full header, validate it. */
+ if (!XLogReaderValidatePageHeader(state, state->readPagePtr,
+ (char *) state->readBuf))
+ {
+ /* That's bad. Force reading the page again. */
+ XLogReaderInvalReadState(state);
+
+ return false;
+ }
+
+ state->page_verified = true;
+
+ XLByteToSeg(state->readPagePtr, state->seg.ws_segno,
+ state->segcxt.ws_segsize);
+ }
+
+ /*
+ * The loaded page may not be the one caller is supposing to read when we
+ * are verifying the first page of new segment. In that case, skip further
+ * verification and immediately load the target page.
+ */
+ if (state->page_verified && pageptr == state->readPagePtr)
+ {
+ /*
+ * calculate additional length for page header keeping the total
+ * length within the block size.
+ */
+ if (!header_inclusive)
+ {
+ uint32 pageHeaderSize =
+ XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+
+ addLen = pageHeaderSize;
+ if (reqLen + pageHeaderSize <= XLOG_BLCKSZ)
+ addLen = pageHeaderSize;
+ else
+ addLen = XLOG_BLCKSZ - reqLen;
+
+ Assert(addLen >= 0);
+ }
+
+ /* Return if we already have it. */
+ if (reqLen + addLen <= state->readLen)
+ return false;
+ }
+
+ /* Data is not in our buffer, request the caller for it. */
XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize);
targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize);
+ Assert((pageptr % XLOG_BLCKSZ) == 0);
- /* check whether we have all the requested data already */
- if (targetSegNo == state->seg.ws_segno &&
- targetPageOff == state->segoff && reqLen <= state->readLen)
- return state->readLen;
+ /*
+ * Every time we request to load new data of a page to the caller, even if
+ * we looked at a part of it before, we need to do verification on the next
+ * invocation as the caller might now be rereading data from a different
+ * source.
+ */
+ state->page_verified = false;
/*
- * Data is not in our buffer.
- *
- * Every time we actually read the segment, even if we looked at parts of
- * it before, we need to do verification as the page_read callback might
- * now be rereading data from a different source.
- *
* Whenever switching to a new WAL segment, we read the first page of the
* file and validate its header, even if that's not where the target
* record is. This is so that we can check the additional identification
* info that is present in the first page's "long" header.
+ * Don't do this if the caller requested the first page in the segment.
*/
if (targetSegNo != state->seg.ws_segno && targetPageOff != 0)
{
- XLogRecPtr targetSegmentPtr = pageptr - targetPageOff;
-
- readLen = state->routine.page_read(state, targetSegmentPtr, XLOG_BLCKSZ,
- state->currRecPtr,
- state->readBuf);
- if (readLen < 0)
- goto err;
-
- /* we can be sure to have enough WAL available, we scrolled back */
- Assert(readLen == XLOG_BLCKSZ);
-
- if (!XLogReaderValidatePageHeader(state, targetSegmentPtr,
- state->readBuf))
- goto err;
+ /*
+ * Then we'll see that the targetSegNo now matches the ws_segno, and
+ * will not come back here, but will request the actual target page.
+ */
+ state->readPagePtr = pageptr - targetPageOff;
+ state->readLen = XLOG_BLCKSZ;
+ return true;
}
/*
- * First, read the requested data length, but at least a short page header
- * so that we can validate it.
+ * Request the caller to load the page. We need at least a short page
+ * header so that we can validate it.
*/
- readLen = state->routine.page_read(state, pageptr, Max(reqLen, SizeOfXLogShortPHD),
- state->currRecPtr,
- state->readBuf);
- if (readLen < 0)
- goto err;
-
- Assert(readLen <= XLOG_BLCKSZ);
-
- /* Do we have enough data to check the header length? */
- if (readLen <= SizeOfXLogShortPHD)
- goto err;
-
- Assert(readLen >= reqLen);
-
- hdr = (XLogPageHeader) state->readBuf;
-
- /* still not enough */
- if (readLen < XLogPageHeaderSize(hdr))
- {
- readLen = state->routine.page_read(state, pageptr, XLogPageHeaderSize(hdr),
- state->currRecPtr,
- state->readBuf);
- if (readLen < 0)
- goto err;
- }
-
- /*
- * Now that we know we have the full header, validate it.
- */
- if (!XLogReaderValidatePageHeader(state, pageptr, (char *) hdr))
- goto err;
-
- /* update read state information */
- state->seg.ws_segno = targetSegNo;
- state->segoff = targetPageOff;
- state->readLen = readLen;
-
- return readLen;
-
-err:
- XLogReaderInvalReadState(state);
- return -1;
+ state->readPagePtr = pageptr;
+ state->readLen = Max(reqLen + addLen, SizeOfXLogShortPHD);
+ return true;
}
/*
@@ -673,9 +716,7 @@ err:
static void
XLogReaderInvalReadState(XLogReaderState *state)
{
- state->seg.ws_segno = 0;
- state->segoff = 0;
- state->readLen = 0;
+ state->readPagePtr = InvalidXLogRecPtr;
}
/*
@@ -956,7 +997,6 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
XLogRecPtr targetPagePtr;
int targetRecOff;
uint32 pageHeaderSize;
- int readLen;
/*
* Compute targetRecOff. It should typically be equal or greater than
@@ -964,7 +1004,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
* that, except when caller has explicitly specified the offset that
* falls somewhere there or when we are skipping multi-page
* continuation record. It doesn't matter though because
- * ReadPageInternal() is prepared to handle that and will read at
+ * XLogNeedData() is prepared to handle that and will read at
* least short page-header worth of data
*/
targetRecOff = tmpRecPtr % XLOG_BLCKSZ;
@@ -972,19 +1012,24 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
/* scroll back to page boundary */
targetPagePtr = tmpRecPtr - targetRecOff;
- /* Read the page containing the record */
- readLen = ReadPageInternal(state, targetPagePtr, targetRecOff);
- if (readLen < 0)
+ while(XLogNeedData(state, targetPagePtr, targetRecOff,
+ targetRecOff != 0))
+ {
+ if (!state->routine.page_read(state, state->readPagePtr,
+ state->readLen,
+ state->ReadRecPtr, state->readBuf))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
header = (XLogPageHeader) state->readBuf;
pageHeaderSize = XLogPageHeaderSize(header);
- /* make sure we have enough data for the page header */
- readLen = ReadPageInternal(state, targetPagePtr, pageHeaderSize);
- if (readLen < 0)
- goto err;
+ /* we should have read the page header */
+ Assert (state->readLen >= pageHeaderSize);
/* skip over potential continuation data */
if (header->xlp_info & XLP_FIRST_IS_CONTRECORD)
diff --git a/src/backend/access/transam/xlogutils.c b/src/backend/access/transam/xlogutils.c
index 322b0e8ff5..1b10b6df20 100644
--- a/src/backend/access/transam/xlogutils.c
+++ b/src/backend/access/transam/xlogutils.c
@@ -689,8 +689,8 @@ XLogTruncateRelation(RelFileNode rnode, ForkNumber forkNum,
void
XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wantLength)
{
- const XLogRecPtr lastReadPage = (state->seg.ws_segno *
- state->segcxt.ws_segsize + state->segoff);
+ const XLogRecPtr lastReadPage = state->seg.ws_segno *
+ state->segcxt.ws_segsize + state->readLen;
Assert(wantPage != InvalidXLogRecPtr && wantPage % XLOG_BLCKSZ == 0);
Assert(wantLength <= XLOG_BLCKSZ);
@@ -828,7 +828,7 @@ wal_segment_close(XLogReaderState *state)
* exists for normal backends, so we have to do a check/sleep/repeat style of
* loop for now.
*/
-int
+bool
read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *cur_page)
{
@@ -930,7 +930,8 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
else if (targetPagePtr + reqLen > read_upto)
{
/* not enough data there */
- return -1;
+ state->readLen = -1;
+ return false;
}
else
{
@@ -948,7 +949,8 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
WALReadRaiseError(&errinfo);
/* number of valid bytes in the buffer */
- return count;
+ state->readLen = count;
+ return true;
}
/*
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index e2477c47e0..bf2711bddd 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -811,7 +811,7 @@ StartReplication(StartReplicationCmd *cmd)
* which has to do a plain sleep/busy loop, because the walsender's latch gets
* set every time WAL is flushed.
*/
-static int
+static bool
logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *cur_page)
{
@@ -831,7 +831,10 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
/* fail if not (implies we are going to shut down) */
if (flushptr < targetPagePtr + reqLen)
- return -1;
+ {
+ state->readLen = -1;
+ return false;
+ }
if (targetPagePtr + XLOG_BLCKSZ <= flushptr)
count = XLOG_BLCKSZ; /* more than one block available */
@@ -859,7 +862,8 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
XLByteToSeg(targetPagePtr, segno, state->segcxt.ws_segsize);
CheckXLogRemoved(segno, state->seg.ws_tli);
- return count;
+ state->readLen = count;
+ return true;
}
/*
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index bc6f976994..0b7e73ae79 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -46,7 +46,7 @@ typedef struct XLogPageReadPrivate
int tliIndex;
} XLogPageReadPrivate;
-static int SimpleXLogPageRead(XLogReaderState *xlogreader,
+static bool SimpleXLogPageRead(XLogReaderState *xlogreader,
XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
@@ -236,7 +236,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
}
/* XLogReader callback function, to read a WAL page */
-static int
+static bool
SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf)
{
@@ -296,7 +296,8 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
if (private->restoreCommand == NULL)
{
pg_log_error("could not open file \"%s\": %m", xlogfpath);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
/*
@@ -309,7 +310,10 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
private->restoreCommand);
if (xlogreadfd < 0)
- return -1;
+ {
+ xlogreader->readLen = -1;
+ return false;
+ }
else
pg_log_debug("using file \"%s\" restored from archive",
xlogfpath);
@@ -325,7 +329,8 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
if (lseek(xlogreadfd, (off_t) targetPageOff, SEEK_SET) < 0)
{
pg_log_error("could not seek in file \"%s\": %m", xlogfpath);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
@@ -338,13 +343,15 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
pg_log_error("could not read file \"%s\": read %d of %zu",
xlogfpath, r, (Size) XLOG_BLCKSZ);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
Assert(targetSegNo == xlogreadsegno);
xlogreader->seg.ws_tli = targetHistory[private->tliIndex].tli;
- return XLOG_BLCKSZ;
+ xlogreader->readLen = XLOG_BLCKSZ;
+ return true;
}
/*
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index d1a0678935..ab0c28d9b7 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -331,7 +331,7 @@ WALDumpCloseSegment(XLogReaderState *state)
}
/* pg_waldump's XLogReaderRoutine->page_read callback */
-static int
+static bool
WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetPtr, char *readBuff)
{
@@ -348,7 +348,8 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
else
{
private->endptr_reached = true;
- return -1;
+ state->readLen = -1;
+ return false;
}
}
@@ -373,7 +374,8 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
(Size) errinfo.wre_req);
}
- return count;
+ state->readLen = count;
+ return true;
}
/*
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index b0f2a6ed43..6c8848e14f 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -57,8 +57,8 @@ typedef struct WALSegmentContext
typedef struct XLogReaderState XLogReaderState;
-/* Function type definitions for various xlogreader interactions */
-typedef int (*XLogPageReadCB) (XLogReaderState *xlogreader,
+/* Function type definition for the read_page callback */
+typedef bool (*XLogPageReadCB) (XLogReaderState *xlogreader,
XLogRecPtr targetPagePtr,
int reqLen,
XLogRecPtr targetRecPtr,
@@ -175,6 +175,20 @@ struct XLogReaderState
XLogRecPtr ReadRecPtr; /* start of last record read */
XLogRecPtr EndRecPtr; /* end+1 of last record read */
+ /* ----------------------------------------
+ * Communication with page reader
+ * readBuf is XLOG_BLCKSZ bytes, valid up to at least readLen bytes.
+ * ----------------------------------------
+ */
+ /* variables to communicate with page reader */
+ XLogRecPtr readPagePtr; /* page pointer to read */
+ int32 readLen; /* bytes requested to reader, or actual bytes
+ * read by reader, which must be larger than
+ * the request, or -1 on error */
+ TimeLineID readPageTLI; /* TLI for data currently in readBuf */
+ char *readBuf; /* buffer to store data */
+ bool page_verified; /* is the page on the buffer verified? */
+
/* ----------------------------------------
* Decoded representation of current record
@@ -201,13 +215,6 @@ struct XLogReaderState
* ----------------------------------------
*/
- /*
- * Buffer for currently read page (XLOG_BLCKSZ bytes, valid up to at least
- * readLen bytes)
- */
- char *readBuf;
- uint32 readLen;
-
/* last read XLOG position for data currently in readBuf */
WALSegmentContext segcxt;
WALOpenSegment seg;
diff --git a/src/include/access/xlogutils.h b/src/include/access/xlogutils.h
index e59b6cf3a9..c287edf206 100644
--- a/src/include/access/xlogutils.h
+++ b/src/include/access/xlogutils.h
@@ -47,7 +47,7 @@ extern Buffer XLogReadBufferExtended(RelFileNode rnode, ForkNumber forknum,
extern Relation CreateFakeRelcacheEntry(RelFileNode rnode);
extern void FreeFakeRelcacheEntry(Relation fakerel);
-extern int read_local_xlog_page(XLogReaderState *state,
+extern bool read_local_xlog_page(XLogReaderState *state,
XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *cur_page);
extern void wal_segment_open(XLogReaderState *state,
--
2.18.4
----Next_Part(Thu_Jul__2_13_53_30_2020_072)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
filename="v15-0002-Move-page-reader-out-of-XLogReadRecord.patch"
^ permalink raw reply [nested|flat] 69+ messages in thread
* [PATCH v17 1/4] Move callback-call from ReadPageInternal to XLogReadRecord.
@ 2019-09-05 11:21 Kyotaro Horiguchi <[email protected]>
0 siblings, 0 replies; 69+ messages in thread
From: Kyotaro Horiguchi @ 2019-09-05 11:21 UTC (permalink / raw)
The current WAL record reader reads page data using a call back
function. Although it is not so problematic alone, it would be a
problem if we are going to do add tasks like encryption which is
performed on page data before WAL reader reads them. To avoid that the
record reader facility has to have a new code path corresponds to
every new callback, this patch separates page reader from WAL record
reading facility by modifying the current WAL record reader to a state
machine.
As the first step of that change, this patch moves the page reader
function out of ReadPageInternal, then the remaining tasks of the
function are taken over by the new function XLogNeedData. As the
result XLogPageRead directly calls the page reader callback function
according to the feedback from XLogNeedData.
---
src/backend/access/transam/xlog.c | 16 +-
src/backend/access/transam/xlogreader.c | 281 ++++++++++++++----------
src/backend/access/transam/xlogutils.c | 12 +-
src/backend/replication/walsender.c | 10 +-
src/bin/pg_rewind/parsexlog.c | 21 +-
src/bin/pg_waldump/pg_waldump.c | 8 +-
src/include/access/xlogreader.h | 25 ++-
src/include/access/xlogutils.h | 2 +-
8 files changed, 222 insertions(+), 153 deletions(-)
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 377afb8732..606328a65a 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -917,7 +917,7 @@ static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath,
static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli,
XLogSource source, bool notfoundOk);
static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, XLogSource source);
-static int XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
+static bool XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
bool fetching_ckpt, XLogRecPtr tliRecPtr);
@@ -4350,7 +4350,6 @@ ReadRecord(XLogReaderState *xlogreader, int emode,
XLogRecord *record;
XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
- /* Pass through parameters to XLogPageRead */
private->fetching_ckpt = fetching_ckpt;
private->emode = emode;
private->randAccess = (xlogreader->ReadRecPtr == InvalidXLogRecPtr);
@@ -11987,7 +11986,7 @@ CancelBackup(void)
* XLogPageRead() to try fetching the record from another source, or to
* sleep and retry.
*/
-static int
+static bool
XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *readBuf)
{
@@ -12046,7 +12045,8 @@ retry:
readLen = 0;
readSource = XLOG_FROM_ANY;
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
}
@@ -12141,7 +12141,8 @@ retry:
goto next_record_is_invalid;
}
- return readLen;
+ xlogreader->readLen = readLen;
+ return true;
next_record_is_invalid:
lastSourceFailed = true;
@@ -12155,8 +12156,9 @@ next_record_is_invalid:
/* In standby-mode, keep trying */
if (StandbyMode)
goto retry;
- else
- return -1;
+
+ xlogreader->readLen = -1;
+ return false;
}
/*
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index 42738eb940..7d04995b7b 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -36,8 +36,8 @@
static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
pg_attribute_printf(2, 3);
static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
-static int ReadPageInternal(XLogReaderState *state, XLogRecPtr pageptr,
- int reqLen);
+static bool XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr,
+ int reqLen, bool header_inclusive);
static void XLogReaderInvalReadState(XLogReaderState *state);
static bool ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
XLogRecPtr PrevRecPtr, XLogRecord *record, bool randAccess);
@@ -276,7 +276,6 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
uint32 targetRecOff;
uint32 pageHeaderSize;
bool gotheader;
- int readOff;
/*
* randAccess indicates whether to verify the previous-record pointer of
@@ -326,14 +325,20 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
* byte to cover the whole record header, or at least the part of it that
* fits on the same page.
*/
- readOff = ReadPageInternal(state, targetPagePtr,
- Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ));
- if (readOff < 0)
+ while (XLogNeedData(state, targetPagePtr,
+ Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+ targetRecOff != 0))
+ {
+ if (!state->routine.page_read(state, state->readPagePtr, state->readLen,
+ RecPtr, state->readBuf))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
/*
- * ReadPageInternal always returns at least the page header, so we can
- * examine it now.
+ * We have at least the page header, so we can examine it now.
*/
pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
if (targetRecOff == 0)
@@ -359,8 +364,8 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
goto err;
}
- /* ReadPageInternal has verified the page header */
- Assert(pageHeaderSize <= readOff);
+ /* XLogNeedData has verified the page header */
+ Assert(pageHeaderSize <= state->readLen);
/*
* Read the record length.
@@ -432,18 +437,27 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
do
{
+ int rest_len = total_len - gotlen;
+
/* Calculate pointer to beginning of next page */
targetPagePtr += XLOG_BLCKSZ;
/* Wait for the next page to become available */
- readOff = ReadPageInternal(state, targetPagePtr,
- Min(total_len - gotlen + SizeOfXLogShortPHD,
- XLOG_BLCKSZ));
+ while (XLogNeedData(state, targetPagePtr,
+ Min(rest_len, XLOG_BLCKSZ),
+ false))
+ {
+ if (!state->routine.page_read(state, state->readPagePtr,
+ state->readLen,
+ state->ReadRecPtr,
+ state->readBuf))
+ break;
+ }
- if (readOff < 0)
+ if (!state->page_verified)
goto err;
- Assert(SizeOfXLogShortPHD <= readOff);
+ Assert(SizeOfXLogShortPHD <= state->readLen);
/* Check that the continuation on next page looks valid */
pageHeader = (XLogPageHeader) state->readBuf;
@@ -473,21 +487,14 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
/* Append the continuation from this page to the buffer */
pageHeaderSize = XLogPageHeaderSize(pageHeader);
- if (readOff < pageHeaderSize)
- readOff = ReadPageInternal(state, targetPagePtr,
- pageHeaderSize);
-
- Assert(pageHeaderSize <= readOff);
+ Assert(pageHeaderSize <= state->readLen);
contdata = (char *) state->readBuf + pageHeaderSize;
len = XLOG_BLCKSZ - pageHeaderSize;
if (pageHeader->xlp_rem_len < len)
len = pageHeader->xlp_rem_len;
- if (readOff < pageHeaderSize + len)
- readOff = ReadPageInternal(state, targetPagePtr,
- pageHeaderSize + len);
-
+ Assert (pageHeaderSize + len <= state->readLen);
memcpy(buffer, (char *) contdata, len);
buffer += len;
gotlen += len;
@@ -517,9 +524,16 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
else
{
/* Wait for the record data to become available */
- readOff = ReadPageInternal(state, targetPagePtr,
- Min(targetRecOff + total_len, XLOG_BLCKSZ));
- if (readOff < 0)
+ while (XLogNeedData(state, targetPagePtr,
+ Min(targetRecOff + total_len, XLOG_BLCKSZ), true))
+ {
+ if (!state->routine.page_read(state, state->readPagePtr,
+ state->readLen,
+ state->ReadRecPtr, state->readBuf))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
/* Record does not cross a page boundary */
@@ -562,109 +576,138 @@ err:
}
/*
- * Read a single xlog page including at least [pageptr, reqLen] of valid data
- * via the page_read() callback.
+ * Checks that an xlog page loaded in state->readBuf is including at least
+ * [pageptr, reqLen] and the page is valid. header_inclusive indicates that
+ * reqLen is calculated including page header length.
*
- * Returns -1 if the required page cannot be read for some reason; errormsg_buf
- * is set in that case (unless the error occurs in the page_read callback).
+ * Returns false if the buffer already contains the requested data, or found
+ * error. state->page_verified is set to true for the former and false for the
+ * latter.
*
- * We fetch the page from a reader-local cache if we know we have the required
- * data and if there hasn't been any error since caching the data.
+ * Otherwise returns true and requests data loaded onto state->readBuf by
+ * state->readPagePtr and state->readLen. The caller shall call this function
+ * again after filling the buffer at least with that portion of data and set
+ * state->readLen to the length of actually loaded data.
+ *
+ * If header_inclusive is false, corrects reqLen internally by adding the
+ * actual page header length and may request caller for new data.
*/
-static int
-ReadPageInternal(XLogReaderState *state, XLogRecPtr pageptr, int reqLen)
+static bool
+XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
+ bool header_inclusive)
{
- int readLen;
uint32 targetPageOff;
XLogSegNo targetSegNo;
- XLogPageHeader hdr;
+ uint32 addLen = 0;
- Assert((pageptr % XLOG_BLCKSZ) == 0);
+ /* Some data is loaded, but page header is not verified yet. */
+ if (!state->page_verified &&
+ !XLogRecPtrIsInvalid(state->readPagePtr) && state->readLen >= 0)
+ {
+ uint32 pageHeaderSize;
+ /* just loaded new data so needs to verify page header */
+
+ /* The caller must have loaded at least page header */
+ Assert (state->readLen >= SizeOfXLogShortPHD);
+
+ /*
+ * We have enough data to check the header length. Recheck the loaded
+ * length against the actual header length.
+ */
+ pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+
+ /* Request more data if we don't have the full header. */
+ if (state->readLen < pageHeaderSize)
+ {
+ state->readLen = pageHeaderSize;
+ return true;
+ }
+
+ /* Now that we know we have the full header, validate it. */
+ if (!XLogReaderValidatePageHeader(state, state->readPagePtr,
+ (char *) state->readBuf))
+ {
+ /* That's bad. Force reading the page again. */
+ XLogReaderInvalReadState(state);
+
+ return false;
+ }
+
+ state->page_verified = true;
+
+ XLByteToSeg(state->readPagePtr, state->seg.ws_segno,
+ state->segcxt.ws_segsize);
+ }
+
+ /*
+ * The loaded page may not be the one caller is supposing to read when we
+ * are verifying the first page of new segment. In that case, skip further
+ * verification and immediately load the target page.
+ */
+ if (state->page_verified && pageptr == state->readPagePtr)
+ {
+ /*
+ * calculate additional length for page header keeping the total
+ * length within the block size.
+ */
+ if (!header_inclusive)
+ {
+ uint32 pageHeaderSize =
+ XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+
+ addLen = pageHeaderSize;
+ if (reqLen + pageHeaderSize <= XLOG_BLCKSZ)
+ addLen = pageHeaderSize;
+ else
+ addLen = XLOG_BLCKSZ - reqLen;
+
+ Assert(addLen >= 0);
+ }
+
+ /* Return if we already have it. */
+ if (reqLen + addLen <= state->readLen)
+ return false;
+ }
+
+ /* Data is not in our buffer, request the caller for it. */
XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize);
targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize);
+ Assert((pageptr % XLOG_BLCKSZ) == 0);
- /* check whether we have all the requested data already */
- if (targetSegNo == state->seg.ws_segno &&
- targetPageOff == state->segoff && reqLen <= state->readLen)
- return state->readLen;
+ /*
+ * Every time we request to load new data of a page to the caller, even if
+ * we looked at a part of it before, we need to do verification on the next
+ * invocation as the caller might now be rereading data from a different
+ * source.
+ */
+ state->page_verified = false;
/*
- * Data is not in our buffer.
- *
- * Every time we actually read the segment, even if we looked at parts of
- * it before, we need to do verification as the page_read callback might
- * now be rereading data from a different source.
- *
* Whenever switching to a new WAL segment, we read the first page of the
* file and validate its header, even if that's not where the target
* record is. This is so that we can check the additional identification
* info that is present in the first page's "long" header.
+ * Don't do this if the caller requested the first page in the segment.
*/
if (targetSegNo != state->seg.ws_segno && targetPageOff != 0)
{
- XLogRecPtr targetSegmentPtr = pageptr - targetPageOff;
-
- readLen = state->routine.page_read(state, targetSegmentPtr, XLOG_BLCKSZ,
- state->currRecPtr,
- state->readBuf);
- if (readLen < 0)
- goto err;
-
- /* we can be sure to have enough WAL available, we scrolled back */
- Assert(readLen == XLOG_BLCKSZ);
-
- if (!XLogReaderValidatePageHeader(state, targetSegmentPtr,
- state->readBuf))
- goto err;
+ /*
+ * Then we'll see that the targetSegNo now matches the ws_segno, and
+ * will not come back here, but will request the actual target page.
+ */
+ state->readPagePtr = pageptr - targetPageOff;
+ state->readLen = XLOG_BLCKSZ;
+ return true;
}
/*
- * First, read the requested data length, but at least a short page header
- * so that we can validate it.
+ * Request the caller to load the page. We need at least a short page
+ * header so that we can validate it.
*/
- readLen = state->routine.page_read(state, pageptr, Max(reqLen, SizeOfXLogShortPHD),
- state->currRecPtr,
- state->readBuf);
- if (readLen < 0)
- goto err;
-
- Assert(readLen <= XLOG_BLCKSZ);
-
- /* Do we have enough data to check the header length? */
- if (readLen <= SizeOfXLogShortPHD)
- goto err;
-
- Assert(readLen >= reqLen);
-
- hdr = (XLogPageHeader) state->readBuf;
-
- /* still not enough */
- if (readLen < XLogPageHeaderSize(hdr))
- {
- readLen = state->routine.page_read(state, pageptr, XLogPageHeaderSize(hdr),
- state->currRecPtr,
- state->readBuf);
- if (readLen < 0)
- goto err;
- }
-
- /*
- * Now that we know we have the full header, validate it.
- */
- if (!XLogReaderValidatePageHeader(state, pageptr, (char *) hdr))
- goto err;
-
- /* update read state information */
- state->seg.ws_segno = targetSegNo;
- state->segoff = targetPageOff;
- state->readLen = readLen;
-
- return readLen;
-
-err:
- XLogReaderInvalReadState(state);
- return -1;
+ state->readPagePtr = pageptr;
+ state->readLen = Max(reqLen + addLen, SizeOfXLogShortPHD);
+ return true;
}
/*
@@ -673,9 +716,7 @@ err:
static void
XLogReaderInvalReadState(XLogReaderState *state)
{
- state->seg.ws_segno = 0;
- state->segoff = 0;
- state->readLen = 0;
+ state->readPagePtr = InvalidXLogRecPtr;
}
/*
@@ -953,7 +994,6 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
XLogRecPtr targetPagePtr;
int targetRecOff;
uint32 pageHeaderSize;
- int readLen;
/*
* Compute targetRecOff. It should typically be equal or greater than
@@ -961,7 +1001,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
* that, except when caller has explicitly specified the offset that
* falls somewhere there or when we are skipping multi-page
* continuation record. It doesn't matter though because
- * ReadPageInternal() is prepared to handle that and will read at
+ * XLogNeedData() is prepared to handle that and will read at
* least short page-header worth of data
*/
targetRecOff = tmpRecPtr % XLOG_BLCKSZ;
@@ -969,19 +1009,24 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
/* scroll back to page boundary */
targetPagePtr = tmpRecPtr - targetRecOff;
- /* Read the page containing the record */
- readLen = ReadPageInternal(state, targetPagePtr, targetRecOff);
- if (readLen < 0)
+ while(XLogNeedData(state, targetPagePtr, targetRecOff,
+ targetRecOff != 0))
+ {
+ if (!state->routine.page_read(state, state->readPagePtr,
+ state->readLen,
+ state->ReadRecPtr, state->readBuf))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
header = (XLogPageHeader) state->readBuf;
pageHeaderSize = XLogPageHeaderSize(header);
- /* make sure we have enough data for the page header */
- readLen = ReadPageInternal(state, targetPagePtr, pageHeaderSize);
- if (readLen < 0)
- goto err;
+ /* we should have read the page header */
+ Assert (state->readLen >= pageHeaderSize);
/* skip over potential continuation data */
if (header->xlp_info & XLP_FIRST_IS_CONTRECORD)
diff --git a/src/backend/access/transam/xlogutils.c b/src/backend/access/transam/xlogutils.c
index d17d660f46..46eda33f25 100644
--- a/src/backend/access/transam/xlogutils.c
+++ b/src/backend/access/transam/xlogutils.c
@@ -686,8 +686,8 @@ XLogTruncateRelation(RelFileNode rnode, ForkNumber forkNum,
void
XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wantLength)
{
- const XLogRecPtr lastReadPage = (state->seg.ws_segno *
- state->segcxt.ws_segsize + state->segoff);
+ const XLogRecPtr lastReadPage = state->seg.ws_segno *
+ state->segcxt.ws_segsize + state->readLen;
Assert(wantPage != InvalidXLogRecPtr && wantPage % XLOG_BLCKSZ == 0);
Assert(wantLength <= XLOG_BLCKSZ);
@@ -824,7 +824,7 @@ wal_segment_close(XLogReaderState *state)
* exists for normal backends, so we have to do a check/sleep/repeat style of
* loop for now.
*/
-int
+bool
read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *cur_page)
{
@@ -926,7 +926,8 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
else if (targetPagePtr + reqLen > read_upto)
{
/* not enough data there */
- return -1;
+ state->readLen = -1;
+ return false;
}
else
{
@@ -944,7 +945,8 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
WALReadRaiseError(&errinfo);
/* number of valid bytes in the buffer */
- return count;
+ state->readLen = count;
+ return true;
}
/*
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index 23baa4498a..c36b91ae5e 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -806,7 +806,7 @@ StartReplication(StartReplicationCmd *cmd)
* which has to do a plain sleep/busy loop, because the walsender's latch gets
* set every time WAL is flushed.
*/
-static int
+static bool
logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *cur_page)
{
@@ -826,7 +826,10 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
/* fail if not (implies we are going to shut down) */
if (flushptr < targetPagePtr + reqLen)
- return -1;
+ {
+ state->readLen = -1;
+ return false;
+ }
if (targetPagePtr + XLOG_BLCKSZ <= flushptr)
count = XLOG_BLCKSZ; /* more than one block available */
@@ -854,7 +857,8 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
XLByteToSeg(targetPagePtr, segno, state->segcxt.ws_segsize);
CheckXLogRemoved(segno, state->seg.ws_tli);
- return count;
+ state->readLen = count;
+ return true;
}
/*
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index 59ebac7d6a..cf119848b0 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -47,7 +47,7 @@ typedef struct XLogPageReadPrivate
int tliIndex;
} XLogPageReadPrivate;
-static int SimpleXLogPageRead(XLogReaderState *xlogreader,
+static bool SimpleXLogPageRead(XLogReaderState *xlogreader,
XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
@@ -246,7 +246,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
}
/* XLogReader callback function, to read a WAL page */
-static int
+static bool
SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf)
{
@@ -306,7 +306,8 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
if (private->restoreCommand == NULL)
{
pg_log_error("could not open file \"%s\": %m", xlogfpath);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
/*
@@ -319,7 +320,10 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
private->restoreCommand);
if (xlogreadfd < 0)
- return -1;
+ {
+ xlogreader->readLen = -1;
+ return false;
+ }
else
pg_log_debug("using file \"%s\" restored from archive",
xlogfpath);
@@ -335,7 +339,8 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
if (lseek(xlogreadfd, (off_t) targetPageOff, SEEK_SET) < 0)
{
pg_log_error("could not seek in file \"%s\": %m", xlogfpath);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
@@ -348,13 +353,15 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
pg_log_error("could not read file \"%s\": read %d of %zu",
xlogfpath, r, (Size) XLOG_BLCKSZ);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
Assert(targetSegNo == xlogreadsegno);
xlogreader->seg.ws_tli = targetHistory[private->tliIndex].tli;
- return XLOG_BLCKSZ;
+ xlogreader->readLen = XLOG_BLCKSZ;
+ return true;
}
/*
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index 610f65e471..0f0e0723a3 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -331,7 +331,7 @@ WALDumpCloseSegment(XLogReaderState *state)
}
/* pg_waldump's XLogReaderRoutine->page_read callback */
-static int
+static bool
WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetPtr, char *readBuff)
{
@@ -348,7 +348,8 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
else
{
private->endptr_reached = true;
- return -1;
+ state->readLen = -1;
+ return false;
}
}
@@ -373,7 +374,8 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
(Size) errinfo.wre_req);
}
- return count;
+ state->readLen = count;
+ return true;
}
/*
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index 21d200d3df..23d9e70a59 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -57,8 +57,8 @@ typedef struct WALSegmentContext
typedef struct XLogReaderState XLogReaderState;
-/* Function type definitions for various xlogreader interactions */
-typedef int (*XLogPageReadCB) (XLogReaderState *xlogreader,
+/* Function type definition for the read_page callback */
+typedef bool (*XLogPageReadCB) (XLogReaderState *xlogreader,
XLogRecPtr targetPagePtr,
int reqLen,
XLogRecPtr targetRecPtr,
@@ -175,6 +175,20 @@ struct XLogReaderState
XLogRecPtr ReadRecPtr; /* start of last record read */
XLogRecPtr EndRecPtr; /* end+1 of last record read */
+ /* ----------------------------------------
+ * Communication with page reader
+ * readBuf is XLOG_BLCKSZ bytes, valid up to at least readLen bytes.
+ * ----------------------------------------
+ */
+ /* variables to communicate with page reader */
+ XLogRecPtr readPagePtr; /* page pointer to read */
+ int32 readLen; /* bytes requested to reader, or actual bytes
+ * read by reader, which must be larger than
+ * the request, or -1 on error */
+ TimeLineID readPageTLI; /* TLI for data currently in readBuf */
+ char *readBuf; /* buffer to store data */
+ bool page_verified; /* is the page on the buffer verified? */
+
/* ----------------------------------------
* Decoded representation of current record
@@ -203,13 +217,6 @@ struct XLogReaderState
* ----------------------------------------
*/
- /*
- * Buffer for currently read page (XLOG_BLCKSZ bytes, valid up to at least
- * readLen bytes)
- */
- char *readBuf;
- uint32 readLen;
-
/* last read XLOG position for data currently in readBuf */
WALSegmentContext segcxt;
WALOpenSegment seg;
diff --git a/src/include/access/xlogutils.h b/src/include/access/xlogutils.h
index 9ac602b674..364a21c4ea 100644
--- a/src/include/access/xlogutils.h
+++ b/src/include/access/xlogutils.h
@@ -47,7 +47,7 @@ extern Buffer XLogReadBufferExtended(RelFileNode rnode, ForkNumber forknum,
extern Relation CreateFakeRelcacheEntry(RelFileNode rnode);
extern void FreeFakeRelcacheEntry(Relation fakerel);
-extern int read_local_xlog_page(XLogReaderState *state,
+extern bool read_local_xlog_page(XLogReaderState *state,
XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *cur_page);
extern void wal_segment_open(XLogReaderState *state,
--
2.27.0
----Next_Part(Thu_Mar__4_11_28_53_2021_014)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
filename="v17-0002-Move-page-reader-out-of-XLogReadRecord.patch"
^ permalink raw reply [nested|flat] 69+ messages in thread
* [PATCH v15 1/4] Move callback-call from ReadPageInternal to XLogReadRecord.
@ 2019-09-05 11:21 Kyotaro Horiguchi <[email protected]>
0 siblings, 0 replies; 69+ messages in thread
From: Kyotaro Horiguchi @ 2019-09-05 11:21 UTC (permalink / raw)
The current WAL record reader reads page data using a call back
function. Although it is not so problematic alone, it would be a
problem if we are going to do add tasks like encryption which is
performed on page data before WAL reader reads them. To avoid that the
record reader facility has to have a new code path corresponds to
every new callback, this patch separates page reader from WAL record
reading facility by modifying the current WAL record reader to a state
machine.
As the first step of that change, this patch moves the page reader
function out of ReadPageInternal, then the remaining tasks of the
function are taken over by the new function XLogNeedData. As the
result XLogPageRead directly calls the page reader callback function
according to the feedback from XLogNeedData.
---
src/backend/access/transam/xlog.c | 16 +-
src/backend/access/transam/xlogreader.c | 281 ++++++++++++++----------
src/backend/access/transam/xlogutils.c | 12 +-
src/backend/replication/walsender.c | 10 +-
src/bin/pg_rewind/parsexlog.c | 21 +-
src/bin/pg_waldump/pg_waldump.c | 8 +-
src/include/access/xlogreader.h | 25 ++-
src/include/access/xlogutils.h | 2 +-
8 files changed, 222 insertions(+), 153 deletions(-)
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index fd93bcfaeb..e570e56a24 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -912,7 +912,7 @@ static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath,
static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli,
XLogSource source, bool notfoundOk);
static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, XLogSource source);
-static int XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
+static bool XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
bool fetching_ckpt, XLogRecPtr tliRecPtr);
@@ -4332,7 +4332,6 @@ ReadRecord(XLogReaderState *xlogreader, int emode,
XLogRecord *record;
XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
- /* Pass through parameters to XLogPageRead */
private->fetching_ckpt = fetching_ckpt;
private->emode = emode;
private->randAccess = (xlogreader->ReadRecPtr == InvalidXLogRecPtr);
@@ -11860,7 +11859,7 @@ CancelBackup(void)
* XLogPageRead() to try fetching the record from another source, or to
* sleep and retry.
*/
-static int
+static bool
XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *readBuf)
{
@@ -11919,7 +11918,8 @@ retry:
readLen = 0;
readSource = XLOG_FROM_ANY;
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
}
@@ -12014,7 +12014,8 @@ retry:
goto next_record_is_invalid;
}
- return readLen;
+ xlogreader->readLen = readLen;
+ return true;
next_record_is_invalid:
lastSourceFailed = true;
@@ -12028,8 +12029,9 @@ next_record_is_invalid:
/* In standby-mode, keep trying */
if (StandbyMode)
goto retry;
- else
- return -1;
+
+ xlogreader->readLen = -1;
+ return false;
}
/*
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index cb76be4f46..3d599325ee 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -36,8 +36,8 @@
static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
pg_attribute_printf(2, 3);
static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
-static int ReadPageInternal(XLogReaderState *state, XLogRecPtr pageptr,
- int reqLen);
+static bool XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr,
+ int reqLen, bool header_inclusive);
static void XLogReaderInvalReadState(XLogReaderState *state);
static bool ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
XLogRecPtr PrevRecPtr, XLogRecord *record, bool randAccess);
@@ -276,7 +276,6 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
uint32 targetRecOff;
uint32 pageHeaderSize;
bool gotheader;
- int readOff;
/*
* randAccess indicates whether to verify the previous-record pointer of
@@ -326,14 +325,20 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
* byte to cover the whole record header, or at least the part of it that
* fits on the same page.
*/
- readOff = ReadPageInternal(state, targetPagePtr,
- Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ));
- if (readOff < 0)
+ while (XLogNeedData(state, targetPagePtr,
+ Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+ targetRecOff != 0))
+ {
+ if (!state->routine.page_read(state, state->readPagePtr, state->readLen,
+ RecPtr, state->readBuf))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
/*
- * ReadPageInternal always returns at least the page header, so we can
- * examine it now.
+ * We have at least the page header, so we can examine it now.
*/
pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
if (targetRecOff == 0)
@@ -359,8 +364,8 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
goto err;
}
- /* ReadPageInternal has verified the page header */
- Assert(pageHeaderSize <= readOff);
+ /* XLogNeedData has verified the page header */
+ Assert(pageHeaderSize <= state->readLen);
/*
* Read the record length.
@@ -433,18 +438,27 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
do
{
+ int rest_len = total_len - gotlen;
+
/* Calculate pointer to beginning of next page */
targetPagePtr += XLOG_BLCKSZ;
/* Wait for the next page to become available */
- readOff = ReadPageInternal(state, targetPagePtr,
- Min(total_len - gotlen + SizeOfXLogShortPHD,
- XLOG_BLCKSZ));
+ while (XLogNeedData(state, targetPagePtr,
+ Min(rest_len, XLOG_BLCKSZ),
+ false))
+ {
+ if (!state->routine.page_read(state, state->readPagePtr,
+ state->readLen,
+ state->ReadRecPtr,
+ state->readBuf))
+ break;
+ }
- if (readOff < 0)
+ if (!state->page_verified)
goto err;
- Assert(SizeOfXLogShortPHD <= readOff);
+ Assert(SizeOfXLogShortPHD <= state->readLen);
/* Check that the continuation on next page looks valid */
pageHeader = (XLogPageHeader) state->readBuf;
@@ -473,21 +487,14 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
/* Append the continuation from this page to the buffer */
pageHeaderSize = XLogPageHeaderSize(pageHeader);
- if (readOff < pageHeaderSize)
- readOff = ReadPageInternal(state, targetPagePtr,
- pageHeaderSize);
-
- Assert(pageHeaderSize <= readOff);
+ Assert(pageHeaderSize <= state->readLen);
contdata = (char *) state->readBuf + pageHeaderSize;
len = XLOG_BLCKSZ - pageHeaderSize;
if (pageHeader->xlp_rem_len < len)
len = pageHeader->xlp_rem_len;
- if (readOff < pageHeaderSize + len)
- readOff = ReadPageInternal(state, targetPagePtr,
- pageHeaderSize + len);
-
+ Assert (pageHeaderSize + len <= state->readLen);
memcpy(buffer, (char *) contdata, len);
buffer += len;
gotlen += len;
@@ -517,9 +524,16 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
else
{
/* Wait for the record data to become available */
- readOff = ReadPageInternal(state, targetPagePtr,
- Min(targetRecOff + total_len, XLOG_BLCKSZ));
- if (readOff < 0)
+ while (XLogNeedData(state, targetPagePtr,
+ Min(targetRecOff + total_len, XLOG_BLCKSZ), true))
+ {
+ if (!state->routine.page_read(state, state->readPagePtr,
+ state->readLen,
+ state->ReadRecPtr, state->readBuf))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
/* Record does not cross a page boundary */
@@ -562,109 +576,138 @@ err:
}
/*
- * Read a single xlog page including at least [pageptr, reqLen] of valid data
- * via the page_read() callback.
+ * Checks that an xlog page loaded in state->readBuf is including at least
+ * [pageptr, reqLen] and the page is valid. header_inclusive indicates that
+ * reqLen is calculated including page header length.
*
- * Returns -1 if the required page cannot be read for some reason; errormsg_buf
- * is set in that case (unless the error occurs in the page_read callback).
+ * Returns false if the buffer already contains the requested data, or found
+ * error. state->page_verified is set to true for the former and false for the
+ * latter.
*
- * We fetch the page from a reader-local cache if we know we have the required
- * data and if there hasn't been any error since caching the data.
+ * Otherwise returns true and requests data loaded onto state->readBuf by
+ * state->readPagePtr and state->readLen. The caller shall call this function
+ * again after filling the buffer at least with that portion of data and set
+ * state->readLen to the length of actually loaded data.
+ *
+ * If header_inclusive is false, corrects reqLen internally by adding the
+ * actual page header length and may request caller for new data.
*/
-static int
-ReadPageInternal(XLogReaderState *state, XLogRecPtr pageptr, int reqLen)
+static bool
+XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
+ bool header_inclusive)
{
- int readLen;
uint32 targetPageOff;
XLogSegNo targetSegNo;
- XLogPageHeader hdr;
+ uint32 addLen = 0;
- Assert((pageptr % XLOG_BLCKSZ) == 0);
+ /* Some data is loaded, but page header is not verified yet. */
+ if (!state->page_verified &&
+ !XLogRecPtrIsInvalid(state->readPagePtr) && state->readLen >= 0)
+ {
+ uint32 pageHeaderSize;
+ /* just loaded new data so needs to verify page header */
+
+ /* The caller must have loaded at least page header */
+ Assert (state->readLen >= SizeOfXLogShortPHD);
+
+ /*
+ * We have enough data to check the header length. Recheck the loaded
+ * length against the actual header length.
+ */
+ pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+
+ /* Request more data if we don't have the full header. */
+ if (state->readLen < pageHeaderSize)
+ {
+ state->readLen = pageHeaderSize;
+ return true;
+ }
+
+ /* Now that we know we have the full header, validate it. */
+ if (!XLogReaderValidatePageHeader(state, state->readPagePtr,
+ (char *) state->readBuf))
+ {
+ /* That's bad. Force reading the page again. */
+ XLogReaderInvalReadState(state);
+
+ return false;
+ }
+
+ state->page_verified = true;
+
+ XLByteToSeg(state->readPagePtr, state->seg.ws_segno,
+ state->segcxt.ws_segsize);
+ }
+
+ /*
+ * The loaded page may not be the one caller is supposing to read when we
+ * are verifying the first page of new segment. In that case, skip further
+ * verification and immediately load the target page.
+ */
+ if (state->page_verified && pageptr == state->readPagePtr)
+ {
+ /*
+ * calculate additional length for page header keeping the total
+ * length within the block size.
+ */
+ if (!header_inclusive)
+ {
+ uint32 pageHeaderSize =
+ XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+
+ addLen = pageHeaderSize;
+ if (reqLen + pageHeaderSize <= XLOG_BLCKSZ)
+ addLen = pageHeaderSize;
+ else
+ addLen = XLOG_BLCKSZ - reqLen;
+
+ Assert(addLen >= 0);
+ }
+
+ /* Return if we already have it. */
+ if (reqLen + addLen <= state->readLen)
+ return false;
+ }
+
+ /* Data is not in our buffer, request the caller for it. */
XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize);
targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize);
+ Assert((pageptr % XLOG_BLCKSZ) == 0);
- /* check whether we have all the requested data already */
- if (targetSegNo == state->seg.ws_segno &&
- targetPageOff == state->segoff && reqLen <= state->readLen)
- return state->readLen;
+ /*
+ * Every time we request to load new data of a page to the caller, even if
+ * we looked at a part of it before, we need to do verification on the next
+ * invocation as the caller might now be rereading data from a different
+ * source.
+ */
+ state->page_verified = false;
/*
- * Data is not in our buffer.
- *
- * Every time we actually read the segment, even if we looked at parts of
- * it before, we need to do verification as the page_read callback might
- * now be rereading data from a different source.
- *
* Whenever switching to a new WAL segment, we read the first page of the
* file and validate its header, even if that's not where the target
* record is. This is so that we can check the additional identification
* info that is present in the first page's "long" header.
+ * Don't do this if the caller requested the first page in the segment.
*/
if (targetSegNo != state->seg.ws_segno && targetPageOff != 0)
{
- XLogRecPtr targetSegmentPtr = pageptr - targetPageOff;
-
- readLen = state->routine.page_read(state, targetSegmentPtr, XLOG_BLCKSZ,
- state->currRecPtr,
- state->readBuf);
- if (readLen < 0)
- goto err;
-
- /* we can be sure to have enough WAL available, we scrolled back */
- Assert(readLen == XLOG_BLCKSZ);
-
- if (!XLogReaderValidatePageHeader(state, targetSegmentPtr,
- state->readBuf))
- goto err;
+ /*
+ * Then we'll see that the targetSegNo now matches the ws_segno, and
+ * will not come back here, but will request the actual target page.
+ */
+ state->readPagePtr = pageptr - targetPageOff;
+ state->readLen = XLOG_BLCKSZ;
+ return true;
}
/*
- * First, read the requested data length, but at least a short page header
- * so that we can validate it.
+ * Request the caller to load the page. We need at least a short page
+ * header so that we can validate it.
*/
- readLen = state->routine.page_read(state, pageptr, Max(reqLen, SizeOfXLogShortPHD),
- state->currRecPtr,
- state->readBuf);
- if (readLen < 0)
- goto err;
-
- Assert(readLen <= XLOG_BLCKSZ);
-
- /* Do we have enough data to check the header length? */
- if (readLen <= SizeOfXLogShortPHD)
- goto err;
-
- Assert(readLen >= reqLen);
-
- hdr = (XLogPageHeader) state->readBuf;
-
- /* still not enough */
- if (readLen < XLogPageHeaderSize(hdr))
- {
- readLen = state->routine.page_read(state, pageptr, XLogPageHeaderSize(hdr),
- state->currRecPtr,
- state->readBuf);
- if (readLen < 0)
- goto err;
- }
-
- /*
- * Now that we know we have the full header, validate it.
- */
- if (!XLogReaderValidatePageHeader(state, pageptr, (char *) hdr))
- goto err;
-
- /* update read state information */
- state->seg.ws_segno = targetSegNo;
- state->segoff = targetPageOff;
- state->readLen = readLen;
-
- return readLen;
-
-err:
- XLogReaderInvalReadState(state);
- return -1;
+ state->readPagePtr = pageptr;
+ state->readLen = Max(reqLen + addLen, SizeOfXLogShortPHD);
+ return true;
}
/*
@@ -673,9 +716,7 @@ err:
static void
XLogReaderInvalReadState(XLogReaderState *state)
{
- state->seg.ws_segno = 0;
- state->segoff = 0;
- state->readLen = 0;
+ state->readPagePtr = InvalidXLogRecPtr;
}
/*
@@ -956,7 +997,6 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
XLogRecPtr targetPagePtr;
int targetRecOff;
uint32 pageHeaderSize;
- int readLen;
/*
* Compute targetRecOff. It should typically be equal or greater than
@@ -964,7 +1004,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
* that, except when caller has explicitly specified the offset that
* falls somewhere there or when we are skipping multi-page
* continuation record. It doesn't matter though because
- * ReadPageInternal() is prepared to handle that and will read at
+ * XLogNeedData() is prepared to handle that and will read at
* least short page-header worth of data
*/
targetRecOff = tmpRecPtr % XLOG_BLCKSZ;
@@ -972,19 +1012,24 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
/* scroll back to page boundary */
targetPagePtr = tmpRecPtr - targetRecOff;
- /* Read the page containing the record */
- readLen = ReadPageInternal(state, targetPagePtr, targetRecOff);
- if (readLen < 0)
+ while(XLogNeedData(state, targetPagePtr, targetRecOff,
+ targetRecOff != 0))
+ {
+ if (!state->routine.page_read(state, state->readPagePtr,
+ state->readLen,
+ state->ReadRecPtr, state->readBuf))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
header = (XLogPageHeader) state->readBuf;
pageHeaderSize = XLogPageHeaderSize(header);
- /* make sure we have enough data for the page header */
- readLen = ReadPageInternal(state, targetPagePtr, pageHeaderSize);
- if (readLen < 0)
- goto err;
+ /* we should have read the page header */
+ Assert (state->readLen >= pageHeaderSize);
/* skip over potential continuation data */
if (header->xlp_info & XLP_FIRST_IS_CONTRECORD)
diff --git a/src/backend/access/transam/xlogutils.c b/src/backend/access/transam/xlogutils.c
index 322b0e8ff5..1b10b6df20 100644
--- a/src/backend/access/transam/xlogutils.c
+++ b/src/backend/access/transam/xlogutils.c
@@ -689,8 +689,8 @@ XLogTruncateRelation(RelFileNode rnode, ForkNumber forkNum,
void
XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wantLength)
{
- const XLogRecPtr lastReadPage = (state->seg.ws_segno *
- state->segcxt.ws_segsize + state->segoff);
+ const XLogRecPtr lastReadPage = state->seg.ws_segno *
+ state->segcxt.ws_segsize + state->readLen;
Assert(wantPage != InvalidXLogRecPtr && wantPage % XLOG_BLCKSZ == 0);
Assert(wantLength <= XLOG_BLCKSZ);
@@ -828,7 +828,7 @@ wal_segment_close(XLogReaderState *state)
* exists for normal backends, so we have to do a check/sleep/repeat style of
* loop for now.
*/
-int
+bool
read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *cur_page)
{
@@ -930,7 +930,8 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
else if (targetPagePtr + reqLen > read_upto)
{
/* not enough data there */
- return -1;
+ state->readLen = -1;
+ return false;
}
else
{
@@ -948,7 +949,8 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
WALReadRaiseError(&errinfo);
/* number of valid bytes in the buffer */
- return count;
+ state->readLen = count;
+ return true;
}
/*
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index e2477c47e0..bf2711bddd 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -811,7 +811,7 @@ StartReplication(StartReplicationCmd *cmd)
* which has to do a plain sleep/busy loop, because the walsender's latch gets
* set every time WAL is flushed.
*/
-static int
+static bool
logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *cur_page)
{
@@ -831,7 +831,10 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
/* fail if not (implies we are going to shut down) */
if (flushptr < targetPagePtr + reqLen)
- return -1;
+ {
+ state->readLen = -1;
+ return false;
+ }
if (targetPagePtr + XLOG_BLCKSZ <= flushptr)
count = XLOG_BLCKSZ; /* more than one block available */
@@ -859,7 +862,8 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
XLByteToSeg(targetPagePtr, segno, state->segcxt.ws_segsize);
CheckXLogRemoved(segno, state->seg.ws_tli);
- return count;
+ state->readLen = count;
+ return true;
}
/*
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index bc6f976994..0b7e73ae79 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -46,7 +46,7 @@ typedef struct XLogPageReadPrivate
int tliIndex;
} XLogPageReadPrivate;
-static int SimpleXLogPageRead(XLogReaderState *xlogreader,
+static bool SimpleXLogPageRead(XLogReaderState *xlogreader,
XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
@@ -236,7 +236,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
}
/* XLogReader callback function, to read a WAL page */
-static int
+static bool
SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf)
{
@@ -296,7 +296,8 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
if (private->restoreCommand == NULL)
{
pg_log_error("could not open file \"%s\": %m", xlogfpath);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
/*
@@ -309,7 +310,10 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
private->restoreCommand);
if (xlogreadfd < 0)
- return -1;
+ {
+ xlogreader->readLen = -1;
+ return false;
+ }
else
pg_log_debug("using file \"%s\" restored from archive",
xlogfpath);
@@ -325,7 +329,8 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
if (lseek(xlogreadfd, (off_t) targetPageOff, SEEK_SET) < 0)
{
pg_log_error("could not seek in file \"%s\": %m", xlogfpath);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
@@ -338,13 +343,15 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
pg_log_error("could not read file \"%s\": read %d of %zu",
xlogfpath, r, (Size) XLOG_BLCKSZ);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
Assert(targetSegNo == xlogreadsegno);
xlogreader->seg.ws_tli = targetHistory[private->tliIndex].tli;
- return XLOG_BLCKSZ;
+ xlogreader->readLen = XLOG_BLCKSZ;
+ return true;
}
/*
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index d1a0678935..ab0c28d9b7 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -331,7 +331,7 @@ WALDumpCloseSegment(XLogReaderState *state)
}
/* pg_waldump's XLogReaderRoutine->page_read callback */
-static int
+static bool
WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetPtr, char *readBuff)
{
@@ -348,7 +348,8 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
else
{
private->endptr_reached = true;
- return -1;
+ state->readLen = -1;
+ return false;
}
}
@@ -373,7 +374,8 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
(Size) errinfo.wre_req);
}
- return count;
+ state->readLen = count;
+ return true;
}
/*
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index b0f2a6ed43..6c8848e14f 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -57,8 +57,8 @@ typedef struct WALSegmentContext
typedef struct XLogReaderState XLogReaderState;
-/* Function type definitions for various xlogreader interactions */
-typedef int (*XLogPageReadCB) (XLogReaderState *xlogreader,
+/* Function type definition for the read_page callback */
+typedef bool (*XLogPageReadCB) (XLogReaderState *xlogreader,
XLogRecPtr targetPagePtr,
int reqLen,
XLogRecPtr targetRecPtr,
@@ -175,6 +175,20 @@ struct XLogReaderState
XLogRecPtr ReadRecPtr; /* start of last record read */
XLogRecPtr EndRecPtr; /* end+1 of last record read */
+ /* ----------------------------------------
+ * Communication with page reader
+ * readBuf is XLOG_BLCKSZ bytes, valid up to at least readLen bytes.
+ * ----------------------------------------
+ */
+ /* variables to communicate with page reader */
+ XLogRecPtr readPagePtr; /* page pointer to read */
+ int32 readLen; /* bytes requested to reader, or actual bytes
+ * read by reader, which must be larger than
+ * the request, or -1 on error */
+ TimeLineID readPageTLI; /* TLI for data currently in readBuf */
+ char *readBuf; /* buffer to store data */
+ bool page_verified; /* is the page on the buffer verified? */
+
/* ----------------------------------------
* Decoded representation of current record
@@ -201,13 +215,6 @@ struct XLogReaderState
* ----------------------------------------
*/
- /*
- * Buffer for currently read page (XLOG_BLCKSZ bytes, valid up to at least
- * readLen bytes)
- */
- char *readBuf;
- uint32 readLen;
-
/* last read XLOG position for data currently in readBuf */
WALSegmentContext segcxt;
WALOpenSegment seg;
diff --git a/src/include/access/xlogutils.h b/src/include/access/xlogutils.h
index e59b6cf3a9..c287edf206 100644
--- a/src/include/access/xlogutils.h
+++ b/src/include/access/xlogutils.h
@@ -47,7 +47,7 @@ extern Buffer XLogReadBufferExtended(RelFileNode rnode, ForkNumber forknum,
extern Relation CreateFakeRelcacheEntry(RelFileNode rnode);
extern void FreeFakeRelcacheEntry(Relation fakerel);
-extern int read_local_xlog_page(XLogReaderState *state,
+extern bool read_local_xlog_page(XLogReaderState *state,
XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *cur_page);
extern void wal_segment_open(XLogReaderState *state,
--
2.18.4
----Next_Part(Thu_Jul__2_13_53_30_2020_072)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
filename="v15-0002-Move-page-reader-out-of-XLogReadRecord.patch"
^ permalink raw reply [nested|flat] 69+ messages in thread
* [PATCH v18 1/5] Move callback-call from ReadPageInternal to XLogReadRecord.
@ 2021-09-30 02:48 Kyotaro Horiguchi <[email protected]>
0 siblings, 0 replies; 69+ messages in thread
From: Kyotaro Horiguchi @ 2021-09-30 02:48 UTC (permalink / raw)
The current WAL record reader reads page data using a call back
function. Redesign the interface so that it asks the caller for more
data when required. This model works better for proposed projects that
encryption, prefetching and other new features that would require
extending the callback interface for each case.
As the first step of that change, this patch moves the page reader
function out of ReadPageInternal(), then the remaining tasks of the
function are taken over by the new function XLogNeedData().
Author: Kyotaro HORIGUCHI <[email protected]>
Author: Heikki Linnakangas <[email protected]>
Reviewed-by: Antonin Houska <[email protected]>
Reviewed-by: Alvaro Herrera <[email protected]>
Reviewed-by: Takashi Menjo <[email protected]>
Reviewed-by: Thomas Munro <[email protected]>
Discussion: https://postgr.es/m/20190418.210257.43726183.horiguchi.kyotaro%40lab.ntt.co.jp
---
src/backend/access/transam/xlog.c | 16 +-
src/backend/access/transam/xlogreader.c | 325 +++++++++++++++---------
src/backend/access/transam/xlogutils.c | 12 +-
src/backend/replication/walsender.c | 10 +-
src/bin/pg_rewind/parsexlog.c | 21 +-
src/bin/pg_waldump/pg_waldump.c | 8 +-
src/include/access/xlogreader.h | 31 ++-
src/include/access/xlogutils.h | 2 +-
8 files changed, 266 insertions(+), 159 deletions(-)
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 26dcc00ac0..1557ceb8c1 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -918,7 +918,7 @@ static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath,
static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli,
XLogSource source, bool notfoundOk);
static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, XLogSource source);
-static int XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
+static bool XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
bool fetching_ckpt, XLogRecPtr tliRecPtr);
@@ -4401,7 +4401,6 @@ ReadRecord(XLogReaderState *xlogreader, int emode,
XLogRecord *record;
XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
- /* Pass through parameters to XLogPageRead */
private->fetching_ckpt = fetching_ckpt;
private->emode = emode;
private->randAccess = (xlogreader->ReadRecPtr == InvalidXLogRecPtr);
@@ -12300,7 +12299,7 @@ CancelBackup(void)
* XLogPageRead() to try fetching the record from another source, or to
* sleep and retry.
*/
-static int
+static bool
XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *readBuf)
{
@@ -12359,7 +12358,8 @@ retry:
readLen = 0;
readSource = XLOG_FROM_ANY;
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
}
@@ -12469,7 +12469,8 @@ retry:
goto next_record_is_invalid;
}
- return readLen;
+ xlogreader->readLen = readLen;
+ return true;
next_record_is_invalid:
lastSourceFailed = true;
@@ -12483,8 +12484,9 @@ next_record_is_invalid:
/* In standby-mode, keep trying */
if (StandbyMode)
goto retry;
- else
- return -1;
+
+ xlogreader->readLen = -1;
+ return false;
}
/*
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index 4b03577dcc..1d9976ecf4 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -39,8 +39,8 @@
static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
pg_attribute_printf(2, 3);
static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
-static int ReadPageInternal(XLogReaderState *state, XLogRecPtr pageptr,
- int reqLen);
+static bool XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr,
+ int reqLen, bool header_inclusive);
static void XLogReaderInvalReadState(XLogReaderState *state);
static bool ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
XLogRecPtr PrevRecPtr, XLogRecord *record, bool randAccess);
@@ -264,8 +264,48 @@ XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
* If the reading fails for some other reason, NULL is also returned, and
* *errormsg is set to a string with details of the failure.
*
+ * Returns XLREAD_NEED_DATA if more data is needed to finish reading the
+ * current record. In that case, state->readPagePtr and state->readLen inform
+ * the desired position and minimum length of data needed. The caller shall
+ * read in the requested data and set state->readBuf to point to a buffer
+ * containing it. The caller must also set state->seg->ws_tli and
+ * state->readLen to indicate the timeline that it was read from, and the
+ * length of data that is now available (which must be >= given readLen),
+ * respectively.
+ *
+ * If invalid data is encountered, returns XLREAD_FAIL with *record being set to
+ * NULL. *errormsg is set to a string with details of the failure.
* The returned pointer (or *errormsg) points to an internal buffer that's
* valid until the next call to XLogReadRecord.
+ *
+ *
+ * This function runs a state machine consists of the following states.
+ *
+ * XLREAD_NEXT_RECORD :
+ * The initial state, if called with valid RecPtr, try to read a record at
+ * that position. If invalid RecPtr is given try to read a record just after
+ * the last one previously read.
+ * This state ens after setting ReadRecPtr. Then goes to XLREAD_TOT_LEN.
+ *
+ * XLREAD_TOT_LEN:
+ * Examining record header. Ends after reading record total
+ * length. recordRemainLen and recordGotLen are initialized.
+ *
+ * XLREAD_FIRST_FRAGMENT:
+ * Reading the first fragment. Ends with finishing reading a single
+ * record. Goes to XLREAD_NEXT_RECORD if that's all or
+ * XLREAD_CONTINUATION if we have continuation.
+
+ * XLREAD_CONTINUATION:
+ * Reading continuation of record. Ends with finishing the whole record then
+ * goes to XLREAD_NEXT_RECORD. During this state, recordRemainLen indicates
+ * how much is left and readRecordBuf holds the partially assert
+ * record.recordContRecPtr points to the beginning of the next page where to
+ * continue.
+ *
+ * If wrong data found in any state, the state machine stays at the current
+ * state. This behavior allows to continue reading a reacord switching among
+ * different souces, while streaming replication.
*/
XLogRecord *
XLogReadRecord(XLogReaderState *state, char **errormsg)
@@ -280,7 +320,6 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
uint32 pageHeaderSize;
bool assembled;
bool gotheader;
- int readOff;
/*
* randAccess indicates whether to verify the previous-record pointer of
@@ -334,14 +373,20 @@ restart:
* byte to cover the whole record header, or at least the part of it that
* fits on the same page.
*/
- readOff = ReadPageInternal(state, targetPagePtr,
- Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ));
- if (readOff < 0)
+ while (XLogNeedData(state, targetPagePtr,
+ Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+ targetRecOff != 0))
+ {
+ if (!state->routine.page_read(state, state->readPagePtr, state->readLen,
+ RecPtr, state->readBuf))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
/*
- * ReadPageInternal always returns at least the page header, so we can
- * examine it now.
+ * We have at least the page header, so we can examine it now.
*/
pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
if (targetRecOff == 0)
@@ -367,8 +412,8 @@ restart:
goto err;
}
- /* ReadPageInternal has verified the page header */
- Assert(pageHeaderSize <= readOff);
+ /* XLogNeedData has verified the page header */
+ Assert(pageHeaderSize <= state->readLen);
/*
* Read the record length.
@@ -442,18 +487,27 @@ restart:
do
{
+ int rest_len = total_len - gotlen;
+
/* Calculate pointer to beginning of next page */
targetPagePtr += XLOG_BLCKSZ;
/* Wait for the next page to become available */
- readOff = ReadPageInternal(state, targetPagePtr,
- Min(total_len - gotlen + SizeOfXLogShortPHD,
- XLOG_BLCKSZ));
+ while (XLogNeedData(state, targetPagePtr,
+ Min(rest_len, XLOG_BLCKSZ),
+ false))
+ {
+ if (!state->routine.page_read(state, state->readPagePtr,
+ state->readLen,
+ state->ReadRecPtr,
+ state->readBuf))
+ break;
+ }
- if (readOff < 0)
+ if (!state->page_verified)
goto err;
- Assert(SizeOfXLogShortPHD <= readOff);
+ Assert(SizeOfXLogShortPHD <= state->readLen);
pageHeader = (XLogPageHeader) state->readBuf;
@@ -500,21 +554,14 @@ restart:
/* Append the continuation from this page to the buffer */
pageHeaderSize = XLogPageHeaderSize(pageHeader);
- if (readOff < pageHeaderSize)
- readOff = ReadPageInternal(state, targetPagePtr,
- pageHeaderSize);
-
- Assert(pageHeaderSize <= readOff);
+ Assert(pageHeaderSize <= state->readLen);
contdata = (char *) state->readBuf + pageHeaderSize;
len = XLOG_BLCKSZ - pageHeaderSize;
if (pageHeader->xlp_rem_len < len)
len = pageHeader->xlp_rem_len;
- if (readOff < pageHeaderSize + len)
- readOff = ReadPageInternal(state, targetPagePtr,
- pageHeaderSize + len);
-
+ Assert(pageHeaderSize + len <= state->readLen);
memcpy(buffer, (char *) contdata, len);
buffer += len;
gotlen += len;
@@ -544,9 +591,16 @@ restart:
else
{
/* Wait for the record data to become available */
- readOff = ReadPageInternal(state, targetPagePtr,
- Min(targetRecOff + total_len, XLOG_BLCKSZ));
- if (readOff < 0)
+ while (XLogNeedData(state, targetPagePtr,
+ Min(targetRecOff + total_len, XLOG_BLCKSZ), true))
+ {
+ if (!state->routine.page_read(state, state->readPagePtr,
+ state->readLen,
+ state->ReadRecPtr, state->readBuf))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
/* Record does not cross a page boundary */
@@ -603,109 +657,138 @@ err:
}
/*
- * Read a single xlog page including at least [pageptr, reqLen] of valid data
- * via the page_read() callback.
+ * Checks that an xlog page loaded in state->readBuf is including at least
+ * [pageptr, reqLen] and the page is valid. header_inclusive indicates that
+ * reqLen is calculated including page header length.
*
- * Returns -1 if the required page cannot be read for some reason; errormsg_buf
- * is set in that case (unless the error occurs in the page_read callback).
+ * Returns false if the buffer already contains the requested data, or found
+ * error. state->page_verified is set to true for the former and false for the
+ * latter.
*
- * We fetch the page from a reader-local cache if we know we have the required
- * data and if there hasn't been any error since caching the data.
+ * Otherwise returns true and requests data loaded onto state->readBuf by
+ * state->readPagePtr and state->readLen. The caller shall call this function
+ * again after filling the buffer at least with that portion of data and set
+ * state->readLen to the length of actually loaded data.
+ *
+ * If header_inclusive is false, corrects reqLen internally by adding the
+ * actual page header length and may request caller for new data.
*/
-static int
-ReadPageInternal(XLogReaderState *state, XLogRecPtr pageptr, int reqLen)
+static bool
+XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
+ bool header_inclusive)
{
- int readLen;
uint32 targetPageOff;
XLogSegNo targetSegNo;
- XLogPageHeader hdr;
+ uint32 addLen = 0;
- Assert((pageptr % XLOG_BLCKSZ) == 0);
+ /* Some data is loaded, but page header is not verified yet. */
+ if (!state->page_verified &&
+ !XLogRecPtrIsInvalid(state->readPagePtr) && state->readLen >= 0)
+ {
+ uint32 pageHeaderSize;
+ /* just loaded new data so needs to verify page header */
+
+ /* The caller must have loaded at least page header */
+ Assert(state->readLen >= SizeOfXLogShortPHD);
+
+ /*
+ * We have enough data to check the header length. Recheck the loaded
+ * length against the actual header length.
+ */
+ pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+
+ /* Request more data if we don't have the full header. */
+ if (state->readLen < pageHeaderSize)
+ {
+ state->readLen = pageHeaderSize;
+ return true;
+ }
+
+ /* Now that we know we have the full header, validate it. */
+ if (!XLogReaderValidatePageHeader(state, state->readPagePtr,
+ (char *) state->readBuf))
+ {
+ /* That's bad. Force reading the page again. */
+ XLogReaderInvalReadState(state);
+
+ return false;
+ }
+
+ state->page_verified = true;
+
+ XLByteToSeg(state->readPagePtr, state->seg.ws_segno,
+ state->segcxt.ws_segsize);
+ }
+
+ /*
+ * The loaded page may not be the one caller is supposing to read when we
+ * are verifying the first page of new segment. In that case, skip further
+ * verification and immediately load the target page.
+ */
+ if (state->page_verified && pageptr == state->readPagePtr)
+ {
+ /*
+ * calculate additional length for page header keeping the total
+ * length within the block size.
+ */
+ if (!header_inclusive)
+ {
+ uint32 pageHeaderSize =
+ XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+
+ addLen = pageHeaderSize;
+ if (reqLen + pageHeaderSize <= XLOG_BLCKSZ)
+ addLen = pageHeaderSize;
+ else
+ addLen = XLOG_BLCKSZ - reqLen;
+
+ Assert(addLen >= 0);
+ }
+
+ /* Return if we already have it. */
+ if (reqLen + addLen <= state->readLen)
+ return false;
+ }
+
+ /* Data is not in our buffer, request the caller for it. */
XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize);
targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize);
+ Assert((pageptr % XLOG_BLCKSZ) == 0);
- /* check whether we have all the requested data already */
- if (targetSegNo == state->seg.ws_segno &&
- targetPageOff == state->segoff && reqLen <= state->readLen)
- return state->readLen;
+ /*
+ * Every time we request to load new data of a page to the caller, even if
+ * we looked at a part of it before, we need to do verification on the
+ * next invocation as the caller might now be rereading data from a
+ * different source.
+ */
+ state->page_verified = false;
/*
- * Data is not in our buffer.
- *
- * Every time we actually read the segment, even if we looked at parts of
- * it before, we need to do verification as the page_read callback might
- * now be rereading data from a different source.
- *
* Whenever switching to a new WAL segment, we read the first page of the
* file and validate its header, even if that's not where the target
* record is. This is so that we can check the additional identification
- * info that is present in the first page's "long" header.
+ * info that is present in the first page's "long" header. Don't do this
+ * if the caller requested the first page in the segment.
*/
if (targetSegNo != state->seg.ws_segno && targetPageOff != 0)
{
- XLogRecPtr targetSegmentPtr = pageptr - targetPageOff;
-
- readLen = state->routine.page_read(state, targetSegmentPtr, XLOG_BLCKSZ,
- state->currRecPtr,
- state->readBuf);
- if (readLen < 0)
- goto err;
-
- /* we can be sure to have enough WAL available, we scrolled back */
- Assert(readLen == XLOG_BLCKSZ);
-
- if (!XLogReaderValidatePageHeader(state, targetSegmentPtr,
- state->readBuf))
- goto err;
+ /*
+ * Then we'll see that the targetSegNo now matches the ws_segno, and
+ * will not come back here, but will request the actual target page.
+ */
+ state->readPagePtr = pageptr - targetPageOff;
+ state->readLen = XLOG_BLCKSZ;
+ return true;
}
/*
- * First, read the requested data length, but at least a short page header
- * so that we can validate it.
+ * Request the caller to load the page. We need at least a short page
+ * header so that we can validate it.
*/
- readLen = state->routine.page_read(state, pageptr, Max(reqLen, SizeOfXLogShortPHD),
- state->currRecPtr,
- state->readBuf);
- if (readLen < 0)
- goto err;
-
- Assert(readLen <= XLOG_BLCKSZ);
-
- /* Do we have enough data to check the header length? */
- if (readLen <= SizeOfXLogShortPHD)
- goto err;
-
- Assert(readLen >= reqLen);
-
- hdr = (XLogPageHeader) state->readBuf;
-
- /* still not enough */
- if (readLen < XLogPageHeaderSize(hdr))
- {
- readLen = state->routine.page_read(state, pageptr, XLogPageHeaderSize(hdr),
- state->currRecPtr,
- state->readBuf);
- if (readLen < 0)
- goto err;
- }
-
- /*
- * Now that we know we have the full header, validate it.
- */
- if (!XLogReaderValidatePageHeader(state, pageptr, (char *) hdr))
- goto err;
-
- /* update read state information */
- state->seg.ws_segno = targetSegNo;
- state->segoff = targetPageOff;
- state->readLen = readLen;
-
- return readLen;
-
-err:
- XLogReaderInvalReadState(state);
- return -1;
+ state->readPagePtr = pageptr;
+ state->readLen = Max(reqLen + addLen, SizeOfXLogShortPHD);
+ return true;
}
/*
@@ -714,9 +797,7 @@ err:
static void
XLogReaderInvalReadState(XLogReaderState *state)
{
- state->seg.ws_segno = 0;
- state->segoff = 0;
- state->readLen = 0;
+ state->readPagePtr = InvalidXLogRecPtr;
}
/*
@@ -994,7 +1075,6 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
XLogRecPtr targetPagePtr;
int targetRecOff;
uint32 pageHeaderSize;
- int readLen;
/*
* Compute targetRecOff. It should typically be equal or greater than
@@ -1002,27 +1082,32 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
* that, except when caller has explicitly specified the offset that
* falls somewhere there or when we are skipping multi-page
* continuation record. It doesn't matter though because
- * ReadPageInternal() is prepared to handle that and will read at
- * least short page-header worth of data
+ * XLogNeedData() is prepared to handle that and will read at least
+ * short page-header worth of data
*/
targetRecOff = tmpRecPtr % XLOG_BLCKSZ;
/* scroll back to page boundary */
targetPagePtr = tmpRecPtr - targetRecOff;
- /* Read the page containing the record */
- readLen = ReadPageInternal(state, targetPagePtr, targetRecOff);
- if (readLen < 0)
+ while (XLogNeedData(state, targetPagePtr, targetRecOff,
+ targetRecOff != 0))
+ {
+ if (!state->routine.page_read(state, state->readPagePtr,
+ state->readLen,
+ state->ReadRecPtr, state->readBuf))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
header = (XLogPageHeader) state->readBuf;
pageHeaderSize = XLogPageHeaderSize(header);
- /* make sure we have enough data for the page header */
- readLen = ReadPageInternal(state, targetPagePtr, pageHeaderSize);
- if (readLen < 0)
- goto err;
+ /* we should have read the page header */
+ Assert(state->readLen >= pageHeaderSize);
/* skip over potential continuation data */
if (header->xlp_info & XLP_FIRST_IS_CONTRECORD)
diff --git a/src/backend/access/transam/xlogutils.c b/src/backend/access/transam/xlogutils.c
index 88a1bfd939..421040f391 100644
--- a/src/backend/access/transam/xlogutils.c
+++ b/src/backend/access/transam/xlogutils.c
@@ -706,8 +706,8 @@ XLogTruncateRelation(RelFileNode rnode, ForkNumber forkNum,
void
XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wantLength)
{
- const XLogRecPtr lastReadPage = (state->seg.ws_segno *
- state->segcxt.ws_segsize + state->segoff);
+ const XLogRecPtr lastReadPage = state->seg.ws_segno *
+ state->segcxt.ws_segsize + state->readLen;
Assert(wantPage != InvalidXLogRecPtr && wantPage % XLOG_BLCKSZ == 0);
Assert(wantLength <= XLOG_BLCKSZ);
@@ -844,7 +844,7 @@ wal_segment_close(XLogReaderState *state)
* exists for normal backends, so we have to do a check/sleep/repeat style of
* loop for now.
*/
-int
+bool
read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *cur_page)
{
@@ -946,7 +946,8 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
else if (targetPagePtr + reqLen > read_upto)
{
/* not enough data there */
- return -1;
+ state->readLen = -1;
+ return false;
}
else
{
@@ -964,7 +965,8 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
WALReadRaiseError(&errinfo);
/* number of valid bytes in the buffer */
- return count;
+ state->readLen = count;
+ return true;
}
/*
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index b811a5c0ef..ddba340653 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -802,7 +802,7 @@ StartReplication(StartReplicationCmd *cmd)
* which has to do a plain sleep/busy loop, because the walsender's latch gets
* set every time WAL is flushed.
*/
-static int
+static bool
logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *cur_page)
{
@@ -822,7 +822,10 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
/* fail if not (implies we are going to shut down) */
if (flushptr < targetPagePtr + reqLen)
- return -1;
+ {
+ state->readLen = -1;
+ return false;
+ }
if (targetPagePtr + XLOG_BLCKSZ <= flushptr)
count = XLOG_BLCKSZ; /* more than one block available */
@@ -850,7 +853,8 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
XLByteToSeg(targetPagePtr, segno, state->segcxt.ws_segsize);
CheckXLogRemoved(segno, state->seg.ws_tli);
- return count;
+ state->readLen = count;
+ return true;
}
/*
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index 59ebac7d6a..cf119848b0 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -47,7 +47,7 @@ typedef struct XLogPageReadPrivate
int tliIndex;
} XLogPageReadPrivate;
-static int SimpleXLogPageRead(XLogReaderState *xlogreader,
+static bool SimpleXLogPageRead(XLogReaderState *xlogreader,
XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
@@ -246,7 +246,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
}
/* XLogReader callback function, to read a WAL page */
-static int
+static bool
SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf)
{
@@ -306,7 +306,8 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
if (private->restoreCommand == NULL)
{
pg_log_error("could not open file \"%s\": %m", xlogfpath);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
/*
@@ -319,7 +320,10 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
private->restoreCommand);
if (xlogreadfd < 0)
- return -1;
+ {
+ xlogreader->readLen = -1;
+ return false;
+ }
else
pg_log_debug("using file \"%s\" restored from archive",
xlogfpath);
@@ -335,7 +339,8 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
if (lseek(xlogreadfd, (off_t) targetPageOff, SEEK_SET) < 0)
{
pg_log_error("could not seek in file \"%s\": %m", xlogfpath);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
@@ -348,13 +353,15 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
pg_log_error("could not read file \"%s\": read %d of %zu",
xlogfpath, r, (Size) XLOG_BLCKSZ);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
Assert(targetSegNo == xlogreadsegno);
xlogreader->seg.ws_tli = targetHistory[private->tliIndex].tli;
- return XLOG_BLCKSZ;
+ xlogreader->readLen = XLOG_BLCKSZ;
+ return true;
}
/*
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index 1e3894b9c4..833a64210b 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -332,7 +332,7 @@ WALDumpCloseSegment(XLogReaderState *state)
}
/* pg_waldump's XLogReaderRoutine->page_read callback */
-static int
+static bool
WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetPtr, char *readBuff)
{
@@ -349,7 +349,8 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
else
{
private->endptr_reached = true;
- return -1;
+ state->readLen = -1;
+ return false;
}
}
@@ -374,7 +375,8 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
errinfo.wre_req);
}
- return count;
+ state->readLen = count;
+ return true;
}
/*
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index de6fd791fe..f3cf4f2f49 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -57,12 +57,12 @@ typedef struct WALSegmentContext
typedef struct XLogReaderState XLogReaderState;
-/* Function type definitions for various xlogreader interactions */
-typedef int (*XLogPageReadCB) (XLogReaderState *xlogreader,
- XLogRecPtr targetPagePtr,
- int reqLen,
- XLogRecPtr targetRecPtr,
- char *readBuf);
+/* Function type definition for the read_page callback */
+typedef bool (*XLogPageReadCB) (XLogReaderState *xlogreader,
+ XLogRecPtr targetPagePtr,
+ int reqLen,
+ XLogRecPtr targetRecPtr,
+ char *readBuf);
typedef void (*WALSegmentOpenCB) (XLogReaderState *xlogreader,
XLogSegNo nextSegNo,
TimeLineID *tli_p);
@@ -185,6 +185,18 @@ struct XLogReaderState
/* Set when XLP_FIRST_IS_OVERWRITE_CONTRECORD is found */
XLogRecPtr overwrittenRecPtr;
+ /* ----------------------------------------
+ * Communication with page reader
+ * readBuf is XLOG_BLCKSZ bytes, valid up to at least readLen bytes.
+ * ----------------------------------------
+ */
+ /* variables to communicate with page reader */
+ XLogRecPtr readPagePtr; /* page pointer to read */
+ int32 readLen; /* bytes requested to reader, or actual bytes
+ * read by reader, which must be larger than
+ * the request, or -1 on error */
+ char *readBuf; /* buffer to store data */
+ bool page_verified; /* is the page on the buffer verified? */
/* ----------------------------------------
* Decoded representation of current record
@@ -213,13 +225,6 @@ struct XLogReaderState
* ----------------------------------------
*/
- /*
- * Buffer for currently read page (XLOG_BLCKSZ bytes, valid up to at least
- * readLen bytes)
- */
- char *readBuf;
- uint32 readLen;
-
/* last read XLOG position for data currently in readBuf */
WALSegmentContext segcxt;
WALOpenSegment seg;
diff --git a/src/include/access/xlogutils.h b/src/include/access/xlogutils.h
index a5cb3d322c..8669f7eeb3 100644
--- a/src/include/access/xlogutils.h
+++ b/src/include/access/xlogutils.h
@@ -89,7 +89,7 @@ extern Buffer XLogReadBufferExtended(RelFileNode rnode, ForkNumber forknum,
extern Relation CreateFakeRelcacheEntry(RelFileNode rnode);
extern void FreeFakeRelcacheEntry(Relation fakerel);
-extern int read_local_xlog_page(XLogReaderState *state,
+extern bool read_local_xlog_page(XLogReaderState *state,
XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *cur_page);
extern void wal_segment_open(XLogReaderState *state,
--
2.27.0
----Next_Part(Thu_Oct__7_17_28_20_2021_531)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
filename="v18-0002-Move-page-reader-out-of-XLogReadRecord.patch"
^ permalink raw reply [nested|flat] 69+ messages in thread
* Re: Reducing output size of nodeToString
@ 2024-02-12 18:03 Matthias van de Meent <[email protected]>
0 siblings, 1 reply; 69+ messages in thread
From: Matthias van de Meent @ 2024-02-12 18:03 UTC (permalink / raw)
To: Robert Haas <[email protected]>; +Cc: Peter Eisentraut <[email protected]>; PostgreSQL Hackers <[email protected]>; Andres Freund <[email protected]>; Michel Pelletier <[email protected]>
On Wed, 31 Jan 2024 at 18:47, Robert Haas <[email protected]> wrote:
>
> On Wed, Jan 31, 2024 at 11:17 AM Matthias van de Meent
> <[email protected]> wrote:
> > I was also thinking about smaller per-attribute expression storage, for index attribute expressions, table default expressions, and functions. Other than that, less memory overhead for the serialized form of these constructs also helps for catalog cache sizes, etc.
> > People complained about the size of a fresh initdb, and I agreed with them, so I started looking at low-hanging fruits, and this is one.
> >
> > I've not done any tests yet on whether it's more performant in general. I'd expect the new code to do a bit better given the extremely verbose nature of the data and the rather complex byte-at-a-time token read method used, but this is currently hypothesis.
> > I do think that serialization itself may be slightly slower, but given that this generally happens only in DDL, and that we have to grow the output buffer less often, this too may still be a net win (but, again, this is an untested hypothesis).
>
> I think we're going to have to have separate formats for debugging and
> storage if we want to get very far here. The current format sucks for
> readability because it's so verbose, and tightening that up where we
> can makes sense to me. For me, that can include things like emitting
> unset location fields for sure, but delta-encoding of bitmap sets is
> more questionable. Turning 1 2 3 4 5 6 7 8 9 10 into 1-10 would be
> fine with me because that is both shorter and more readable, but
> turning 2 4 6 8 10 into 2 2 2 2 2 is way worse for a human reader.
> Such optimizations might make sense in a format that is designed for
> computer processing only but not one that has to serve multiple
> purposes.
I suppose so, yes. I've removed the delta-encoding from the
serialization of bitmapsets in the attached patchset.
Peter E. and I spoke about this patchset at FOSDEM PGDay, too. I said
to him that I wouldn't mind if this patchset was only partly applied:
The gains for most of the changes are definitely worth it even if some
others don't get in.
I think it'd be a nice QoL and storage improvement if even only (say)
the first two patches were committed, though the typmod default
markings (or alternatively, using a typedef-ed TypMod and one more
type-specific serialization handler) would also be a good improvement
without introducing too many "common value = default = omitted"
considerations that would reduce debugability.
Attached is patchset v2, which contains the improvements from these patches:
0001 has the "omit defaults" for the current types.
-23.5%pt / -35.1%pt (toasted / raw)
0002+0003 has new #defined type "Location" for those fields in Nodes
that point into (or have sizes of) query texts, and adds
infrastructure to conditionally omit them at all (see previous
discussions)
-3.5%pt / -6.3%pt
0004 has new #defined type TypeMod as alias for int32, that uses a
default value of -1 for (de)serialization purposes.
-3.0%pt / -6.1%pt
0005 updates Const node serialization to omit `:constvalue` if the
value is null.
+0.1%pt / -0.1%pt [^0]
0006 does run-length encoding for bitmaps and the various typed
integer lists, using "+int" as indicators of a run of a certain
length, excluding the start value.
Bitmaps, IntLists and XidLists are based on runs with increments
of 1 (so, a notation (i 1 +3) means (i 1 2 3 4), while OidLists are
based on runs with no increments (so, (o 1 +3) means (o 1 1 1 1).
-2.5%pt / -0.6%pt
0007 does add some select custom 'default' values, in that the
varnosyn and varattnosyn fields now treat the value of varno and
varattno as their default values.
This reduces the size of lists of Vars significantly and has a
very meaningful impact on the size of the compressed data (the default
pg_rewrite dataset contains some 10.8k Var nodes).
-10.4%pt / 9.7%pt
Total for the full applied patchset:
55.5% smaller data in pg_rewrite.ev_action before TOAST
45.7% smaller data in pg_rewrite.ev_action after applying TOAST
Toast relation size, as fraction of the main pg_rewrite table:
select pg_relation_size(2838) *1.0 / pg_relation_size('pg_rewrite');
master: 4.7
0007: 1.3
Kind regards,
Matthias van de Meent
Neon (https://neon.tech)
[^0]: The small difference in size for patch 0005 is presumably due to
low occurrance of NULL-valued Const nodes. Additionally, the inline vs
out-of-line TOASTed data and data (not) fitting on the last blocks of
each relation are likely to cause the change in total compression
ratio. If we had more null-valued Const nodes in pg_rewrite, the ratio
would presumably have been better than this.
PS: query I used for my data collection, + combined data:
select 'master' as "version"
, pg_database_size('template0') as "template0"
, pg_total_relation_size('pg_rewrite') as "pg_rewrite"
, sum(pg_column_size(ev_action)) as "toasted"
, sum(octet_length(ev_action)) as "raw";
version | template0 | pg_rewrite | toasted | raw
---------+-----------+------------+---------+---------
master | 7537167 | 770048 | 574003 | 3002556
0001 | 7348751 | 630784 | 438852 | 1946364
0002 | 7242255 | 573440 | 403160 | 1840404
0003 | 7242255 | 573440 | 402325 | 1838367
0004 | 7225871 | 557056 | 384888 | 1652287
0005 | 7234063 | 565248 | 385678 | 1648717
0006 | 7217679 | 548864 | 371256 | 1627733
0007 | 7143951 | 475136 | 311255 | 1337496
Attachments:
[application/octet-stream] v2-0005-nodeToString-omit-serializing-NULL-datums-in-Cons.patch (1.9K, ../../CAEze2WhTQ1LGtpyobcLFvda61BDtkiYvCfyuUA9-Mi9iwd-gyg@mail.gmail.com/2-v2-0005-nodeToString-omit-serializing-NULL-datums-in-Cons.patch)
download | inline diff:
From 234342d8c729911b17c7f69f38fac63f71b10ee0 Mon Sep 17 00:00:00 2001
From: Matthias van de Meent <[email protected]>
Date: Thu, 8 Feb 2024 20:24:22 +0100
Subject: [PATCH v2 5/7] nodeToString: omit serializing NULL datums in Const
nodes
This saves some bytes in certain cases, and aligns its serialization conditions
with other field's serialization conditions.
---
src/backend/nodes/outfuncs.c | 8 ++++----
src/backend/nodes/readfuncs.c | 14 ++++++++++----
2 files changed, 14 insertions(+), 8 deletions(-)
diff --git a/src/backend/nodes/outfuncs.c b/src/backend/nodes/outfuncs.c
index 3973c0e489..1a17eafd57 100644
--- a/src/backend/nodes/outfuncs.c
+++ b/src/backend/nodes/outfuncs.c
@@ -499,11 +499,11 @@ _outConst(StringInfo str, const Const *node, bool omitLocation)
WRITE_BOOL_FIELD(constisnull);
WRITE_LOCATION_FIELD(location);
- appendStringInfoString(str, " :constvalue ");
- if (node->constisnull)
- appendStringInfoString(str, "<>");
- else
+ if (!node->constisnull)
+ {
+ appendStringInfoString(str, " :constvalue ");
outDatum(str, node->constvalue, node->constlen, node->constbyval);
+ }
}
static void
diff --git a/src/backend/nodes/readfuncs.c b/src/backend/nodes/readfuncs.c
index 44ab140799..4cdbad9e7e 100644
--- a/src/backend/nodes/readfuncs.c
+++ b/src/backend/nodes/readfuncs.c
@@ -363,11 +363,17 @@ _readConst(void)
READ_BOOL_FIELD(constisnull);
READ_LOCATION_FIELD(location);
- token = pg_strtok(&length); /* skip :constvalue */
- if (local_node->constisnull)
- token = pg_strtok(&length); /* skip "<>" */
- else
+ if (pg_strtoken_next(":constvalue"))
+ {
+ token = pg_strtok(&length); /* skip :constvalue */
+ Assert(strncmp(token, ":constvalue", sizeof(":constvalue") - 1) == 0);
local_node->constvalue = readDatum(local_node->constbyval);
+ }
+ else
+ {
+ /* value was omitted */
+ Assert(local_node->constisnull);
+ }
READ_DONE();
}
--
2.40.1
[application/octet-stream] v2-0001-pg_node_tree-Omit-serialization-of-fields-with-de.patch (23.0K, ../../CAEze2WhTQ1LGtpyobcLFvda61BDtkiYvCfyuUA9-Mi9iwd-gyg@mail.gmail.com/3-v2-0001-pg_node_tree-Omit-serialization-of-fields-with-de.patch)
download | inline diff:
From 805c534ab2f632c7db68321f858f4bb4b689d63a Mon Sep 17 00:00:00 2001
From: Matthias van de Meent <[email protected]>
Date: Tue, 2 Jan 2024 23:55:02 +0100
Subject: [PATCH v2 1/7] pg_node_tree: Omit serialization of fields with
default values.
Often, the values in nodes are their default values. By not serializing
those fields and inserting the defaults during deserialization, we
reduce the size of pg_node_tree attributes seen in e.g. pg_rewrite by a
significant factor.
In passing, we fix a test that had a strict dependency on the
serialization of pg_node_tree; we now do the checks in a more generic
manner, making it more stable and ensuring its stability in future work.
---
src/backend/nodes/outfuncs.c | 175 ++++++++++++++++++----
src/backend/nodes/read.c | 45 ++++++
src/backend/nodes/readfuncs.c | 131 +++++++++++++---
src/include/nodes/readfuncs.h | 1 +
src/test/regress/expected/rowsecurity.out | 5 +-
src/test/regress/sql/rowsecurity.sql | 5 +-
6 files changed, 309 insertions(+), 53 deletions(-)
diff --git a/src/backend/nodes/outfuncs.c b/src/backend/nodes/outfuncs.c
index 03f67b6850..50da80ee34 100644
--- a/src/backend/nodes/outfuncs.c
+++ b/src/backend/nodes/outfuncs.c
@@ -41,94 +41,203 @@ static void outDouble(StringInfo str, double d);
appendStringInfoString(str, nodelabel)
/* Write an integer field (anything written as ":fldname %d") */
+#define WRITE_INT_FIELD_DEFAULT(fldname, default) \
+ do { \
+ if (node->fldname != default) \
+ appendStringInfo(str, " :" CppAsString(fldname) " %d", \
+ node->fldname); \
+ } while (0)
#define WRITE_INT_FIELD(fldname) \
- appendStringInfo(str, " :" CppAsString(fldname) " %d", node->fldname)
+ WRITE_INT_FIELD_DEFAULT(fldname, 0)
/* Write an unsigned integer field (anything written as ":fldname %u") */
+#define WRITE_UINT_FIELD_DEFAULT(fldname, default) \
+ do { \
+ if (node->fldname != default) \
+ appendStringInfo(str, " :" CppAsString(fldname) " %u", \
+ node->fldname); \
+ } while (0)
#define WRITE_UINT_FIELD(fldname) \
- appendStringInfo(str, " :" CppAsString(fldname) " %u", node->fldname)
+ WRITE_UINT_FIELD_DEFAULT(fldname, 0)
/* Write an unsigned integer field (anything written with UINT64_FORMAT) */
+#define WRITE_UINT64_FIELD_DEFAULT(fldname, default) \
+ do { \
+ if (node->fldname != default) \
+ appendStringInfo(str, " :" CppAsString(fldname) " " UINT64_FORMAT, \
+ node->fldname); \
+ } while (0)
#define WRITE_UINT64_FIELD(fldname) \
- appendStringInfo(str, " :" CppAsString(fldname) " " UINT64_FORMAT, \
- node->fldname)
+ WRITE_UINT64_FIELD_DEFAULT(fldname, 0)
/* Write an OID field (don't hard-wire assumption that OID is same as uint) */
+#define WRITE_OID_FIELD_DEFAULT(fldname, default) \
+ do { \
+ if (node->fldname != default) \
+ appendStringInfo(str, " :" CppAsString(fldname) " %u", \
+ node->fldname); \
+ } while (0)
#define WRITE_OID_FIELD(fldname) \
- appendStringInfo(str, " :" CppAsString(fldname) " %u", node->fldname)
+ WRITE_OID_FIELD_DEFAULT(fldname, 0)
/* Write a long-integer field */
+#define WRITE_LONG_FIELD_DEFAULT(fldname, default) \
+ do { \
+ if (node->fldname != default) \
+ appendStringInfo(str, " :" CppAsString(fldname) " %ld", \
+ node->fldname); \
+ } while (0)
#define WRITE_LONG_FIELD(fldname) \
- appendStringInfo(str, " :" CppAsString(fldname) " %ld", node->fldname)
+ WRITE_LONG_FIELD_DEFAULT(fldname, 0)
+
/* Write a char field (ie, one ascii character) */
+#define WRITE_CHAR_FIELD_DEFAULT(fldname, default) \
+ do { \
+ if (node->fldname != default) \
+ { \
+ appendStringInfoString(str, " :" CppAsString(fldname) " "); \
+ outChar(str, node->fldname); \
+ } \
+ } while (0)
#define WRITE_CHAR_FIELD(fldname) \
- (appendStringInfo(str, " :" CppAsString(fldname) " "), \
- outChar(str, node->fldname))
+ WRITE_CHAR_FIELD_DEFAULT(fldname, '\0')
/* Write an enumerated-type field as an integer code */
+#define WRITE_ENUM_FIELD_DEFAULT(fldname, enumtype, default) \
+ do { \
+ if (node->fldname != default) \
+ appendStringInfo(str, " :" CppAsString(fldname) " %d", \
+ (int) node->fldname); \
+ } while (0)
#define WRITE_ENUM_FIELD(fldname, enumtype) \
- appendStringInfo(str, " :" CppAsString(fldname) " %d", \
- (int) node->fldname)
+ WRITE_ENUM_FIELD_DEFAULT(fldname, enumtype, 0)
/* Write a float field (actually, they're double) */
+#define WRITE_FLOAT_FIELD_DEFAULT(fldname, default) \
+ do { \
+ if (node->fldname != default) \
+ { \
+ appendStringInfoString(str, " :" CppAsString(fldname) " "); \
+ outDouble(str, node->fldname); \
+ } \
+ } while (0)
#define WRITE_FLOAT_FIELD(fldname) \
- (appendStringInfo(str, " :" CppAsString(fldname) " "), \
- outDouble(str, node->fldname))
+ WRITE_FLOAT_FIELD_DEFAULT(fldname, 0.0)
/* Write a boolean field */
+#define WRITE_BOOL_FIELD_DEFAULT(fldname, default) \
+ do { \
+ if (node->fldname != default) \
+ appendStringInfo(str, " :" CppAsString(fldname) " %s", \
+ booltostr(node->fldname)); \
+ } while (0)
#define WRITE_BOOL_FIELD(fldname) \
- appendStringInfo(str, " :" CppAsString(fldname) " %s", \
- booltostr(node->fldname))
+ WRITE_BOOL_FIELD_DEFAULT(fldname, false)
+
+/*
+ * Non-null defaults of by-ref types are exceedingly rare (if not generally
+ * nonexistent), so we don't (yet) have a specialized macro for non-NULL
+ * defaults omission.
+ */
/* Write a character-string (possibly NULL) field */
#define WRITE_STRING_FIELD(fldname) \
- (appendStringInfoString(str, " :" CppAsString(fldname) " "), \
- outToken(str, node->fldname))
+ do { \
+ if (node->fldname != NULL) \
+ { \
+ appendStringInfoString(str, " :" CppAsString(fldname) " "); \
+ outToken(str, node->fldname); \
+ } \
+ } while (0)
/* Write a parse location field (actually same as INT case) */
#define WRITE_LOCATION_FIELD(fldname) \
- appendStringInfo(str, " :" CppAsString(fldname) " %d", node->fldname)
+ do { \
+ if (node->fldname != -1) \
+ appendStringInfo(str, " :" CppAsString(fldname) " %d", \
+ node->fldname); \
+ } while (0)
/* Write a Node field */
#define WRITE_NODE_FIELD(fldname) \
- (appendStringInfoString(str, " :" CppAsString(fldname) " "), \
- outNode(str, node->fldname))
+ do { \
+ if (node->fldname != NULL) \
+ { \
+ appendStringInfoString(str, " :" CppAsString(fldname) " "); \
+ outNode(str, node->fldname); \
+ } \
+ } while (0)
/* Write a bitmapset field */
#define WRITE_BITMAPSET_FIELD(fldname) \
- (appendStringInfoString(str, " :" CppAsString(fldname) " "), \
- outBitmapset(str, node->fldname))
+ do { \
+ if (node->fldname != NULL) \
+ { \
+ appendStringInfoString(str, " :" CppAsString(fldname) " "); \
+ outBitmapset(str, node->fldname); \
+ } \
+ } while (0)
/* Write a variable-length array (not a List) of Node pointers */
#define WRITE_NODE_ARRAY(fldname, len) \
- (appendStringInfoString(str, " :" CppAsString(fldname) " "), \
- writeNodeArray(str, (const Node * const *) node->fldname, len))
+ do { \
+ if (node->fldname != NULL) \
+ { \
+ appendStringInfoString(str, " :" CppAsString(fldname) " "); \
+ writeNodeArray(str, (const Node * const *) node->fldname, len); \
+ } \
+ } while (0)
/* Write a variable-length array of AttrNumber */
#define WRITE_ATTRNUMBER_ARRAY(fldname, len) \
- (appendStringInfoString(str, " :" CppAsString(fldname) " "), \
- writeAttrNumberCols(str, node->fldname, len))
+ do { \
+ if (node->fldname != NULL) \
+ { \
+ appendStringInfoString(str, " :" CppAsString(fldname) " "); \
+ writeAttrNumberCols(str, node->fldname, len); \
+ } \
+ } while (0)
/* Write a variable-length array of Oid */
#define WRITE_OID_ARRAY(fldname, len) \
- (appendStringInfoString(str, " :" CppAsString(fldname) " "), \
- writeOidCols(str, node->fldname, len))
+ do { \
+ if (node->fldname != NULL) \
+ { \
+ appendStringInfoString(str, " :" CppAsString(fldname) " "); \
+ writeOidCols(str, node->fldname, len); \
+ } \
+ } while (0)
/* Write a variable-length array of Index */
#define WRITE_INDEX_ARRAY(fldname, len) \
- (appendStringInfoString(str, " :" CppAsString(fldname) " "), \
- writeIndexCols(str, node->fldname, len))
+ do { \
+ if (node->fldname != NULL) \
+ { \
+ appendStringInfoString(str, " :" CppAsString(fldname) " "); \
+ writeIndexCols(str, node->fldname, len); \
+ } \
+ } while (0)
/* Write a variable-length array of int */
#define WRITE_INT_ARRAY(fldname, len) \
- (appendStringInfoString(str, " :" CppAsString(fldname) " "), \
- writeIntCols(str, node->fldname, len))
+ do { \
+ if (node->fldname != NULL) \
+ { \
+ appendStringInfoString(str, " :" CppAsString(fldname) " "); \
+ writeIntCols(str, node->fldname, len); \
+ } \
+ } while (0)
/* Write a variable-length array of bool */
#define WRITE_BOOL_ARRAY(fldname, len) \
- (appendStringInfoString(str, " :" CppAsString(fldname) " "), \
- writeBoolCols(str, node->fldname, len))
+ do { \
+ if (node->fldname != NULL) \
+ { \
+ appendStringInfoString(str, " :" CppAsString(fldname) " "); \
+ writeBoolCols(str, node->fldname, len); \
+ } \
+ } while (0)
#define booltostr(x) ((x) ? "true" : "false")
diff --git a/src/backend/nodes/read.c b/src/backend/nodes/read.c
index 969d0ec199..14360307a3 100644
--- a/src/backend/nodes/read.c
+++ b/src/backend/nodes/read.c
@@ -205,6 +205,51 @@ pg_strtok(int *length)
return ret_str;
}
+/*
+ * Check if the next token is 'expect_token'.
+ *
+ * It handles similar to pg_strtok, except that this does not consume the
+ * next token, and has special casing for some less common tokens.
+ */
+bool
+pg_strtoken_next(const char *expect_token)
+{
+ const char *local_str; /* working pointer to string */
+ Size expect_len = strlen(expect_token);
+ char next_char;
+
+ local_str = pg_strtok_ptr;
+
+ while (*local_str == ' ' || *local_str == '\n' || *local_str == '\t')
+ local_str++;
+
+ if (*local_str == '\0')
+ return false; /* no more tokens */
+
+ Assert(expect_len > 0);
+
+ next_char = local_str[expect_len];
+
+ /* check if the next few bytes match the token */
+ if (strncmp(local_str, expect_token, expect_len) != 0)
+ return false;
+
+ /*
+ * Check that the token was actually terminated at the end of the
+ * expected token with a character that is a separate token.
+ * Otherwise, we'd get positive matches for mathing the token of "is"
+ * against a local_str of "isn't", which is clearly wrong.
+ */
+ return (next_char == '\0' ||
+ next_char == ' ' ||
+ next_char == '\n' ||
+ next_char == '\t' ||
+ next_char == '(' ||
+ next_char == ')' ||
+ next_char == '{' ||
+ next_char == '}');
+}
+
/*
* debackslash -
* create a palloc'd string holding the given token.
diff --git a/src/backend/nodes/readfuncs.c b/src/backend/nodes/readfuncs.c
index cfb552fde7..47f4ba2695 100644
--- a/src/backend/nodes/readfuncs.c
+++ b/src/backend/nodes/readfuncs.c
@@ -56,112 +56,207 @@
READ_LOCALS_NO_FIELDS(nodeTypeName); \
READ_TEMP_LOCALS()
+/* a scaffold function to read an optionally-omitted field */
+#define READ_OPT_SCAFFOLD(fldname, read_field_code, default_value) \
+ do { \
+ if (pg_strtoken_next(":" CppAsString(fldname))) \
+ { \
+ read_field_code; \
+ } \
+ else \
+ local_node->fldname = default_value; \
+ } while (0)
+
/* Read an integer field (anything written as ":fldname %d") */
-#define READ_INT_FIELD(fldname) \
+#define READ_INT_FIELD_DIRECT(fldname) \
token = pg_strtok(&length); /* skip :fldname */ \
+ Assert(strncmp(token, ":" CppAsString(fldname), length) == 0); \
token = pg_strtok(&length); /* get field value */ \
local_node->fldname = atoi(token)
+#define READ_INT_FIELD_DEFAULT(fldname, default_value) \
+ READ_OPT_SCAFFOLD(fldname, READ_INT_FIELD_DIRECT(fldname), default_value)
+#define READ_INT_FIELD(fldname) \
+ READ_INT_FIELD_DEFAULT(fldname, 0)
/* Read an unsigned integer field (anything written as ":fldname %u") */
-#define READ_UINT_FIELD(fldname) \
+#define READ_UINT_FIELD_DIRECT(fldname) \
token = pg_strtok(&length); /* skip :fldname */ \
+ Assert(strncmp(token, ":" CppAsString(fldname), length) == 0); \
token = pg_strtok(&length); /* get field value */ \
local_node->fldname = atoui(token)
+#define READ_UINT_FIELD_DEFAULT(fldname, default_value) \
+ READ_OPT_SCAFFOLD(fldname, READ_UINT_FIELD_DIRECT(fldname), default_value)
+#define READ_UINT_FIELD(fldname) \
+ READ_UINT_FIELD_DEFAULT(fldname, 0)
/* Read an unsigned integer field (anything written using UINT64_FORMAT) */
-#define READ_UINT64_FIELD(fldname) \
+#define READ_UINT64_FIELD_DIRECT(fldname) \
token = pg_strtok(&length); /* skip :fldname */ \
+ Assert(strncmp(token, ":" CppAsString(fldname), length) == 0); \
token = pg_strtok(&length); /* get field value */ \
local_node->fldname = strtou64(token, NULL, 10)
+#define READ_UINT64_FIELD_DEFAULT(fldname, default_value) \
+ READ_OPT_SCAFFOLD(fldname, READ_UINT64_FIELD_DIRECT(fldname), default_value)
+#define READ_UINT64_FIELD(fldname) \
+ READ_UINT64_FIELD_DEFAULT(fldname, 0)
/* Read a long integer field (anything written as ":fldname %ld") */
-#define READ_LONG_FIELD(fldname) \
+#define READ_LONG_FIELD_DIRECT(fldname) \
token = pg_strtok(&length); /* skip :fldname */ \
+ Assert(strncmp(token, ":" CppAsString(fldname), length) == 0); \
token = pg_strtok(&length); /* get field value */ \
local_node->fldname = atol(token)
+#define READ_LONG_FIELD_DEFAULT(fldname, default_value) \
+ READ_OPT_SCAFFOLD(fldname, READ_LONG_FIELD_DIRECT(fldname), default_value)
+#define READ_LONG_FIELD(fldname) \
+ READ_LONG_FIELD_DEFAULT(fldname, 0)
/* Read an OID field (don't hard-wire assumption that OID is same as uint) */
-#define READ_OID_FIELD(fldname) \
+#define READ_OID_FIELD_DIRECT(fldname) \
token = pg_strtok(&length); /* skip :fldname */ \
+ Assert(strncmp(token, ":" CppAsString(fldname), length) == 0); \
token = pg_strtok(&length); /* get field value */ \
local_node->fldname = atooid(token)
+#define READ_OID_FIELD_DEFAULT(fldname, default_value) \
+ READ_OPT_SCAFFOLD(fldname, READ_OID_FIELD_DIRECT(fldname), default_value)
+#define READ_OID_FIELD(fldname) \
+ READ_OID_FIELD_DEFAULT(fldname, 0)
/* Read a char field (ie, one ascii character) */
-#define READ_CHAR_FIELD(fldname) \
+#define READ_CHAR_FIELD_DIRECT(fldname) \
token = pg_strtok(&length); /* skip :fldname */ \
+ Assert(strncmp(token, ":" CppAsString(fldname), length) == 0); \
token = pg_strtok(&length); /* get field value */ \
/* avoid overhead of calling debackslash() for one char */ \
local_node->fldname = (length == 0) ? '\0' : (token[0] == '\\' ? token[1] : token[0])
+#define READ_CHAR_FIELD_DEFAULT(fldname, default_value) \
+ READ_OPT_SCAFFOLD(fldname, READ_CHAR_FIELD_DIRECT(fldname), default_value)
+#define READ_CHAR_FIELD(fldname) \
+ READ_CHAR_FIELD_DEFAULT(fldname, '\0')
/* Read an enumerated-type field that was written as an integer code */
-#define READ_ENUM_FIELD(fldname, enumtype) \
+#define READ_ENUM_FIELD_DIRECT(fldname, enumtype) \
token = pg_strtok(&length); /* skip :fldname */ \
+ Assert(strncmp(token, ":" CppAsString(fldname), length) == 0); \
token = pg_strtok(&length); /* get field value */ \
local_node->fldname = (enumtype) atoi(token)
+#define READ_ENUM_FIELD_DEFAULT(fldname, enumtype, default_value) \
+ READ_OPT_SCAFFOLD(fldname, READ_ENUM_FIELD_DIRECT(fldname, enumtype), default_value)
+#define READ_ENUM_FIELD(fldname, enumtype) \
+ READ_ENUM_FIELD_DEFAULT(fldname, enumtype, 0)
/* Read a float field */
-#define READ_FLOAT_FIELD(fldname) \
+#define READ_FLOAT_FIELD_DIRECT(fldname) \
token = pg_strtok(&length); /* skip :fldname */ \
+ Assert(strncmp(token, ":" CppAsString(fldname), length) == 0); \
token = pg_strtok(&length); /* get field value */ \
local_node->fldname = atof(token)
+#define READ_FLOAT_FIELD_DEFAULT(fldname, default_value) \
+ READ_OPT_SCAFFOLD(fldname, READ_FLOAT_FIELD_DIRECT(fldname), default_value)
+#define READ_FLOAT_FIELD(fldname) \
+ READ_FLOAT_FIELD_DEFAULT(fldname, 0.0)
/* Read a boolean field */
-#define READ_BOOL_FIELD(fldname) \
+#define READ_BOOL_FIELD_DIRECT(fldname) \
token = pg_strtok(&length); /* skip :fldname */ \
+ Assert(strncmp(token, ":" CppAsString(fldname), length) == 0); \
token = pg_strtok(&length); /* get field value */ \
local_node->fldname = strtobool(token)
+#define READ_BOOL_FIELD_DEFAULT(fldname, default_value) \
+ READ_OPT_SCAFFOLD(fldname, READ_BOOL_FIELD_DIRECT(fldname), default_value)
+#define READ_BOOL_FIELD(fldname) \
+ READ_BOOL_FIELD_DEFAULT(fldname, false)
/* Read a character-string field */
-#define READ_STRING_FIELD(fldname) \
+#define READ_STRING_FIELD_DIRECT(fldname) \
token = pg_strtok(&length); /* skip :fldname */ \
+ Assert(strncmp(token, ":" CppAsString(fldname), length) == 0); \
token = pg_strtok(&length); /* get field value */ \
local_node->fldname = nullable_string(token, length)
+/* see WRITE_STRING_FIELD in outfuncs.c */
+#define READ_STRING_FIELD(fldname) \
+ READ_OPT_SCAFFOLD(fldname, READ_STRING_FIELD_DIRECT(fldname), NULL)
/* Read a parse location field (and possibly throw away the value) */
#ifdef WRITE_READ_PARSE_PLAN_TREES
-#define READ_LOCATION_FIELD(fldname) \
+#define READ_LOCATION_FIELD_DIRECT(fldname) \
token = pg_strtok(&length); /* skip :fldname */ \
+ Assert(strncmp(token, ":" CppAsString(fldname), length) == 0); \
token = pg_strtok(&length); /* get field value */ \
local_node->fldname = restore_location_fields ? atoi(token) : -1
#else
-#define READ_LOCATION_FIELD(fldname) \
+#define READ_LOCATION_FIELD_DIRECT(fldname) \
token = pg_strtok(&length); /* skip :fldname */ \
+ Assert(strncmp(token, ":" CppAsString(fldname), length) == 0); \
token = pg_strtok(&length); /* get field value */ \
(void) token; /* in case not used elsewhere */ \
local_node->fldname = -1 /* set field to "unknown" */
#endif
+/* The default Location field value is -1 ('unknown') */
+#define READ_LOCATION_FIELD(fldname) \
+ READ_OPT_SCAFFOLD(fldname, READ_LOCATION_FIELD_DIRECT(fldname), -1)
/* Read a Node field */
-#define READ_NODE_FIELD(fldname) \
+#define READ_NODE_FIELD_DIRECT(fldname) \
token = pg_strtok(&length); /* skip :fldname */ \
+ Assert(strncmp(token, ":" CppAsString(fldname), length) == 0); \
(void) token; /* in case not used elsewhere */ \
local_node->fldname = nodeRead(NULL, 0)
+#define READ_NODE_FIELD_DEFAULT(fldname, default_value) \
+ READ_OPT_SCAFFOLD(fldname, READ_NODE_FIELD_DIRECT(fldname), default_value)
+#define READ_NODE_FIELD(fldname) \
+ READ_NODE_FIELD_DEFAULT(fldname, NULL)
/* Read a bitmapset field */
-#define READ_BITMAPSET_FIELD(fldname) \
+#define READ_BITMAPSET_FIELD_DIRECT(fldname) \
token = pg_strtok(&length); /* skip :fldname */ \
+ Assert(strncmp(token, ":" CppAsString(fldname), length) == 0); \
(void) token; /* in case not used elsewhere */ \
local_node->fldname = _readBitmapset()
+#define READ_BITMAPSET_FIELD_DEFAULT(fldname, default_value) \
+ READ_OPT_SCAFFOLD(fldname, READ_BITMAPSET_FIELD_DIRECT(fldname), default_value)
+#define READ_BITMAPSET_FIELD(fldname) \
+ READ_BITMAPSET_FIELD_DEFAULT(fldname, NULL)
/* Read an attribute number array */
-#define READ_ATTRNUMBER_ARRAY(fldname, len) \
+#define READ_ATTRNUMBER_ARRAY_DIRECT(fldname, len) \
token = pg_strtok(&length); /* skip :fldname */ \
+ Assert(strncmp(token, ":" CppAsString(fldname), length) == 0); \
local_node->fldname = readAttrNumberCols(len)
+#define READ_ATTRNUMBER_ARRAY_DEFAULT(fldname, len, default_value) \
+ READ_OPT_SCAFFOLD(fldname, READ_ATTRNUMBER_ARRAY_DIRECT(fldname, len), default_value)
+#define READ_ATTRNUMBER_ARRAY(fldname, len) \
+ READ_ATTRNUMBER_ARRAY_DEFAULT(fldname, len, NULL)
/* Read an oid array */
-#define READ_OID_ARRAY(fldname, len) \
+#define READ_OID_ARRAY_DIRECT(fldname, len) \
token = pg_strtok(&length); /* skip :fldname */ \
+ Assert(strncmp(token, ":" CppAsString(fldname), length) == 0); \
local_node->fldname = readOidCols(len)
+#define READ_OID_ARRAY_DEFAULT(fldname, len, default_value) \
+ READ_OPT_SCAFFOLD(fldname, READ_OID_ARRAY_DIRECT(fldname, len), default_value)
+#define READ_OID_ARRAY(fldname, len) \
+ READ_OID_ARRAY_DEFAULT(fldname, len, NULL)
/* Read an int array */
-#define READ_INT_ARRAY(fldname, len) \
+#define READ_INT_ARRAY_DIRECT(fldname, len) \
token = pg_strtok(&length); /* skip :fldname */ \
+ Assert(strncmp(token, ":" CppAsString(fldname), length) == 0); \
local_node->fldname = readIntCols(len)
+#define READ_INT_ARRAY_DEFAULT(fldname, len, default_value) \
+ READ_OPT_SCAFFOLD(fldname, READ_INT_ARRAY_DIRECT(fldname, len), default_value)
+#define READ_INT_ARRAY(fldname, len) \
+ READ_INT_ARRAY_DEFAULT(fldname, len, NULL)
/* Read a bool array */
-#define READ_BOOL_ARRAY(fldname, len) \
+#define READ_BOOL_ARRAY_DIRECT(fldname, len) \
token = pg_strtok(&length); /* skip :fldname */ \
+ Assert(strncmp(token, ":" CppAsString(fldname), length) == 0); \
local_node->fldname = readBoolCols(len)
+#define READ_BOOL_ARRAY_DEFAULT(fldname, len, default_value) \
+ READ_OPT_SCAFFOLD(fldname, READ_BOOL_ARRAY_DIRECT(fldname, len), default_value)
+#define READ_BOOL_ARRAY(fldname, len) \
+ READ_BOOL_ARRAY_DEFAULT(fldname, len, NULL)
/* Routine exit */
#define READ_DONE() \
diff --git a/src/include/nodes/readfuncs.h b/src/include/nodes/readfuncs.h
index 8466038ed0..dab4547db2 100644
--- a/src/include/nodes/readfuncs.h
+++ b/src/include/nodes/readfuncs.h
@@ -27,6 +27,7 @@ extern PGDLLIMPORT bool restore_location_fields;
* prototypes for functions in read.c (the lisp token parser)
*/
extern const char *pg_strtok(int *length);
+extern bool pg_strtoken_next(const char *expect_token);
extern char *debackslash(const char *token, int length);
extern void *nodeRead(const char *token, int tok_len);
diff --git a/src/test/regress/expected/rowsecurity.out b/src/test/regress/expected/rowsecurity.out
index 6988128aa4..a69aa40f82 100644
--- a/src/test/regress/expected/rowsecurity.out
+++ b/src/test/regress/expected/rowsecurity.out
@@ -3937,7 +3937,10 @@ CREATE TABLE coll_t (c) AS VALUES ('bar'::text);
CREATE POLICY coll_p ON coll_t USING (c < ('foo'::text COLLATE "C"));
ALTER TABLE coll_t ENABLE ROW LEVEL SECURITY;
GRANT SELECT ON coll_t TO regress_rls_alice;
-SELECT (string_to_array(polqual, ':'))[7] AS inputcollid FROM pg_policy WHERE polrelid = 'coll_t'::regclass;
+SELECT split AS inputcollid
+FROM pg_policy,
+ lateral unnest(string_to_array(polqual, ':')) as split
+WHERE polrelid = 'coll_t'::regclass and split LIKE '%inputcollid%';
inputcollid
------------------
inputcollid 950
diff --git a/src/test/regress/sql/rowsecurity.sql b/src/test/regress/sql/rowsecurity.sql
index dec7340538..cb7a4c776b 100644
--- a/src/test/regress/sql/rowsecurity.sql
+++ b/src/test/regress/sql/rowsecurity.sql
@@ -1732,7 +1732,10 @@ CREATE TABLE coll_t (c) AS VALUES ('bar'::text);
CREATE POLICY coll_p ON coll_t USING (c < ('foo'::text COLLATE "C"));
ALTER TABLE coll_t ENABLE ROW LEVEL SECURITY;
GRANT SELECT ON coll_t TO regress_rls_alice;
-SELECT (string_to_array(polqual, ':'))[7] AS inputcollid FROM pg_policy WHERE polrelid = 'coll_t'::regclass;
+SELECT split AS inputcollid
+FROM pg_policy,
+ lateral unnest(string_to_array(polqual, ':')) as split
+WHERE polrelid = 'coll_t'::regclass and split LIKE '%inputcollid%';
SET SESSION AUTHORIZATION regress_rls_alice;
SELECT * FROM coll_t;
ROLLBACK;
--
2.40.1
[application/octet-stream] v2-0003-gen_node_support.pl-Mark-location-fields-as-type-.patch (26.4K, ../../CAEze2WhTQ1LGtpyobcLFvda61BDtkiYvCfyuUA9-Mi9iwd-gyg@mail.gmail.com/4-v2-0003-gen_node_support.pl-Mark-location-fields-as-type-.patch)
download | inline diff:
From a7ccc6187131559aa4e5e5e9b02c938b21e64045 Mon Sep 17 00:00:00 2001
From: Matthias van de Meent <[email protected]>
Date: Thu, 8 Feb 2024 17:15:22 +0100
Subject: [PATCH v2 3/7] gen_node_support.pl: Mark location fields as type
alias Location
Instead of the rather ugly type=int + name ~= location$, we now have a
special type for offset pointers or sizes that are only relevant when a
query text is included, which decreases the complexity required in
gen_node_support.pl for handling these values.
---
src/backend/nodes/gen_node_support.pl | 6 +-
src/include/nodes/parsenodes.h | 92 +++++++++++++--------------
src/include/nodes/primnodes.h | 72 +++++++++++----------
3 files changed, 86 insertions(+), 84 deletions(-)
diff --git a/src/backend/nodes/gen_node_support.pl b/src/backend/nodes/gen_node_support.pl
index 487f6f7728..ec850c3484 100644
--- a/src/backend/nodes/gen_node_support.pl
+++ b/src/backend/nodes/gen_node_support.pl
@@ -777,7 +777,7 @@ _equal${n}(const $n *a, const $n *b)
print $eff "\tCOMPARE_BITMAPSET_FIELD($f);\n"
unless $equal_ignore;
}
- elsif ($t eq 'int' && $f =~ 'location$')
+ elsif ($t eq 'Location')
{
print $cff "\tCOPY_LOCATION_FIELD($f);\n" unless $copy_ignore;
print $eff "\tCOMPARE_LOCATION_FIELD($f);\n" unless $equal_ignore;
@@ -1010,7 +1010,7 @@ _read${n}(void)
print $off "\tWRITE_BOOL_FIELD($f);\n";
print $rff "\tREAD_BOOL_FIELD($f);\n" unless $no_read;
}
- elsif ($t eq 'int' && $f =~ 'location$')
+ elsif ($t eq 'Location')
{
print $off "\tWRITE_LOCATION_FIELD($f);\n";
print $rff "\tREAD_LOCATION_FIELD($f);\n" unless $no_read;
@@ -1303,7 +1303,7 @@ _jumble${n}(JumbleState *jstate, Node *node)
print $jff "\tJUMBLE_NODE($f);\n"
unless $query_jumble_ignore;
}
- elsif ($t eq 'int' && $f =~ 'location$')
+ elsif ($t eq 'Location')
{
# Track the node's location only if directly requested.
if ($query_jumble_location)
diff --git a/src/include/nodes/parsenodes.h b/src/include/nodes/parsenodes.h
index 476d55dd24..dd8aa30aaf 100644
--- a/src/include/nodes/parsenodes.h
+++ b/src/include/nodes/parsenodes.h
@@ -225,9 +225,9 @@ typedef struct Query
* both be -1 meaning "unknown".
*/
/* start location, or -1 if unknown */
- int stmt_location;
+ Location stmt_location;
/* length in bytes; 0 means "rest of string" */
- int stmt_len pg_node_attr(query_jumble_ignore);
+ Location stmt_len pg_node_attr(query_jumble_ignore);
} Query;
@@ -262,7 +262,7 @@ typedef struct TypeName
List *typmods; /* type modifier expression(s) */
int32 typemod; /* prespecified type modifier */
List *arrayBounds; /* array bounds */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} TypeName;
/*
@@ -282,7 +282,7 @@ typedef struct ColumnRef
{
NodeTag type;
List *fields; /* field names (String nodes) or A_Star */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} ColumnRef;
/*
@@ -292,7 +292,7 @@ typedef struct ParamRef
{
NodeTag type;
int number; /* the number of the parameter */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} ParamRef;
/*
@@ -325,7 +325,7 @@ typedef struct A_Expr
List *name; /* possibly-qualified name of operator */
Node *lexpr; /* left argument, or NULL if none */
Node *rexpr; /* right argument, or NULL if none */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} A_Expr;
/*
@@ -351,7 +351,7 @@ typedef struct A_Const
NodeTag type;
union ValUnion val;
bool isnull; /* SQL NULL constant */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} A_Const;
/*
@@ -362,7 +362,7 @@ typedef struct TypeCast
NodeTag type;
Node *arg; /* the expression being casted */
TypeName *typeName; /* the target type */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} TypeCast;
/*
@@ -373,7 +373,7 @@ typedef struct CollateClause
NodeTag type;
Node *arg; /* input expression */
List *collname; /* possibly-qualified collation name */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} CollateClause;
/*
@@ -393,7 +393,7 @@ typedef struct RoleSpec
NodeTag type;
RoleSpecType roletype; /* Type of this rolespec */
char *rolename; /* filled only for ROLESPEC_CSTRING */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} RoleSpec;
/*
@@ -423,7 +423,7 @@ typedef struct FuncCall
bool agg_distinct; /* arguments were labeled DISTINCT */
bool func_variadic; /* last argument was labeled VARIADIC */
CoercionForm funcformat; /* how to display this node */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} FuncCall;
/*
@@ -480,7 +480,7 @@ typedef struct A_ArrayExpr
{
NodeTag type;
List *elements; /* array element expressions */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} A_ArrayExpr;
/*
@@ -507,7 +507,7 @@ typedef struct ResTarget
char *name; /* column name or NULL */
List *indirection; /* subscripts, field names, and '*', or NIL */
Node *val; /* the value expression to compute or assign */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} ResTarget;
/*
@@ -537,7 +537,7 @@ typedef struct SortBy
SortByDir sortby_dir; /* ASC/DESC/USING/default */
SortByNulls sortby_nulls; /* NULLS FIRST/LAST */
List *useOp; /* name of op to use, if SORTBY_USING */
- int location; /* operator location, or -1 if none/unknown */
+ Location location; /* operator location, or -1 if none/unknown */
} SortBy;
/*
@@ -558,7 +558,7 @@ typedef struct WindowDef
int frameOptions; /* frame_clause options, see below */
Node *startOffset; /* expression for starting bound, if any */
Node *endOffset; /* expression for ending bound, if any */
- int location; /* parse location, or -1 if none/unknown */
+ Location location; /* parse location, or -1 if none/unknown */
} WindowDef;
/*
@@ -648,7 +648,7 @@ typedef struct RangeTableFunc
List *namespaces; /* list of namespaces as ResTarget */
List *columns; /* list of RangeTableFuncCol */
Alias *alias; /* table alias & optional column aliases */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} RangeTableFunc;
/*
@@ -666,7 +666,7 @@ typedef struct RangeTableFuncCol
bool is_not_null; /* does it have NOT NULL? */
Node *colexpr; /* column filter expression */
Node *coldefexpr; /* column default value expression */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} RangeTableFuncCol;
/*
@@ -686,7 +686,7 @@ typedef struct RangeTableSample
List *method; /* sampling method name (possibly qualified) */
List *args; /* argument(s) for sampling method */
Node *repeatable; /* REPEATABLE expression, or NULL if none */
- int location; /* method name location, or -1 if unknown */
+ Location location; /* method name location, or -1 if unknown */
} RangeTableSample;
/*
@@ -729,7 +729,7 @@ typedef struct ColumnDef
Oid collOid; /* collation OID (InvalidOid if not set) */
List *constraints; /* other constraints on column */
List *fdwoptions; /* per-column FDW options */
- int location; /* parse location, or -1 if none/unknown */
+ Location location; /* parse location, or -1 if none/unknown */
} ColumnDef;
/*
@@ -803,7 +803,7 @@ typedef struct DefElem
Node *arg; /* typically Integer, Float, String, or
* TypeName */
DefElemAction defaction; /* unspecified action, or SET/ADD/DROP */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} DefElem;
/*
@@ -833,7 +833,7 @@ typedef struct XmlSerialize
Node *expr;
TypeName *typeName;
bool indent; /* [NO] INDENT */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} XmlSerialize;
/* Partitioning related definitions */
@@ -851,7 +851,7 @@ typedef struct PartitionElem
Node *expr; /* expression to partition on, or NULL */
List *collation; /* name of collation; NIL = default */
List *opclass; /* name of desired opclass; NIL = default */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} PartitionElem;
typedef enum PartitionStrategy
@@ -871,7 +871,7 @@ typedef struct PartitionSpec
NodeTag type;
PartitionStrategy strategy;
List *partParams; /* List of PartitionElems */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} PartitionSpec;
/*
@@ -898,7 +898,7 @@ struct PartitionBoundSpec
List *lowerdatums; /* List of PartitionRangeDatums */
List *upperdatums; /* List of PartitionRangeDatums */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
};
/*
@@ -921,7 +921,7 @@ typedef struct PartitionRangeDatum
Node *value; /* Const (or A_Const in raw tree), if kind is
* PARTITION_RANGE_DATUM_VALUE, else NULL */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} PartitionRangeDatum;
/*
@@ -1454,7 +1454,7 @@ typedef struct GroupingSet
NodeTag type;
GroupingSetKind kind pg_node_attr(query_jumble_ignore);
List *content;
- int location;
+ Location location;
} GroupingSet;
/*
@@ -1542,7 +1542,7 @@ typedef struct WithClause
NodeTag type;
List *ctes; /* list of CommonTableExprs */
bool recursive; /* true = WITH RECURSIVE */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} WithClause;
/*
@@ -1557,7 +1557,7 @@ typedef struct InferClause
List *indexElems; /* IndexElems to infer unique index */
Node *whereClause; /* qualification (partial-index predicate) */
char *conname; /* Constraint name, or NULL if unnamed */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} InferClause;
/*
@@ -1573,7 +1573,7 @@ typedef struct OnConflictClause
InferClause *infer; /* Optional index inference clause */
List *targetList; /* the target list (of ResTarget) */
Node *whereClause; /* qualifications */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} OnConflictClause;
/*
@@ -1594,7 +1594,7 @@ typedef struct CTESearchClause
List *search_col_list;
bool search_breadth_first;
char *search_seq_column;
- int location;
+ Location location;
} CTESearchClause;
typedef struct CTECycleClause
@@ -1605,7 +1605,7 @@ typedef struct CTECycleClause
Node *cycle_mark_value;
Node *cycle_mark_default;
char *cycle_path_column;
- int location;
+ Location location;
/* These fields are set during parse analysis: */
Oid cycle_mark_type; /* common type of _value and _default */
int cycle_mark_typmod;
@@ -1629,7 +1629,7 @@ typedef struct CommonTableExpr
Node *ctequery; /* the CTE's subquery */
CTESearchClause *search_clause pg_node_attr(query_jumble_ignore);
CTECycleClause *cycle_clause pg_node_attr(query_jumble_ignore);
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
/* These fields are set during parse analysis: */
/* is this CTE actually recursive? */
bool cterecursive pg_node_attr(query_jumble_ignore);
@@ -1725,7 +1725,7 @@ typedef struct JsonParseExpr
JsonValueExpr *expr; /* string expression */
JsonOutput *output; /* RETURNING clause, if specified */
bool unique_keys; /* WITH UNIQUE KEYS? */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} JsonParseExpr;
/*
@@ -1737,7 +1737,7 @@ typedef struct JsonScalarExpr
NodeTag type;
Expr *expr; /* scalar expression */
JsonOutput *output; /* RETURNING clause, if specified */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} JsonScalarExpr;
/*
@@ -1749,7 +1749,7 @@ typedef struct JsonSerializeExpr
NodeTag type;
JsonValueExpr *expr; /* json value expression */
JsonOutput *output; /* RETURNING clause, if specified */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} JsonSerializeExpr;
/*
@@ -1763,7 +1763,7 @@ typedef struct JsonObjectConstructor
JsonOutput *output; /* RETURNING clause, if specified */
bool absent_on_null; /* skip NULL values? */
bool unique; /* check key uniqueness? */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} JsonObjectConstructor;
/*
@@ -1776,7 +1776,7 @@ typedef struct JsonArrayConstructor
List *exprs; /* list of JsonValueExpr elements */
JsonOutput *output; /* RETURNING clause, if specified */
bool absent_on_null; /* skip NULL elements? */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} JsonArrayConstructor;
/*
@@ -1790,7 +1790,7 @@ typedef struct JsonArrayQueryConstructor
JsonOutput *output; /* RETURNING clause, if specified */
JsonFormat *format; /* FORMAT clause for subquery, if specified */
bool absent_on_null; /* skip NULL elements? */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} JsonArrayQueryConstructor;
/*
@@ -1805,7 +1805,7 @@ typedef struct JsonAggConstructor
Node *agg_filter; /* FILTER clause, if any */
List *agg_order; /* ORDER BY clause, if any */
struct WindowDef *over; /* OVER clause, if any */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} JsonAggConstructor;
/*
@@ -1859,8 +1859,8 @@ typedef struct RawStmt
NodeTag type;
Node *stmt; /* raw parse tree */
- int stmt_location; /* start location, or -1 if unknown */
- int stmt_len; /* length in bytes; 0 means "rest of string" */
+ Location stmt_location; /* start location, or -1 if unknown */
+ Location stmt_len; /* length in bytes; 0 means "rest of string" */
} RawStmt;
/*****************************************************************************
@@ -2067,7 +2067,7 @@ typedef struct PLAssignStmt
List *indirection; /* subscripts and field names, if any */
int nnames; /* number of names to use in ColumnRef */
SelectStmt *val; /* the PL/pgSQL expression to assign */
- int location; /* name's token location, or -1 if unknown */
+ Location location; /* name's token location, or -1 if unknown */
} PLAssignStmt;
@@ -2575,7 +2575,7 @@ typedef struct Constraint
char *conname; /* Constraint name, or NULL if unnamed */
bool deferrable; /* DEFERRABLE? */
bool initdeferred; /* INITIALLY DEFERRED? */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
/* Fields used for constraints with expressions (CHECK and DEFAULT): */
bool is_no_inherit; /* is constraint non-inheritable? */
@@ -3528,7 +3528,7 @@ typedef struct TransactionStmt
char *gid pg_node_attr(query_jumble_ignore);
bool chain; /* AND CHAIN option */
/* token location, or -1 if unknown */
- int location pg_node_attr(query_jumble_location);
+ Location location pg_node_attr(query_jumble_location);
} TransactionStmt;
/* ----------------------
@@ -3914,7 +3914,7 @@ typedef struct DeallocateStmt
/* true if DEALLOCATE ALL */
bool isall;
/* token location, or -1 if unknown */
- int location pg_node_attr(query_jumble_location);
+ Location location pg_node_attr(query_jumble_location);
} DeallocateStmt;
/*
@@ -4002,7 +4002,7 @@ typedef struct PublicationObjSpec
PublicationObjSpecType pubobjtype; /* type of this publication object */
char *name;
PublicationTable *pubtable;
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} PublicationObjSpec;
typedef struct CreatePublicationStmt
diff --git a/src/include/nodes/primnodes.h b/src/include/nodes/primnodes.h
index 4a154606d2..557be05657 100644
--- a/src/include/nodes/primnodes.h
+++ b/src/include/nodes/primnodes.h
@@ -30,6 +30,8 @@ typedef enum OverridingKind
} OverridingKind;
+#define Location int
+
/* ----------------------------------------------------------------
* node definitions
* ----------------------------------------------------------------
@@ -91,7 +93,7 @@ typedef struct RangeVar
Alias *alias;
/* token location, or -1 if unknown */
- int location;
+ Location location;
} RangeVar;
/*
@@ -128,7 +130,7 @@ typedef struct TableFunc
/* counts from 0; -1 if none specified */
int ordinalitycol pg_node_attr(query_jumble_ignore);
/* token location, or -1 if unknown */
- int location;
+ Location location;
} TableFunc;
/*
@@ -276,7 +278,7 @@ typedef struct Var
AttrNumber varattnosyn pg_node_attr(equal_ignore, query_jumble_ignore);
/* token location, or -1 if unknown */
- int location;
+ Location location;
} Var;
/*
@@ -318,7 +320,7 @@ typedef struct Const
* token location, or -1 if unknown. All constants are tracked as
* locations in query jumbling, to be marked as parameters.
*/
- int location pg_node_attr(query_jumble_location);
+ Location location pg_node_attr(query_jumble_location);
} Const;
/*
@@ -367,7 +369,7 @@ typedef struct Param
/* OID of collation, or InvalidOid if none */
Oid paramcollid pg_node_attr(query_jumble_ignore);
/* token location, or -1 if unknown */
- int location;
+ Location location;
} Param;
/*
@@ -490,7 +492,7 @@ typedef struct Aggref
int aggtransno pg_node_attr(query_jumble_ignore);
/* token location, or -1 if unknown */
- int location;
+ Location location;
} Aggref;
/*
@@ -537,7 +539,7 @@ typedef struct GroupingFunc
Index agglevelsup;
/* token location */
- int location;
+ Location location;
} GroupingFunc;
/*
@@ -568,7 +570,7 @@ typedef struct WindowFunc
/* is function a simple aggregate? */
bool winagg pg_node_attr(query_jumble_ignore);
/* token location, or -1 if unknown */
- int location;
+ Location location;
} WindowFunc;
/*
@@ -702,7 +704,7 @@ typedef struct FuncExpr
/* arguments to the function */
List *args;
/* token location, or -1 if unknown */
- int location;
+ Location location;
} FuncExpr;
/*
@@ -729,7 +731,7 @@ typedef struct NamedArgExpr
/* argument's number in positional notation */
int argnumber;
/* argument name location, or -1 if unknown */
- int location;
+ Location location;
} NamedArgExpr;
/*
@@ -771,7 +773,7 @@ typedef struct OpExpr
List *args;
/* token location, or -1 if unknown */
- int location;
+ Location location;
} OpExpr;
/*
@@ -851,7 +853,7 @@ typedef struct ScalarArrayOpExpr
List *args;
/* token location, or -1 if unknown */
- int location;
+ Location location;
} ScalarArrayOpExpr;
/*
@@ -873,7 +875,7 @@ typedef struct BoolExpr
Expr xpr;
BoolExprType boolop;
List *args; /* arguments to this expression */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} BoolExpr;
/*
@@ -950,7 +952,7 @@ typedef struct SubLink
List *operName pg_node_attr(query_jumble_ignore);
/* subselect as Query* or raw parsetree */
Node *subselect;
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} SubLink;
/*
@@ -1124,7 +1126,7 @@ typedef struct RelabelType
Oid resultcollid pg_node_attr(query_jumble_ignore);
/* how to display this node */
CoercionForm relabelformat pg_node_attr(query_jumble_ignore);
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} RelabelType;
/* ----------------
@@ -1146,7 +1148,7 @@ typedef struct CoerceViaIO
Oid resultcollid pg_node_attr(query_jumble_ignore);
/* how to display this node */
CoercionForm coerceformat pg_node_attr(query_jumble_ignore);
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} CoerceViaIO;
/* ----------------
@@ -1174,7 +1176,7 @@ typedef struct ArrayCoerceExpr
Oid resultcollid pg_node_attr(query_jumble_ignore);
/* how to display this node */
CoercionForm coerceformat pg_node_attr(query_jumble_ignore);
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} ArrayCoerceExpr;
/* ----------------
@@ -1198,7 +1200,7 @@ typedef struct ConvertRowtypeExpr
/* Like RowExpr, we deliberately omit a typmod and collation here */
/* how to display this node */
CoercionForm convertformat pg_node_attr(query_jumble_ignore);
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} ConvertRowtypeExpr;
/*----------
@@ -1213,7 +1215,7 @@ typedef struct CollateExpr
Expr xpr;
Expr *arg; /* input expression */
Oid collOid; /* collation's OID */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} CollateExpr;
/*----------
@@ -1248,7 +1250,7 @@ typedef struct CaseExpr
Expr *arg; /* implicit equality comparison argument */
List *args; /* the arguments (list of WHEN clauses) */
Expr *defresult; /* the default result (ELSE clause) */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} CaseExpr;
/*
@@ -1259,7 +1261,7 @@ typedef struct CaseWhen
Expr xpr;
Expr *expr; /* condition expression */
Expr *result; /* substitution result */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} CaseWhen;
/*
@@ -1316,7 +1318,7 @@ typedef struct ArrayExpr
/* true if elements are sub-arrays */
bool multidims pg_node_attr(query_jumble_ignore);
/* token location, or -1 if unknown */
- int location;
+ Location location;
} ArrayExpr;
/*
@@ -1367,7 +1369,7 @@ typedef struct RowExpr
/* list of String, or NIL */
List *colnames pg_node_attr(query_jumble_ignore);
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} RowExpr;
/*
@@ -1426,7 +1428,7 @@ typedef struct CoalesceExpr
/* the arguments */
List *args;
/* token location, or -1 if unknown */
- int location;
+ Location location;
} CoalesceExpr;
/*
@@ -1452,7 +1454,7 @@ typedef struct MinMaxExpr
/* the arguments */
List *args;
/* token location, or -1 if unknown */
- int location;
+ Location location;
} MinMaxExpr;
/*
@@ -1496,7 +1498,7 @@ typedef struct SQLValueFunction
*/
Oid type pg_node_attr(query_jumble_ignore);
int32 typmod;
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} SQLValueFunction;
/*
@@ -1549,7 +1551,7 @@ typedef struct XmlExpr
Oid type pg_node_attr(query_jumble_ignore);
int32 typmod pg_node_attr(query_jumble_ignore);
/* token location, or -1 if unknown */
- int location;
+ Location location;
} XmlExpr;
/*
@@ -1585,7 +1587,7 @@ typedef struct JsonFormat
NodeTag type;
JsonFormatType format_type; /* format type */
JsonEncoding encoding; /* JSON encoding */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} JsonFormat;
/*
@@ -1641,7 +1643,7 @@ typedef struct JsonConstructorExpr
JsonReturning *returning; /* RETURNING clause */
bool absent_on_null; /* ABSENT ON NULL? */
bool unique; /* WITH UNIQUE KEYS? (JSON_OBJECT[AGG] only) */
- int location;
+ Location location;
} JsonConstructorExpr;
/*
@@ -1667,7 +1669,7 @@ typedef struct JsonIsPredicate
JsonFormat *format; /* FORMAT clause, if specified */
JsonValueType item_type; /* JSON item type */
bool unique_keys; /* check key uniqueness? */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} JsonIsPredicate;
/* ----------------
@@ -1701,7 +1703,7 @@ typedef struct NullTest
NullTestType nulltesttype; /* IS NULL, IS NOT NULL */
/* T to perform field-by-field null checks */
bool argisrow pg_node_attr(query_jumble_ignore);
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} NullTest;
/*
@@ -1723,7 +1725,7 @@ typedef struct BooleanTest
Expr xpr;
Expr *arg; /* input expression */
BoolTestType booltesttype; /* test type */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} BooleanTest;
@@ -1765,7 +1767,7 @@ typedef struct CoerceToDomain
Oid resultcollid pg_node_attr(query_jumble_ignore);
/* how to display this node */
CoercionForm coercionformat pg_node_attr(query_jumble_ignore);
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} CoerceToDomain;
/*
@@ -1787,7 +1789,7 @@ typedef struct CoerceToDomainValue
/* collation for the substituted value */
Oid collation pg_node_attr(query_jumble_ignore);
/* token location, or -1 if unknown */
- int location;
+ Location location;
} CoerceToDomainValue;
/*
@@ -1807,7 +1809,7 @@ typedef struct SetToDefault
/* collation for the substituted value */
Oid collation pg_node_attr(query_jumble_ignore);
/* token location, or -1 if unknown */
- int location;
+ Location location;
} SetToDefault;
/*
--
2.40.1
[application/octet-stream] v2-0002-pg_node_tree-Don-t-store-query-text-locations-in-.patch (19.2K, ../../CAEze2WhTQ1LGtpyobcLFvda61BDtkiYvCfyuUA9-Mi9iwd-gyg@mail.gmail.com/5-v2-0002-pg_node_tree-Don-t-store-query-text-locations-in-.patch)
download | inline diff:
From 0e1858ed65ab3def775b50497b4f4a84452765d4 Mon Sep 17 00:00:00 2001
From: Matthias van de Meent <[email protected]>
Date: Wed, 3 Jan 2024 01:39:42 +0100
Subject: [PATCH v2 2/7] pg_node_tree: Don't store query text locations in
pg_node_tree fields.
We don't store original query texts, so any lingering "location" value
can only be useful in forensic debugging. In normal operation, however,
a non-default value will show up as measurable overhead in
serialization, just omit serialization, saving several 10s of kBs.
---
src/backend/catalog/heap.c | 9 ++--
src/backend/catalog/index.c | 4 +-
src/backend/catalog/pg_attrdef.c | 6 ++-
src/backend/catalog/pg_proc.c | 10 +++-
src/backend/catalog/pg_publication.c | 6 ++-
src/backend/commands/policy.c | 11 +++-
src/backend/commands/statscmds.c | 2 +-
src/backend/commands/trigger.c | 3 +-
src/backend/commands/typecmds.c | 8 +--
src/backend/nodes/gen_node_support.pl | 4 +-
src/backend/nodes/outfuncs.c | 77 ++++++++++++++++-----------
src/backend/rewrite/rewriteDefine.c | 4 +-
src/include/nodes/nodes.h | 4 +-
13 files changed, 95 insertions(+), 53 deletions(-)
diff --git a/src/backend/catalog/heap.c b/src/backend/catalog/heap.c
index c73f7bcd01..5c13fdab77 100644
--- a/src/backend/catalog/heap.c
+++ b/src/backend/catalog/heap.c
@@ -2060,9 +2060,9 @@ StoreRelCheck(Relation rel, const char *ccname, Node *expr,
Oid constrOid;
/*
- * Flatten expression to string form for storage.
+ * Flatten expression to string form for storage, without query refs.
*/
- ccbin = nodeToString(expr);
+ ccbin = nodeToStringNoQLocs(expr);
/*
* Find columns of rel that are used in expr
@@ -3676,7 +3676,7 @@ StorePartitionKey(Relation rel,
{
char *exprString;
- exprString = nodeToString(partexprs);
+ exprString = nodeToStringNoQLocs(partexprs);
partexprDatum = CStringGetTextDatum(exprString);
pfree(exprString);
}
@@ -3834,7 +3834,8 @@ StorePartitionBound(Relation rel, Relation parent, PartitionBoundSpec *bound)
memset(new_val, 0, sizeof(new_val));
memset(new_null, false, sizeof(new_null));
memset(new_repl, false, sizeof(new_repl));
- new_val[Anum_pg_class_relpartbound - 1] = CStringGetTextDatum(nodeToString(bound));
+ new_val[Anum_pg_class_relpartbound - 1] =
+ CStringGetTextDatum(nodeToStringNoQLocs(bound));
new_null[Anum_pg_class_relpartbound - 1] = false;
new_repl[Anum_pg_class_relpartbound - 1] = true;
newtuple = heap_modify_tuple(tuple, RelationGetDescr(classRel),
diff --git a/src/backend/catalog/index.c b/src/backend/catalog/index.c
index 4b88a9cb87..3d5f4e53d5 100644
--- a/src/backend/catalog/index.c
+++ b/src/backend/catalog/index.c
@@ -587,7 +587,7 @@ UpdateIndexRelation(Oid indexoid,
{
char *exprsString;
- exprsString = nodeToString(indexInfo->ii_Expressions);
+ exprsString = nodeToStringNoQLocs(indexInfo->ii_Expressions);
exprsDatum = CStringGetTextDatum(exprsString);
pfree(exprsString);
}
@@ -602,7 +602,7 @@ UpdateIndexRelation(Oid indexoid,
{
char *predString;
- predString = nodeToString(make_ands_explicit(indexInfo->ii_Predicate));
+ predString = nodeToStringNoQLocs(make_ands_explicit(indexInfo->ii_Predicate));
predDatum = CStringGetTextDatum(predString);
pfree(predString);
}
diff --git a/src/backend/catalog/pg_attrdef.c b/src/backend/catalog/pg_attrdef.c
index 003ae70b4d..a900c9bb28 100644
--- a/src/backend/catalog/pg_attrdef.c
+++ b/src/backend/catalog/pg_attrdef.c
@@ -23,6 +23,7 @@
#include "catalog/objectaccess.h"
#include "catalog/pg_attrdef.h"
#include "executor/executor.h"
+#include "nodes/nodeFuncs.h"
#include "optimizer/optimizer.h"
#include "utils/array.h"
#include "utils/builtins.h"
@@ -62,9 +63,10 @@ StoreAttrDefault(Relation rel, AttrNumber attnum,
adrel = table_open(AttrDefaultRelationId, RowExclusiveLock);
/*
- * Flatten expression to string form for storage.
+ * Flatten expression to string form for storage, without references to
+ * the original query string.
*/
- adbin = nodeToString(expr);
+ adbin = nodeToStringNoQLocs(expr);
/*
* Make the pg_attrdef entry.
diff --git a/src/backend/catalog/pg_proc.c b/src/backend/catalog/pg_proc.c
index b581d334d3..c5790a2224 100644
--- a/src/backend/catalog/pg_proc.c
+++ b/src/backend/catalog/pg_proc.c
@@ -331,7 +331,10 @@ ProcedureCreate(const char *procedureName,
else
nulls[Anum_pg_proc_proargnames - 1] = true;
if (parameterDefaults != NIL)
- values[Anum_pg_proc_proargdefaults - 1] = CStringGetTextDatum(nodeToString(parameterDefaults));
+ {
+ values[Anum_pg_proc_proargdefaults - 1] =
+ CStringGetTextDatum(nodeToStringNoQLocs(parameterDefaults));
+ }
else
nulls[Anum_pg_proc_proargdefaults - 1] = true;
if (trftypes != PointerGetDatum(NULL))
@@ -344,7 +347,10 @@ ProcedureCreate(const char *procedureName,
else
nulls[Anum_pg_proc_probin - 1] = true;
if (prosqlbody)
- values[Anum_pg_proc_prosqlbody - 1] = CStringGetTextDatum(nodeToString(prosqlbody));
+ {
+ values[Anum_pg_proc_prosqlbody - 1] =
+ CStringGetTextDatum(nodeToStringNoQLocs(prosqlbody));
+ }
else
nulls[Anum_pg_proc_prosqlbody - 1] = true;
if (proconfig != PointerGetDatum(NULL))
diff --git a/src/backend/catalog/pg_publication.c b/src/backend/catalog/pg_publication.c
index b98b0ce0ae..b201313430 100644
--- a/src/backend/catalog/pg_publication.c
+++ b/src/backend/catalog/pg_publication.c
@@ -36,6 +36,7 @@
#include "commands/publicationcmds.h"
#include "funcapi.h"
#include "miscadmin.h"
+#include "nodes/nodeFuncs.h"
#include "utils/array.h"
#include "utils/builtins.h"
#include "utils/catcache.h"
@@ -422,7 +423,10 @@ publication_add_relation(Oid pubid, PublicationRelInfo *pri,
/* Add qualifications, if available */
if (pri->whereClause != NULL)
- values[Anum_pg_publication_rel_prqual - 1] = CStringGetTextDatum(nodeToString(pri->whereClause));
+ {
+ values[Anum_pg_publication_rel_prqual - 1] =
+ CStringGetTextDatum(nodeToStringNoQLocs(pri->whereClause));
+ }
else
nulls[Anum_pg_publication_rel_prqual - 1] = true;
diff --git a/src/backend/commands/policy.c b/src/backend/commands/policy.c
index 596326e5ec..1e6842bf41 100644
--- a/src/backend/commands/policy.c
+++ b/src/backend/commands/policy.c
@@ -30,6 +30,7 @@
#include "commands/policy.h"
#include "miscadmin.h"
#include "nodes/makefuncs.h"
+#include "nodes/nodeFuncs.h"
#include "nodes/pg_list.h"
#include "parser/parse_clause.h"
#include "parser/parse_collate.h"
@@ -701,13 +702,19 @@ CreatePolicy(CreatePolicyStmt *stmt)
/* Add qual if present. */
if (qual)
- values[Anum_pg_policy_polqual - 1] = CStringGetTextDatum(nodeToString(qual));
+ {
+ values[Anum_pg_policy_polqual - 1] =
+ CStringGetTextDatum(nodeToStringNoQLocs(qual));
+ }
else
isnull[Anum_pg_policy_polqual - 1] = true;
/* Add WITH CHECK qual if present */
if (with_check_qual)
- values[Anum_pg_policy_polwithcheck - 1] = CStringGetTextDatum(nodeToString(with_check_qual));
+ {
+ values[Anum_pg_policy_polwithcheck - 1] =
+ CStringGetTextDatum(nodeToStringNoQLocs(with_check_qual));
+ }
else
isnull[Anum_pg_policy_polwithcheck - 1] = true;
diff --git a/src/backend/commands/statscmds.c b/src/backend/commands/statscmds.c
index b1a9c74bd6..58e8133f93 100644
--- a/src/backend/commands/statscmds.c
+++ b/src/backend/commands/statscmds.c
@@ -477,7 +477,7 @@ CreateStatistics(CreateStatsStmt *stmt)
{
char *exprsString;
- exprsString = nodeToString(stxexprs);
+ exprsString = nodeToStringNoQLocs(stxexprs);
exprsDatum = CStringGetTextDatum(exprsString);
pfree(exprsString);
}
diff --git a/src/backend/commands/trigger.c b/src/backend/commands/trigger.c
index c344ff0944..6a3dc13a67 100644
--- a/src/backend/commands/trigger.c
+++ b/src/backend/commands/trigger.c
@@ -39,6 +39,7 @@
#include "miscadmin.h"
#include "nodes/bitmapset.h"
#include "nodes/makefuncs.h"
+#include "nodes/nodeFuncs.h"
#include "optimizer/optimizer.h"
#include "parser/parse_clause.h"
#include "parser/parse_collate.h"
@@ -674,7 +675,7 @@ CreateTriggerFiringOn(CreateTrigStmt *stmt, const char *queryString,
/* we'll need the rtable for recordDependencyOnExpr */
whenRtable = pstate->p_rtable;
- qual = nodeToString(whenClause);
+ qual = nodeToStringNoQLocs(whenClause);
free_parsestate(pstate);
}
diff --git a/src/backend/commands/typecmds.c b/src/backend/commands/typecmds.c
index a400fb39f6..00180a54b9 100644
--- a/src/backend/commands/typecmds.c
+++ b/src/backend/commands/typecmds.c
@@ -58,6 +58,7 @@
#include "executor/executor.h"
#include "miscadmin.h"
#include "nodes/makefuncs.h"
+#include "nodes/nodeFuncs.h"
#include "optimizer/optimizer.h"
#include "parser/parse_coerce.h"
#include "parser/parse_collate.h"
@@ -924,7 +925,7 @@ DefineDomain(CreateDomainStmt *stmt)
defaultValue =
deparse_expression(defaultExpr,
NIL, false, false);
- defaultValueBin = nodeToString(defaultExpr);
+ defaultValueBin = nodeToStringNoQLocs(defaultExpr);
}
}
else
@@ -3506,9 +3507,10 @@ domainAddConstraint(Oid domainOid, Oid domainNamespace, Oid baseTypeOid,
errmsg("cannot use table references in domain check constraint")));
/*
- * Convert to string form for storage.
+ * Convert to string form for storage, without references to the original
+ * query text.
*/
- ccbin = nodeToString(expr);
+ ccbin = nodeToStringNoQLocs(expr);
/*
* Store the constraint in pg_constraint
diff --git a/src/backend/nodes/gen_node_support.pl b/src/backend/nodes/gen_node_support.pl
index 2f0a59bc87..487f6f7728 100644
--- a/src/backend/nodes/gen_node_support.pl
+++ b/src/backend/nodes/gen_node_support.pl
@@ -921,7 +921,7 @@ foreach my $n (@node_types)
my $N = uc $n;
print $ofs "\t\t\tcase T_${n}:\n"
- . "\t\t\t\t_out${n}(str, obj);\n"
+ . "\t\t\t\t_out${n}(str, obj, omitLocation);\n"
. "\t\t\t\tbreak;\n";
print $rfs "\tif (MATCH(\"$N\", "
@@ -933,7 +933,7 @@ foreach my $n (@node_types)
print $off "
static void
-_out${n}(StringInfo str, const $n *node)
+_out${n}(StringInfo str, const $n *node, bool omitLocation)
{
\tWRITE_NODE_TYPE(\"$N\");
diff --git a/src/backend/nodes/outfuncs.c b/src/backend/nodes/outfuncs.c
index 50da80ee34..eca3160104 100644
--- a/src/backend/nodes/outfuncs.c
+++ b/src/backend/nodes/outfuncs.c
@@ -154,7 +154,7 @@ static void outDouble(StringInfo str, double d);
/* Write a parse location field (actually same as INT case) */
#define WRITE_LOCATION_FIELD(fldname) \
do { \
- if (node->fldname != -1) \
+ if (node->fldname != -1 && !omitLocation) \
appendStringInfo(str, " :" CppAsString(fldname) " %d", \
node->fldname); \
} while (0)
@@ -165,7 +165,7 @@ static void outDouble(StringInfo str, double d);
if (node->fldname != NULL) \
{ \
appendStringInfoString(str, " :" CppAsString(fldname) " "); \
- outNode(str, node->fldname); \
+ outNode(str, node->fldname, omitLocation); \
} \
} while (0)
@@ -185,7 +185,8 @@ static void outDouble(StringInfo str, double d);
if (node->fldname != NULL) \
{ \
appendStringInfoString(str, " :" CppAsString(fldname) " "); \
- writeNodeArray(str, (const Node * const *) node->fldname, len); \
+ writeNodeArray(str, (const Node * const *) node->fldname, len, \
+ omitLocation); \
} \
} while (0)
@@ -358,7 +359,8 @@ WRITE_SCALAR_ARRAY(writeBoolCols, bool, " %s", booltostr)
* quite use appendStringInfo() in the loop.
*/
static void
-writeNodeArray(StringInfo str, const Node *const *arr, int len)
+writeNodeArray(StringInfo str, const Node *const *arr, int len,
+ bool omitLocation)
{
if (arr != NULL)
{
@@ -366,7 +368,7 @@ writeNodeArray(StringInfo str, const Node *const *arr, int len)
for (int i = 0; i < len; i++)
{
appendStringInfoChar(str, ' ');
- outNode(str, arr[i]);
+ outNode(str, arr[i], omitLocation);
}
appendStringInfoChar(str, ')');
}
@@ -378,7 +380,7 @@ writeNodeArray(StringInfo str, const Node *const *arr, int len)
* Print a List.
*/
static void
-_outList(StringInfo str, const List *node)
+_outList(StringInfo str, const List *node, bool omitLocation)
{
const ListCell *lc;
@@ -400,7 +402,7 @@ _outList(StringInfo str, const List *node)
*/
if (IsA(node, List))
{
- outNode(str, lfirst(lc));
+ outNode(str, lfirst(lc), omitLocation);
if (lnext(node, lc))
appendStringInfoChar(str, ' ');
}
@@ -485,7 +487,7 @@ outDatum(StringInfo str, Datum value, int typlen, bool typbyval)
*/
static void
-_outConst(StringInfo str, const Const *node)
+_outConst(StringInfo str, const Const *node, bool omitLocation)
{
WRITE_NODE_TYPE("CONST");
@@ -505,7 +507,7 @@ _outConst(StringInfo str, const Const *node)
}
static void
-_outBoolExpr(StringInfo str, const BoolExpr *node)
+_outBoolExpr(StringInfo str, const BoolExpr *node, bool omitLocation)
{
char *opstr = NULL;
@@ -532,7 +534,8 @@ _outBoolExpr(StringInfo str, const BoolExpr *node)
}
static void
-_outForeignKeyOptInfo(StringInfo str, const ForeignKeyOptInfo *node)
+_outForeignKeyOptInfo(StringInfo str, const ForeignKeyOptInfo *node,
+ bool omitLocation)
{
int i;
@@ -558,7 +561,8 @@ _outForeignKeyOptInfo(StringInfo str, const ForeignKeyOptInfo *node)
}
static void
-_outEquivalenceClass(StringInfo str, const EquivalenceClass *node)
+_outEquivalenceClass(StringInfo str, const EquivalenceClass *node,
+ bool omitLocation)
{
/*
* To simplify reading, we just chase up to the topmost merged EC and
@@ -584,7 +588,8 @@ _outEquivalenceClass(StringInfo str, const EquivalenceClass *node)
}
static void
-_outExtensibleNode(StringInfo str, const ExtensibleNode *node)
+_outExtensibleNode(StringInfo str, const ExtensibleNode *node,
+ bool omitLocation)
{
const ExtensibleNodeMethods *methods;
@@ -599,7 +604,8 @@ _outExtensibleNode(StringInfo str, const ExtensibleNode *node)
}
static void
-_outRangeTblEntry(StringInfo str, const RangeTblEntry *node)
+_outRangeTblEntry(StringInfo str, const RangeTblEntry *node,
+ bool omitLocation)
{
WRITE_NODE_TYPE("RANGETBLENTRY");
@@ -679,7 +685,7 @@ _outRangeTblEntry(StringInfo str, const RangeTblEntry *node)
}
static void
-_outA_Expr(StringInfo str, const A_Expr *node)
+_outA_Expr(StringInfo str, const A_Expr *node, bool omitLocation)
{
WRITE_NODE_TYPE("A_EXPR");
@@ -751,13 +757,13 @@ _outA_Expr(StringInfo str, const A_Expr *node)
}
static void
-_outInteger(StringInfo str, const Integer *node)
+_outInteger(StringInfo str, const Integer *node, bool omitLocation)
{
appendStringInfo(str, "%d", node->ival);
}
static void
-_outFloat(StringInfo str, const Float *node)
+_outFloat(StringInfo str, const Float *node, bool omitLocation)
{
/*
* We assume the value is a valid numeric literal and so does not need
@@ -767,13 +773,13 @@ _outFloat(StringInfo str, const Float *node)
}
static void
-_outBoolean(StringInfo str, const Boolean *node)
+_outBoolean(StringInfo str, const Boolean *node, bool omitLocation)
{
appendStringInfoString(str, node->boolval ? "true" : "false");
}
static void
-_outString(StringInfo str, const String *node)
+_outString(StringInfo str, const String *node, bool omitLocation)
{
/*
* We use outToken to provide escaping of the string's content, but we
@@ -787,14 +793,14 @@ _outString(StringInfo str, const String *node)
}
static void
-_outBitString(StringInfo str, const BitString *node)
+_outBitString(StringInfo str, const BitString *node, bool omitLocation)
{
/* internal representation already has leading 'b' */
appendStringInfoString(str, node->bsval);
}
static void
-_outA_Const(StringInfo str, const A_Const *node)
+_outA_Const(StringInfo str, const A_Const *node, bool omitLocation)
{
WRITE_NODE_TYPE("A_CONST");
@@ -803,13 +809,13 @@ _outA_Const(StringInfo str, const A_Const *node)
else
{
appendStringInfoString(str, " :val ");
- outNode(str, &node->val);
+ outNode(str, &node->val, omitLocation);
}
WRITE_LOCATION_FIELD(location);
}
static void
-_outConstraint(StringInfo str, const Constraint *node)
+_outConstraint(StringInfo str, const Constraint *node, bool omitLocation)
{
WRITE_NODE_TYPE("CONSTRAINT");
@@ -942,7 +948,7 @@ _outConstraint(StringInfo str, const Constraint *node)
* converts a Node into ascii string and append it to 'str'
*/
void
-outNode(StringInfo str, const void *obj)
+outNode(StringInfo str, const void *obj, bool omitLocation)
{
/* Guard against stack overflow due to overly complex expressions */
check_stack_depth();
@@ -951,18 +957,18 @@ outNode(StringInfo str, const void *obj)
appendStringInfoString(str, "<>");
else if (IsA(obj, List) || IsA(obj, IntList) || IsA(obj, OidList) ||
IsA(obj, XidList))
- _outList(str, obj);
+ _outList(str, obj, omitLocation);
/* nodeRead does not want to see { } around these! */
else if (IsA(obj, Integer))
- _outInteger(str, (Integer *) obj);
+ _outInteger(str, (Integer *) obj, omitLocation);
else if (IsA(obj, Float))
- _outFloat(str, (Float *) obj);
+ _outFloat(str, (Float *) obj, omitLocation);
else if (IsA(obj, Boolean))
- _outBoolean(str, (Boolean *) obj);
+ _outBoolean(str, (Boolean *) obj, omitLocation);
else if (IsA(obj, String))
- _outString(str, (String *) obj);
+ _outString(str, (String *) obj, omitLocation);
else if (IsA(obj, BitString))
- _outBitString(str, (BitString *) obj);
+ _outBitString(str, (BitString *) obj, omitLocation);
else if (IsA(obj, Bitmapset))
outBitmapset(str, (Bitmapset *) obj);
else
@@ -997,7 +1003,18 @@ nodeToString(const void *obj)
/* see stringinfo.h for an explanation of this maneuver */
initStringInfo(&str);
- outNode(&str, obj);
+ outNode(&str, obj, false);
+ return str.data;
+}
+
+char *
+nodeToStringNoQLocs(const void *obj)
+{
+ StringInfoData str;
+
+ /* see stringinfo.h for an explanation of this maneuver */
+ initStringInfo(&str);
+ outNode(&str, obj, true);
return str.data;
}
diff --git a/src/backend/rewrite/rewriteDefine.c b/src/backend/rewrite/rewriteDefine.c
index b449244a53..6302cd1472 100644
--- a/src/backend/rewrite/rewriteDefine.c
+++ b/src/backend/rewrite/rewriteDefine.c
@@ -64,8 +64,8 @@ InsertRule(const char *rulname,
List *action,
bool replace)
{
- char *evqual = nodeToString(event_qual);
- char *actiontree = nodeToString((Node *) action);
+ char *evqual = nodeToStringNoQLocs(event_qual);
+ char *actiontree = nodeToStringNoQLocs((Node *) action);
Datum values[Natts_pg_rewrite];
bool nulls[Natts_pg_rewrite] = {0};
NameData rname;
diff --git a/src/include/nodes/nodes.h b/src/include/nodes/nodes.h
index 2969dd831b..f7adb5e767 100644
--- a/src/include/nodes/nodes.h
+++ b/src/include/nodes/nodes.h
@@ -188,13 +188,15 @@ castNodeImpl(NodeTag type, void *ptr)
struct Bitmapset; /* not to include bitmapset.h here */
struct StringInfoData; /* not to include stringinfo.h here */
-extern void outNode(struct StringInfoData *str, const void *obj);
+extern void outNode(struct StringInfoData *str, const void *obj,
+ bool omitLocation);
extern void outToken(struct StringInfoData *str, const char *s);
extern void outBitmapset(struct StringInfoData *str,
const struct Bitmapset *bms);
extern void outDatum(struct StringInfoData *str, uintptr_t value,
int typlen, bool typbyval);
extern char *nodeToString(const void *obj);
+extern char *nodeToStringNoQLocs(const void *obj);
extern char *bmsToString(const struct Bitmapset *bms);
/*
--
2.40.1
[application/octet-stream] v2-0004-gen_node_support.pl-Add-a-TypMod-type-for-signall.patch (10.4K, ../../CAEze2WhTQ1LGtpyobcLFvda61BDtkiYvCfyuUA9-Mi9iwd-gyg@mail.gmail.com/6-v2-0004-gen_node_support.pl-Add-a-TypMod-type-for-signall.patch)
download | inline diff:
From b0e559b5d0f037a8a835b5f502b79453833c8620 Mon Sep 17 00:00:00 2001
From: Matthias van de Meent <[email protected]>
Date: Thu, 8 Feb 2024 17:37:39 +0100
Subject: [PATCH v2 4/7] gen_node_support.pl: Add a TypMod type for signalling
TypMod behaviour
Like Location, TypMod has its own default (-1). By using a type, we can
omit adding pg_node_attribute(default(-1)) markers to every typmod-valued
field, whilst still getting the benefits of a smaller size in serialization.
---
src/backend/nodes/gen_node_support.pl | 8 ++++++-
src/backend/nodes/outfuncs.c | 2 +-
src/backend/nodes/readfuncs.c | 2 +-
src/include/nodes/parsenodes.h | 4 ++--
src/include/nodes/pathnodes.h | 2 +-
src/include/nodes/primnodes.h | 33 ++++++++++++++-------------
6 files changed, 29 insertions(+), 22 deletions(-)
diff --git a/src/backend/nodes/gen_node_support.pl b/src/backend/nodes/gen_node_support.pl
index ec850c3484..fda8c3a494 100644
--- a/src/backend/nodes/gen_node_support.pl
+++ b/src/backend/nodes/gen_node_support.pl
@@ -135,7 +135,8 @@ my @nodetag_only;
# types that are copied by straight assignment
my @scalar_types = qw(
bits32 bool char double int int8 int16 int32 int64 long uint8 uint16 uint32 uint64
- AclMode AttrNumber Cardinality Cost Index Oid RelFileNumber Selectivity Size StrategyNumber SubTransactionId TimeLineID XLogRecPtr
+ AclMode AttrNumber Cardinality Cost Index Oid RelFileNumber Selectivity Size StrategyNumber
+ SubTransactionId TimeLineID XLogRecPtr TypMod
);
# collect enum types
@@ -1015,6 +1016,11 @@ _read${n}(void)
print $off "\tWRITE_LOCATION_FIELD($f);\n";
print $rff "\tREAD_LOCATION_FIELD($f);\n" unless $no_read;
}
+ elsif ($t eq 'TypMod')
+ {
+ print $off "\tWRITE_INT_FIELD_DEFAULT($f, -1);\n";
+ print $rff "\tREAD_INT_FIELD_DEFAULT($f, -1);\n" unless $no_read;
+ }
elsif ($t eq 'int'
|| $t eq 'int16'
|| $t eq 'int32'
diff --git a/src/backend/nodes/outfuncs.c b/src/backend/nodes/outfuncs.c
index eca3160104..3973c0e489 100644
--- a/src/backend/nodes/outfuncs.c
+++ b/src/backend/nodes/outfuncs.c
@@ -492,7 +492,7 @@ _outConst(StringInfo str, const Const *node, bool omitLocation)
WRITE_NODE_TYPE("CONST");
WRITE_OID_FIELD(consttype);
- WRITE_INT_FIELD(consttypmod);
+ WRITE_INT_FIELD_DEFAULT(consttypmod, -1);
WRITE_OID_FIELD(constcollid);
WRITE_INT_FIELD(constlen);
WRITE_BOOL_FIELD(constbyval);
diff --git a/src/backend/nodes/readfuncs.c b/src/backend/nodes/readfuncs.c
index 47f4ba2695..44ab140799 100644
--- a/src/backend/nodes/readfuncs.c
+++ b/src/backend/nodes/readfuncs.c
@@ -356,7 +356,7 @@ _readConst(void)
READ_LOCALS(Const);
READ_OID_FIELD(consttype);
- READ_INT_FIELD(consttypmod);
+ READ_INT_FIELD_DEFAULT(consttypmod, -1);
READ_OID_FIELD(constcollid);
READ_INT_FIELD(constlen);
READ_BOOL_FIELD(constbyval);
diff --git a/src/include/nodes/parsenodes.h b/src/include/nodes/parsenodes.h
index dd8aa30aaf..52d8504a9d 100644
--- a/src/include/nodes/parsenodes.h
+++ b/src/include/nodes/parsenodes.h
@@ -260,7 +260,7 @@ typedef struct TypeName
bool setof; /* is a set? */
bool pct_type; /* %TYPE specified? */
List *typmods; /* type modifier expression(s) */
- int32 typemod; /* prespecified type modifier */
+ TypMod typemod; /* prespecified type modifier */
List *arrayBounds; /* array bounds */
Location location; /* token location, or -1 if unknown */
} TypeName;
@@ -1608,7 +1608,7 @@ typedef struct CTECycleClause
Location location;
/* These fields are set during parse analysis: */
Oid cycle_mark_type; /* common type of _value and _default */
- int cycle_mark_typmod;
+ TypMod cycle_mark_typmod;
Oid cycle_mark_collation;
Oid cycle_mark_neop; /* <> operator for type */
} CTECycleClause;
diff --git a/src/include/nodes/pathnodes.h b/src/include/nodes/pathnodes.h
index 534692bee1..f7c2496a7f 100644
--- a/src/include/nodes/pathnodes.h
+++ b/src/include/nodes/pathnodes.h
@@ -3394,7 +3394,7 @@ typedef struct AggTransInfo
Oid aggtranstype;
/* Additional data about transtype */
- int32 aggtranstypmod;
+ TypMod aggtranstypmod;
int transtypeLen;
bool transtypeByVal;
diff --git a/src/include/nodes/primnodes.h b/src/include/nodes/primnodes.h
index 557be05657..94282497d7 100644
--- a/src/include/nodes/primnodes.h
+++ b/src/include/nodes/primnodes.h
@@ -29,8 +29,9 @@ typedef enum OverridingKind
OVERRIDING_SYSTEM_VALUE,
} OverridingKind;
-
+/* Type aliases for gen_node_support */
#define Location int
+#define TypMod int32
/* ----------------------------------------------------------------
* node definitions
@@ -250,7 +251,7 @@ typedef struct Var
/* pg_type OID for the type of this var */
Oid vartype pg_node_attr(query_jumble_ignore);
/* pg_attribute typmod value */
- int32 vartypmod pg_node_attr(query_jumble_ignore);
+ TypMod vartypmod pg_node_attr(query_jumble_ignore);
/* OID of collation, or InvalidOid if none */
Oid varcollid pg_node_attr(query_jumble_ignore);
@@ -299,7 +300,7 @@ typedef struct Const
/* pg_type OID of the constant's datatype */
Oid consttype;
/* typmod value, if any */
- int32 consttypmod pg_node_attr(query_jumble_ignore);
+ TypMod consttypmod pg_node_attr(query_jumble_ignore);
/* OID of collation, or InvalidOid if none */
Oid constcollid pg_node_attr(query_jumble_ignore);
/* typlen of the constant's datatype */
@@ -365,7 +366,7 @@ typedef struct Param
int paramid; /* numeric ID for parameter */
Oid paramtype; /* pg_type OID of parameter's datatype */
/* typmod value, if known */
- int32 paramtypmod pg_node_attr(query_jumble_ignore);
+ TypMod paramtypmod pg_node_attr(query_jumble_ignore);
/* OID of collation, or InvalidOid if none */
Oid paramcollid pg_node_attr(query_jumble_ignore);
/* token location, or -1 if unknown */
@@ -623,7 +624,7 @@ typedef struct SubscriptingRef
/* type of the SubscriptingRef's result */
Oid refrestype pg_node_attr(query_jumble_ignore);
/* typmod of the result */
- int32 reftypmod pg_node_attr(query_jumble_ignore);
+ TypMod reftypmod pg_node_attr(query_jumble_ignore);
/* collation of result, or InvalidOid if none */
Oid refcollid pg_node_attr(query_jumble_ignore);
/* expressions that evaluate to upper container indexes */
@@ -1009,7 +1010,7 @@ typedef struct SubPlan
char *plan_name; /* A name assigned during planning */
/* Extra data useful for determining subplan's output type: */
Oid firstColType; /* Type of first column of subplan result */
- int32 firstColTypmod; /* Typmod of first column of subplan result */
+ TypMod firstColTypmod; /* Typmod of first column of subplan result */
Oid firstColCollation; /* Collation of first column of subplan
* result */
/* Information about execution strategy: */
@@ -1067,7 +1068,7 @@ typedef struct FieldSelect
/* type of the field (result type of this node) */
Oid resulttype pg_node_attr(query_jumble_ignore);
/* output typmod (usually -1) */
- int32 resulttypmod pg_node_attr(query_jumble_ignore);
+ TypMod resulttypmod pg_node_attr(query_jumble_ignore);
/* OID of collation of the field */
Oid resultcollid pg_node_attr(query_jumble_ignore);
} FieldSelect;
@@ -1121,7 +1122,7 @@ typedef struct RelabelType
Expr *arg; /* input expression */
Oid resulttype; /* output type of coercion expression */
/* output typmod (usually -1) */
- int32 resulttypmod pg_node_attr(query_jumble_ignore);
+ TypMod resulttypmod pg_node_attr(query_jumble_ignore);
/* OID of collation, or InvalidOid if none */
Oid resultcollid pg_node_attr(query_jumble_ignore);
/* how to display this node */
@@ -1171,7 +1172,7 @@ typedef struct ArrayCoerceExpr
Expr *elemexpr; /* expression representing per-element work */
Oid resulttype; /* output type of coercion (an array type) */
/* output typmod (also element typmod) */
- int32 resulttypmod pg_node_attr(query_jumble_ignore);
+ TypMod resulttypmod pg_node_attr(query_jumble_ignore);
/* OID of collation, or InvalidOid if none */
Oid resultcollid pg_node_attr(query_jumble_ignore);
/* how to display this node */
@@ -1291,7 +1292,7 @@ typedef struct CaseTestExpr
Expr xpr;
Oid typeId; /* type for substituted value */
/* typemod for substituted value */
- int32 typeMod pg_node_attr(query_jumble_ignore);
+ TypMod typeMod pg_node_attr(query_jumble_ignore);
/* collation for the substituted value */
Oid collation pg_node_attr(query_jumble_ignore);
} CaseTestExpr;
@@ -1497,7 +1498,7 @@ typedef struct SQLValueFunction
* include this Oid in the query jumbling.
*/
Oid type pg_node_attr(query_jumble_ignore);
- int32 typmod;
+ TypMod typmod;
Location location; /* token location, or -1 if unknown */
} SQLValueFunction;
@@ -1549,7 +1550,7 @@ typedef struct XmlExpr
bool indent;
/* target type/typmod for XMLSERIALIZE */
Oid type pg_node_attr(query_jumble_ignore);
- int32 typmod pg_node_attr(query_jumble_ignore);
+ TypMod typmod pg_node_attr(query_jumble_ignore);
/* token location, or -1 if unknown */
Location location;
} XmlExpr;
@@ -1599,7 +1600,7 @@ typedef struct JsonReturning
NodeTag type;
JsonFormat *format; /* output JSON format */
Oid typid; /* target type Oid */
- int32 typmod; /* target type modifier */
+ TypMod typmod; /* target type modifier */
} JsonReturning;
/*
@@ -1762,7 +1763,7 @@ typedef struct CoerceToDomain
Expr *arg; /* input expression */
Oid resulttype; /* domain type ID (result type) */
/* output typmod (currently always -1) */
- int32 resulttypmod pg_node_attr(query_jumble_ignore);
+ TypMod resulttypmod pg_node_attr(query_jumble_ignore);
/* OID of collation, or InvalidOid if none */
Oid resultcollid pg_node_attr(query_jumble_ignore);
/* how to display this node */
@@ -1785,7 +1786,7 @@ typedef struct CoerceToDomainValue
/* type for substituted value */
Oid typeId;
/* typemod for substituted value */
- int32 typeMod pg_node_attr(query_jumble_ignore);
+ TypMod typeMod pg_node_attr(query_jumble_ignore);
/* collation for the substituted value */
Oid collation pg_node_attr(query_jumble_ignore);
/* token location, or -1 if unknown */
@@ -1805,7 +1806,7 @@ typedef struct SetToDefault
/* type for substituted value */
Oid typeId;
/* typemod for substituted value */
- int32 typeMod pg_node_attr(query_jumble_ignore);
+ TypMod typeMod pg_node_attr(query_jumble_ignore);
/* collation for the substituted value */
Oid collation pg_node_attr(query_jumble_ignore);
/* token location, or -1 if unknown */
--
2.40.1
[application/octet-stream] v2-0007-gen_node_support.pl-Optimize-serialization-of-fie.patch (9.2K, ../../CAEze2WhTQ1LGtpyobcLFvda61BDtkiYvCfyuUA9-Mi9iwd-gyg@mail.gmail.com/7-v2-0007-gen_node_support.pl-Optimize-serialization-of-fie.patch)
download | inline diff:
From bbc18ad0708da345c770b4fc6e26960ff4316e7f Mon Sep 17 00:00:00 2001
From: Matthias van de Meent <[email protected]>
Date: Mon, 12 Feb 2024 17:12:02 +0100
Subject: [PATCH v2 7/7] gen_node_support.pl: Optimize serialization of fields
with copied values
The Var node's [syn] fields often contain the same data as their non-syn
counterparts. We invent a new pg_node_attr()ibute that represents this
relation, which allows us to omit these fields from serialization when they
are indeed copies of the original fields.
---
src/backend/nodes/gen_node_support.pl | 94 ++++++++++++++++++---------
src/backend/nodes/outfuncs.c | 3 +
src/backend/nodes/readfuncs.c | 3 +
src/include/nodes/primnodes.h | 4 +-
4 files changed, 70 insertions(+), 34 deletions(-)
diff --git a/src/backend/nodes/gen_node_support.pl b/src/backend/nodes/gen_node_support.pl
index fda8c3a494..66661a3881 100644
--- a/src/backend/nodes/gen_node_support.pl
+++ b/src/backend/nodes/gen_node_support.pl
@@ -363,6 +363,10 @@ foreach my $infile (@ARGV)
{
$manual_nodetag_number{$in_struct} = $1;
}
+ elsif ($attr =~ /^default_ref\(([\w\d."'-]+)\)$/)
+ {
+ # Unused at the node level
+ }
else
{
die
@@ -437,7 +441,7 @@ foreach my $infile (@ARGV)
}
# normal struct field
elsif ($line =~
- /^\s*(.+)\s*\b(\w+)(\[[\w\s+]+\])?\s*(?:pg_node_attr\(([\w(), ]*)\))?;/
+ /^\s*(.+)\s*\b(\w+)(\[[\w\s+]+\])?\s*(?:pg_node_attr\(([\w\d(), ."'-]*)\))?;/
)
{
if ($is_node_struct)
@@ -469,6 +473,7 @@ foreach my $infile (@ARGV)
if ( $attr !~ /^array_size\(\w+\)$/
&& $attr !~ /^copy_as\(\w+\)$/
&& $attr !~ /^read_as\(\w+\)$/
+ && $attr !~ /^default_ref\([\w\d."'-]+\)$/
&& !elem $attr,
qw(copy_as_scalar
equal_as_scalar
@@ -495,7 +500,7 @@ foreach my $infile (@ARGV)
}
# function pointer field
elsif ($line =~
- /^\s*([\w\s*]+)\s*\(\*(\w+)\)\s*\((.*)\)\s*(?:pg_node_attr\(([\w(), ]*)\))?;/
+ /^\s*([\w\s*]+)\s*\(\*(\w+)\)\s*\((.*)\)\s*(?:pg_node_attr\(([\w\d(), ."'-]*)\))?;/
)
{
if ($is_node_struct)
@@ -970,6 +975,15 @@ _read${n}(void)
my $array_size_field;
my $read_as_field;
my $read_write_ignore = 0;
+ # Type read/write macro suffix, "" for normal use, or "_DEFAULT" when
+ # value argument.
+ my $s = "";
+ # Default parameter to read/write macro. Includes comma separator so
+ # that MACRO_NAME$s($fieldname$defaultparam) is the full complete
+ # read/write expression for essentially all types.
+ # Note that this (currently) only works for scalar values.
+ my $d = "";
+
foreach my $a (@a)
{
if ($a =~ /^array_size\(([\w.]+)\)$/)
@@ -988,6 +1002,19 @@ _read${n}(void)
{
$read_write_ignore = 1;
}
+ elsif ($a =~ /^default_ref\(([\w\d+."'-]+)\)$/)
+ {
+ $s = "_DEFAULT";
+ $d = ", NODE_FIELD($1)";
+ }
+ }
+
+ if ($s eq "_DEFAULT")
+ {
+ die "custom defaults for non-scalar fields are not supported\n\tat $n.$f"
+ unless (elem $t, @scalar_types or elem $t, @enum_types);
+ die "custom defaults for Location fields are not supported\n\tat $n.$f"
+ if ($t eq "Location");
}
if ($read_write_ignore)
@@ -1008,8 +1035,8 @@ _read${n}(void)
# select instructions by field type
if ($t eq 'bool')
{
- print $off "\tWRITE_BOOL_FIELD($f);\n";
- print $rff "\tREAD_BOOL_FIELD($f);\n" unless $no_read;
+ print $off "\tWRITE_BOOL_FIELD$s($f$d);\n";
+ print $rff "\tREAD_BOOL_FIELD$s($f$d);\n" unless $no_read;
}
elsif ($t eq 'Location')
{
@@ -1018,8 +1045,11 @@ _read${n}(void)
}
elsif ($t eq 'TypMod')
{
- print $off "\tWRITE_INT_FIELD_DEFAULT($f, -1);\n";
- print $rff "\tREAD_INT_FIELD_DEFAULT($f, -1);\n" unless $no_read;
+ # The default value of a TypMod fields is -1, rather than 0
+ # for normal int fields.
+ $d = ", -1" if ($d eq "");
+ print $off "\tWRITE_INT_FIELD_DEFAULT($f$d);\n";
+ print $rff "\tREAD_INT_FIELD_DEFAULT($f$d);\n" unless $no_read;
}
elsif ($t eq 'int'
|| $t eq 'int16'
@@ -1027,8 +1057,8 @@ _read${n}(void)
|| $t eq 'AttrNumber'
|| $t eq 'StrategyNumber')
{
- print $off "\tWRITE_INT_FIELD($f);\n";
- print $rff "\tREAD_INT_FIELD($f);\n" unless $no_read;
+ print $off "\tWRITE_INT_FIELD$s($f$d);\n";
+ print $rff "\tREAD_INT_FIELD$s($f$d);\n" unless $no_read;
}
elsif ($t eq 'uint32'
|| $t eq 'bits32'
@@ -1036,56 +1066,56 @@ _read${n}(void)
|| $t eq 'Index'
|| $t eq 'SubTransactionId')
{
- print $off "\tWRITE_UINT_FIELD($f);\n";
- print $rff "\tREAD_UINT_FIELD($f);\n" unless $no_read;
+ print $off "\tWRITE_UINT_FIELD$s($f$d);\n";
+ print $rff "\tREAD_UINT_FIELD$s($f$d);\n" unless $no_read;
}
elsif ($t eq 'uint64'
|| $t eq 'AclMode')
{
- print $off "\tWRITE_UINT64_FIELD($f);\n";
- print $rff "\tREAD_UINT64_FIELD($f);\n" unless $no_read;
+ print $off "\tWRITE_UINT64_FIELD$s($f$d);\n";
+ print $rff "\tREAD_UINT64_FIELD$s($f$d);\n" unless $no_read;
}
elsif ($t eq 'Oid' || $t eq 'RelFileNumber')
{
- print $off "\tWRITE_OID_FIELD($f);\n";
- print $rff "\tREAD_OID_FIELD($f);\n" unless $no_read;
+ print $off "\tWRITE_OID_FIELD$s($f$d);\n";
+ print $rff "\tREAD_OID_FIELD$s($f$d);\n" unless $no_read;
}
elsif ($t eq 'long')
{
- print $off "\tWRITE_LONG_FIELD($f);\n";
- print $rff "\tREAD_LONG_FIELD($f);\n" unless $no_read;
+ print $off "\tWRITE_LONG_FIELD$s($f$d);\n";
+ print $rff "\tREAD_LONG_FIELD$s($f$d);\n" unless $no_read;
}
elsif ($t eq 'char')
{
- print $off "\tWRITE_CHAR_FIELD($f);\n";
- print $rff "\tREAD_CHAR_FIELD($f);\n" unless $no_read;
+ print $off "\tWRITE_CHAR_FIELD$s($f$d);\n";
+ print $rff "\tREAD_CHAR_FIELD$s($f$d);\n" unless $no_read;
}
elsif ($t eq 'double')
{
- print $off "\tWRITE_FLOAT_FIELD($f);\n";
- print $rff "\tREAD_FLOAT_FIELD($f);\n" unless $no_read;
+ print $off "\tWRITE_FLOAT_FIELD$s($f$d);\n";
+ print $rff "\tREAD_FLOAT_FIELD$s($f$d);\n" unless $no_read;
}
elsif ($t eq 'Cardinality')
{
- print $off "\tWRITE_FLOAT_FIELD($f);\n";
- print $rff "\tREAD_FLOAT_FIELD($f);\n" unless $no_read;
+ print $off "\tWRITE_FLOAT_FIELD$s($f$d);\n";
+ print $rff "\tREAD_FLOAT_FIELD$s($f$d);\n" unless $no_read;
}
elsif ($t eq 'Cost')
{
- print $off "\tWRITE_FLOAT_FIELD($f);\n";
- print $rff "\tREAD_FLOAT_FIELD($f);\n" unless $no_read;
+ print $off "\tWRITE_FLOAT_FIELD$s($f$d);\n";
+ print $rff "\tREAD_FLOAT_FIELD$s($f$d);\n" unless $no_read;
}
elsif ($t eq 'QualCost')
{
- print $off "\tWRITE_FLOAT_FIELD($f.startup);\n";
- print $off "\tWRITE_FLOAT_FIELD($f.per_tuple);\n";
- print $rff "\tREAD_FLOAT_FIELD($f.startup);\n" unless $no_read;
- print $rff "\tREAD_FLOAT_FIELD($f.per_tuple);\n" unless $no_read;
+ print $off "\tWRITE_FLOAT_FIELD$s($f.startup$d);\n";
+ print $off "\tWRITE_FLOAT_FIELD$s($f.per_tuple$d);\n";
+ print $rff "\tREAD_FLOAT_FIELD$s($f.startup$d);\n" unless $no_read;
+ print $rff "\tREAD_FLOAT_FIELD$s($f.per_tuple$d);\n" unless $no_read;
}
elsif ($t eq 'Selectivity')
{
- print $off "\tWRITE_FLOAT_FIELD($f);\n";
- print $rff "\tREAD_FLOAT_FIELD($f);\n" unless $no_read;
+ print $off "\tWRITE_FLOAT_FIELD$s($f$d);\n";
+ print $rff "\tREAD_FLOAT_FIELD$s($f$d);\n" unless $no_read;
}
elsif ($t eq 'char*')
{
@@ -1099,8 +1129,8 @@ _read${n}(void)
}
elsif (elem $t, @enum_types)
{
- print $off "\tWRITE_ENUM_FIELD($f, $t);\n";
- print $rff "\tREAD_ENUM_FIELD($f, $t);\n" unless $no_read;
+ print $off "\tWRITE_ENUM_FIELD$s($f, $t$d);\n";
+ print $rff "\tREAD_ENUM_FIELD$s($f, $t$d);\n" unless $no_read;
}
# arrays of scalar types
elsif ($t =~ /^(\w+)(\*|\[\w+\])$/ and elem $1, @scalar_types)
diff --git a/src/backend/nodes/outfuncs.c b/src/backend/nodes/outfuncs.c
index f135753fb4..75a7703b62 100644
--- a/src/backend/nodes/outfuncs.c
+++ b/src/backend/nodes/outfuncs.c
@@ -240,6 +240,9 @@ static void outDouble(StringInfo str, double d);
} \
} while (0)
+/* Accessing a node's field goes like this */
+#define NODE_FIELD(fldname) (node->fldname)
+
#define booltostr(x) ((x) ? "true" : "false")
diff --git a/src/backend/nodes/readfuncs.c b/src/backend/nodes/readfuncs.c
index 55e40f5603..4d11696e68 100644
--- a/src/backend/nodes/readfuncs.c
+++ b/src/backend/nodes/readfuncs.c
@@ -258,6 +258,9 @@
#define READ_BOOL_ARRAY(fldname, len) \
READ_BOOL_ARRAY_DEFAULT(fldname, len, NULL)
+/* Accessing a node's field goes like this */
+#define NODE_FIELD(fldname) (local_node->fldname)
+
/* Routine exit */
#define READ_DONE() \
return local_node
diff --git a/src/include/nodes/primnodes.h b/src/include/nodes/primnodes.h
index 94282497d7..9dd60ad5c5 100644
--- a/src/include/nodes/primnodes.h
+++ b/src/include/nodes/primnodes.h
@@ -274,9 +274,9 @@ typedef struct Var
* their varno/varattno match.
*/
/* syntactic relation index (0 if unknown) */
- Index varnosyn pg_node_attr(equal_ignore, query_jumble_ignore);
+ Index varnosyn pg_node_attr(equal_ignore, query_jumble_ignore, default_ref(varno));
/* syntactic attribute number */
- AttrNumber varattnosyn pg_node_attr(equal_ignore, query_jumble_ignore);
+ AttrNumber varattnosyn pg_node_attr(equal_ignore, query_jumble_ignore, default_ref(varattno));
/* token location, or -1 if unknown */
Location location;
--
2.40.1
[application/octet-stream] v2-0006-nodeToString-Apply-RLE-on-Bitmapset-and-numeric-L.patch (7.9K, ../../CAEze2WhTQ1LGtpyobcLFvda61BDtkiYvCfyuUA9-Mi9iwd-gyg@mail.gmail.com/8-v2-0006-nodeToString-Apply-RLE-on-Bitmapset-and-numeric-L.patch)
download | inline diff:
From bcdeab761040004e0bd42ca1ae54a3993ca18c42 Mon Sep 17 00:00:00 2001
From: Matthias van de Meent <[email protected]>
Date: Sat, 10 Feb 2024 00:57:19 +0100
Subject: [PATCH v2 6/7] nodeToString: Apply RLE on Bitmapset and numeric List
types
This reduces the size of full serialized queries in pg_rewrite by several %,
reducing overhead in the system.
---
src/backend/nodes/outfuncs.c | 96 +++++++++++++++++++++++++++++++----
src/backend/nodes/read.c | 53 +++++++++++++++++--
src/backend/nodes/readfuncs.c | 17 ++++++-
3 files changed, 150 insertions(+), 16 deletions(-)
diff --git a/src/backend/nodes/outfuncs.c b/src/backend/nodes/outfuncs.c
index 1a17eafd57..f135753fb4 100644
--- a/src/backend/nodes/outfuncs.c
+++ b/src/backend/nodes/outfuncs.c
@@ -378,23 +378,64 @@ writeNodeArray(StringInfo str, const Node *const *arr, int len,
/*
* Print a List.
+ *
+ * Notes:
+ * NodeList is formatted as (<node1> <node2> ...)
+ *
+ * OidList is formatted as (o int +int int int ...), with +int indicating N
+ * successive identical values.
+ *
+ * IntList and XidList are formatted as (i/x int +int int int ...), with +int
+ * indicating N successively increasing values. (i 9 +3) is thus equivalent
+ * to (i 9 10 11 12).
*/
static void
_outList(StringInfo str, const List *node, bool omitLocation)
{
const ListCell *lc;
+ union LCSurrogate {
+ int32 i;
+ Oid o;
+ TransactionId x;
+ } previous = { .i = 0};
+ int run = 0;
+ int run_delta;
+ const char *fmt;
appendStringInfoChar(str, '(');
if (IsA(node, IntList))
+ {
appendStringInfoChar(str, 'i');
+ fmt = " %d";
+ run_delta = 1;
+ }
else if (IsA(node, OidList))
+ {
appendStringInfoChar(str, 'o');
+ fmt = " %u";
+ run_delta = 0;
+ }
else if (IsA(node, XidList))
+ {
appendStringInfoChar(str, 'x');
+ fmt = " %u";
+ run_delta = 1;
+ }
+ else if (IsA(node, List))
+ {
+ /* silence the compiler about uninitialized variables */
+ fmt = "";
+ run_delta = 0;
+ }
+ else
+ elog(ERROR, "unrecognized list node type: %d",
+ (int) node->type);
foreach(lc, node)
{
+ union LCSurrogate val = {.i = 0};
+
/*
* For the sake of backward compatibility, we emit a slightly
* different whitespace format for lists of nodes vs. other types of
@@ -405,26 +446,41 @@ _outList(StringInfo str, const List *node, bool omitLocation)
outNode(str, lfirst(lc), omitLocation);
if (lnext(node, lc))
appendStringInfoChar(str, ' ');
+ continue;
}
else if (IsA(node, IntList))
- appendStringInfo(str, " %d", lfirst_int(lc));
+ val.i = lfirst_int(lc);
else if (IsA(node, OidList))
- appendStringInfo(str, " %u", lfirst_oid(lc));
+ val.o = lfirst_oid(lc);
else if (IsA(node, XidList))
- appendStringInfo(str, " %u", lfirst_xid(lc));
+ val.x = lfirst_xid(lc);
+
+ if (val.i == previous.i + run_delta)
+ run += 1;
else
- elog(ERROR, "unrecognized list node type: %d",
- (int) node->type);
+ {
+ if (run > 0)
+ appendStringInfo(str, " +%d", run);
+ run = 0;
+ appendStringInfo(str, fmt, val.i);
+ }
+ previous = val;
}
- appendStringInfoChar(str, ')');
-}
+ if (run > 0)
+ appendStringInfo(str, " +%d", run);
+
+ appendStringInfoChar(str, ')');}
/*
* outBitmapset -
* converts a bitmap set of integers
*
- * Note: the output format is "(b int int ...)", similar to an integer List.
+ * Note: the output format is "(b int int +int ...)", similar to an
+ * integer List. Note that consecutive runs of incremental values are
+ * indicated by [... int +int ...], where the first int is the first set bit,
+ * and the int that's tagged with the '+' sign that follows is the number of
+ * subsequent set bits (resulting in a run of N+1 set bits).
*
* We export this function for use by extensions that define extensible nodes.
* That's somewhat historical, though, because calling outNode() will work.
@@ -432,13 +488,31 @@ _outList(StringInfo str, const List *node, bool omitLocation)
void
outBitmapset(StringInfo str, const Bitmapset *bms)
{
- int x;
+ int x = 0;
+ int prev = -2;
+ int run = 0;
appendStringInfoChar(str, '(');
appendStringInfoChar(str, 'b');
- x = -1;
+
while ((x = bms_next_member(bms, x)) >= 0)
- appendStringInfo(str, " %d", x);
+ {
+ if (x - prev == 1)
+ run++;
+ else
+ {
+ if (run != 0)
+ appendStringInfo(str, " +%d", run);
+
+ run = 0;
+ appendStringInfo(str, " %d", x);
+ }
+ prev = x;
+ }
+
+ if (run > 0)
+ appendStringInfo(str, " +%d", run);
+
appendStringInfoChar(str, ')');
}
diff --git a/src/backend/nodes/read.c b/src/backend/nodes/read.c
index 14360307a3..ad3ab990a2 100644
--- a/src/backend/nodes/read.c
+++ b/src/backend/nodes/read.c
@@ -402,6 +402,8 @@ nodeRead(const char *token, int tok_len)
elog(ERROR, "unterminated List structure");
if (tok_len == 1 && token[0] == 'i')
{
+ int prev = 0;
+
/* List of integers */
for (;;)
{
@@ -417,12 +419,23 @@ nodeRead(const char *token, int tok_len)
if (endptr != token + tok_len)
elog(ERROR, "unrecognized integer: \"%.*s\"",
tok_len, token);
- l = lappend_int(l, val);
+
+ if (token[0] == '+')
+ {
+ for (int i = 0; i < val; i++)
+ l = lappend_int(l, ++prev);
+ }
+ else
+ {
+ prev = val;
+ l = lappend_int(l, val);
+ }
}
result = (Node *) l;
}
else if (tok_len == 1 && token[0] == 'o')
{
+ Oid prev = 0;
/* List of OIDs */
for (;;)
{
@@ -438,12 +451,23 @@ nodeRead(const char *token, int tok_len)
if (endptr != token + tok_len)
elog(ERROR, "unrecognized OID: \"%.*s\"",
tok_len, token);
- l = lappend_oid(l, val);
+
+ if (token[0] == '+')
+ {
+ for (int i = 0; i < val; i++)
+ l = lappend_oid(l, prev);
+ }
+ else
+ {
+ prev = val;
+ l = lappend_oid(l, val);
+ }
}
result = (Node *) l;
}
else if (tok_len == 1 && token[0] == 'x')
{
+ TransactionId prev = 0;
/* List of TransactionIds */
for (;;)
{
@@ -459,7 +483,17 @@ nodeRead(const char *token, int tok_len)
if (endptr != token + tok_len)
elog(ERROR, "unrecognized Xid: \"%.*s\"",
tok_len, token);
- l = lappend_xid(l, val);
+
+ if (token[0] == '+')
+ {
+ for (int i = 0; i < val; i++)
+ l = lappend_xid(l, ++prev);
+ }
+ else
+ {
+ prev = val;
+ l = lappend_xid(l, val);
+ }
}
result = (Node *) l;
}
@@ -467,6 +501,7 @@ nodeRead(const char *token, int tok_len)
{
/* Bitmapset -- see also _readBitmapset() */
Bitmapset *bms = NULL;
+ int prev = -2;
for (;;)
{
@@ -482,7 +517,17 @@ nodeRead(const char *token, int tok_len)
if (endptr != token + tok_len)
elog(ERROR, "unrecognized integer: \"%.*s\"",
tok_len, token);
- bms = bms_add_member(bms, val);
+
+ if (token[0] == '+')
+ {
+ for (int i = 0; i < val; i++)
+ bms = bms_add_member(bms, ++prev);
+ }
+ else
+ {
+ bms = bms_add_member(bms, val);
+ prev = val;
+ }
}
result = (Node *) bms;
}
diff --git a/src/backend/nodes/readfuncs.c b/src/backend/nodes/readfuncs.c
index 4cdbad9e7e..55e40f5603 100644
--- a/src/backend/nodes/readfuncs.c
+++ b/src/backend/nodes/readfuncs.c
@@ -298,6 +298,7 @@ static Bitmapset *
_readBitmapset(void)
{
Bitmapset *result = NULL;
+ int prev = -2;
READ_TEMP_LOCALS();
@@ -326,7 +327,21 @@ _readBitmapset(void)
val = (int) strtol(token, &endptr, 10);
if (endptr != token + length)
elog(ERROR, "unrecognized integer: \"%.*s\"", length, token);
- result = bms_add_member(result, val);
+
+ if (token[0] == '+')
+ {
+ Assert(prev >= 0);
+ for (int i = 0; i < val; i++)
+ {
+ prev++;
+ result = bms_add_member(result, prev);
+ }
+ }
+ else
+ {
+ result = bms_add_member(result, val);
+ prev = val;
+ }
}
return result;
--
2.40.1
^ permalink raw reply [nested|flat] 69+ messages in thread
* Re: Reducing output size of nodeToString
@ 2024-02-12 19:32 Matthias van de Meent <[email protected]>
parent: Matthias van de Meent <[email protected]>
0 siblings, 1 reply; 69+ messages in thread
From: Matthias van de Meent @ 2024-02-12 19:32 UTC (permalink / raw)
To: ; +Cc: Robert Haas <[email protected]>; Peter Eisentraut <[email protected]>; PostgreSQL Hackers <[email protected]>; Andres Freund <[email protected]>; Michel Pelletier <[email protected]>
On Mon, 12 Feb 2024 at 19:03, Matthias van de Meent
<[email protected]> wrote:
> Attached is patchset v2, which contains the improvements from these patches:
Attached v3, which fixes an out-of-bounds read in pg_strtoken_next,
detected by asan, that was a likely cause of the problems in CFBot's
FreeBSD regression tests.
Kind regards,
Matthias van de Meent
Neon (https://neon.tech)
Attachments:
[application/octet-stream] v3-0002-pg_node_tree-Don-t-store-query-text-locations-in-.patch (19.2K, ../../CAEze2Wg==idqY7UX_+kqOZ6DNVWaqVtXV0dHqXW6oGevFfa91w@mail.gmail.com/2-v3-0002-pg_node_tree-Don-t-store-query-text-locations-in-.patch)
download | inline diff:
From 798e172eb62a71e408bf59a86c1c1d20ced5148f Mon Sep 17 00:00:00 2001
From: Matthias van de Meent <[email protected]>
Date: Wed, 3 Jan 2024 01:39:42 +0100
Subject: [PATCH v3 2/7] pg_node_tree: Don't store query text locations in
pg_node_tree fields.
We don't store original query texts, so any lingering "location" value
can only be useful in forensic debugging. In normal operation, however,
a non-default value will show up as measurable overhead in
serialization, just omit serialization, saving several 10s of kBs.
---
src/backend/catalog/heap.c | 9 ++--
src/backend/catalog/index.c | 4 +-
src/backend/catalog/pg_attrdef.c | 6 ++-
src/backend/catalog/pg_proc.c | 10 +++-
src/backend/catalog/pg_publication.c | 6 ++-
src/backend/commands/policy.c | 11 +++-
src/backend/commands/statscmds.c | 2 +-
src/backend/commands/trigger.c | 3 +-
src/backend/commands/typecmds.c | 8 +--
src/backend/nodes/gen_node_support.pl | 4 +-
src/backend/nodes/outfuncs.c | 77 ++++++++++++++++-----------
src/backend/rewrite/rewriteDefine.c | 4 +-
src/include/nodes/nodes.h | 4 +-
13 files changed, 95 insertions(+), 53 deletions(-)
diff --git a/src/backend/catalog/heap.c b/src/backend/catalog/heap.c
index c73f7bcd01..5c13fdab77 100644
--- a/src/backend/catalog/heap.c
+++ b/src/backend/catalog/heap.c
@@ -2060,9 +2060,9 @@ StoreRelCheck(Relation rel, const char *ccname, Node *expr,
Oid constrOid;
/*
- * Flatten expression to string form for storage.
+ * Flatten expression to string form for storage, without query refs.
*/
- ccbin = nodeToString(expr);
+ ccbin = nodeToStringNoQLocs(expr);
/*
* Find columns of rel that are used in expr
@@ -3676,7 +3676,7 @@ StorePartitionKey(Relation rel,
{
char *exprString;
- exprString = nodeToString(partexprs);
+ exprString = nodeToStringNoQLocs(partexprs);
partexprDatum = CStringGetTextDatum(exprString);
pfree(exprString);
}
@@ -3834,7 +3834,8 @@ StorePartitionBound(Relation rel, Relation parent, PartitionBoundSpec *bound)
memset(new_val, 0, sizeof(new_val));
memset(new_null, false, sizeof(new_null));
memset(new_repl, false, sizeof(new_repl));
- new_val[Anum_pg_class_relpartbound - 1] = CStringGetTextDatum(nodeToString(bound));
+ new_val[Anum_pg_class_relpartbound - 1] =
+ CStringGetTextDatum(nodeToStringNoQLocs(bound));
new_null[Anum_pg_class_relpartbound - 1] = false;
new_repl[Anum_pg_class_relpartbound - 1] = true;
newtuple = heap_modify_tuple(tuple, RelationGetDescr(classRel),
diff --git a/src/backend/catalog/index.c b/src/backend/catalog/index.c
index 4b88a9cb87..3d5f4e53d5 100644
--- a/src/backend/catalog/index.c
+++ b/src/backend/catalog/index.c
@@ -587,7 +587,7 @@ UpdateIndexRelation(Oid indexoid,
{
char *exprsString;
- exprsString = nodeToString(indexInfo->ii_Expressions);
+ exprsString = nodeToStringNoQLocs(indexInfo->ii_Expressions);
exprsDatum = CStringGetTextDatum(exprsString);
pfree(exprsString);
}
@@ -602,7 +602,7 @@ UpdateIndexRelation(Oid indexoid,
{
char *predString;
- predString = nodeToString(make_ands_explicit(indexInfo->ii_Predicate));
+ predString = nodeToStringNoQLocs(make_ands_explicit(indexInfo->ii_Predicate));
predDatum = CStringGetTextDatum(predString);
pfree(predString);
}
diff --git a/src/backend/catalog/pg_attrdef.c b/src/backend/catalog/pg_attrdef.c
index 003ae70b4d..a900c9bb28 100644
--- a/src/backend/catalog/pg_attrdef.c
+++ b/src/backend/catalog/pg_attrdef.c
@@ -23,6 +23,7 @@
#include "catalog/objectaccess.h"
#include "catalog/pg_attrdef.h"
#include "executor/executor.h"
+#include "nodes/nodeFuncs.h"
#include "optimizer/optimizer.h"
#include "utils/array.h"
#include "utils/builtins.h"
@@ -62,9 +63,10 @@ StoreAttrDefault(Relation rel, AttrNumber attnum,
adrel = table_open(AttrDefaultRelationId, RowExclusiveLock);
/*
- * Flatten expression to string form for storage.
+ * Flatten expression to string form for storage, without references to
+ * the original query string.
*/
- adbin = nodeToString(expr);
+ adbin = nodeToStringNoQLocs(expr);
/*
* Make the pg_attrdef entry.
diff --git a/src/backend/catalog/pg_proc.c b/src/backend/catalog/pg_proc.c
index b581d334d3..c5790a2224 100644
--- a/src/backend/catalog/pg_proc.c
+++ b/src/backend/catalog/pg_proc.c
@@ -331,7 +331,10 @@ ProcedureCreate(const char *procedureName,
else
nulls[Anum_pg_proc_proargnames - 1] = true;
if (parameterDefaults != NIL)
- values[Anum_pg_proc_proargdefaults - 1] = CStringGetTextDatum(nodeToString(parameterDefaults));
+ {
+ values[Anum_pg_proc_proargdefaults - 1] =
+ CStringGetTextDatum(nodeToStringNoQLocs(parameterDefaults));
+ }
else
nulls[Anum_pg_proc_proargdefaults - 1] = true;
if (trftypes != PointerGetDatum(NULL))
@@ -344,7 +347,10 @@ ProcedureCreate(const char *procedureName,
else
nulls[Anum_pg_proc_probin - 1] = true;
if (prosqlbody)
- values[Anum_pg_proc_prosqlbody - 1] = CStringGetTextDatum(nodeToString(prosqlbody));
+ {
+ values[Anum_pg_proc_prosqlbody - 1] =
+ CStringGetTextDatum(nodeToStringNoQLocs(prosqlbody));
+ }
else
nulls[Anum_pg_proc_prosqlbody - 1] = true;
if (proconfig != PointerGetDatum(NULL))
diff --git a/src/backend/catalog/pg_publication.c b/src/backend/catalog/pg_publication.c
index b98b0ce0ae..b201313430 100644
--- a/src/backend/catalog/pg_publication.c
+++ b/src/backend/catalog/pg_publication.c
@@ -36,6 +36,7 @@
#include "commands/publicationcmds.h"
#include "funcapi.h"
#include "miscadmin.h"
+#include "nodes/nodeFuncs.h"
#include "utils/array.h"
#include "utils/builtins.h"
#include "utils/catcache.h"
@@ -422,7 +423,10 @@ publication_add_relation(Oid pubid, PublicationRelInfo *pri,
/* Add qualifications, if available */
if (pri->whereClause != NULL)
- values[Anum_pg_publication_rel_prqual - 1] = CStringGetTextDatum(nodeToString(pri->whereClause));
+ {
+ values[Anum_pg_publication_rel_prqual - 1] =
+ CStringGetTextDatum(nodeToStringNoQLocs(pri->whereClause));
+ }
else
nulls[Anum_pg_publication_rel_prqual - 1] = true;
diff --git a/src/backend/commands/policy.c b/src/backend/commands/policy.c
index 596326e5ec..1e6842bf41 100644
--- a/src/backend/commands/policy.c
+++ b/src/backend/commands/policy.c
@@ -30,6 +30,7 @@
#include "commands/policy.h"
#include "miscadmin.h"
#include "nodes/makefuncs.h"
+#include "nodes/nodeFuncs.h"
#include "nodes/pg_list.h"
#include "parser/parse_clause.h"
#include "parser/parse_collate.h"
@@ -701,13 +702,19 @@ CreatePolicy(CreatePolicyStmt *stmt)
/* Add qual if present. */
if (qual)
- values[Anum_pg_policy_polqual - 1] = CStringGetTextDatum(nodeToString(qual));
+ {
+ values[Anum_pg_policy_polqual - 1] =
+ CStringGetTextDatum(nodeToStringNoQLocs(qual));
+ }
else
isnull[Anum_pg_policy_polqual - 1] = true;
/* Add WITH CHECK qual if present */
if (with_check_qual)
- values[Anum_pg_policy_polwithcheck - 1] = CStringGetTextDatum(nodeToString(with_check_qual));
+ {
+ values[Anum_pg_policy_polwithcheck - 1] =
+ CStringGetTextDatum(nodeToStringNoQLocs(with_check_qual));
+ }
else
isnull[Anum_pg_policy_polwithcheck - 1] = true;
diff --git a/src/backend/commands/statscmds.c b/src/backend/commands/statscmds.c
index b1a9c74bd6..58e8133f93 100644
--- a/src/backend/commands/statscmds.c
+++ b/src/backend/commands/statscmds.c
@@ -477,7 +477,7 @@ CreateStatistics(CreateStatsStmt *stmt)
{
char *exprsString;
- exprsString = nodeToString(stxexprs);
+ exprsString = nodeToStringNoQLocs(stxexprs);
exprsDatum = CStringGetTextDatum(exprsString);
pfree(exprsString);
}
diff --git a/src/backend/commands/trigger.c b/src/backend/commands/trigger.c
index c344ff0944..6a3dc13a67 100644
--- a/src/backend/commands/trigger.c
+++ b/src/backend/commands/trigger.c
@@ -39,6 +39,7 @@
#include "miscadmin.h"
#include "nodes/bitmapset.h"
#include "nodes/makefuncs.h"
+#include "nodes/nodeFuncs.h"
#include "optimizer/optimizer.h"
#include "parser/parse_clause.h"
#include "parser/parse_collate.h"
@@ -674,7 +675,7 @@ CreateTriggerFiringOn(CreateTrigStmt *stmt, const char *queryString,
/* we'll need the rtable for recordDependencyOnExpr */
whenRtable = pstate->p_rtable;
- qual = nodeToString(whenClause);
+ qual = nodeToStringNoQLocs(whenClause);
free_parsestate(pstate);
}
diff --git a/src/backend/commands/typecmds.c b/src/backend/commands/typecmds.c
index a400fb39f6..00180a54b9 100644
--- a/src/backend/commands/typecmds.c
+++ b/src/backend/commands/typecmds.c
@@ -58,6 +58,7 @@
#include "executor/executor.h"
#include "miscadmin.h"
#include "nodes/makefuncs.h"
+#include "nodes/nodeFuncs.h"
#include "optimizer/optimizer.h"
#include "parser/parse_coerce.h"
#include "parser/parse_collate.h"
@@ -924,7 +925,7 @@ DefineDomain(CreateDomainStmt *stmt)
defaultValue =
deparse_expression(defaultExpr,
NIL, false, false);
- defaultValueBin = nodeToString(defaultExpr);
+ defaultValueBin = nodeToStringNoQLocs(defaultExpr);
}
}
else
@@ -3506,9 +3507,10 @@ domainAddConstraint(Oid domainOid, Oid domainNamespace, Oid baseTypeOid,
errmsg("cannot use table references in domain check constraint")));
/*
- * Convert to string form for storage.
+ * Convert to string form for storage, without references to the original
+ * query text.
*/
- ccbin = nodeToString(expr);
+ ccbin = nodeToStringNoQLocs(expr);
/*
* Store the constraint in pg_constraint
diff --git a/src/backend/nodes/gen_node_support.pl b/src/backend/nodes/gen_node_support.pl
index 2f0a59bc87..487f6f7728 100644
--- a/src/backend/nodes/gen_node_support.pl
+++ b/src/backend/nodes/gen_node_support.pl
@@ -921,7 +921,7 @@ foreach my $n (@node_types)
my $N = uc $n;
print $ofs "\t\t\tcase T_${n}:\n"
- . "\t\t\t\t_out${n}(str, obj);\n"
+ . "\t\t\t\t_out${n}(str, obj, omitLocation);\n"
. "\t\t\t\tbreak;\n";
print $rfs "\tif (MATCH(\"$N\", "
@@ -933,7 +933,7 @@ foreach my $n (@node_types)
print $off "
static void
-_out${n}(StringInfo str, const $n *node)
+_out${n}(StringInfo str, const $n *node, bool omitLocation)
{
\tWRITE_NODE_TYPE(\"$N\");
diff --git a/src/backend/nodes/outfuncs.c b/src/backend/nodes/outfuncs.c
index 50da80ee34..eca3160104 100644
--- a/src/backend/nodes/outfuncs.c
+++ b/src/backend/nodes/outfuncs.c
@@ -154,7 +154,7 @@ static void outDouble(StringInfo str, double d);
/* Write a parse location field (actually same as INT case) */
#define WRITE_LOCATION_FIELD(fldname) \
do { \
- if (node->fldname != -1) \
+ if (node->fldname != -1 && !omitLocation) \
appendStringInfo(str, " :" CppAsString(fldname) " %d", \
node->fldname); \
} while (0)
@@ -165,7 +165,7 @@ static void outDouble(StringInfo str, double d);
if (node->fldname != NULL) \
{ \
appendStringInfoString(str, " :" CppAsString(fldname) " "); \
- outNode(str, node->fldname); \
+ outNode(str, node->fldname, omitLocation); \
} \
} while (0)
@@ -185,7 +185,8 @@ static void outDouble(StringInfo str, double d);
if (node->fldname != NULL) \
{ \
appendStringInfoString(str, " :" CppAsString(fldname) " "); \
- writeNodeArray(str, (const Node * const *) node->fldname, len); \
+ writeNodeArray(str, (const Node * const *) node->fldname, len, \
+ omitLocation); \
} \
} while (0)
@@ -358,7 +359,8 @@ WRITE_SCALAR_ARRAY(writeBoolCols, bool, " %s", booltostr)
* quite use appendStringInfo() in the loop.
*/
static void
-writeNodeArray(StringInfo str, const Node *const *arr, int len)
+writeNodeArray(StringInfo str, const Node *const *arr, int len,
+ bool omitLocation)
{
if (arr != NULL)
{
@@ -366,7 +368,7 @@ writeNodeArray(StringInfo str, const Node *const *arr, int len)
for (int i = 0; i < len; i++)
{
appendStringInfoChar(str, ' ');
- outNode(str, arr[i]);
+ outNode(str, arr[i], omitLocation);
}
appendStringInfoChar(str, ')');
}
@@ -378,7 +380,7 @@ writeNodeArray(StringInfo str, const Node *const *arr, int len)
* Print a List.
*/
static void
-_outList(StringInfo str, const List *node)
+_outList(StringInfo str, const List *node, bool omitLocation)
{
const ListCell *lc;
@@ -400,7 +402,7 @@ _outList(StringInfo str, const List *node)
*/
if (IsA(node, List))
{
- outNode(str, lfirst(lc));
+ outNode(str, lfirst(lc), omitLocation);
if (lnext(node, lc))
appendStringInfoChar(str, ' ');
}
@@ -485,7 +487,7 @@ outDatum(StringInfo str, Datum value, int typlen, bool typbyval)
*/
static void
-_outConst(StringInfo str, const Const *node)
+_outConst(StringInfo str, const Const *node, bool omitLocation)
{
WRITE_NODE_TYPE("CONST");
@@ -505,7 +507,7 @@ _outConst(StringInfo str, const Const *node)
}
static void
-_outBoolExpr(StringInfo str, const BoolExpr *node)
+_outBoolExpr(StringInfo str, const BoolExpr *node, bool omitLocation)
{
char *opstr = NULL;
@@ -532,7 +534,8 @@ _outBoolExpr(StringInfo str, const BoolExpr *node)
}
static void
-_outForeignKeyOptInfo(StringInfo str, const ForeignKeyOptInfo *node)
+_outForeignKeyOptInfo(StringInfo str, const ForeignKeyOptInfo *node,
+ bool omitLocation)
{
int i;
@@ -558,7 +561,8 @@ _outForeignKeyOptInfo(StringInfo str, const ForeignKeyOptInfo *node)
}
static void
-_outEquivalenceClass(StringInfo str, const EquivalenceClass *node)
+_outEquivalenceClass(StringInfo str, const EquivalenceClass *node,
+ bool omitLocation)
{
/*
* To simplify reading, we just chase up to the topmost merged EC and
@@ -584,7 +588,8 @@ _outEquivalenceClass(StringInfo str, const EquivalenceClass *node)
}
static void
-_outExtensibleNode(StringInfo str, const ExtensibleNode *node)
+_outExtensibleNode(StringInfo str, const ExtensibleNode *node,
+ bool omitLocation)
{
const ExtensibleNodeMethods *methods;
@@ -599,7 +604,8 @@ _outExtensibleNode(StringInfo str, const ExtensibleNode *node)
}
static void
-_outRangeTblEntry(StringInfo str, const RangeTblEntry *node)
+_outRangeTblEntry(StringInfo str, const RangeTblEntry *node,
+ bool omitLocation)
{
WRITE_NODE_TYPE("RANGETBLENTRY");
@@ -679,7 +685,7 @@ _outRangeTblEntry(StringInfo str, const RangeTblEntry *node)
}
static void
-_outA_Expr(StringInfo str, const A_Expr *node)
+_outA_Expr(StringInfo str, const A_Expr *node, bool omitLocation)
{
WRITE_NODE_TYPE("A_EXPR");
@@ -751,13 +757,13 @@ _outA_Expr(StringInfo str, const A_Expr *node)
}
static void
-_outInteger(StringInfo str, const Integer *node)
+_outInteger(StringInfo str, const Integer *node, bool omitLocation)
{
appendStringInfo(str, "%d", node->ival);
}
static void
-_outFloat(StringInfo str, const Float *node)
+_outFloat(StringInfo str, const Float *node, bool omitLocation)
{
/*
* We assume the value is a valid numeric literal and so does not need
@@ -767,13 +773,13 @@ _outFloat(StringInfo str, const Float *node)
}
static void
-_outBoolean(StringInfo str, const Boolean *node)
+_outBoolean(StringInfo str, const Boolean *node, bool omitLocation)
{
appendStringInfoString(str, node->boolval ? "true" : "false");
}
static void
-_outString(StringInfo str, const String *node)
+_outString(StringInfo str, const String *node, bool omitLocation)
{
/*
* We use outToken to provide escaping of the string's content, but we
@@ -787,14 +793,14 @@ _outString(StringInfo str, const String *node)
}
static void
-_outBitString(StringInfo str, const BitString *node)
+_outBitString(StringInfo str, const BitString *node, bool omitLocation)
{
/* internal representation already has leading 'b' */
appendStringInfoString(str, node->bsval);
}
static void
-_outA_Const(StringInfo str, const A_Const *node)
+_outA_Const(StringInfo str, const A_Const *node, bool omitLocation)
{
WRITE_NODE_TYPE("A_CONST");
@@ -803,13 +809,13 @@ _outA_Const(StringInfo str, const A_Const *node)
else
{
appendStringInfoString(str, " :val ");
- outNode(str, &node->val);
+ outNode(str, &node->val, omitLocation);
}
WRITE_LOCATION_FIELD(location);
}
static void
-_outConstraint(StringInfo str, const Constraint *node)
+_outConstraint(StringInfo str, const Constraint *node, bool omitLocation)
{
WRITE_NODE_TYPE("CONSTRAINT");
@@ -942,7 +948,7 @@ _outConstraint(StringInfo str, const Constraint *node)
* converts a Node into ascii string and append it to 'str'
*/
void
-outNode(StringInfo str, const void *obj)
+outNode(StringInfo str, const void *obj, bool omitLocation)
{
/* Guard against stack overflow due to overly complex expressions */
check_stack_depth();
@@ -951,18 +957,18 @@ outNode(StringInfo str, const void *obj)
appendStringInfoString(str, "<>");
else if (IsA(obj, List) || IsA(obj, IntList) || IsA(obj, OidList) ||
IsA(obj, XidList))
- _outList(str, obj);
+ _outList(str, obj, omitLocation);
/* nodeRead does not want to see { } around these! */
else if (IsA(obj, Integer))
- _outInteger(str, (Integer *) obj);
+ _outInteger(str, (Integer *) obj, omitLocation);
else if (IsA(obj, Float))
- _outFloat(str, (Float *) obj);
+ _outFloat(str, (Float *) obj, omitLocation);
else if (IsA(obj, Boolean))
- _outBoolean(str, (Boolean *) obj);
+ _outBoolean(str, (Boolean *) obj, omitLocation);
else if (IsA(obj, String))
- _outString(str, (String *) obj);
+ _outString(str, (String *) obj, omitLocation);
else if (IsA(obj, BitString))
- _outBitString(str, (BitString *) obj);
+ _outBitString(str, (BitString *) obj, omitLocation);
else if (IsA(obj, Bitmapset))
outBitmapset(str, (Bitmapset *) obj);
else
@@ -997,7 +1003,18 @@ nodeToString(const void *obj)
/* see stringinfo.h for an explanation of this maneuver */
initStringInfo(&str);
- outNode(&str, obj);
+ outNode(&str, obj, false);
+ return str.data;
+}
+
+char *
+nodeToStringNoQLocs(const void *obj)
+{
+ StringInfoData str;
+
+ /* see stringinfo.h for an explanation of this maneuver */
+ initStringInfo(&str);
+ outNode(&str, obj, true);
return str.data;
}
diff --git a/src/backend/rewrite/rewriteDefine.c b/src/backend/rewrite/rewriteDefine.c
index b449244a53..6302cd1472 100644
--- a/src/backend/rewrite/rewriteDefine.c
+++ b/src/backend/rewrite/rewriteDefine.c
@@ -64,8 +64,8 @@ InsertRule(const char *rulname,
List *action,
bool replace)
{
- char *evqual = nodeToString(event_qual);
- char *actiontree = nodeToString((Node *) action);
+ char *evqual = nodeToStringNoQLocs(event_qual);
+ char *actiontree = nodeToStringNoQLocs((Node *) action);
Datum values[Natts_pg_rewrite];
bool nulls[Natts_pg_rewrite] = {0};
NameData rname;
diff --git a/src/include/nodes/nodes.h b/src/include/nodes/nodes.h
index 2969dd831b..f7adb5e767 100644
--- a/src/include/nodes/nodes.h
+++ b/src/include/nodes/nodes.h
@@ -188,13 +188,15 @@ castNodeImpl(NodeTag type, void *ptr)
struct Bitmapset; /* not to include bitmapset.h here */
struct StringInfoData; /* not to include stringinfo.h here */
-extern void outNode(struct StringInfoData *str, const void *obj);
+extern void outNode(struct StringInfoData *str, const void *obj,
+ bool omitLocation);
extern void outToken(struct StringInfoData *str, const char *s);
extern void outBitmapset(struct StringInfoData *str,
const struct Bitmapset *bms);
extern void outDatum(struct StringInfoData *str, uintptr_t value,
int typlen, bool typbyval);
extern char *nodeToString(const void *obj);
+extern char *nodeToStringNoQLocs(const void *obj);
extern char *bmsToString(const struct Bitmapset *bms);
/*
--
2.40.1
[application/octet-stream] v3-0004-gen_node_support.pl-Add-a-TypMod-type-for-signall.patch (10.4K, ../../CAEze2Wg==idqY7UX_+kqOZ6DNVWaqVtXV0dHqXW6oGevFfa91w@mail.gmail.com/3-v3-0004-gen_node_support.pl-Add-a-TypMod-type-for-signall.patch)
download | inline diff:
From 13bcebe36548b9cb96eaf3b6e24f6049c14757ac Mon Sep 17 00:00:00 2001
From: Matthias van de Meent <[email protected]>
Date: Thu, 8 Feb 2024 17:37:39 +0100
Subject: [PATCH v3 4/7] gen_node_support.pl: Add a TypMod type for signalling
TypMod behaviour
Like Location, TypMod has its own default (-1). By using a type, we can
omit adding pg_node_attribute(default(-1)) markers to every typmod-valued
field, whilst still getting the benefits of a smaller size in serialization.
---
src/backend/nodes/gen_node_support.pl | 8 ++++++-
src/backend/nodes/outfuncs.c | 2 +-
src/backend/nodes/readfuncs.c | 2 +-
src/include/nodes/parsenodes.h | 4 ++--
src/include/nodes/pathnodes.h | 2 +-
src/include/nodes/primnodes.h | 33 ++++++++++++++-------------
6 files changed, 29 insertions(+), 22 deletions(-)
diff --git a/src/backend/nodes/gen_node_support.pl b/src/backend/nodes/gen_node_support.pl
index ec850c3484..fda8c3a494 100644
--- a/src/backend/nodes/gen_node_support.pl
+++ b/src/backend/nodes/gen_node_support.pl
@@ -135,7 +135,8 @@ my @nodetag_only;
# types that are copied by straight assignment
my @scalar_types = qw(
bits32 bool char double int int8 int16 int32 int64 long uint8 uint16 uint32 uint64
- AclMode AttrNumber Cardinality Cost Index Oid RelFileNumber Selectivity Size StrategyNumber SubTransactionId TimeLineID XLogRecPtr
+ AclMode AttrNumber Cardinality Cost Index Oid RelFileNumber Selectivity Size StrategyNumber
+ SubTransactionId TimeLineID XLogRecPtr TypMod
);
# collect enum types
@@ -1015,6 +1016,11 @@ _read${n}(void)
print $off "\tWRITE_LOCATION_FIELD($f);\n";
print $rff "\tREAD_LOCATION_FIELD($f);\n" unless $no_read;
}
+ elsif ($t eq 'TypMod')
+ {
+ print $off "\tWRITE_INT_FIELD_DEFAULT($f, -1);\n";
+ print $rff "\tREAD_INT_FIELD_DEFAULT($f, -1);\n" unless $no_read;
+ }
elsif ($t eq 'int'
|| $t eq 'int16'
|| $t eq 'int32'
diff --git a/src/backend/nodes/outfuncs.c b/src/backend/nodes/outfuncs.c
index eca3160104..3973c0e489 100644
--- a/src/backend/nodes/outfuncs.c
+++ b/src/backend/nodes/outfuncs.c
@@ -492,7 +492,7 @@ _outConst(StringInfo str, const Const *node, bool omitLocation)
WRITE_NODE_TYPE("CONST");
WRITE_OID_FIELD(consttype);
- WRITE_INT_FIELD(consttypmod);
+ WRITE_INT_FIELD_DEFAULT(consttypmod, -1);
WRITE_OID_FIELD(constcollid);
WRITE_INT_FIELD(constlen);
WRITE_BOOL_FIELD(constbyval);
diff --git a/src/backend/nodes/readfuncs.c b/src/backend/nodes/readfuncs.c
index 47f4ba2695..44ab140799 100644
--- a/src/backend/nodes/readfuncs.c
+++ b/src/backend/nodes/readfuncs.c
@@ -356,7 +356,7 @@ _readConst(void)
READ_LOCALS(Const);
READ_OID_FIELD(consttype);
- READ_INT_FIELD(consttypmod);
+ READ_INT_FIELD_DEFAULT(consttypmod, -1);
READ_OID_FIELD(constcollid);
READ_INT_FIELD(constlen);
READ_BOOL_FIELD(constbyval);
diff --git a/src/include/nodes/parsenodes.h b/src/include/nodes/parsenodes.h
index dd8aa30aaf..52d8504a9d 100644
--- a/src/include/nodes/parsenodes.h
+++ b/src/include/nodes/parsenodes.h
@@ -260,7 +260,7 @@ typedef struct TypeName
bool setof; /* is a set? */
bool pct_type; /* %TYPE specified? */
List *typmods; /* type modifier expression(s) */
- int32 typemod; /* prespecified type modifier */
+ TypMod typemod; /* prespecified type modifier */
List *arrayBounds; /* array bounds */
Location location; /* token location, or -1 if unknown */
} TypeName;
@@ -1608,7 +1608,7 @@ typedef struct CTECycleClause
Location location;
/* These fields are set during parse analysis: */
Oid cycle_mark_type; /* common type of _value and _default */
- int cycle_mark_typmod;
+ TypMod cycle_mark_typmod;
Oid cycle_mark_collation;
Oid cycle_mark_neop; /* <> operator for type */
} CTECycleClause;
diff --git a/src/include/nodes/pathnodes.h b/src/include/nodes/pathnodes.h
index 534692bee1..f7c2496a7f 100644
--- a/src/include/nodes/pathnodes.h
+++ b/src/include/nodes/pathnodes.h
@@ -3394,7 +3394,7 @@ typedef struct AggTransInfo
Oid aggtranstype;
/* Additional data about transtype */
- int32 aggtranstypmod;
+ TypMod aggtranstypmod;
int transtypeLen;
bool transtypeByVal;
diff --git a/src/include/nodes/primnodes.h b/src/include/nodes/primnodes.h
index 557be05657..94282497d7 100644
--- a/src/include/nodes/primnodes.h
+++ b/src/include/nodes/primnodes.h
@@ -29,8 +29,9 @@ typedef enum OverridingKind
OVERRIDING_SYSTEM_VALUE,
} OverridingKind;
-
+/* Type aliases for gen_node_support */
#define Location int
+#define TypMod int32
/* ----------------------------------------------------------------
* node definitions
@@ -250,7 +251,7 @@ typedef struct Var
/* pg_type OID for the type of this var */
Oid vartype pg_node_attr(query_jumble_ignore);
/* pg_attribute typmod value */
- int32 vartypmod pg_node_attr(query_jumble_ignore);
+ TypMod vartypmod pg_node_attr(query_jumble_ignore);
/* OID of collation, or InvalidOid if none */
Oid varcollid pg_node_attr(query_jumble_ignore);
@@ -299,7 +300,7 @@ typedef struct Const
/* pg_type OID of the constant's datatype */
Oid consttype;
/* typmod value, if any */
- int32 consttypmod pg_node_attr(query_jumble_ignore);
+ TypMod consttypmod pg_node_attr(query_jumble_ignore);
/* OID of collation, or InvalidOid if none */
Oid constcollid pg_node_attr(query_jumble_ignore);
/* typlen of the constant's datatype */
@@ -365,7 +366,7 @@ typedef struct Param
int paramid; /* numeric ID for parameter */
Oid paramtype; /* pg_type OID of parameter's datatype */
/* typmod value, if known */
- int32 paramtypmod pg_node_attr(query_jumble_ignore);
+ TypMod paramtypmod pg_node_attr(query_jumble_ignore);
/* OID of collation, or InvalidOid if none */
Oid paramcollid pg_node_attr(query_jumble_ignore);
/* token location, or -1 if unknown */
@@ -623,7 +624,7 @@ typedef struct SubscriptingRef
/* type of the SubscriptingRef's result */
Oid refrestype pg_node_attr(query_jumble_ignore);
/* typmod of the result */
- int32 reftypmod pg_node_attr(query_jumble_ignore);
+ TypMod reftypmod pg_node_attr(query_jumble_ignore);
/* collation of result, or InvalidOid if none */
Oid refcollid pg_node_attr(query_jumble_ignore);
/* expressions that evaluate to upper container indexes */
@@ -1009,7 +1010,7 @@ typedef struct SubPlan
char *plan_name; /* A name assigned during planning */
/* Extra data useful for determining subplan's output type: */
Oid firstColType; /* Type of first column of subplan result */
- int32 firstColTypmod; /* Typmod of first column of subplan result */
+ TypMod firstColTypmod; /* Typmod of first column of subplan result */
Oid firstColCollation; /* Collation of first column of subplan
* result */
/* Information about execution strategy: */
@@ -1067,7 +1068,7 @@ typedef struct FieldSelect
/* type of the field (result type of this node) */
Oid resulttype pg_node_attr(query_jumble_ignore);
/* output typmod (usually -1) */
- int32 resulttypmod pg_node_attr(query_jumble_ignore);
+ TypMod resulttypmod pg_node_attr(query_jumble_ignore);
/* OID of collation of the field */
Oid resultcollid pg_node_attr(query_jumble_ignore);
} FieldSelect;
@@ -1121,7 +1122,7 @@ typedef struct RelabelType
Expr *arg; /* input expression */
Oid resulttype; /* output type of coercion expression */
/* output typmod (usually -1) */
- int32 resulttypmod pg_node_attr(query_jumble_ignore);
+ TypMod resulttypmod pg_node_attr(query_jumble_ignore);
/* OID of collation, or InvalidOid if none */
Oid resultcollid pg_node_attr(query_jumble_ignore);
/* how to display this node */
@@ -1171,7 +1172,7 @@ typedef struct ArrayCoerceExpr
Expr *elemexpr; /* expression representing per-element work */
Oid resulttype; /* output type of coercion (an array type) */
/* output typmod (also element typmod) */
- int32 resulttypmod pg_node_attr(query_jumble_ignore);
+ TypMod resulttypmod pg_node_attr(query_jumble_ignore);
/* OID of collation, or InvalidOid if none */
Oid resultcollid pg_node_attr(query_jumble_ignore);
/* how to display this node */
@@ -1291,7 +1292,7 @@ typedef struct CaseTestExpr
Expr xpr;
Oid typeId; /* type for substituted value */
/* typemod for substituted value */
- int32 typeMod pg_node_attr(query_jumble_ignore);
+ TypMod typeMod pg_node_attr(query_jumble_ignore);
/* collation for the substituted value */
Oid collation pg_node_attr(query_jumble_ignore);
} CaseTestExpr;
@@ -1497,7 +1498,7 @@ typedef struct SQLValueFunction
* include this Oid in the query jumbling.
*/
Oid type pg_node_attr(query_jumble_ignore);
- int32 typmod;
+ TypMod typmod;
Location location; /* token location, or -1 if unknown */
} SQLValueFunction;
@@ -1549,7 +1550,7 @@ typedef struct XmlExpr
bool indent;
/* target type/typmod for XMLSERIALIZE */
Oid type pg_node_attr(query_jumble_ignore);
- int32 typmod pg_node_attr(query_jumble_ignore);
+ TypMod typmod pg_node_attr(query_jumble_ignore);
/* token location, or -1 if unknown */
Location location;
} XmlExpr;
@@ -1599,7 +1600,7 @@ typedef struct JsonReturning
NodeTag type;
JsonFormat *format; /* output JSON format */
Oid typid; /* target type Oid */
- int32 typmod; /* target type modifier */
+ TypMod typmod; /* target type modifier */
} JsonReturning;
/*
@@ -1762,7 +1763,7 @@ typedef struct CoerceToDomain
Expr *arg; /* input expression */
Oid resulttype; /* domain type ID (result type) */
/* output typmod (currently always -1) */
- int32 resulttypmod pg_node_attr(query_jumble_ignore);
+ TypMod resulttypmod pg_node_attr(query_jumble_ignore);
/* OID of collation, or InvalidOid if none */
Oid resultcollid pg_node_attr(query_jumble_ignore);
/* how to display this node */
@@ -1785,7 +1786,7 @@ typedef struct CoerceToDomainValue
/* type for substituted value */
Oid typeId;
/* typemod for substituted value */
- int32 typeMod pg_node_attr(query_jumble_ignore);
+ TypMod typeMod pg_node_attr(query_jumble_ignore);
/* collation for the substituted value */
Oid collation pg_node_attr(query_jumble_ignore);
/* token location, or -1 if unknown */
@@ -1805,7 +1806,7 @@ typedef struct SetToDefault
/* type for substituted value */
Oid typeId;
/* typemod for substituted value */
- int32 typeMod pg_node_attr(query_jumble_ignore);
+ TypMod typeMod pg_node_attr(query_jumble_ignore);
/* collation for the substituted value */
Oid collation pg_node_attr(query_jumble_ignore);
/* token location, or -1 if unknown */
--
2.40.1
[application/octet-stream] v3-0005-nodeToString-omit-serializing-NULL-datums-in-Cons.patch (1.9K, ../../CAEze2Wg==idqY7UX_+kqOZ6DNVWaqVtXV0dHqXW6oGevFfa91w@mail.gmail.com/4-v3-0005-nodeToString-omit-serializing-NULL-datums-in-Cons.patch)
download | inline diff:
From 5d6307a2a08110bfc15293ea569238f6d25f0fe4 Mon Sep 17 00:00:00 2001
From: Matthias van de Meent <[email protected]>
Date: Thu, 8 Feb 2024 20:24:22 +0100
Subject: [PATCH v3 5/7] nodeToString: omit serializing NULL datums in Const
nodes
This saves some bytes in certain cases, and aligns its serialization conditions
with other field's serialization conditions.
---
src/backend/nodes/outfuncs.c | 8 ++++----
src/backend/nodes/readfuncs.c | 14 ++++++++++----
2 files changed, 14 insertions(+), 8 deletions(-)
diff --git a/src/backend/nodes/outfuncs.c b/src/backend/nodes/outfuncs.c
index 3973c0e489..1a17eafd57 100644
--- a/src/backend/nodes/outfuncs.c
+++ b/src/backend/nodes/outfuncs.c
@@ -499,11 +499,11 @@ _outConst(StringInfo str, const Const *node, bool omitLocation)
WRITE_BOOL_FIELD(constisnull);
WRITE_LOCATION_FIELD(location);
- appendStringInfoString(str, " :constvalue ");
- if (node->constisnull)
- appendStringInfoString(str, "<>");
- else
+ if (!node->constisnull)
+ {
+ appendStringInfoString(str, " :constvalue ");
outDatum(str, node->constvalue, node->constlen, node->constbyval);
+ }
}
static void
diff --git a/src/backend/nodes/readfuncs.c b/src/backend/nodes/readfuncs.c
index 44ab140799..4cdbad9e7e 100644
--- a/src/backend/nodes/readfuncs.c
+++ b/src/backend/nodes/readfuncs.c
@@ -363,11 +363,17 @@ _readConst(void)
READ_BOOL_FIELD(constisnull);
READ_LOCATION_FIELD(location);
- token = pg_strtok(&length); /* skip :constvalue */
- if (local_node->constisnull)
- token = pg_strtok(&length); /* skip "<>" */
- else
+ if (pg_strtoken_next(":constvalue"))
+ {
+ token = pg_strtok(&length); /* skip :constvalue */
+ Assert(strncmp(token, ":constvalue", sizeof(":constvalue") - 1) == 0);
local_node->constvalue = readDatum(local_node->constbyval);
+ }
+ else
+ {
+ /* value was omitted */
+ Assert(local_node->constisnull);
+ }
READ_DONE();
}
--
2.40.1
[application/octet-stream] v3-0003-gen_node_support.pl-Mark-location-fields-as-type-.patch (26.4K, ../../CAEze2Wg==idqY7UX_+kqOZ6DNVWaqVtXV0dHqXW6oGevFfa91w@mail.gmail.com/5-v3-0003-gen_node_support.pl-Mark-location-fields-as-type-.patch)
download | inline diff:
From 5830c5ec1127975ee32dd6470eb946d8393667b7 Mon Sep 17 00:00:00 2001
From: Matthias van de Meent <[email protected]>
Date: Thu, 8 Feb 2024 17:15:22 +0100
Subject: [PATCH v3 3/7] gen_node_support.pl: Mark location fields as type
alias Location
Instead of the rather ugly type=int + name ~= location$, we now have a
special type for offset pointers or sizes that are only relevant when a
query text is included, which decreases the complexity required in
gen_node_support.pl for handling these values.
---
src/backend/nodes/gen_node_support.pl | 6 +-
src/include/nodes/parsenodes.h | 92 +++++++++++++--------------
src/include/nodes/primnodes.h | 72 +++++++++++----------
3 files changed, 86 insertions(+), 84 deletions(-)
diff --git a/src/backend/nodes/gen_node_support.pl b/src/backend/nodes/gen_node_support.pl
index 487f6f7728..ec850c3484 100644
--- a/src/backend/nodes/gen_node_support.pl
+++ b/src/backend/nodes/gen_node_support.pl
@@ -777,7 +777,7 @@ _equal${n}(const $n *a, const $n *b)
print $eff "\tCOMPARE_BITMAPSET_FIELD($f);\n"
unless $equal_ignore;
}
- elsif ($t eq 'int' && $f =~ 'location$')
+ elsif ($t eq 'Location')
{
print $cff "\tCOPY_LOCATION_FIELD($f);\n" unless $copy_ignore;
print $eff "\tCOMPARE_LOCATION_FIELD($f);\n" unless $equal_ignore;
@@ -1010,7 +1010,7 @@ _read${n}(void)
print $off "\tWRITE_BOOL_FIELD($f);\n";
print $rff "\tREAD_BOOL_FIELD($f);\n" unless $no_read;
}
- elsif ($t eq 'int' && $f =~ 'location$')
+ elsif ($t eq 'Location')
{
print $off "\tWRITE_LOCATION_FIELD($f);\n";
print $rff "\tREAD_LOCATION_FIELD($f);\n" unless $no_read;
@@ -1303,7 +1303,7 @@ _jumble${n}(JumbleState *jstate, Node *node)
print $jff "\tJUMBLE_NODE($f);\n"
unless $query_jumble_ignore;
}
- elsif ($t eq 'int' && $f =~ 'location$')
+ elsif ($t eq 'Location')
{
# Track the node's location only if directly requested.
if ($query_jumble_location)
diff --git a/src/include/nodes/parsenodes.h b/src/include/nodes/parsenodes.h
index 476d55dd24..dd8aa30aaf 100644
--- a/src/include/nodes/parsenodes.h
+++ b/src/include/nodes/parsenodes.h
@@ -225,9 +225,9 @@ typedef struct Query
* both be -1 meaning "unknown".
*/
/* start location, or -1 if unknown */
- int stmt_location;
+ Location stmt_location;
/* length in bytes; 0 means "rest of string" */
- int stmt_len pg_node_attr(query_jumble_ignore);
+ Location stmt_len pg_node_attr(query_jumble_ignore);
} Query;
@@ -262,7 +262,7 @@ typedef struct TypeName
List *typmods; /* type modifier expression(s) */
int32 typemod; /* prespecified type modifier */
List *arrayBounds; /* array bounds */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} TypeName;
/*
@@ -282,7 +282,7 @@ typedef struct ColumnRef
{
NodeTag type;
List *fields; /* field names (String nodes) or A_Star */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} ColumnRef;
/*
@@ -292,7 +292,7 @@ typedef struct ParamRef
{
NodeTag type;
int number; /* the number of the parameter */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} ParamRef;
/*
@@ -325,7 +325,7 @@ typedef struct A_Expr
List *name; /* possibly-qualified name of operator */
Node *lexpr; /* left argument, or NULL if none */
Node *rexpr; /* right argument, or NULL if none */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} A_Expr;
/*
@@ -351,7 +351,7 @@ typedef struct A_Const
NodeTag type;
union ValUnion val;
bool isnull; /* SQL NULL constant */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} A_Const;
/*
@@ -362,7 +362,7 @@ typedef struct TypeCast
NodeTag type;
Node *arg; /* the expression being casted */
TypeName *typeName; /* the target type */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} TypeCast;
/*
@@ -373,7 +373,7 @@ typedef struct CollateClause
NodeTag type;
Node *arg; /* input expression */
List *collname; /* possibly-qualified collation name */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} CollateClause;
/*
@@ -393,7 +393,7 @@ typedef struct RoleSpec
NodeTag type;
RoleSpecType roletype; /* Type of this rolespec */
char *rolename; /* filled only for ROLESPEC_CSTRING */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} RoleSpec;
/*
@@ -423,7 +423,7 @@ typedef struct FuncCall
bool agg_distinct; /* arguments were labeled DISTINCT */
bool func_variadic; /* last argument was labeled VARIADIC */
CoercionForm funcformat; /* how to display this node */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} FuncCall;
/*
@@ -480,7 +480,7 @@ typedef struct A_ArrayExpr
{
NodeTag type;
List *elements; /* array element expressions */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} A_ArrayExpr;
/*
@@ -507,7 +507,7 @@ typedef struct ResTarget
char *name; /* column name or NULL */
List *indirection; /* subscripts, field names, and '*', or NIL */
Node *val; /* the value expression to compute or assign */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} ResTarget;
/*
@@ -537,7 +537,7 @@ typedef struct SortBy
SortByDir sortby_dir; /* ASC/DESC/USING/default */
SortByNulls sortby_nulls; /* NULLS FIRST/LAST */
List *useOp; /* name of op to use, if SORTBY_USING */
- int location; /* operator location, or -1 if none/unknown */
+ Location location; /* operator location, or -1 if none/unknown */
} SortBy;
/*
@@ -558,7 +558,7 @@ typedef struct WindowDef
int frameOptions; /* frame_clause options, see below */
Node *startOffset; /* expression for starting bound, if any */
Node *endOffset; /* expression for ending bound, if any */
- int location; /* parse location, or -1 if none/unknown */
+ Location location; /* parse location, or -1 if none/unknown */
} WindowDef;
/*
@@ -648,7 +648,7 @@ typedef struct RangeTableFunc
List *namespaces; /* list of namespaces as ResTarget */
List *columns; /* list of RangeTableFuncCol */
Alias *alias; /* table alias & optional column aliases */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} RangeTableFunc;
/*
@@ -666,7 +666,7 @@ typedef struct RangeTableFuncCol
bool is_not_null; /* does it have NOT NULL? */
Node *colexpr; /* column filter expression */
Node *coldefexpr; /* column default value expression */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} RangeTableFuncCol;
/*
@@ -686,7 +686,7 @@ typedef struct RangeTableSample
List *method; /* sampling method name (possibly qualified) */
List *args; /* argument(s) for sampling method */
Node *repeatable; /* REPEATABLE expression, or NULL if none */
- int location; /* method name location, or -1 if unknown */
+ Location location; /* method name location, or -1 if unknown */
} RangeTableSample;
/*
@@ -729,7 +729,7 @@ typedef struct ColumnDef
Oid collOid; /* collation OID (InvalidOid if not set) */
List *constraints; /* other constraints on column */
List *fdwoptions; /* per-column FDW options */
- int location; /* parse location, or -1 if none/unknown */
+ Location location; /* parse location, or -1 if none/unknown */
} ColumnDef;
/*
@@ -803,7 +803,7 @@ typedef struct DefElem
Node *arg; /* typically Integer, Float, String, or
* TypeName */
DefElemAction defaction; /* unspecified action, or SET/ADD/DROP */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} DefElem;
/*
@@ -833,7 +833,7 @@ typedef struct XmlSerialize
Node *expr;
TypeName *typeName;
bool indent; /* [NO] INDENT */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} XmlSerialize;
/* Partitioning related definitions */
@@ -851,7 +851,7 @@ typedef struct PartitionElem
Node *expr; /* expression to partition on, or NULL */
List *collation; /* name of collation; NIL = default */
List *opclass; /* name of desired opclass; NIL = default */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} PartitionElem;
typedef enum PartitionStrategy
@@ -871,7 +871,7 @@ typedef struct PartitionSpec
NodeTag type;
PartitionStrategy strategy;
List *partParams; /* List of PartitionElems */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} PartitionSpec;
/*
@@ -898,7 +898,7 @@ struct PartitionBoundSpec
List *lowerdatums; /* List of PartitionRangeDatums */
List *upperdatums; /* List of PartitionRangeDatums */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
};
/*
@@ -921,7 +921,7 @@ typedef struct PartitionRangeDatum
Node *value; /* Const (or A_Const in raw tree), if kind is
* PARTITION_RANGE_DATUM_VALUE, else NULL */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} PartitionRangeDatum;
/*
@@ -1454,7 +1454,7 @@ typedef struct GroupingSet
NodeTag type;
GroupingSetKind kind pg_node_attr(query_jumble_ignore);
List *content;
- int location;
+ Location location;
} GroupingSet;
/*
@@ -1542,7 +1542,7 @@ typedef struct WithClause
NodeTag type;
List *ctes; /* list of CommonTableExprs */
bool recursive; /* true = WITH RECURSIVE */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} WithClause;
/*
@@ -1557,7 +1557,7 @@ typedef struct InferClause
List *indexElems; /* IndexElems to infer unique index */
Node *whereClause; /* qualification (partial-index predicate) */
char *conname; /* Constraint name, or NULL if unnamed */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} InferClause;
/*
@@ -1573,7 +1573,7 @@ typedef struct OnConflictClause
InferClause *infer; /* Optional index inference clause */
List *targetList; /* the target list (of ResTarget) */
Node *whereClause; /* qualifications */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} OnConflictClause;
/*
@@ -1594,7 +1594,7 @@ typedef struct CTESearchClause
List *search_col_list;
bool search_breadth_first;
char *search_seq_column;
- int location;
+ Location location;
} CTESearchClause;
typedef struct CTECycleClause
@@ -1605,7 +1605,7 @@ typedef struct CTECycleClause
Node *cycle_mark_value;
Node *cycle_mark_default;
char *cycle_path_column;
- int location;
+ Location location;
/* These fields are set during parse analysis: */
Oid cycle_mark_type; /* common type of _value and _default */
int cycle_mark_typmod;
@@ -1629,7 +1629,7 @@ typedef struct CommonTableExpr
Node *ctequery; /* the CTE's subquery */
CTESearchClause *search_clause pg_node_attr(query_jumble_ignore);
CTECycleClause *cycle_clause pg_node_attr(query_jumble_ignore);
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
/* These fields are set during parse analysis: */
/* is this CTE actually recursive? */
bool cterecursive pg_node_attr(query_jumble_ignore);
@@ -1725,7 +1725,7 @@ typedef struct JsonParseExpr
JsonValueExpr *expr; /* string expression */
JsonOutput *output; /* RETURNING clause, if specified */
bool unique_keys; /* WITH UNIQUE KEYS? */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} JsonParseExpr;
/*
@@ -1737,7 +1737,7 @@ typedef struct JsonScalarExpr
NodeTag type;
Expr *expr; /* scalar expression */
JsonOutput *output; /* RETURNING clause, if specified */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} JsonScalarExpr;
/*
@@ -1749,7 +1749,7 @@ typedef struct JsonSerializeExpr
NodeTag type;
JsonValueExpr *expr; /* json value expression */
JsonOutput *output; /* RETURNING clause, if specified */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} JsonSerializeExpr;
/*
@@ -1763,7 +1763,7 @@ typedef struct JsonObjectConstructor
JsonOutput *output; /* RETURNING clause, if specified */
bool absent_on_null; /* skip NULL values? */
bool unique; /* check key uniqueness? */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} JsonObjectConstructor;
/*
@@ -1776,7 +1776,7 @@ typedef struct JsonArrayConstructor
List *exprs; /* list of JsonValueExpr elements */
JsonOutput *output; /* RETURNING clause, if specified */
bool absent_on_null; /* skip NULL elements? */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} JsonArrayConstructor;
/*
@@ -1790,7 +1790,7 @@ typedef struct JsonArrayQueryConstructor
JsonOutput *output; /* RETURNING clause, if specified */
JsonFormat *format; /* FORMAT clause for subquery, if specified */
bool absent_on_null; /* skip NULL elements? */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} JsonArrayQueryConstructor;
/*
@@ -1805,7 +1805,7 @@ typedef struct JsonAggConstructor
Node *agg_filter; /* FILTER clause, if any */
List *agg_order; /* ORDER BY clause, if any */
struct WindowDef *over; /* OVER clause, if any */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} JsonAggConstructor;
/*
@@ -1859,8 +1859,8 @@ typedef struct RawStmt
NodeTag type;
Node *stmt; /* raw parse tree */
- int stmt_location; /* start location, or -1 if unknown */
- int stmt_len; /* length in bytes; 0 means "rest of string" */
+ Location stmt_location; /* start location, or -1 if unknown */
+ Location stmt_len; /* length in bytes; 0 means "rest of string" */
} RawStmt;
/*****************************************************************************
@@ -2067,7 +2067,7 @@ typedef struct PLAssignStmt
List *indirection; /* subscripts and field names, if any */
int nnames; /* number of names to use in ColumnRef */
SelectStmt *val; /* the PL/pgSQL expression to assign */
- int location; /* name's token location, or -1 if unknown */
+ Location location; /* name's token location, or -1 if unknown */
} PLAssignStmt;
@@ -2575,7 +2575,7 @@ typedef struct Constraint
char *conname; /* Constraint name, or NULL if unnamed */
bool deferrable; /* DEFERRABLE? */
bool initdeferred; /* INITIALLY DEFERRED? */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
/* Fields used for constraints with expressions (CHECK and DEFAULT): */
bool is_no_inherit; /* is constraint non-inheritable? */
@@ -3528,7 +3528,7 @@ typedef struct TransactionStmt
char *gid pg_node_attr(query_jumble_ignore);
bool chain; /* AND CHAIN option */
/* token location, or -1 if unknown */
- int location pg_node_attr(query_jumble_location);
+ Location location pg_node_attr(query_jumble_location);
} TransactionStmt;
/* ----------------------
@@ -3914,7 +3914,7 @@ typedef struct DeallocateStmt
/* true if DEALLOCATE ALL */
bool isall;
/* token location, or -1 if unknown */
- int location pg_node_attr(query_jumble_location);
+ Location location pg_node_attr(query_jumble_location);
} DeallocateStmt;
/*
@@ -4002,7 +4002,7 @@ typedef struct PublicationObjSpec
PublicationObjSpecType pubobjtype; /* type of this publication object */
char *name;
PublicationTable *pubtable;
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} PublicationObjSpec;
typedef struct CreatePublicationStmt
diff --git a/src/include/nodes/primnodes.h b/src/include/nodes/primnodes.h
index 4a154606d2..557be05657 100644
--- a/src/include/nodes/primnodes.h
+++ b/src/include/nodes/primnodes.h
@@ -30,6 +30,8 @@ typedef enum OverridingKind
} OverridingKind;
+#define Location int
+
/* ----------------------------------------------------------------
* node definitions
* ----------------------------------------------------------------
@@ -91,7 +93,7 @@ typedef struct RangeVar
Alias *alias;
/* token location, or -1 if unknown */
- int location;
+ Location location;
} RangeVar;
/*
@@ -128,7 +130,7 @@ typedef struct TableFunc
/* counts from 0; -1 if none specified */
int ordinalitycol pg_node_attr(query_jumble_ignore);
/* token location, or -1 if unknown */
- int location;
+ Location location;
} TableFunc;
/*
@@ -276,7 +278,7 @@ typedef struct Var
AttrNumber varattnosyn pg_node_attr(equal_ignore, query_jumble_ignore);
/* token location, or -1 if unknown */
- int location;
+ Location location;
} Var;
/*
@@ -318,7 +320,7 @@ typedef struct Const
* token location, or -1 if unknown. All constants are tracked as
* locations in query jumbling, to be marked as parameters.
*/
- int location pg_node_attr(query_jumble_location);
+ Location location pg_node_attr(query_jumble_location);
} Const;
/*
@@ -367,7 +369,7 @@ typedef struct Param
/* OID of collation, or InvalidOid if none */
Oid paramcollid pg_node_attr(query_jumble_ignore);
/* token location, or -1 if unknown */
- int location;
+ Location location;
} Param;
/*
@@ -490,7 +492,7 @@ typedef struct Aggref
int aggtransno pg_node_attr(query_jumble_ignore);
/* token location, or -1 if unknown */
- int location;
+ Location location;
} Aggref;
/*
@@ -537,7 +539,7 @@ typedef struct GroupingFunc
Index agglevelsup;
/* token location */
- int location;
+ Location location;
} GroupingFunc;
/*
@@ -568,7 +570,7 @@ typedef struct WindowFunc
/* is function a simple aggregate? */
bool winagg pg_node_attr(query_jumble_ignore);
/* token location, or -1 if unknown */
- int location;
+ Location location;
} WindowFunc;
/*
@@ -702,7 +704,7 @@ typedef struct FuncExpr
/* arguments to the function */
List *args;
/* token location, or -1 if unknown */
- int location;
+ Location location;
} FuncExpr;
/*
@@ -729,7 +731,7 @@ typedef struct NamedArgExpr
/* argument's number in positional notation */
int argnumber;
/* argument name location, or -1 if unknown */
- int location;
+ Location location;
} NamedArgExpr;
/*
@@ -771,7 +773,7 @@ typedef struct OpExpr
List *args;
/* token location, or -1 if unknown */
- int location;
+ Location location;
} OpExpr;
/*
@@ -851,7 +853,7 @@ typedef struct ScalarArrayOpExpr
List *args;
/* token location, or -1 if unknown */
- int location;
+ Location location;
} ScalarArrayOpExpr;
/*
@@ -873,7 +875,7 @@ typedef struct BoolExpr
Expr xpr;
BoolExprType boolop;
List *args; /* arguments to this expression */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} BoolExpr;
/*
@@ -950,7 +952,7 @@ typedef struct SubLink
List *operName pg_node_attr(query_jumble_ignore);
/* subselect as Query* or raw parsetree */
Node *subselect;
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} SubLink;
/*
@@ -1124,7 +1126,7 @@ typedef struct RelabelType
Oid resultcollid pg_node_attr(query_jumble_ignore);
/* how to display this node */
CoercionForm relabelformat pg_node_attr(query_jumble_ignore);
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} RelabelType;
/* ----------------
@@ -1146,7 +1148,7 @@ typedef struct CoerceViaIO
Oid resultcollid pg_node_attr(query_jumble_ignore);
/* how to display this node */
CoercionForm coerceformat pg_node_attr(query_jumble_ignore);
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} CoerceViaIO;
/* ----------------
@@ -1174,7 +1176,7 @@ typedef struct ArrayCoerceExpr
Oid resultcollid pg_node_attr(query_jumble_ignore);
/* how to display this node */
CoercionForm coerceformat pg_node_attr(query_jumble_ignore);
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} ArrayCoerceExpr;
/* ----------------
@@ -1198,7 +1200,7 @@ typedef struct ConvertRowtypeExpr
/* Like RowExpr, we deliberately omit a typmod and collation here */
/* how to display this node */
CoercionForm convertformat pg_node_attr(query_jumble_ignore);
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} ConvertRowtypeExpr;
/*----------
@@ -1213,7 +1215,7 @@ typedef struct CollateExpr
Expr xpr;
Expr *arg; /* input expression */
Oid collOid; /* collation's OID */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} CollateExpr;
/*----------
@@ -1248,7 +1250,7 @@ typedef struct CaseExpr
Expr *arg; /* implicit equality comparison argument */
List *args; /* the arguments (list of WHEN clauses) */
Expr *defresult; /* the default result (ELSE clause) */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} CaseExpr;
/*
@@ -1259,7 +1261,7 @@ typedef struct CaseWhen
Expr xpr;
Expr *expr; /* condition expression */
Expr *result; /* substitution result */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} CaseWhen;
/*
@@ -1316,7 +1318,7 @@ typedef struct ArrayExpr
/* true if elements are sub-arrays */
bool multidims pg_node_attr(query_jumble_ignore);
/* token location, or -1 if unknown */
- int location;
+ Location location;
} ArrayExpr;
/*
@@ -1367,7 +1369,7 @@ typedef struct RowExpr
/* list of String, or NIL */
List *colnames pg_node_attr(query_jumble_ignore);
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} RowExpr;
/*
@@ -1426,7 +1428,7 @@ typedef struct CoalesceExpr
/* the arguments */
List *args;
/* token location, or -1 if unknown */
- int location;
+ Location location;
} CoalesceExpr;
/*
@@ -1452,7 +1454,7 @@ typedef struct MinMaxExpr
/* the arguments */
List *args;
/* token location, or -1 if unknown */
- int location;
+ Location location;
} MinMaxExpr;
/*
@@ -1496,7 +1498,7 @@ typedef struct SQLValueFunction
*/
Oid type pg_node_attr(query_jumble_ignore);
int32 typmod;
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} SQLValueFunction;
/*
@@ -1549,7 +1551,7 @@ typedef struct XmlExpr
Oid type pg_node_attr(query_jumble_ignore);
int32 typmod pg_node_attr(query_jumble_ignore);
/* token location, or -1 if unknown */
- int location;
+ Location location;
} XmlExpr;
/*
@@ -1585,7 +1587,7 @@ typedef struct JsonFormat
NodeTag type;
JsonFormatType format_type; /* format type */
JsonEncoding encoding; /* JSON encoding */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} JsonFormat;
/*
@@ -1641,7 +1643,7 @@ typedef struct JsonConstructorExpr
JsonReturning *returning; /* RETURNING clause */
bool absent_on_null; /* ABSENT ON NULL? */
bool unique; /* WITH UNIQUE KEYS? (JSON_OBJECT[AGG] only) */
- int location;
+ Location location;
} JsonConstructorExpr;
/*
@@ -1667,7 +1669,7 @@ typedef struct JsonIsPredicate
JsonFormat *format; /* FORMAT clause, if specified */
JsonValueType item_type; /* JSON item type */
bool unique_keys; /* check key uniqueness? */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} JsonIsPredicate;
/* ----------------
@@ -1701,7 +1703,7 @@ typedef struct NullTest
NullTestType nulltesttype; /* IS NULL, IS NOT NULL */
/* T to perform field-by-field null checks */
bool argisrow pg_node_attr(query_jumble_ignore);
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} NullTest;
/*
@@ -1723,7 +1725,7 @@ typedef struct BooleanTest
Expr xpr;
Expr *arg; /* input expression */
BoolTestType booltesttype; /* test type */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} BooleanTest;
@@ -1765,7 +1767,7 @@ typedef struct CoerceToDomain
Oid resultcollid pg_node_attr(query_jumble_ignore);
/* how to display this node */
CoercionForm coercionformat pg_node_attr(query_jumble_ignore);
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} CoerceToDomain;
/*
@@ -1787,7 +1789,7 @@ typedef struct CoerceToDomainValue
/* collation for the substituted value */
Oid collation pg_node_attr(query_jumble_ignore);
/* token location, or -1 if unknown */
- int location;
+ Location location;
} CoerceToDomainValue;
/*
@@ -1807,7 +1809,7 @@ typedef struct SetToDefault
/* collation for the substituted value */
Oid collation pg_node_attr(query_jumble_ignore);
/* token location, or -1 if unknown */
- int location;
+ Location location;
} SetToDefault;
/*
--
2.40.1
[application/octet-stream] v3-0006-nodeToString-Apply-RLE-on-Bitmapset-and-numeric-L.patch (7.9K, ../../CAEze2Wg==idqY7UX_+kqOZ6DNVWaqVtXV0dHqXW6oGevFfa91w@mail.gmail.com/6-v3-0006-nodeToString-Apply-RLE-on-Bitmapset-and-numeric-L.patch)
download | inline diff:
From e62f7a1b15dc58a081a1c724a8207526b7037bf0 Mon Sep 17 00:00:00 2001
From: Matthias van de Meent <[email protected]>
Date: Sat, 10 Feb 2024 00:57:19 +0100
Subject: [PATCH v3 6/7] nodeToString: Apply RLE on Bitmapset and numeric List
types
This reduces the size of full serialized queries in pg_rewrite by several %,
reducing overhead in the system.
---
src/backend/nodes/outfuncs.c | 96 +++++++++++++++++++++++++++++++----
src/backend/nodes/read.c | 53 +++++++++++++++++--
src/backend/nodes/readfuncs.c | 17 ++++++-
3 files changed, 150 insertions(+), 16 deletions(-)
diff --git a/src/backend/nodes/outfuncs.c b/src/backend/nodes/outfuncs.c
index 1a17eafd57..f135753fb4 100644
--- a/src/backend/nodes/outfuncs.c
+++ b/src/backend/nodes/outfuncs.c
@@ -378,23 +378,64 @@ writeNodeArray(StringInfo str, const Node *const *arr, int len,
/*
* Print a List.
+ *
+ * Notes:
+ * NodeList is formatted as (<node1> <node2> ...)
+ *
+ * OidList is formatted as (o int +int int int ...), with +int indicating N
+ * successive identical values.
+ *
+ * IntList and XidList are formatted as (i/x int +int int int ...), with +int
+ * indicating N successively increasing values. (i 9 +3) is thus equivalent
+ * to (i 9 10 11 12).
*/
static void
_outList(StringInfo str, const List *node, bool omitLocation)
{
const ListCell *lc;
+ union LCSurrogate {
+ int32 i;
+ Oid o;
+ TransactionId x;
+ } previous = { .i = 0};
+ int run = 0;
+ int run_delta;
+ const char *fmt;
appendStringInfoChar(str, '(');
if (IsA(node, IntList))
+ {
appendStringInfoChar(str, 'i');
+ fmt = " %d";
+ run_delta = 1;
+ }
else if (IsA(node, OidList))
+ {
appendStringInfoChar(str, 'o');
+ fmt = " %u";
+ run_delta = 0;
+ }
else if (IsA(node, XidList))
+ {
appendStringInfoChar(str, 'x');
+ fmt = " %u";
+ run_delta = 1;
+ }
+ else if (IsA(node, List))
+ {
+ /* silence the compiler about uninitialized variables */
+ fmt = "";
+ run_delta = 0;
+ }
+ else
+ elog(ERROR, "unrecognized list node type: %d",
+ (int) node->type);
foreach(lc, node)
{
+ union LCSurrogate val = {.i = 0};
+
/*
* For the sake of backward compatibility, we emit a slightly
* different whitespace format for lists of nodes vs. other types of
@@ -405,26 +446,41 @@ _outList(StringInfo str, const List *node, bool omitLocation)
outNode(str, lfirst(lc), omitLocation);
if (lnext(node, lc))
appendStringInfoChar(str, ' ');
+ continue;
}
else if (IsA(node, IntList))
- appendStringInfo(str, " %d", lfirst_int(lc));
+ val.i = lfirst_int(lc);
else if (IsA(node, OidList))
- appendStringInfo(str, " %u", lfirst_oid(lc));
+ val.o = lfirst_oid(lc);
else if (IsA(node, XidList))
- appendStringInfo(str, " %u", lfirst_xid(lc));
+ val.x = lfirst_xid(lc);
+
+ if (val.i == previous.i + run_delta)
+ run += 1;
else
- elog(ERROR, "unrecognized list node type: %d",
- (int) node->type);
+ {
+ if (run > 0)
+ appendStringInfo(str, " +%d", run);
+ run = 0;
+ appendStringInfo(str, fmt, val.i);
+ }
+ previous = val;
}
- appendStringInfoChar(str, ')');
-}
+ if (run > 0)
+ appendStringInfo(str, " +%d", run);
+
+ appendStringInfoChar(str, ')');}
/*
* outBitmapset -
* converts a bitmap set of integers
*
- * Note: the output format is "(b int int ...)", similar to an integer List.
+ * Note: the output format is "(b int int +int ...)", similar to an
+ * integer List. Note that consecutive runs of incremental values are
+ * indicated by [... int +int ...], where the first int is the first set bit,
+ * and the int that's tagged with the '+' sign that follows is the number of
+ * subsequent set bits (resulting in a run of N+1 set bits).
*
* We export this function for use by extensions that define extensible nodes.
* That's somewhat historical, though, because calling outNode() will work.
@@ -432,13 +488,31 @@ _outList(StringInfo str, const List *node, bool omitLocation)
void
outBitmapset(StringInfo str, const Bitmapset *bms)
{
- int x;
+ int x = 0;
+ int prev = -2;
+ int run = 0;
appendStringInfoChar(str, '(');
appendStringInfoChar(str, 'b');
- x = -1;
+
while ((x = bms_next_member(bms, x)) >= 0)
- appendStringInfo(str, " %d", x);
+ {
+ if (x - prev == 1)
+ run++;
+ else
+ {
+ if (run != 0)
+ appendStringInfo(str, " +%d", run);
+
+ run = 0;
+ appendStringInfo(str, " %d", x);
+ }
+ prev = x;
+ }
+
+ if (run > 0)
+ appendStringInfo(str, " +%d", run);
+
appendStringInfoChar(str, ')');
}
diff --git a/src/backend/nodes/read.c b/src/backend/nodes/read.c
index d9dcb5865a..34dac773d8 100644
--- a/src/backend/nodes/read.c
+++ b/src/backend/nodes/read.c
@@ -403,6 +403,8 @@ nodeRead(const char *token, int tok_len)
elog(ERROR, "unterminated List structure");
if (tok_len == 1 && token[0] == 'i')
{
+ int prev = 0;
+
/* List of integers */
for (;;)
{
@@ -418,12 +420,23 @@ nodeRead(const char *token, int tok_len)
if (endptr != token + tok_len)
elog(ERROR, "unrecognized integer: \"%.*s\"",
tok_len, token);
- l = lappend_int(l, val);
+
+ if (token[0] == '+')
+ {
+ for (int i = 0; i < val; i++)
+ l = lappend_int(l, ++prev);
+ }
+ else
+ {
+ prev = val;
+ l = lappend_int(l, val);
+ }
}
result = (Node *) l;
}
else if (tok_len == 1 && token[0] == 'o')
{
+ Oid prev = 0;
/* List of OIDs */
for (;;)
{
@@ -439,12 +452,23 @@ nodeRead(const char *token, int tok_len)
if (endptr != token + tok_len)
elog(ERROR, "unrecognized OID: \"%.*s\"",
tok_len, token);
- l = lappend_oid(l, val);
+
+ if (token[0] == '+')
+ {
+ for (int i = 0; i < val; i++)
+ l = lappend_oid(l, prev);
+ }
+ else
+ {
+ prev = val;
+ l = lappend_oid(l, val);
+ }
}
result = (Node *) l;
}
else if (tok_len == 1 && token[0] == 'x')
{
+ TransactionId prev = 0;
/* List of TransactionIds */
for (;;)
{
@@ -460,7 +484,17 @@ nodeRead(const char *token, int tok_len)
if (endptr != token + tok_len)
elog(ERROR, "unrecognized Xid: \"%.*s\"",
tok_len, token);
- l = lappend_xid(l, val);
+
+ if (token[0] == '+')
+ {
+ for (int i = 0; i < val; i++)
+ l = lappend_xid(l, ++prev);
+ }
+ else
+ {
+ prev = val;
+ l = lappend_xid(l, val);
+ }
}
result = (Node *) l;
}
@@ -468,6 +502,7 @@ nodeRead(const char *token, int tok_len)
{
/* Bitmapset -- see also _readBitmapset() */
Bitmapset *bms = NULL;
+ int prev = -2;
for (;;)
{
@@ -483,7 +518,17 @@ nodeRead(const char *token, int tok_len)
if (endptr != token + tok_len)
elog(ERROR, "unrecognized integer: \"%.*s\"",
tok_len, token);
- bms = bms_add_member(bms, val);
+
+ if (token[0] == '+')
+ {
+ for (int i = 0; i < val; i++)
+ bms = bms_add_member(bms, ++prev);
+ }
+ else
+ {
+ bms = bms_add_member(bms, val);
+ prev = val;
+ }
}
result = (Node *) bms;
}
diff --git a/src/backend/nodes/readfuncs.c b/src/backend/nodes/readfuncs.c
index 4cdbad9e7e..55e40f5603 100644
--- a/src/backend/nodes/readfuncs.c
+++ b/src/backend/nodes/readfuncs.c
@@ -298,6 +298,7 @@ static Bitmapset *
_readBitmapset(void)
{
Bitmapset *result = NULL;
+ int prev = -2;
READ_TEMP_LOCALS();
@@ -326,7 +327,21 @@ _readBitmapset(void)
val = (int) strtol(token, &endptr, 10);
if (endptr != token + length)
elog(ERROR, "unrecognized integer: \"%.*s\"", length, token);
- result = bms_add_member(result, val);
+
+ if (token[0] == '+')
+ {
+ Assert(prev >= 0);
+ for (int i = 0; i < val; i++)
+ {
+ prev++;
+ result = bms_add_member(result, prev);
+ }
+ }
+ else
+ {
+ result = bms_add_member(result, val);
+ prev = val;
+ }
}
return result;
--
2.40.1
[application/octet-stream] v3-0007-gen_node_support.pl-Optimize-serialization-of-fie.patch (9.2K, ../../CAEze2Wg==idqY7UX_+kqOZ6DNVWaqVtXV0dHqXW6oGevFfa91w@mail.gmail.com/7-v3-0007-gen_node_support.pl-Optimize-serialization-of-fie.patch)
download | inline diff:
From 705cfa1818765f8971a49a0dc7a9d9b590c963f8 Mon Sep 17 00:00:00 2001
From: Matthias van de Meent <[email protected]>
Date: Mon, 12 Feb 2024 17:12:02 +0100
Subject: [PATCH v3 7/7] gen_node_support.pl: Optimize serialization of fields
with copied values
The Var node's [syn] fields often contain the same data as their non-syn
counterparts. We invent a new pg_node_attr()ibute that represents this
relation, which allows us to omit these fields from serialization when they
are indeed copies of the original fields.
---
src/backend/nodes/gen_node_support.pl | 94 ++++++++++++++++++---------
src/backend/nodes/outfuncs.c | 3 +
src/backend/nodes/readfuncs.c | 3 +
src/include/nodes/primnodes.h | 4 +-
4 files changed, 70 insertions(+), 34 deletions(-)
diff --git a/src/backend/nodes/gen_node_support.pl b/src/backend/nodes/gen_node_support.pl
index fda8c3a494..66661a3881 100644
--- a/src/backend/nodes/gen_node_support.pl
+++ b/src/backend/nodes/gen_node_support.pl
@@ -363,6 +363,10 @@ foreach my $infile (@ARGV)
{
$manual_nodetag_number{$in_struct} = $1;
}
+ elsif ($attr =~ /^default_ref\(([\w\d."'-]+)\)$/)
+ {
+ # Unused at the node level
+ }
else
{
die
@@ -437,7 +441,7 @@ foreach my $infile (@ARGV)
}
# normal struct field
elsif ($line =~
- /^\s*(.+)\s*\b(\w+)(\[[\w\s+]+\])?\s*(?:pg_node_attr\(([\w(), ]*)\))?;/
+ /^\s*(.+)\s*\b(\w+)(\[[\w\s+]+\])?\s*(?:pg_node_attr\(([\w\d(), ."'-]*)\))?;/
)
{
if ($is_node_struct)
@@ -469,6 +473,7 @@ foreach my $infile (@ARGV)
if ( $attr !~ /^array_size\(\w+\)$/
&& $attr !~ /^copy_as\(\w+\)$/
&& $attr !~ /^read_as\(\w+\)$/
+ && $attr !~ /^default_ref\([\w\d."'-]+\)$/
&& !elem $attr,
qw(copy_as_scalar
equal_as_scalar
@@ -495,7 +500,7 @@ foreach my $infile (@ARGV)
}
# function pointer field
elsif ($line =~
- /^\s*([\w\s*]+)\s*\(\*(\w+)\)\s*\((.*)\)\s*(?:pg_node_attr\(([\w(), ]*)\))?;/
+ /^\s*([\w\s*]+)\s*\(\*(\w+)\)\s*\((.*)\)\s*(?:pg_node_attr\(([\w\d(), ."'-]*)\))?;/
)
{
if ($is_node_struct)
@@ -970,6 +975,15 @@ _read${n}(void)
my $array_size_field;
my $read_as_field;
my $read_write_ignore = 0;
+ # Type read/write macro suffix, "" for normal use, or "_DEFAULT" when
+ # value argument.
+ my $s = "";
+ # Default parameter to read/write macro. Includes comma separator so
+ # that MACRO_NAME$s($fieldname$defaultparam) is the full complete
+ # read/write expression for essentially all types.
+ # Note that this (currently) only works for scalar values.
+ my $d = "";
+
foreach my $a (@a)
{
if ($a =~ /^array_size\(([\w.]+)\)$/)
@@ -988,6 +1002,19 @@ _read${n}(void)
{
$read_write_ignore = 1;
}
+ elsif ($a =~ /^default_ref\(([\w\d+."'-]+)\)$/)
+ {
+ $s = "_DEFAULT";
+ $d = ", NODE_FIELD($1)";
+ }
+ }
+
+ if ($s eq "_DEFAULT")
+ {
+ die "custom defaults for non-scalar fields are not supported\n\tat $n.$f"
+ unless (elem $t, @scalar_types or elem $t, @enum_types);
+ die "custom defaults for Location fields are not supported\n\tat $n.$f"
+ if ($t eq "Location");
}
if ($read_write_ignore)
@@ -1008,8 +1035,8 @@ _read${n}(void)
# select instructions by field type
if ($t eq 'bool')
{
- print $off "\tWRITE_BOOL_FIELD($f);\n";
- print $rff "\tREAD_BOOL_FIELD($f);\n" unless $no_read;
+ print $off "\tWRITE_BOOL_FIELD$s($f$d);\n";
+ print $rff "\tREAD_BOOL_FIELD$s($f$d);\n" unless $no_read;
}
elsif ($t eq 'Location')
{
@@ -1018,8 +1045,11 @@ _read${n}(void)
}
elsif ($t eq 'TypMod')
{
- print $off "\tWRITE_INT_FIELD_DEFAULT($f, -1);\n";
- print $rff "\tREAD_INT_FIELD_DEFAULT($f, -1);\n" unless $no_read;
+ # The default value of a TypMod fields is -1, rather than 0
+ # for normal int fields.
+ $d = ", -1" if ($d eq "");
+ print $off "\tWRITE_INT_FIELD_DEFAULT($f$d);\n";
+ print $rff "\tREAD_INT_FIELD_DEFAULT($f$d);\n" unless $no_read;
}
elsif ($t eq 'int'
|| $t eq 'int16'
@@ -1027,8 +1057,8 @@ _read${n}(void)
|| $t eq 'AttrNumber'
|| $t eq 'StrategyNumber')
{
- print $off "\tWRITE_INT_FIELD($f);\n";
- print $rff "\tREAD_INT_FIELD($f);\n" unless $no_read;
+ print $off "\tWRITE_INT_FIELD$s($f$d);\n";
+ print $rff "\tREAD_INT_FIELD$s($f$d);\n" unless $no_read;
}
elsif ($t eq 'uint32'
|| $t eq 'bits32'
@@ -1036,56 +1066,56 @@ _read${n}(void)
|| $t eq 'Index'
|| $t eq 'SubTransactionId')
{
- print $off "\tWRITE_UINT_FIELD($f);\n";
- print $rff "\tREAD_UINT_FIELD($f);\n" unless $no_read;
+ print $off "\tWRITE_UINT_FIELD$s($f$d);\n";
+ print $rff "\tREAD_UINT_FIELD$s($f$d);\n" unless $no_read;
}
elsif ($t eq 'uint64'
|| $t eq 'AclMode')
{
- print $off "\tWRITE_UINT64_FIELD($f);\n";
- print $rff "\tREAD_UINT64_FIELD($f);\n" unless $no_read;
+ print $off "\tWRITE_UINT64_FIELD$s($f$d);\n";
+ print $rff "\tREAD_UINT64_FIELD$s($f$d);\n" unless $no_read;
}
elsif ($t eq 'Oid' || $t eq 'RelFileNumber')
{
- print $off "\tWRITE_OID_FIELD($f);\n";
- print $rff "\tREAD_OID_FIELD($f);\n" unless $no_read;
+ print $off "\tWRITE_OID_FIELD$s($f$d);\n";
+ print $rff "\tREAD_OID_FIELD$s($f$d);\n" unless $no_read;
}
elsif ($t eq 'long')
{
- print $off "\tWRITE_LONG_FIELD($f);\n";
- print $rff "\tREAD_LONG_FIELD($f);\n" unless $no_read;
+ print $off "\tWRITE_LONG_FIELD$s($f$d);\n";
+ print $rff "\tREAD_LONG_FIELD$s($f$d);\n" unless $no_read;
}
elsif ($t eq 'char')
{
- print $off "\tWRITE_CHAR_FIELD($f);\n";
- print $rff "\tREAD_CHAR_FIELD($f);\n" unless $no_read;
+ print $off "\tWRITE_CHAR_FIELD$s($f$d);\n";
+ print $rff "\tREAD_CHAR_FIELD$s($f$d);\n" unless $no_read;
}
elsif ($t eq 'double')
{
- print $off "\tWRITE_FLOAT_FIELD($f);\n";
- print $rff "\tREAD_FLOAT_FIELD($f);\n" unless $no_read;
+ print $off "\tWRITE_FLOAT_FIELD$s($f$d);\n";
+ print $rff "\tREAD_FLOAT_FIELD$s($f$d);\n" unless $no_read;
}
elsif ($t eq 'Cardinality')
{
- print $off "\tWRITE_FLOAT_FIELD($f);\n";
- print $rff "\tREAD_FLOAT_FIELD($f);\n" unless $no_read;
+ print $off "\tWRITE_FLOAT_FIELD$s($f$d);\n";
+ print $rff "\tREAD_FLOAT_FIELD$s($f$d);\n" unless $no_read;
}
elsif ($t eq 'Cost')
{
- print $off "\tWRITE_FLOAT_FIELD($f);\n";
- print $rff "\tREAD_FLOAT_FIELD($f);\n" unless $no_read;
+ print $off "\tWRITE_FLOAT_FIELD$s($f$d);\n";
+ print $rff "\tREAD_FLOAT_FIELD$s($f$d);\n" unless $no_read;
}
elsif ($t eq 'QualCost')
{
- print $off "\tWRITE_FLOAT_FIELD($f.startup);\n";
- print $off "\tWRITE_FLOAT_FIELD($f.per_tuple);\n";
- print $rff "\tREAD_FLOAT_FIELD($f.startup);\n" unless $no_read;
- print $rff "\tREAD_FLOAT_FIELD($f.per_tuple);\n" unless $no_read;
+ print $off "\tWRITE_FLOAT_FIELD$s($f.startup$d);\n";
+ print $off "\tWRITE_FLOAT_FIELD$s($f.per_tuple$d);\n";
+ print $rff "\tREAD_FLOAT_FIELD$s($f.startup$d);\n" unless $no_read;
+ print $rff "\tREAD_FLOAT_FIELD$s($f.per_tuple$d);\n" unless $no_read;
}
elsif ($t eq 'Selectivity')
{
- print $off "\tWRITE_FLOAT_FIELD($f);\n";
- print $rff "\tREAD_FLOAT_FIELD($f);\n" unless $no_read;
+ print $off "\tWRITE_FLOAT_FIELD$s($f$d);\n";
+ print $rff "\tREAD_FLOAT_FIELD$s($f$d);\n" unless $no_read;
}
elsif ($t eq 'char*')
{
@@ -1099,8 +1129,8 @@ _read${n}(void)
}
elsif (elem $t, @enum_types)
{
- print $off "\tWRITE_ENUM_FIELD($f, $t);\n";
- print $rff "\tREAD_ENUM_FIELD($f, $t);\n" unless $no_read;
+ print $off "\tWRITE_ENUM_FIELD$s($f, $t$d);\n";
+ print $rff "\tREAD_ENUM_FIELD$s($f, $t$d);\n" unless $no_read;
}
# arrays of scalar types
elsif ($t =~ /^(\w+)(\*|\[\w+\])$/ and elem $1, @scalar_types)
diff --git a/src/backend/nodes/outfuncs.c b/src/backend/nodes/outfuncs.c
index f135753fb4..75a7703b62 100644
--- a/src/backend/nodes/outfuncs.c
+++ b/src/backend/nodes/outfuncs.c
@@ -240,6 +240,9 @@ static void outDouble(StringInfo str, double d);
} \
} while (0)
+/* Accessing a node's field goes like this */
+#define NODE_FIELD(fldname) (node->fldname)
+
#define booltostr(x) ((x) ? "true" : "false")
diff --git a/src/backend/nodes/readfuncs.c b/src/backend/nodes/readfuncs.c
index 55e40f5603..4d11696e68 100644
--- a/src/backend/nodes/readfuncs.c
+++ b/src/backend/nodes/readfuncs.c
@@ -258,6 +258,9 @@
#define READ_BOOL_ARRAY(fldname, len) \
READ_BOOL_ARRAY_DEFAULT(fldname, len, NULL)
+/* Accessing a node's field goes like this */
+#define NODE_FIELD(fldname) (local_node->fldname)
+
/* Routine exit */
#define READ_DONE() \
return local_node
diff --git a/src/include/nodes/primnodes.h b/src/include/nodes/primnodes.h
index 94282497d7..9dd60ad5c5 100644
--- a/src/include/nodes/primnodes.h
+++ b/src/include/nodes/primnodes.h
@@ -274,9 +274,9 @@ typedef struct Var
* their varno/varattno match.
*/
/* syntactic relation index (0 if unknown) */
- Index varnosyn pg_node_attr(equal_ignore, query_jumble_ignore);
+ Index varnosyn pg_node_attr(equal_ignore, query_jumble_ignore, default_ref(varno));
/* syntactic attribute number */
- AttrNumber varattnosyn pg_node_attr(equal_ignore, query_jumble_ignore);
+ AttrNumber varattnosyn pg_node_attr(equal_ignore, query_jumble_ignore, default_ref(varattno));
/* token location, or -1 if unknown */
Location location;
--
2.40.1
[application/octet-stream] v3-0001-pg_node_tree-Omit-serialization-of-fields-with-de.patch (23.0K, ../../CAEze2Wg==idqY7UX_+kqOZ6DNVWaqVtXV0dHqXW6oGevFfa91w@mail.gmail.com/8-v3-0001-pg_node_tree-Omit-serialization-of-fields-with-de.patch)
download | inline diff:
From b53eabb4d26d04ff527db9e82cccb15b5271f58e Mon Sep 17 00:00:00 2001
From: Matthias van de Meent <[email protected]>
Date: Tue, 2 Jan 2024 23:55:02 +0100
Subject: [PATCH v3 1/7] pg_node_tree: Omit serialization of fields with
default values.
Often, the values in nodes are their default values. By not serializing
those fields and inserting the defaults during deserialization, we
reduce the size of pg_node_tree attributes seen in e.g. pg_rewrite by a
significant factor.
In passing, we fix a test that had a strict dependency on the
serialization of pg_node_tree; we now do the checks in a more generic
manner, making it more stable and ensuring its stability in future work.
---
src/backend/nodes/outfuncs.c | 175 ++++++++++++++++++----
src/backend/nodes/read.c | 45 ++++++
src/backend/nodes/readfuncs.c | 131 +++++++++++++---
src/include/nodes/readfuncs.h | 1 +
src/test/regress/expected/rowsecurity.out | 5 +-
src/test/regress/sql/rowsecurity.sql | 5 +-
6 files changed, 309 insertions(+), 53 deletions(-)
diff --git a/src/backend/nodes/outfuncs.c b/src/backend/nodes/outfuncs.c
index 03f67b6850..50da80ee34 100644
--- a/src/backend/nodes/outfuncs.c
+++ b/src/backend/nodes/outfuncs.c
@@ -41,94 +41,203 @@ static void outDouble(StringInfo str, double d);
appendStringInfoString(str, nodelabel)
/* Write an integer field (anything written as ":fldname %d") */
+#define WRITE_INT_FIELD_DEFAULT(fldname, default) \
+ do { \
+ if (node->fldname != default) \
+ appendStringInfo(str, " :" CppAsString(fldname) " %d", \
+ node->fldname); \
+ } while (0)
#define WRITE_INT_FIELD(fldname) \
- appendStringInfo(str, " :" CppAsString(fldname) " %d", node->fldname)
+ WRITE_INT_FIELD_DEFAULT(fldname, 0)
/* Write an unsigned integer field (anything written as ":fldname %u") */
+#define WRITE_UINT_FIELD_DEFAULT(fldname, default) \
+ do { \
+ if (node->fldname != default) \
+ appendStringInfo(str, " :" CppAsString(fldname) " %u", \
+ node->fldname); \
+ } while (0)
#define WRITE_UINT_FIELD(fldname) \
- appendStringInfo(str, " :" CppAsString(fldname) " %u", node->fldname)
+ WRITE_UINT_FIELD_DEFAULT(fldname, 0)
/* Write an unsigned integer field (anything written with UINT64_FORMAT) */
+#define WRITE_UINT64_FIELD_DEFAULT(fldname, default) \
+ do { \
+ if (node->fldname != default) \
+ appendStringInfo(str, " :" CppAsString(fldname) " " UINT64_FORMAT, \
+ node->fldname); \
+ } while (0)
#define WRITE_UINT64_FIELD(fldname) \
- appendStringInfo(str, " :" CppAsString(fldname) " " UINT64_FORMAT, \
- node->fldname)
+ WRITE_UINT64_FIELD_DEFAULT(fldname, 0)
/* Write an OID field (don't hard-wire assumption that OID is same as uint) */
+#define WRITE_OID_FIELD_DEFAULT(fldname, default) \
+ do { \
+ if (node->fldname != default) \
+ appendStringInfo(str, " :" CppAsString(fldname) " %u", \
+ node->fldname); \
+ } while (0)
#define WRITE_OID_FIELD(fldname) \
- appendStringInfo(str, " :" CppAsString(fldname) " %u", node->fldname)
+ WRITE_OID_FIELD_DEFAULT(fldname, 0)
/* Write a long-integer field */
+#define WRITE_LONG_FIELD_DEFAULT(fldname, default) \
+ do { \
+ if (node->fldname != default) \
+ appendStringInfo(str, " :" CppAsString(fldname) " %ld", \
+ node->fldname); \
+ } while (0)
#define WRITE_LONG_FIELD(fldname) \
- appendStringInfo(str, " :" CppAsString(fldname) " %ld", node->fldname)
+ WRITE_LONG_FIELD_DEFAULT(fldname, 0)
+
/* Write a char field (ie, one ascii character) */
+#define WRITE_CHAR_FIELD_DEFAULT(fldname, default) \
+ do { \
+ if (node->fldname != default) \
+ { \
+ appendStringInfoString(str, " :" CppAsString(fldname) " "); \
+ outChar(str, node->fldname); \
+ } \
+ } while (0)
#define WRITE_CHAR_FIELD(fldname) \
- (appendStringInfo(str, " :" CppAsString(fldname) " "), \
- outChar(str, node->fldname))
+ WRITE_CHAR_FIELD_DEFAULT(fldname, '\0')
/* Write an enumerated-type field as an integer code */
+#define WRITE_ENUM_FIELD_DEFAULT(fldname, enumtype, default) \
+ do { \
+ if (node->fldname != default) \
+ appendStringInfo(str, " :" CppAsString(fldname) " %d", \
+ (int) node->fldname); \
+ } while (0)
#define WRITE_ENUM_FIELD(fldname, enumtype) \
- appendStringInfo(str, " :" CppAsString(fldname) " %d", \
- (int) node->fldname)
+ WRITE_ENUM_FIELD_DEFAULT(fldname, enumtype, 0)
/* Write a float field (actually, they're double) */
+#define WRITE_FLOAT_FIELD_DEFAULT(fldname, default) \
+ do { \
+ if (node->fldname != default) \
+ { \
+ appendStringInfoString(str, " :" CppAsString(fldname) " "); \
+ outDouble(str, node->fldname); \
+ } \
+ } while (0)
#define WRITE_FLOAT_FIELD(fldname) \
- (appendStringInfo(str, " :" CppAsString(fldname) " "), \
- outDouble(str, node->fldname))
+ WRITE_FLOAT_FIELD_DEFAULT(fldname, 0.0)
/* Write a boolean field */
+#define WRITE_BOOL_FIELD_DEFAULT(fldname, default) \
+ do { \
+ if (node->fldname != default) \
+ appendStringInfo(str, " :" CppAsString(fldname) " %s", \
+ booltostr(node->fldname)); \
+ } while (0)
#define WRITE_BOOL_FIELD(fldname) \
- appendStringInfo(str, " :" CppAsString(fldname) " %s", \
- booltostr(node->fldname))
+ WRITE_BOOL_FIELD_DEFAULT(fldname, false)
+
+/*
+ * Non-null defaults of by-ref types are exceedingly rare (if not generally
+ * nonexistent), so we don't (yet) have a specialized macro for non-NULL
+ * defaults omission.
+ */
/* Write a character-string (possibly NULL) field */
#define WRITE_STRING_FIELD(fldname) \
- (appendStringInfoString(str, " :" CppAsString(fldname) " "), \
- outToken(str, node->fldname))
+ do { \
+ if (node->fldname != NULL) \
+ { \
+ appendStringInfoString(str, " :" CppAsString(fldname) " "); \
+ outToken(str, node->fldname); \
+ } \
+ } while (0)
/* Write a parse location field (actually same as INT case) */
#define WRITE_LOCATION_FIELD(fldname) \
- appendStringInfo(str, " :" CppAsString(fldname) " %d", node->fldname)
+ do { \
+ if (node->fldname != -1) \
+ appendStringInfo(str, " :" CppAsString(fldname) " %d", \
+ node->fldname); \
+ } while (0)
/* Write a Node field */
#define WRITE_NODE_FIELD(fldname) \
- (appendStringInfoString(str, " :" CppAsString(fldname) " "), \
- outNode(str, node->fldname))
+ do { \
+ if (node->fldname != NULL) \
+ { \
+ appendStringInfoString(str, " :" CppAsString(fldname) " "); \
+ outNode(str, node->fldname); \
+ } \
+ } while (0)
/* Write a bitmapset field */
#define WRITE_BITMAPSET_FIELD(fldname) \
- (appendStringInfoString(str, " :" CppAsString(fldname) " "), \
- outBitmapset(str, node->fldname))
+ do { \
+ if (node->fldname != NULL) \
+ { \
+ appendStringInfoString(str, " :" CppAsString(fldname) " "); \
+ outBitmapset(str, node->fldname); \
+ } \
+ } while (0)
/* Write a variable-length array (not a List) of Node pointers */
#define WRITE_NODE_ARRAY(fldname, len) \
- (appendStringInfoString(str, " :" CppAsString(fldname) " "), \
- writeNodeArray(str, (const Node * const *) node->fldname, len))
+ do { \
+ if (node->fldname != NULL) \
+ { \
+ appendStringInfoString(str, " :" CppAsString(fldname) " "); \
+ writeNodeArray(str, (const Node * const *) node->fldname, len); \
+ } \
+ } while (0)
/* Write a variable-length array of AttrNumber */
#define WRITE_ATTRNUMBER_ARRAY(fldname, len) \
- (appendStringInfoString(str, " :" CppAsString(fldname) " "), \
- writeAttrNumberCols(str, node->fldname, len))
+ do { \
+ if (node->fldname != NULL) \
+ { \
+ appendStringInfoString(str, " :" CppAsString(fldname) " "); \
+ writeAttrNumberCols(str, node->fldname, len); \
+ } \
+ } while (0)
/* Write a variable-length array of Oid */
#define WRITE_OID_ARRAY(fldname, len) \
- (appendStringInfoString(str, " :" CppAsString(fldname) " "), \
- writeOidCols(str, node->fldname, len))
+ do { \
+ if (node->fldname != NULL) \
+ { \
+ appendStringInfoString(str, " :" CppAsString(fldname) " "); \
+ writeOidCols(str, node->fldname, len); \
+ } \
+ } while (0)
/* Write a variable-length array of Index */
#define WRITE_INDEX_ARRAY(fldname, len) \
- (appendStringInfoString(str, " :" CppAsString(fldname) " "), \
- writeIndexCols(str, node->fldname, len))
+ do { \
+ if (node->fldname != NULL) \
+ { \
+ appendStringInfoString(str, " :" CppAsString(fldname) " "); \
+ writeIndexCols(str, node->fldname, len); \
+ } \
+ } while (0)
/* Write a variable-length array of int */
#define WRITE_INT_ARRAY(fldname, len) \
- (appendStringInfoString(str, " :" CppAsString(fldname) " "), \
- writeIntCols(str, node->fldname, len))
+ do { \
+ if (node->fldname != NULL) \
+ { \
+ appendStringInfoString(str, " :" CppAsString(fldname) " "); \
+ writeIntCols(str, node->fldname, len); \
+ } \
+ } while (0)
/* Write a variable-length array of bool */
#define WRITE_BOOL_ARRAY(fldname, len) \
- (appendStringInfoString(str, " :" CppAsString(fldname) " "), \
- writeBoolCols(str, node->fldname, len))
+ do { \
+ if (node->fldname != NULL) \
+ { \
+ appendStringInfoString(str, " :" CppAsString(fldname) " "); \
+ writeBoolCols(str, node->fldname, len); \
+ } \
+ } while (0)
#define booltostr(x) ((x) ? "true" : "false")
diff --git a/src/backend/nodes/read.c b/src/backend/nodes/read.c
index 969d0ec199..a4cd3a8932 100644
--- a/src/backend/nodes/read.c
+++ b/src/backend/nodes/read.c
@@ -205,6 +205,51 @@ pg_strtok(int *length)
return ret_str;
}
+/*
+ * Check if the next token is 'expect_token'.
+ *
+ * It handles similar to pg_strtok, except that this does not consume the
+ * next token, and has special casing for some less common tokens.
+ */
+bool
+pg_strtoken_next(const char *expect_token)
+{
+ const char *local_str; /* working pointer to string */
+ Size expect_len = strlen(expect_token);
+ char next_char;
+
+ local_str = pg_strtok_ptr;
+
+ while (*local_str == ' ' || *local_str == '\n' || *local_str == '\t')
+ local_str++;
+
+ if (*local_str == '\0')
+ return false; /* no more tokens */
+
+ Assert(expect_len > 0);
+
+ /* check if the next few bytes match the token */
+ if (strncmp(local_str, expect_token, expect_len) != 0)
+ return false;
+
+ next_char = local_str[expect_len];
+
+ /*
+ * Check that the token was actually terminated at the end of the
+ * expected token with a character that is a separate token.
+ * Otherwise, we'd get positive matches for mathing the token of "is"
+ * against a local_str of "isn't", which is clearly wrong.
+ */
+ return (next_char == '\0' ||
+ next_char == ' ' ||
+ next_char == '\n' ||
+ next_char == '\t' ||
+ next_char == '(' ||
+ next_char == ')' ||
+ next_char == '{' ||
+ next_char == '}');
+}
+
/*
* debackslash -
* create a palloc'd string holding the given token.
diff --git a/src/backend/nodes/readfuncs.c b/src/backend/nodes/readfuncs.c
index cfb552fde7..47f4ba2695 100644
--- a/src/backend/nodes/readfuncs.c
+++ b/src/backend/nodes/readfuncs.c
@@ -56,112 +56,207 @@
READ_LOCALS_NO_FIELDS(nodeTypeName); \
READ_TEMP_LOCALS()
+/* a scaffold function to read an optionally-omitted field */
+#define READ_OPT_SCAFFOLD(fldname, read_field_code, default_value) \
+ do { \
+ if (pg_strtoken_next(":" CppAsString(fldname))) \
+ { \
+ read_field_code; \
+ } \
+ else \
+ local_node->fldname = default_value; \
+ } while (0)
+
/* Read an integer field (anything written as ":fldname %d") */
-#define READ_INT_FIELD(fldname) \
+#define READ_INT_FIELD_DIRECT(fldname) \
token = pg_strtok(&length); /* skip :fldname */ \
+ Assert(strncmp(token, ":" CppAsString(fldname), length) == 0); \
token = pg_strtok(&length); /* get field value */ \
local_node->fldname = atoi(token)
+#define READ_INT_FIELD_DEFAULT(fldname, default_value) \
+ READ_OPT_SCAFFOLD(fldname, READ_INT_FIELD_DIRECT(fldname), default_value)
+#define READ_INT_FIELD(fldname) \
+ READ_INT_FIELD_DEFAULT(fldname, 0)
/* Read an unsigned integer field (anything written as ":fldname %u") */
-#define READ_UINT_FIELD(fldname) \
+#define READ_UINT_FIELD_DIRECT(fldname) \
token = pg_strtok(&length); /* skip :fldname */ \
+ Assert(strncmp(token, ":" CppAsString(fldname), length) == 0); \
token = pg_strtok(&length); /* get field value */ \
local_node->fldname = atoui(token)
+#define READ_UINT_FIELD_DEFAULT(fldname, default_value) \
+ READ_OPT_SCAFFOLD(fldname, READ_UINT_FIELD_DIRECT(fldname), default_value)
+#define READ_UINT_FIELD(fldname) \
+ READ_UINT_FIELD_DEFAULT(fldname, 0)
/* Read an unsigned integer field (anything written using UINT64_FORMAT) */
-#define READ_UINT64_FIELD(fldname) \
+#define READ_UINT64_FIELD_DIRECT(fldname) \
token = pg_strtok(&length); /* skip :fldname */ \
+ Assert(strncmp(token, ":" CppAsString(fldname), length) == 0); \
token = pg_strtok(&length); /* get field value */ \
local_node->fldname = strtou64(token, NULL, 10)
+#define READ_UINT64_FIELD_DEFAULT(fldname, default_value) \
+ READ_OPT_SCAFFOLD(fldname, READ_UINT64_FIELD_DIRECT(fldname), default_value)
+#define READ_UINT64_FIELD(fldname) \
+ READ_UINT64_FIELD_DEFAULT(fldname, 0)
/* Read a long integer field (anything written as ":fldname %ld") */
-#define READ_LONG_FIELD(fldname) \
+#define READ_LONG_FIELD_DIRECT(fldname) \
token = pg_strtok(&length); /* skip :fldname */ \
+ Assert(strncmp(token, ":" CppAsString(fldname), length) == 0); \
token = pg_strtok(&length); /* get field value */ \
local_node->fldname = atol(token)
+#define READ_LONG_FIELD_DEFAULT(fldname, default_value) \
+ READ_OPT_SCAFFOLD(fldname, READ_LONG_FIELD_DIRECT(fldname), default_value)
+#define READ_LONG_FIELD(fldname) \
+ READ_LONG_FIELD_DEFAULT(fldname, 0)
/* Read an OID field (don't hard-wire assumption that OID is same as uint) */
-#define READ_OID_FIELD(fldname) \
+#define READ_OID_FIELD_DIRECT(fldname) \
token = pg_strtok(&length); /* skip :fldname */ \
+ Assert(strncmp(token, ":" CppAsString(fldname), length) == 0); \
token = pg_strtok(&length); /* get field value */ \
local_node->fldname = atooid(token)
+#define READ_OID_FIELD_DEFAULT(fldname, default_value) \
+ READ_OPT_SCAFFOLD(fldname, READ_OID_FIELD_DIRECT(fldname), default_value)
+#define READ_OID_FIELD(fldname) \
+ READ_OID_FIELD_DEFAULT(fldname, 0)
/* Read a char field (ie, one ascii character) */
-#define READ_CHAR_FIELD(fldname) \
+#define READ_CHAR_FIELD_DIRECT(fldname) \
token = pg_strtok(&length); /* skip :fldname */ \
+ Assert(strncmp(token, ":" CppAsString(fldname), length) == 0); \
token = pg_strtok(&length); /* get field value */ \
/* avoid overhead of calling debackslash() for one char */ \
local_node->fldname = (length == 0) ? '\0' : (token[0] == '\\' ? token[1] : token[0])
+#define READ_CHAR_FIELD_DEFAULT(fldname, default_value) \
+ READ_OPT_SCAFFOLD(fldname, READ_CHAR_FIELD_DIRECT(fldname), default_value)
+#define READ_CHAR_FIELD(fldname) \
+ READ_CHAR_FIELD_DEFAULT(fldname, '\0')
/* Read an enumerated-type field that was written as an integer code */
-#define READ_ENUM_FIELD(fldname, enumtype) \
+#define READ_ENUM_FIELD_DIRECT(fldname, enumtype) \
token = pg_strtok(&length); /* skip :fldname */ \
+ Assert(strncmp(token, ":" CppAsString(fldname), length) == 0); \
token = pg_strtok(&length); /* get field value */ \
local_node->fldname = (enumtype) atoi(token)
+#define READ_ENUM_FIELD_DEFAULT(fldname, enumtype, default_value) \
+ READ_OPT_SCAFFOLD(fldname, READ_ENUM_FIELD_DIRECT(fldname, enumtype), default_value)
+#define READ_ENUM_FIELD(fldname, enumtype) \
+ READ_ENUM_FIELD_DEFAULT(fldname, enumtype, 0)
/* Read a float field */
-#define READ_FLOAT_FIELD(fldname) \
+#define READ_FLOAT_FIELD_DIRECT(fldname) \
token = pg_strtok(&length); /* skip :fldname */ \
+ Assert(strncmp(token, ":" CppAsString(fldname), length) == 0); \
token = pg_strtok(&length); /* get field value */ \
local_node->fldname = atof(token)
+#define READ_FLOAT_FIELD_DEFAULT(fldname, default_value) \
+ READ_OPT_SCAFFOLD(fldname, READ_FLOAT_FIELD_DIRECT(fldname), default_value)
+#define READ_FLOAT_FIELD(fldname) \
+ READ_FLOAT_FIELD_DEFAULT(fldname, 0.0)
/* Read a boolean field */
-#define READ_BOOL_FIELD(fldname) \
+#define READ_BOOL_FIELD_DIRECT(fldname) \
token = pg_strtok(&length); /* skip :fldname */ \
+ Assert(strncmp(token, ":" CppAsString(fldname), length) == 0); \
token = pg_strtok(&length); /* get field value */ \
local_node->fldname = strtobool(token)
+#define READ_BOOL_FIELD_DEFAULT(fldname, default_value) \
+ READ_OPT_SCAFFOLD(fldname, READ_BOOL_FIELD_DIRECT(fldname), default_value)
+#define READ_BOOL_FIELD(fldname) \
+ READ_BOOL_FIELD_DEFAULT(fldname, false)
/* Read a character-string field */
-#define READ_STRING_FIELD(fldname) \
+#define READ_STRING_FIELD_DIRECT(fldname) \
token = pg_strtok(&length); /* skip :fldname */ \
+ Assert(strncmp(token, ":" CppAsString(fldname), length) == 0); \
token = pg_strtok(&length); /* get field value */ \
local_node->fldname = nullable_string(token, length)
+/* see WRITE_STRING_FIELD in outfuncs.c */
+#define READ_STRING_FIELD(fldname) \
+ READ_OPT_SCAFFOLD(fldname, READ_STRING_FIELD_DIRECT(fldname), NULL)
/* Read a parse location field (and possibly throw away the value) */
#ifdef WRITE_READ_PARSE_PLAN_TREES
-#define READ_LOCATION_FIELD(fldname) \
+#define READ_LOCATION_FIELD_DIRECT(fldname) \
token = pg_strtok(&length); /* skip :fldname */ \
+ Assert(strncmp(token, ":" CppAsString(fldname), length) == 0); \
token = pg_strtok(&length); /* get field value */ \
local_node->fldname = restore_location_fields ? atoi(token) : -1
#else
-#define READ_LOCATION_FIELD(fldname) \
+#define READ_LOCATION_FIELD_DIRECT(fldname) \
token = pg_strtok(&length); /* skip :fldname */ \
+ Assert(strncmp(token, ":" CppAsString(fldname), length) == 0); \
token = pg_strtok(&length); /* get field value */ \
(void) token; /* in case not used elsewhere */ \
local_node->fldname = -1 /* set field to "unknown" */
#endif
+/* The default Location field value is -1 ('unknown') */
+#define READ_LOCATION_FIELD(fldname) \
+ READ_OPT_SCAFFOLD(fldname, READ_LOCATION_FIELD_DIRECT(fldname), -1)
/* Read a Node field */
-#define READ_NODE_FIELD(fldname) \
+#define READ_NODE_FIELD_DIRECT(fldname) \
token = pg_strtok(&length); /* skip :fldname */ \
+ Assert(strncmp(token, ":" CppAsString(fldname), length) == 0); \
(void) token; /* in case not used elsewhere */ \
local_node->fldname = nodeRead(NULL, 0)
+#define READ_NODE_FIELD_DEFAULT(fldname, default_value) \
+ READ_OPT_SCAFFOLD(fldname, READ_NODE_FIELD_DIRECT(fldname), default_value)
+#define READ_NODE_FIELD(fldname) \
+ READ_NODE_FIELD_DEFAULT(fldname, NULL)
/* Read a bitmapset field */
-#define READ_BITMAPSET_FIELD(fldname) \
+#define READ_BITMAPSET_FIELD_DIRECT(fldname) \
token = pg_strtok(&length); /* skip :fldname */ \
+ Assert(strncmp(token, ":" CppAsString(fldname), length) == 0); \
(void) token; /* in case not used elsewhere */ \
local_node->fldname = _readBitmapset()
+#define READ_BITMAPSET_FIELD_DEFAULT(fldname, default_value) \
+ READ_OPT_SCAFFOLD(fldname, READ_BITMAPSET_FIELD_DIRECT(fldname), default_value)
+#define READ_BITMAPSET_FIELD(fldname) \
+ READ_BITMAPSET_FIELD_DEFAULT(fldname, NULL)
/* Read an attribute number array */
-#define READ_ATTRNUMBER_ARRAY(fldname, len) \
+#define READ_ATTRNUMBER_ARRAY_DIRECT(fldname, len) \
token = pg_strtok(&length); /* skip :fldname */ \
+ Assert(strncmp(token, ":" CppAsString(fldname), length) == 0); \
local_node->fldname = readAttrNumberCols(len)
+#define READ_ATTRNUMBER_ARRAY_DEFAULT(fldname, len, default_value) \
+ READ_OPT_SCAFFOLD(fldname, READ_ATTRNUMBER_ARRAY_DIRECT(fldname, len), default_value)
+#define READ_ATTRNUMBER_ARRAY(fldname, len) \
+ READ_ATTRNUMBER_ARRAY_DEFAULT(fldname, len, NULL)
/* Read an oid array */
-#define READ_OID_ARRAY(fldname, len) \
+#define READ_OID_ARRAY_DIRECT(fldname, len) \
token = pg_strtok(&length); /* skip :fldname */ \
+ Assert(strncmp(token, ":" CppAsString(fldname), length) == 0); \
local_node->fldname = readOidCols(len)
+#define READ_OID_ARRAY_DEFAULT(fldname, len, default_value) \
+ READ_OPT_SCAFFOLD(fldname, READ_OID_ARRAY_DIRECT(fldname, len), default_value)
+#define READ_OID_ARRAY(fldname, len) \
+ READ_OID_ARRAY_DEFAULT(fldname, len, NULL)
/* Read an int array */
-#define READ_INT_ARRAY(fldname, len) \
+#define READ_INT_ARRAY_DIRECT(fldname, len) \
token = pg_strtok(&length); /* skip :fldname */ \
+ Assert(strncmp(token, ":" CppAsString(fldname), length) == 0); \
local_node->fldname = readIntCols(len)
+#define READ_INT_ARRAY_DEFAULT(fldname, len, default_value) \
+ READ_OPT_SCAFFOLD(fldname, READ_INT_ARRAY_DIRECT(fldname, len), default_value)
+#define READ_INT_ARRAY(fldname, len) \
+ READ_INT_ARRAY_DEFAULT(fldname, len, NULL)
/* Read a bool array */
-#define READ_BOOL_ARRAY(fldname, len) \
+#define READ_BOOL_ARRAY_DIRECT(fldname, len) \
token = pg_strtok(&length); /* skip :fldname */ \
+ Assert(strncmp(token, ":" CppAsString(fldname), length) == 0); \
local_node->fldname = readBoolCols(len)
+#define READ_BOOL_ARRAY_DEFAULT(fldname, len, default_value) \
+ READ_OPT_SCAFFOLD(fldname, READ_BOOL_ARRAY_DIRECT(fldname, len), default_value)
+#define READ_BOOL_ARRAY(fldname, len) \
+ READ_BOOL_ARRAY_DEFAULT(fldname, len, NULL)
/* Routine exit */
#define READ_DONE() \
diff --git a/src/include/nodes/readfuncs.h b/src/include/nodes/readfuncs.h
index 8466038ed0..dab4547db2 100644
--- a/src/include/nodes/readfuncs.h
+++ b/src/include/nodes/readfuncs.h
@@ -27,6 +27,7 @@ extern PGDLLIMPORT bool restore_location_fields;
* prototypes for functions in read.c (the lisp token parser)
*/
extern const char *pg_strtok(int *length);
+extern bool pg_strtoken_next(const char *expect_token);
extern char *debackslash(const char *token, int length);
extern void *nodeRead(const char *token, int tok_len);
diff --git a/src/test/regress/expected/rowsecurity.out b/src/test/regress/expected/rowsecurity.out
index 6988128aa4..a69aa40f82 100644
--- a/src/test/regress/expected/rowsecurity.out
+++ b/src/test/regress/expected/rowsecurity.out
@@ -3937,7 +3937,10 @@ CREATE TABLE coll_t (c) AS VALUES ('bar'::text);
CREATE POLICY coll_p ON coll_t USING (c < ('foo'::text COLLATE "C"));
ALTER TABLE coll_t ENABLE ROW LEVEL SECURITY;
GRANT SELECT ON coll_t TO regress_rls_alice;
-SELECT (string_to_array(polqual, ':'))[7] AS inputcollid FROM pg_policy WHERE polrelid = 'coll_t'::regclass;
+SELECT split AS inputcollid
+FROM pg_policy,
+ lateral unnest(string_to_array(polqual, ':')) as split
+WHERE polrelid = 'coll_t'::regclass and split LIKE '%inputcollid%';
inputcollid
------------------
inputcollid 950
diff --git a/src/test/regress/sql/rowsecurity.sql b/src/test/regress/sql/rowsecurity.sql
index dec7340538..cb7a4c776b 100644
--- a/src/test/regress/sql/rowsecurity.sql
+++ b/src/test/regress/sql/rowsecurity.sql
@@ -1732,7 +1732,10 @@ CREATE TABLE coll_t (c) AS VALUES ('bar'::text);
CREATE POLICY coll_p ON coll_t USING (c < ('foo'::text COLLATE "C"));
ALTER TABLE coll_t ENABLE ROW LEVEL SECURITY;
GRANT SELECT ON coll_t TO regress_rls_alice;
-SELECT (string_to_array(polqual, ':'))[7] AS inputcollid FROM pg_policy WHERE polrelid = 'coll_t'::regclass;
+SELECT split AS inputcollid
+FROM pg_policy,
+ lateral unnest(string_to_array(polqual, ':')) as split
+WHERE polrelid = 'coll_t'::regclass and split LIKE '%inputcollid%';
SET SESSION AUTHORIZATION regress_rls_alice;
SELECT * FROM coll_t;
ROLLBACK;
--
2.40.1
^ permalink raw reply [nested|flat] 69+ messages in thread
* Re: Reducing output size of nodeToString
@ 2024-02-12 23:10 Matthias van de Meent <[email protected]>
parent: Matthias van de Meent <[email protected]>
0 siblings, 1 reply; 69+ messages in thread
From: Matthias van de Meent @ 2024-02-12 23:10 UTC (permalink / raw)
To: ; +Cc: Robert Haas <[email protected]>; Peter Eisentraut <[email protected]>; PostgreSQL Hackers <[email protected]>; Andres Freund <[email protected]>; Michel Pelletier <[email protected]>
On Mon, 12 Feb 2024 at 20:32, Matthias van de Meent
<[email protected]> wrote:
>
> On Mon, 12 Feb 2024 at 19:03, Matthias van de Meent
> <[email protected]> wrote:
> > Attached is patchset v2, which contains the improvements from these patches:
>
> Attached v3, which fixes an out-of-bounds read in pg_strtoken_next,
> detected by asan, that was a likely cause of the problems in CFBot's
> FreeBSD regression tests.
Apparently that was caused by issues in my updated bitmapset
serializer; where I used bms_next_member(..., x=0) as first iteration
thus skipping the first bit. This didn't show up earlier because that
bit is not exercised in PG's builtin views, but is exercised when
WRITE_READ_PARSE_PLAN_TREES is defined (as on the FreeBSD CI job).
Trivial fix in the attached v4 of the patchset, with some fixes for
other assertions that'd get some exercise in non-pg_node_tree paths in
the WRITE_READ configuration.
Attachments:
[application/octet-stream] v4-0001-pg_node_tree-Omit-serialization-of-fields-with-de.patch (23.0K, ../../CAEze2WhR_99QHboeXU73u8tc_N4zSnkoyhpm8ZuRS6gh49ZGjw@mail.gmail.com/2-v4-0001-pg_node_tree-Omit-serialization-of-fields-with-de.patch)
download | inline diff:
From b53eabb4d26d04ff527db9e82cccb15b5271f58e Mon Sep 17 00:00:00 2001
From: Matthias van de Meent <[email protected]>
Date: Tue, 2 Jan 2024 23:55:02 +0100
Subject: [PATCH v4 1/7] pg_node_tree: Omit serialization of fields with
default values.
Often, the values in nodes are their default values. By not serializing
those fields and inserting the defaults during deserialization, we
reduce the size of pg_node_tree attributes seen in e.g. pg_rewrite by a
significant factor.
In passing, we fix a test that had a strict dependency on the
serialization of pg_node_tree; we now do the checks in a more generic
manner, making it more stable and ensuring its stability in future work.
---
src/backend/nodes/outfuncs.c | 175 ++++++++++++++++++----
src/backend/nodes/read.c | 45 ++++++
src/backend/nodes/readfuncs.c | 131 +++++++++++++---
src/include/nodes/readfuncs.h | 1 +
src/test/regress/expected/rowsecurity.out | 5 +-
src/test/regress/sql/rowsecurity.sql | 5 +-
6 files changed, 309 insertions(+), 53 deletions(-)
diff --git a/src/backend/nodes/outfuncs.c b/src/backend/nodes/outfuncs.c
index 03f67b6850..50da80ee34 100644
--- a/src/backend/nodes/outfuncs.c
+++ b/src/backend/nodes/outfuncs.c
@@ -41,94 +41,203 @@ static void outDouble(StringInfo str, double d);
appendStringInfoString(str, nodelabel)
/* Write an integer field (anything written as ":fldname %d") */
+#define WRITE_INT_FIELD_DEFAULT(fldname, default) \
+ do { \
+ if (node->fldname != default) \
+ appendStringInfo(str, " :" CppAsString(fldname) " %d", \
+ node->fldname); \
+ } while (0)
#define WRITE_INT_FIELD(fldname) \
- appendStringInfo(str, " :" CppAsString(fldname) " %d", node->fldname)
+ WRITE_INT_FIELD_DEFAULT(fldname, 0)
/* Write an unsigned integer field (anything written as ":fldname %u") */
+#define WRITE_UINT_FIELD_DEFAULT(fldname, default) \
+ do { \
+ if (node->fldname != default) \
+ appendStringInfo(str, " :" CppAsString(fldname) " %u", \
+ node->fldname); \
+ } while (0)
#define WRITE_UINT_FIELD(fldname) \
- appendStringInfo(str, " :" CppAsString(fldname) " %u", node->fldname)
+ WRITE_UINT_FIELD_DEFAULT(fldname, 0)
/* Write an unsigned integer field (anything written with UINT64_FORMAT) */
+#define WRITE_UINT64_FIELD_DEFAULT(fldname, default) \
+ do { \
+ if (node->fldname != default) \
+ appendStringInfo(str, " :" CppAsString(fldname) " " UINT64_FORMAT, \
+ node->fldname); \
+ } while (0)
#define WRITE_UINT64_FIELD(fldname) \
- appendStringInfo(str, " :" CppAsString(fldname) " " UINT64_FORMAT, \
- node->fldname)
+ WRITE_UINT64_FIELD_DEFAULT(fldname, 0)
/* Write an OID field (don't hard-wire assumption that OID is same as uint) */
+#define WRITE_OID_FIELD_DEFAULT(fldname, default) \
+ do { \
+ if (node->fldname != default) \
+ appendStringInfo(str, " :" CppAsString(fldname) " %u", \
+ node->fldname); \
+ } while (0)
#define WRITE_OID_FIELD(fldname) \
- appendStringInfo(str, " :" CppAsString(fldname) " %u", node->fldname)
+ WRITE_OID_FIELD_DEFAULT(fldname, 0)
/* Write a long-integer field */
+#define WRITE_LONG_FIELD_DEFAULT(fldname, default) \
+ do { \
+ if (node->fldname != default) \
+ appendStringInfo(str, " :" CppAsString(fldname) " %ld", \
+ node->fldname); \
+ } while (0)
#define WRITE_LONG_FIELD(fldname) \
- appendStringInfo(str, " :" CppAsString(fldname) " %ld", node->fldname)
+ WRITE_LONG_FIELD_DEFAULT(fldname, 0)
+
/* Write a char field (ie, one ascii character) */
+#define WRITE_CHAR_FIELD_DEFAULT(fldname, default) \
+ do { \
+ if (node->fldname != default) \
+ { \
+ appendStringInfoString(str, " :" CppAsString(fldname) " "); \
+ outChar(str, node->fldname); \
+ } \
+ } while (0)
#define WRITE_CHAR_FIELD(fldname) \
- (appendStringInfo(str, " :" CppAsString(fldname) " "), \
- outChar(str, node->fldname))
+ WRITE_CHAR_FIELD_DEFAULT(fldname, '\0')
/* Write an enumerated-type field as an integer code */
+#define WRITE_ENUM_FIELD_DEFAULT(fldname, enumtype, default) \
+ do { \
+ if (node->fldname != default) \
+ appendStringInfo(str, " :" CppAsString(fldname) " %d", \
+ (int) node->fldname); \
+ } while (0)
#define WRITE_ENUM_FIELD(fldname, enumtype) \
- appendStringInfo(str, " :" CppAsString(fldname) " %d", \
- (int) node->fldname)
+ WRITE_ENUM_FIELD_DEFAULT(fldname, enumtype, 0)
/* Write a float field (actually, they're double) */
+#define WRITE_FLOAT_FIELD_DEFAULT(fldname, default) \
+ do { \
+ if (node->fldname != default) \
+ { \
+ appendStringInfoString(str, " :" CppAsString(fldname) " "); \
+ outDouble(str, node->fldname); \
+ } \
+ } while (0)
#define WRITE_FLOAT_FIELD(fldname) \
- (appendStringInfo(str, " :" CppAsString(fldname) " "), \
- outDouble(str, node->fldname))
+ WRITE_FLOAT_FIELD_DEFAULT(fldname, 0.0)
/* Write a boolean field */
+#define WRITE_BOOL_FIELD_DEFAULT(fldname, default) \
+ do { \
+ if (node->fldname != default) \
+ appendStringInfo(str, " :" CppAsString(fldname) " %s", \
+ booltostr(node->fldname)); \
+ } while (0)
#define WRITE_BOOL_FIELD(fldname) \
- appendStringInfo(str, " :" CppAsString(fldname) " %s", \
- booltostr(node->fldname))
+ WRITE_BOOL_FIELD_DEFAULT(fldname, false)
+
+/*
+ * Non-null defaults of by-ref types are exceedingly rare (if not generally
+ * nonexistent), so we don't (yet) have a specialized macro for non-NULL
+ * defaults omission.
+ */
/* Write a character-string (possibly NULL) field */
#define WRITE_STRING_FIELD(fldname) \
- (appendStringInfoString(str, " :" CppAsString(fldname) " "), \
- outToken(str, node->fldname))
+ do { \
+ if (node->fldname != NULL) \
+ { \
+ appendStringInfoString(str, " :" CppAsString(fldname) " "); \
+ outToken(str, node->fldname); \
+ } \
+ } while (0)
/* Write a parse location field (actually same as INT case) */
#define WRITE_LOCATION_FIELD(fldname) \
- appendStringInfo(str, " :" CppAsString(fldname) " %d", node->fldname)
+ do { \
+ if (node->fldname != -1) \
+ appendStringInfo(str, " :" CppAsString(fldname) " %d", \
+ node->fldname); \
+ } while (0)
/* Write a Node field */
#define WRITE_NODE_FIELD(fldname) \
- (appendStringInfoString(str, " :" CppAsString(fldname) " "), \
- outNode(str, node->fldname))
+ do { \
+ if (node->fldname != NULL) \
+ { \
+ appendStringInfoString(str, " :" CppAsString(fldname) " "); \
+ outNode(str, node->fldname); \
+ } \
+ } while (0)
/* Write a bitmapset field */
#define WRITE_BITMAPSET_FIELD(fldname) \
- (appendStringInfoString(str, " :" CppAsString(fldname) " "), \
- outBitmapset(str, node->fldname))
+ do { \
+ if (node->fldname != NULL) \
+ { \
+ appendStringInfoString(str, " :" CppAsString(fldname) " "); \
+ outBitmapset(str, node->fldname); \
+ } \
+ } while (0)
/* Write a variable-length array (not a List) of Node pointers */
#define WRITE_NODE_ARRAY(fldname, len) \
- (appendStringInfoString(str, " :" CppAsString(fldname) " "), \
- writeNodeArray(str, (const Node * const *) node->fldname, len))
+ do { \
+ if (node->fldname != NULL) \
+ { \
+ appendStringInfoString(str, " :" CppAsString(fldname) " "); \
+ writeNodeArray(str, (const Node * const *) node->fldname, len); \
+ } \
+ } while (0)
/* Write a variable-length array of AttrNumber */
#define WRITE_ATTRNUMBER_ARRAY(fldname, len) \
- (appendStringInfoString(str, " :" CppAsString(fldname) " "), \
- writeAttrNumberCols(str, node->fldname, len))
+ do { \
+ if (node->fldname != NULL) \
+ { \
+ appendStringInfoString(str, " :" CppAsString(fldname) " "); \
+ writeAttrNumberCols(str, node->fldname, len); \
+ } \
+ } while (0)
/* Write a variable-length array of Oid */
#define WRITE_OID_ARRAY(fldname, len) \
- (appendStringInfoString(str, " :" CppAsString(fldname) " "), \
- writeOidCols(str, node->fldname, len))
+ do { \
+ if (node->fldname != NULL) \
+ { \
+ appendStringInfoString(str, " :" CppAsString(fldname) " "); \
+ writeOidCols(str, node->fldname, len); \
+ } \
+ } while (0)
/* Write a variable-length array of Index */
#define WRITE_INDEX_ARRAY(fldname, len) \
- (appendStringInfoString(str, " :" CppAsString(fldname) " "), \
- writeIndexCols(str, node->fldname, len))
+ do { \
+ if (node->fldname != NULL) \
+ { \
+ appendStringInfoString(str, " :" CppAsString(fldname) " "); \
+ writeIndexCols(str, node->fldname, len); \
+ } \
+ } while (0)
/* Write a variable-length array of int */
#define WRITE_INT_ARRAY(fldname, len) \
- (appendStringInfoString(str, " :" CppAsString(fldname) " "), \
- writeIntCols(str, node->fldname, len))
+ do { \
+ if (node->fldname != NULL) \
+ { \
+ appendStringInfoString(str, " :" CppAsString(fldname) " "); \
+ writeIntCols(str, node->fldname, len); \
+ } \
+ } while (0)
/* Write a variable-length array of bool */
#define WRITE_BOOL_ARRAY(fldname, len) \
- (appendStringInfoString(str, " :" CppAsString(fldname) " "), \
- writeBoolCols(str, node->fldname, len))
+ do { \
+ if (node->fldname != NULL) \
+ { \
+ appendStringInfoString(str, " :" CppAsString(fldname) " "); \
+ writeBoolCols(str, node->fldname, len); \
+ } \
+ } while (0)
#define booltostr(x) ((x) ? "true" : "false")
diff --git a/src/backend/nodes/read.c b/src/backend/nodes/read.c
index 969d0ec199..a4cd3a8932 100644
--- a/src/backend/nodes/read.c
+++ b/src/backend/nodes/read.c
@@ -205,6 +205,51 @@ pg_strtok(int *length)
return ret_str;
}
+/*
+ * Check if the next token is 'expect_token'.
+ *
+ * It handles similar to pg_strtok, except that this does not consume the
+ * next token, and has special casing for some less common tokens.
+ */
+bool
+pg_strtoken_next(const char *expect_token)
+{
+ const char *local_str; /* working pointer to string */
+ Size expect_len = strlen(expect_token);
+ char next_char;
+
+ local_str = pg_strtok_ptr;
+
+ while (*local_str == ' ' || *local_str == '\n' || *local_str == '\t')
+ local_str++;
+
+ if (*local_str == '\0')
+ return false; /* no more tokens */
+
+ Assert(expect_len > 0);
+
+ /* check if the next few bytes match the token */
+ if (strncmp(local_str, expect_token, expect_len) != 0)
+ return false;
+
+ next_char = local_str[expect_len];
+
+ /*
+ * Check that the token was actually terminated at the end of the
+ * expected token with a character that is a separate token.
+ * Otherwise, we'd get positive matches for mathing the token of "is"
+ * against a local_str of "isn't", which is clearly wrong.
+ */
+ return (next_char == '\0' ||
+ next_char == ' ' ||
+ next_char == '\n' ||
+ next_char == '\t' ||
+ next_char == '(' ||
+ next_char == ')' ||
+ next_char == '{' ||
+ next_char == '}');
+}
+
/*
* debackslash -
* create a palloc'd string holding the given token.
diff --git a/src/backend/nodes/readfuncs.c b/src/backend/nodes/readfuncs.c
index cfb552fde7..47f4ba2695 100644
--- a/src/backend/nodes/readfuncs.c
+++ b/src/backend/nodes/readfuncs.c
@@ -56,112 +56,207 @@
READ_LOCALS_NO_FIELDS(nodeTypeName); \
READ_TEMP_LOCALS()
+/* a scaffold function to read an optionally-omitted field */
+#define READ_OPT_SCAFFOLD(fldname, read_field_code, default_value) \
+ do { \
+ if (pg_strtoken_next(":" CppAsString(fldname))) \
+ { \
+ read_field_code; \
+ } \
+ else \
+ local_node->fldname = default_value; \
+ } while (0)
+
/* Read an integer field (anything written as ":fldname %d") */
-#define READ_INT_FIELD(fldname) \
+#define READ_INT_FIELD_DIRECT(fldname) \
token = pg_strtok(&length); /* skip :fldname */ \
+ Assert(strncmp(token, ":" CppAsString(fldname), length) == 0); \
token = pg_strtok(&length); /* get field value */ \
local_node->fldname = atoi(token)
+#define READ_INT_FIELD_DEFAULT(fldname, default_value) \
+ READ_OPT_SCAFFOLD(fldname, READ_INT_FIELD_DIRECT(fldname), default_value)
+#define READ_INT_FIELD(fldname) \
+ READ_INT_FIELD_DEFAULT(fldname, 0)
/* Read an unsigned integer field (anything written as ":fldname %u") */
-#define READ_UINT_FIELD(fldname) \
+#define READ_UINT_FIELD_DIRECT(fldname) \
token = pg_strtok(&length); /* skip :fldname */ \
+ Assert(strncmp(token, ":" CppAsString(fldname), length) == 0); \
token = pg_strtok(&length); /* get field value */ \
local_node->fldname = atoui(token)
+#define READ_UINT_FIELD_DEFAULT(fldname, default_value) \
+ READ_OPT_SCAFFOLD(fldname, READ_UINT_FIELD_DIRECT(fldname), default_value)
+#define READ_UINT_FIELD(fldname) \
+ READ_UINT_FIELD_DEFAULT(fldname, 0)
/* Read an unsigned integer field (anything written using UINT64_FORMAT) */
-#define READ_UINT64_FIELD(fldname) \
+#define READ_UINT64_FIELD_DIRECT(fldname) \
token = pg_strtok(&length); /* skip :fldname */ \
+ Assert(strncmp(token, ":" CppAsString(fldname), length) == 0); \
token = pg_strtok(&length); /* get field value */ \
local_node->fldname = strtou64(token, NULL, 10)
+#define READ_UINT64_FIELD_DEFAULT(fldname, default_value) \
+ READ_OPT_SCAFFOLD(fldname, READ_UINT64_FIELD_DIRECT(fldname), default_value)
+#define READ_UINT64_FIELD(fldname) \
+ READ_UINT64_FIELD_DEFAULT(fldname, 0)
/* Read a long integer field (anything written as ":fldname %ld") */
-#define READ_LONG_FIELD(fldname) \
+#define READ_LONG_FIELD_DIRECT(fldname) \
token = pg_strtok(&length); /* skip :fldname */ \
+ Assert(strncmp(token, ":" CppAsString(fldname), length) == 0); \
token = pg_strtok(&length); /* get field value */ \
local_node->fldname = atol(token)
+#define READ_LONG_FIELD_DEFAULT(fldname, default_value) \
+ READ_OPT_SCAFFOLD(fldname, READ_LONG_FIELD_DIRECT(fldname), default_value)
+#define READ_LONG_FIELD(fldname) \
+ READ_LONG_FIELD_DEFAULT(fldname, 0)
/* Read an OID field (don't hard-wire assumption that OID is same as uint) */
-#define READ_OID_FIELD(fldname) \
+#define READ_OID_FIELD_DIRECT(fldname) \
token = pg_strtok(&length); /* skip :fldname */ \
+ Assert(strncmp(token, ":" CppAsString(fldname), length) == 0); \
token = pg_strtok(&length); /* get field value */ \
local_node->fldname = atooid(token)
+#define READ_OID_FIELD_DEFAULT(fldname, default_value) \
+ READ_OPT_SCAFFOLD(fldname, READ_OID_FIELD_DIRECT(fldname), default_value)
+#define READ_OID_FIELD(fldname) \
+ READ_OID_FIELD_DEFAULT(fldname, 0)
/* Read a char field (ie, one ascii character) */
-#define READ_CHAR_FIELD(fldname) \
+#define READ_CHAR_FIELD_DIRECT(fldname) \
token = pg_strtok(&length); /* skip :fldname */ \
+ Assert(strncmp(token, ":" CppAsString(fldname), length) == 0); \
token = pg_strtok(&length); /* get field value */ \
/* avoid overhead of calling debackslash() for one char */ \
local_node->fldname = (length == 0) ? '\0' : (token[0] == '\\' ? token[1] : token[0])
+#define READ_CHAR_FIELD_DEFAULT(fldname, default_value) \
+ READ_OPT_SCAFFOLD(fldname, READ_CHAR_FIELD_DIRECT(fldname), default_value)
+#define READ_CHAR_FIELD(fldname) \
+ READ_CHAR_FIELD_DEFAULT(fldname, '\0')
/* Read an enumerated-type field that was written as an integer code */
-#define READ_ENUM_FIELD(fldname, enumtype) \
+#define READ_ENUM_FIELD_DIRECT(fldname, enumtype) \
token = pg_strtok(&length); /* skip :fldname */ \
+ Assert(strncmp(token, ":" CppAsString(fldname), length) == 0); \
token = pg_strtok(&length); /* get field value */ \
local_node->fldname = (enumtype) atoi(token)
+#define READ_ENUM_FIELD_DEFAULT(fldname, enumtype, default_value) \
+ READ_OPT_SCAFFOLD(fldname, READ_ENUM_FIELD_DIRECT(fldname, enumtype), default_value)
+#define READ_ENUM_FIELD(fldname, enumtype) \
+ READ_ENUM_FIELD_DEFAULT(fldname, enumtype, 0)
/* Read a float field */
-#define READ_FLOAT_FIELD(fldname) \
+#define READ_FLOAT_FIELD_DIRECT(fldname) \
token = pg_strtok(&length); /* skip :fldname */ \
+ Assert(strncmp(token, ":" CppAsString(fldname), length) == 0); \
token = pg_strtok(&length); /* get field value */ \
local_node->fldname = atof(token)
+#define READ_FLOAT_FIELD_DEFAULT(fldname, default_value) \
+ READ_OPT_SCAFFOLD(fldname, READ_FLOAT_FIELD_DIRECT(fldname), default_value)
+#define READ_FLOAT_FIELD(fldname) \
+ READ_FLOAT_FIELD_DEFAULT(fldname, 0.0)
/* Read a boolean field */
-#define READ_BOOL_FIELD(fldname) \
+#define READ_BOOL_FIELD_DIRECT(fldname) \
token = pg_strtok(&length); /* skip :fldname */ \
+ Assert(strncmp(token, ":" CppAsString(fldname), length) == 0); \
token = pg_strtok(&length); /* get field value */ \
local_node->fldname = strtobool(token)
+#define READ_BOOL_FIELD_DEFAULT(fldname, default_value) \
+ READ_OPT_SCAFFOLD(fldname, READ_BOOL_FIELD_DIRECT(fldname), default_value)
+#define READ_BOOL_FIELD(fldname) \
+ READ_BOOL_FIELD_DEFAULT(fldname, false)
/* Read a character-string field */
-#define READ_STRING_FIELD(fldname) \
+#define READ_STRING_FIELD_DIRECT(fldname) \
token = pg_strtok(&length); /* skip :fldname */ \
+ Assert(strncmp(token, ":" CppAsString(fldname), length) == 0); \
token = pg_strtok(&length); /* get field value */ \
local_node->fldname = nullable_string(token, length)
+/* see WRITE_STRING_FIELD in outfuncs.c */
+#define READ_STRING_FIELD(fldname) \
+ READ_OPT_SCAFFOLD(fldname, READ_STRING_FIELD_DIRECT(fldname), NULL)
/* Read a parse location field (and possibly throw away the value) */
#ifdef WRITE_READ_PARSE_PLAN_TREES
-#define READ_LOCATION_FIELD(fldname) \
+#define READ_LOCATION_FIELD_DIRECT(fldname) \
token = pg_strtok(&length); /* skip :fldname */ \
+ Assert(strncmp(token, ":" CppAsString(fldname), length) == 0); \
token = pg_strtok(&length); /* get field value */ \
local_node->fldname = restore_location_fields ? atoi(token) : -1
#else
-#define READ_LOCATION_FIELD(fldname) \
+#define READ_LOCATION_FIELD_DIRECT(fldname) \
token = pg_strtok(&length); /* skip :fldname */ \
+ Assert(strncmp(token, ":" CppAsString(fldname), length) == 0); \
token = pg_strtok(&length); /* get field value */ \
(void) token; /* in case not used elsewhere */ \
local_node->fldname = -1 /* set field to "unknown" */
#endif
+/* The default Location field value is -1 ('unknown') */
+#define READ_LOCATION_FIELD(fldname) \
+ READ_OPT_SCAFFOLD(fldname, READ_LOCATION_FIELD_DIRECT(fldname), -1)
/* Read a Node field */
-#define READ_NODE_FIELD(fldname) \
+#define READ_NODE_FIELD_DIRECT(fldname) \
token = pg_strtok(&length); /* skip :fldname */ \
+ Assert(strncmp(token, ":" CppAsString(fldname), length) == 0); \
(void) token; /* in case not used elsewhere */ \
local_node->fldname = nodeRead(NULL, 0)
+#define READ_NODE_FIELD_DEFAULT(fldname, default_value) \
+ READ_OPT_SCAFFOLD(fldname, READ_NODE_FIELD_DIRECT(fldname), default_value)
+#define READ_NODE_FIELD(fldname) \
+ READ_NODE_FIELD_DEFAULT(fldname, NULL)
/* Read a bitmapset field */
-#define READ_BITMAPSET_FIELD(fldname) \
+#define READ_BITMAPSET_FIELD_DIRECT(fldname) \
token = pg_strtok(&length); /* skip :fldname */ \
+ Assert(strncmp(token, ":" CppAsString(fldname), length) == 0); \
(void) token; /* in case not used elsewhere */ \
local_node->fldname = _readBitmapset()
+#define READ_BITMAPSET_FIELD_DEFAULT(fldname, default_value) \
+ READ_OPT_SCAFFOLD(fldname, READ_BITMAPSET_FIELD_DIRECT(fldname), default_value)
+#define READ_BITMAPSET_FIELD(fldname) \
+ READ_BITMAPSET_FIELD_DEFAULT(fldname, NULL)
/* Read an attribute number array */
-#define READ_ATTRNUMBER_ARRAY(fldname, len) \
+#define READ_ATTRNUMBER_ARRAY_DIRECT(fldname, len) \
token = pg_strtok(&length); /* skip :fldname */ \
+ Assert(strncmp(token, ":" CppAsString(fldname), length) == 0); \
local_node->fldname = readAttrNumberCols(len)
+#define READ_ATTRNUMBER_ARRAY_DEFAULT(fldname, len, default_value) \
+ READ_OPT_SCAFFOLD(fldname, READ_ATTRNUMBER_ARRAY_DIRECT(fldname, len), default_value)
+#define READ_ATTRNUMBER_ARRAY(fldname, len) \
+ READ_ATTRNUMBER_ARRAY_DEFAULT(fldname, len, NULL)
/* Read an oid array */
-#define READ_OID_ARRAY(fldname, len) \
+#define READ_OID_ARRAY_DIRECT(fldname, len) \
token = pg_strtok(&length); /* skip :fldname */ \
+ Assert(strncmp(token, ":" CppAsString(fldname), length) == 0); \
local_node->fldname = readOidCols(len)
+#define READ_OID_ARRAY_DEFAULT(fldname, len, default_value) \
+ READ_OPT_SCAFFOLD(fldname, READ_OID_ARRAY_DIRECT(fldname, len), default_value)
+#define READ_OID_ARRAY(fldname, len) \
+ READ_OID_ARRAY_DEFAULT(fldname, len, NULL)
/* Read an int array */
-#define READ_INT_ARRAY(fldname, len) \
+#define READ_INT_ARRAY_DIRECT(fldname, len) \
token = pg_strtok(&length); /* skip :fldname */ \
+ Assert(strncmp(token, ":" CppAsString(fldname), length) == 0); \
local_node->fldname = readIntCols(len)
+#define READ_INT_ARRAY_DEFAULT(fldname, len, default_value) \
+ READ_OPT_SCAFFOLD(fldname, READ_INT_ARRAY_DIRECT(fldname, len), default_value)
+#define READ_INT_ARRAY(fldname, len) \
+ READ_INT_ARRAY_DEFAULT(fldname, len, NULL)
/* Read a bool array */
-#define READ_BOOL_ARRAY(fldname, len) \
+#define READ_BOOL_ARRAY_DIRECT(fldname, len) \
token = pg_strtok(&length); /* skip :fldname */ \
+ Assert(strncmp(token, ":" CppAsString(fldname), length) == 0); \
local_node->fldname = readBoolCols(len)
+#define READ_BOOL_ARRAY_DEFAULT(fldname, len, default_value) \
+ READ_OPT_SCAFFOLD(fldname, READ_BOOL_ARRAY_DIRECT(fldname, len), default_value)
+#define READ_BOOL_ARRAY(fldname, len) \
+ READ_BOOL_ARRAY_DEFAULT(fldname, len, NULL)
/* Routine exit */
#define READ_DONE() \
diff --git a/src/include/nodes/readfuncs.h b/src/include/nodes/readfuncs.h
index 8466038ed0..dab4547db2 100644
--- a/src/include/nodes/readfuncs.h
+++ b/src/include/nodes/readfuncs.h
@@ -27,6 +27,7 @@ extern PGDLLIMPORT bool restore_location_fields;
* prototypes for functions in read.c (the lisp token parser)
*/
extern const char *pg_strtok(int *length);
+extern bool pg_strtoken_next(const char *expect_token);
extern char *debackslash(const char *token, int length);
extern void *nodeRead(const char *token, int tok_len);
diff --git a/src/test/regress/expected/rowsecurity.out b/src/test/regress/expected/rowsecurity.out
index 6988128aa4..a69aa40f82 100644
--- a/src/test/regress/expected/rowsecurity.out
+++ b/src/test/regress/expected/rowsecurity.out
@@ -3937,7 +3937,10 @@ CREATE TABLE coll_t (c) AS VALUES ('bar'::text);
CREATE POLICY coll_p ON coll_t USING (c < ('foo'::text COLLATE "C"));
ALTER TABLE coll_t ENABLE ROW LEVEL SECURITY;
GRANT SELECT ON coll_t TO regress_rls_alice;
-SELECT (string_to_array(polqual, ':'))[7] AS inputcollid FROM pg_policy WHERE polrelid = 'coll_t'::regclass;
+SELECT split AS inputcollid
+FROM pg_policy,
+ lateral unnest(string_to_array(polqual, ':')) as split
+WHERE polrelid = 'coll_t'::regclass and split LIKE '%inputcollid%';
inputcollid
------------------
inputcollid 950
diff --git a/src/test/regress/sql/rowsecurity.sql b/src/test/regress/sql/rowsecurity.sql
index dec7340538..cb7a4c776b 100644
--- a/src/test/regress/sql/rowsecurity.sql
+++ b/src/test/regress/sql/rowsecurity.sql
@@ -1732,7 +1732,10 @@ CREATE TABLE coll_t (c) AS VALUES ('bar'::text);
CREATE POLICY coll_p ON coll_t USING (c < ('foo'::text COLLATE "C"));
ALTER TABLE coll_t ENABLE ROW LEVEL SECURITY;
GRANT SELECT ON coll_t TO regress_rls_alice;
-SELECT (string_to_array(polqual, ':'))[7] AS inputcollid FROM pg_policy WHERE polrelid = 'coll_t'::regclass;
+SELECT split AS inputcollid
+FROM pg_policy,
+ lateral unnest(string_to_array(polqual, ':')) as split
+WHERE polrelid = 'coll_t'::regclass and split LIKE '%inputcollid%';
SET SESSION AUTHORIZATION regress_rls_alice;
SELECT * FROM coll_t;
ROLLBACK;
--
2.40.1
[application/octet-stream] v4-0002-pg_node_tree-Don-t-store-query-text-locations-in-.patch (19.2K, ../../CAEze2WhR_99QHboeXU73u8tc_N4zSnkoyhpm8ZuRS6gh49ZGjw@mail.gmail.com/3-v4-0002-pg_node_tree-Don-t-store-query-text-locations-in-.patch)
download | inline diff:
From 7856b66c5443164661a746d211b01779833efdfb Mon Sep 17 00:00:00 2001
From: Matthias van de Meent <[email protected]>
Date: Wed, 3 Jan 2024 01:39:42 +0100
Subject: [PATCH v4 2/7] pg_node_tree: Don't store query text locations in
pg_node_tree fields.
We don't store original query texts, so any lingering "location" value
can only be useful in forensic debugging. In normal operation, however,
a non-default value will show up as measurable overhead in
serialization, just omit serialization, saving several 10s of kBs.
---
src/backend/catalog/heap.c | 9 ++--
src/backend/catalog/index.c | 4 +-
src/backend/catalog/pg_attrdef.c | 6 ++-
src/backend/catalog/pg_proc.c | 10 +++-
src/backend/catalog/pg_publication.c | 6 ++-
src/backend/commands/policy.c | 11 +++-
src/backend/commands/statscmds.c | 2 +-
src/backend/commands/trigger.c | 3 +-
src/backend/commands/typecmds.c | 8 +--
src/backend/nodes/gen_node_support.pl | 4 +-
src/backend/nodes/outfuncs.c | 77 ++++++++++++++++-----------
src/backend/rewrite/rewriteDefine.c | 4 +-
src/include/nodes/nodes.h | 4 +-
13 files changed, 95 insertions(+), 53 deletions(-)
diff --git a/src/backend/catalog/heap.c b/src/backend/catalog/heap.c
index c73f7bcd01..5c13fdab77 100644
--- a/src/backend/catalog/heap.c
+++ b/src/backend/catalog/heap.c
@@ -2060,9 +2060,9 @@ StoreRelCheck(Relation rel, const char *ccname, Node *expr,
Oid constrOid;
/*
- * Flatten expression to string form for storage.
+ * Flatten expression to string form for storage, without query refs.
*/
- ccbin = nodeToString(expr);
+ ccbin = nodeToStringNoQLocs(expr);
/*
* Find columns of rel that are used in expr
@@ -3676,7 +3676,7 @@ StorePartitionKey(Relation rel,
{
char *exprString;
- exprString = nodeToString(partexprs);
+ exprString = nodeToStringNoQLocs(partexprs);
partexprDatum = CStringGetTextDatum(exprString);
pfree(exprString);
}
@@ -3834,7 +3834,8 @@ StorePartitionBound(Relation rel, Relation parent, PartitionBoundSpec *bound)
memset(new_val, 0, sizeof(new_val));
memset(new_null, false, sizeof(new_null));
memset(new_repl, false, sizeof(new_repl));
- new_val[Anum_pg_class_relpartbound - 1] = CStringGetTextDatum(nodeToString(bound));
+ new_val[Anum_pg_class_relpartbound - 1] =
+ CStringGetTextDatum(nodeToStringNoQLocs(bound));
new_null[Anum_pg_class_relpartbound - 1] = false;
new_repl[Anum_pg_class_relpartbound - 1] = true;
newtuple = heap_modify_tuple(tuple, RelationGetDescr(classRel),
diff --git a/src/backend/catalog/index.c b/src/backend/catalog/index.c
index 4b88a9cb87..3d5f4e53d5 100644
--- a/src/backend/catalog/index.c
+++ b/src/backend/catalog/index.c
@@ -587,7 +587,7 @@ UpdateIndexRelation(Oid indexoid,
{
char *exprsString;
- exprsString = nodeToString(indexInfo->ii_Expressions);
+ exprsString = nodeToStringNoQLocs(indexInfo->ii_Expressions);
exprsDatum = CStringGetTextDatum(exprsString);
pfree(exprsString);
}
@@ -602,7 +602,7 @@ UpdateIndexRelation(Oid indexoid,
{
char *predString;
- predString = nodeToString(make_ands_explicit(indexInfo->ii_Predicate));
+ predString = nodeToStringNoQLocs(make_ands_explicit(indexInfo->ii_Predicate));
predDatum = CStringGetTextDatum(predString);
pfree(predString);
}
diff --git a/src/backend/catalog/pg_attrdef.c b/src/backend/catalog/pg_attrdef.c
index 003ae70b4d..a900c9bb28 100644
--- a/src/backend/catalog/pg_attrdef.c
+++ b/src/backend/catalog/pg_attrdef.c
@@ -23,6 +23,7 @@
#include "catalog/objectaccess.h"
#include "catalog/pg_attrdef.h"
#include "executor/executor.h"
+#include "nodes/nodeFuncs.h"
#include "optimizer/optimizer.h"
#include "utils/array.h"
#include "utils/builtins.h"
@@ -62,9 +63,10 @@ StoreAttrDefault(Relation rel, AttrNumber attnum,
adrel = table_open(AttrDefaultRelationId, RowExclusiveLock);
/*
- * Flatten expression to string form for storage.
+ * Flatten expression to string form for storage, without references to
+ * the original query string.
*/
- adbin = nodeToString(expr);
+ adbin = nodeToStringNoQLocs(expr);
/*
* Make the pg_attrdef entry.
diff --git a/src/backend/catalog/pg_proc.c b/src/backend/catalog/pg_proc.c
index b581d334d3..c5790a2224 100644
--- a/src/backend/catalog/pg_proc.c
+++ b/src/backend/catalog/pg_proc.c
@@ -331,7 +331,10 @@ ProcedureCreate(const char *procedureName,
else
nulls[Anum_pg_proc_proargnames - 1] = true;
if (parameterDefaults != NIL)
- values[Anum_pg_proc_proargdefaults - 1] = CStringGetTextDatum(nodeToString(parameterDefaults));
+ {
+ values[Anum_pg_proc_proargdefaults - 1] =
+ CStringGetTextDatum(nodeToStringNoQLocs(parameterDefaults));
+ }
else
nulls[Anum_pg_proc_proargdefaults - 1] = true;
if (trftypes != PointerGetDatum(NULL))
@@ -344,7 +347,10 @@ ProcedureCreate(const char *procedureName,
else
nulls[Anum_pg_proc_probin - 1] = true;
if (prosqlbody)
- values[Anum_pg_proc_prosqlbody - 1] = CStringGetTextDatum(nodeToString(prosqlbody));
+ {
+ values[Anum_pg_proc_prosqlbody - 1] =
+ CStringGetTextDatum(nodeToStringNoQLocs(prosqlbody));
+ }
else
nulls[Anum_pg_proc_prosqlbody - 1] = true;
if (proconfig != PointerGetDatum(NULL))
diff --git a/src/backend/catalog/pg_publication.c b/src/backend/catalog/pg_publication.c
index b98b0ce0ae..b201313430 100644
--- a/src/backend/catalog/pg_publication.c
+++ b/src/backend/catalog/pg_publication.c
@@ -36,6 +36,7 @@
#include "commands/publicationcmds.h"
#include "funcapi.h"
#include "miscadmin.h"
+#include "nodes/nodeFuncs.h"
#include "utils/array.h"
#include "utils/builtins.h"
#include "utils/catcache.h"
@@ -422,7 +423,10 @@ publication_add_relation(Oid pubid, PublicationRelInfo *pri,
/* Add qualifications, if available */
if (pri->whereClause != NULL)
- values[Anum_pg_publication_rel_prqual - 1] = CStringGetTextDatum(nodeToString(pri->whereClause));
+ {
+ values[Anum_pg_publication_rel_prqual - 1] =
+ CStringGetTextDatum(nodeToStringNoQLocs(pri->whereClause));
+ }
else
nulls[Anum_pg_publication_rel_prqual - 1] = true;
diff --git a/src/backend/commands/policy.c b/src/backend/commands/policy.c
index 596326e5ec..1e6842bf41 100644
--- a/src/backend/commands/policy.c
+++ b/src/backend/commands/policy.c
@@ -30,6 +30,7 @@
#include "commands/policy.h"
#include "miscadmin.h"
#include "nodes/makefuncs.h"
+#include "nodes/nodeFuncs.h"
#include "nodes/pg_list.h"
#include "parser/parse_clause.h"
#include "parser/parse_collate.h"
@@ -701,13 +702,19 @@ CreatePolicy(CreatePolicyStmt *stmt)
/* Add qual if present. */
if (qual)
- values[Anum_pg_policy_polqual - 1] = CStringGetTextDatum(nodeToString(qual));
+ {
+ values[Anum_pg_policy_polqual - 1] =
+ CStringGetTextDatum(nodeToStringNoQLocs(qual));
+ }
else
isnull[Anum_pg_policy_polqual - 1] = true;
/* Add WITH CHECK qual if present */
if (with_check_qual)
- values[Anum_pg_policy_polwithcheck - 1] = CStringGetTextDatum(nodeToString(with_check_qual));
+ {
+ values[Anum_pg_policy_polwithcheck - 1] =
+ CStringGetTextDatum(nodeToStringNoQLocs(with_check_qual));
+ }
else
isnull[Anum_pg_policy_polwithcheck - 1] = true;
diff --git a/src/backend/commands/statscmds.c b/src/backend/commands/statscmds.c
index b1a9c74bd6..58e8133f93 100644
--- a/src/backend/commands/statscmds.c
+++ b/src/backend/commands/statscmds.c
@@ -477,7 +477,7 @@ CreateStatistics(CreateStatsStmt *stmt)
{
char *exprsString;
- exprsString = nodeToString(stxexprs);
+ exprsString = nodeToStringNoQLocs(stxexprs);
exprsDatum = CStringGetTextDatum(exprsString);
pfree(exprsString);
}
diff --git a/src/backend/commands/trigger.c b/src/backend/commands/trigger.c
index c344ff0944..6a3dc13a67 100644
--- a/src/backend/commands/trigger.c
+++ b/src/backend/commands/trigger.c
@@ -39,6 +39,7 @@
#include "miscadmin.h"
#include "nodes/bitmapset.h"
#include "nodes/makefuncs.h"
+#include "nodes/nodeFuncs.h"
#include "optimizer/optimizer.h"
#include "parser/parse_clause.h"
#include "parser/parse_collate.h"
@@ -674,7 +675,7 @@ CreateTriggerFiringOn(CreateTrigStmt *stmt, const char *queryString,
/* we'll need the rtable for recordDependencyOnExpr */
whenRtable = pstate->p_rtable;
- qual = nodeToString(whenClause);
+ qual = nodeToStringNoQLocs(whenClause);
free_parsestate(pstate);
}
diff --git a/src/backend/commands/typecmds.c b/src/backend/commands/typecmds.c
index a400fb39f6..00180a54b9 100644
--- a/src/backend/commands/typecmds.c
+++ b/src/backend/commands/typecmds.c
@@ -58,6 +58,7 @@
#include "executor/executor.h"
#include "miscadmin.h"
#include "nodes/makefuncs.h"
+#include "nodes/nodeFuncs.h"
#include "optimizer/optimizer.h"
#include "parser/parse_coerce.h"
#include "parser/parse_collate.h"
@@ -924,7 +925,7 @@ DefineDomain(CreateDomainStmt *stmt)
defaultValue =
deparse_expression(defaultExpr,
NIL, false, false);
- defaultValueBin = nodeToString(defaultExpr);
+ defaultValueBin = nodeToStringNoQLocs(defaultExpr);
}
}
else
@@ -3506,9 +3507,10 @@ domainAddConstraint(Oid domainOid, Oid domainNamespace, Oid baseTypeOid,
errmsg("cannot use table references in domain check constraint")));
/*
- * Convert to string form for storage.
+ * Convert to string form for storage, without references to the original
+ * query text.
*/
- ccbin = nodeToString(expr);
+ ccbin = nodeToStringNoQLocs(expr);
/*
* Store the constraint in pg_constraint
diff --git a/src/backend/nodes/gen_node_support.pl b/src/backend/nodes/gen_node_support.pl
index 2f0a59bc87..487f6f7728 100644
--- a/src/backend/nodes/gen_node_support.pl
+++ b/src/backend/nodes/gen_node_support.pl
@@ -921,7 +921,7 @@ foreach my $n (@node_types)
my $N = uc $n;
print $ofs "\t\t\tcase T_${n}:\n"
- . "\t\t\t\t_out${n}(str, obj);\n"
+ . "\t\t\t\t_out${n}(str, obj, omitLocation);\n"
. "\t\t\t\tbreak;\n";
print $rfs "\tif (MATCH(\"$N\", "
@@ -933,7 +933,7 @@ foreach my $n (@node_types)
print $off "
static void
-_out${n}(StringInfo str, const $n *node)
+_out${n}(StringInfo str, const $n *node, bool omitLocation)
{
\tWRITE_NODE_TYPE(\"$N\");
diff --git a/src/backend/nodes/outfuncs.c b/src/backend/nodes/outfuncs.c
index 50da80ee34..eca3160104 100644
--- a/src/backend/nodes/outfuncs.c
+++ b/src/backend/nodes/outfuncs.c
@@ -154,7 +154,7 @@ static void outDouble(StringInfo str, double d);
/* Write a parse location field (actually same as INT case) */
#define WRITE_LOCATION_FIELD(fldname) \
do { \
- if (node->fldname != -1) \
+ if (node->fldname != -1 && !omitLocation) \
appendStringInfo(str, " :" CppAsString(fldname) " %d", \
node->fldname); \
} while (0)
@@ -165,7 +165,7 @@ static void outDouble(StringInfo str, double d);
if (node->fldname != NULL) \
{ \
appendStringInfoString(str, " :" CppAsString(fldname) " "); \
- outNode(str, node->fldname); \
+ outNode(str, node->fldname, omitLocation); \
} \
} while (0)
@@ -185,7 +185,8 @@ static void outDouble(StringInfo str, double d);
if (node->fldname != NULL) \
{ \
appendStringInfoString(str, " :" CppAsString(fldname) " "); \
- writeNodeArray(str, (const Node * const *) node->fldname, len); \
+ writeNodeArray(str, (const Node * const *) node->fldname, len, \
+ omitLocation); \
} \
} while (0)
@@ -358,7 +359,8 @@ WRITE_SCALAR_ARRAY(writeBoolCols, bool, " %s", booltostr)
* quite use appendStringInfo() in the loop.
*/
static void
-writeNodeArray(StringInfo str, const Node *const *arr, int len)
+writeNodeArray(StringInfo str, const Node *const *arr, int len,
+ bool omitLocation)
{
if (arr != NULL)
{
@@ -366,7 +368,7 @@ writeNodeArray(StringInfo str, const Node *const *arr, int len)
for (int i = 0; i < len; i++)
{
appendStringInfoChar(str, ' ');
- outNode(str, arr[i]);
+ outNode(str, arr[i], omitLocation);
}
appendStringInfoChar(str, ')');
}
@@ -378,7 +380,7 @@ writeNodeArray(StringInfo str, const Node *const *arr, int len)
* Print a List.
*/
static void
-_outList(StringInfo str, const List *node)
+_outList(StringInfo str, const List *node, bool omitLocation)
{
const ListCell *lc;
@@ -400,7 +402,7 @@ _outList(StringInfo str, const List *node)
*/
if (IsA(node, List))
{
- outNode(str, lfirst(lc));
+ outNode(str, lfirst(lc), omitLocation);
if (lnext(node, lc))
appendStringInfoChar(str, ' ');
}
@@ -485,7 +487,7 @@ outDatum(StringInfo str, Datum value, int typlen, bool typbyval)
*/
static void
-_outConst(StringInfo str, const Const *node)
+_outConst(StringInfo str, const Const *node, bool omitLocation)
{
WRITE_NODE_TYPE("CONST");
@@ -505,7 +507,7 @@ _outConst(StringInfo str, const Const *node)
}
static void
-_outBoolExpr(StringInfo str, const BoolExpr *node)
+_outBoolExpr(StringInfo str, const BoolExpr *node, bool omitLocation)
{
char *opstr = NULL;
@@ -532,7 +534,8 @@ _outBoolExpr(StringInfo str, const BoolExpr *node)
}
static void
-_outForeignKeyOptInfo(StringInfo str, const ForeignKeyOptInfo *node)
+_outForeignKeyOptInfo(StringInfo str, const ForeignKeyOptInfo *node,
+ bool omitLocation)
{
int i;
@@ -558,7 +561,8 @@ _outForeignKeyOptInfo(StringInfo str, const ForeignKeyOptInfo *node)
}
static void
-_outEquivalenceClass(StringInfo str, const EquivalenceClass *node)
+_outEquivalenceClass(StringInfo str, const EquivalenceClass *node,
+ bool omitLocation)
{
/*
* To simplify reading, we just chase up to the topmost merged EC and
@@ -584,7 +588,8 @@ _outEquivalenceClass(StringInfo str, const EquivalenceClass *node)
}
static void
-_outExtensibleNode(StringInfo str, const ExtensibleNode *node)
+_outExtensibleNode(StringInfo str, const ExtensibleNode *node,
+ bool omitLocation)
{
const ExtensibleNodeMethods *methods;
@@ -599,7 +604,8 @@ _outExtensibleNode(StringInfo str, const ExtensibleNode *node)
}
static void
-_outRangeTblEntry(StringInfo str, const RangeTblEntry *node)
+_outRangeTblEntry(StringInfo str, const RangeTblEntry *node,
+ bool omitLocation)
{
WRITE_NODE_TYPE("RANGETBLENTRY");
@@ -679,7 +685,7 @@ _outRangeTblEntry(StringInfo str, const RangeTblEntry *node)
}
static void
-_outA_Expr(StringInfo str, const A_Expr *node)
+_outA_Expr(StringInfo str, const A_Expr *node, bool omitLocation)
{
WRITE_NODE_TYPE("A_EXPR");
@@ -751,13 +757,13 @@ _outA_Expr(StringInfo str, const A_Expr *node)
}
static void
-_outInteger(StringInfo str, const Integer *node)
+_outInteger(StringInfo str, const Integer *node, bool omitLocation)
{
appendStringInfo(str, "%d", node->ival);
}
static void
-_outFloat(StringInfo str, const Float *node)
+_outFloat(StringInfo str, const Float *node, bool omitLocation)
{
/*
* We assume the value is a valid numeric literal and so does not need
@@ -767,13 +773,13 @@ _outFloat(StringInfo str, const Float *node)
}
static void
-_outBoolean(StringInfo str, const Boolean *node)
+_outBoolean(StringInfo str, const Boolean *node, bool omitLocation)
{
appendStringInfoString(str, node->boolval ? "true" : "false");
}
static void
-_outString(StringInfo str, const String *node)
+_outString(StringInfo str, const String *node, bool omitLocation)
{
/*
* We use outToken to provide escaping of the string's content, but we
@@ -787,14 +793,14 @@ _outString(StringInfo str, const String *node)
}
static void
-_outBitString(StringInfo str, const BitString *node)
+_outBitString(StringInfo str, const BitString *node, bool omitLocation)
{
/* internal representation already has leading 'b' */
appendStringInfoString(str, node->bsval);
}
static void
-_outA_Const(StringInfo str, const A_Const *node)
+_outA_Const(StringInfo str, const A_Const *node, bool omitLocation)
{
WRITE_NODE_TYPE("A_CONST");
@@ -803,13 +809,13 @@ _outA_Const(StringInfo str, const A_Const *node)
else
{
appendStringInfoString(str, " :val ");
- outNode(str, &node->val);
+ outNode(str, &node->val, omitLocation);
}
WRITE_LOCATION_FIELD(location);
}
static void
-_outConstraint(StringInfo str, const Constraint *node)
+_outConstraint(StringInfo str, const Constraint *node, bool omitLocation)
{
WRITE_NODE_TYPE("CONSTRAINT");
@@ -942,7 +948,7 @@ _outConstraint(StringInfo str, const Constraint *node)
* converts a Node into ascii string and append it to 'str'
*/
void
-outNode(StringInfo str, const void *obj)
+outNode(StringInfo str, const void *obj, bool omitLocation)
{
/* Guard against stack overflow due to overly complex expressions */
check_stack_depth();
@@ -951,18 +957,18 @@ outNode(StringInfo str, const void *obj)
appendStringInfoString(str, "<>");
else if (IsA(obj, List) || IsA(obj, IntList) || IsA(obj, OidList) ||
IsA(obj, XidList))
- _outList(str, obj);
+ _outList(str, obj, omitLocation);
/* nodeRead does not want to see { } around these! */
else if (IsA(obj, Integer))
- _outInteger(str, (Integer *) obj);
+ _outInteger(str, (Integer *) obj, omitLocation);
else if (IsA(obj, Float))
- _outFloat(str, (Float *) obj);
+ _outFloat(str, (Float *) obj, omitLocation);
else if (IsA(obj, Boolean))
- _outBoolean(str, (Boolean *) obj);
+ _outBoolean(str, (Boolean *) obj, omitLocation);
else if (IsA(obj, String))
- _outString(str, (String *) obj);
+ _outString(str, (String *) obj, omitLocation);
else if (IsA(obj, BitString))
- _outBitString(str, (BitString *) obj);
+ _outBitString(str, (BitString *) obj, omitLocation);
else if (IsA(obj, Bitmapset))
outBitmapset(str, (Bitmapset *) obj);
else
@@ -997,7 +1003,18 @@ nodeToString(const void *obj)
/* see stringinfo.h for an explanation of this maneuver */
initStringInfo(&str);
- outNode(&str, obj);
+ outNode(&str, obj, false);
+ return str.data;
+}
+
+char *
+nodeToStringNoQLocs(const void *obj)
+{
+ StringInfoData str;
+
+ /* see stringinfo.h for an explanation of this maneuver */
+ initStringInfo(&str);
+ outNode(&str, obj, true);
return str.data;
}
diff --git a/src/backend/rewrite/rewriteDefine.c b/src/backend/rewrite/rewriteDefine.c
index b449244a53..6302cd1472 100644
--- a/src/backend/rewrite/rewriteDefine.c
+++ b/src/backend/rewrite/rewriteDefine.c
@@ -64,8 +64,8 @@ InsertRule(const char *rulname,
List *action,
bool replace)
{
- char *evqual = nodeToString(event_qual);
- char *actiontree = nodeToString((Node *) action);
+ char *evqual = nodeToStringNoQLocs(event_qual);
+ char *actiontree = nodeToStringNoQLocs((Node *) action);
Datum values[Natts_pg_rewrite];
bool nulls[Natts_pg_rewrite] = {0};
NameData rname;
diff --git a/src/include/nodes/nodes.h b/src/include/nodes/nodes.h
index 2969dd831b..f7adb5e767 100644
--- a/src/include/nodes/nodes.h
+++ b/src/include/nodes/nodes.h
@@ -188,13 +188,15 @@ castNodeImpl(NodeTag type, void *ptr)
struct Bitmapset; /* not to include bitmapset.h here */
struct StringInfoData; /* not to include stringinfo.h here */
-extern void outNode(struct StringInfoData *str, const void *obj);
+extern void outNode(struct StringInfoData *str, const void *obj,
+ bool omitLocation);
extern void outToken(struct StringInfoData *str, const char *s);
extern void outBitmapset(struct StringInfoData *str,
const struct Bitmapset *bms);
extern void outDatum(struct StringInfoData *str, uintptr_t value,
int typlen, bool typbyval);
extern char *nodeToString(const void *obj);
+extern char *nodeToStringNoQLocs(const void *obj);
extern char *bmsToString(const struct Bitmapset *bms);
/*
--
2.40.1
[application/octet-stream] v4-0005-nodeToString-omit-serializing-NULL-datums-in-Cons.patch (1.9K, ../../CAEze2WhR_99QHboeXU73u8tc_N4zSnkoyhpm8ZuRS6gh49ZGjw@mail.gmail.com/4-v4-0005-nodeToString-omit-serializing-NULL-datums-in-Cons.patch)
download | inline diff:
From a1b210c0d3f7329a409c1e2a09558ac33256c318 Mon Sep 17 00:00:00 2001
From: Matthias van de Meent <[email protected]>
Date: Thu, 8 Feb 2024 20:24:22 +0100
Subject: [PATCH v4 5/7] nodeToString: omit serializing NULL datums in Const
nodes
This saves some bytes in certain cases, and aligns its serialization conditions
with other field's serialization conditions.
---
src/backend/nodes/outfuncs.c | 8 ++++----
src/backend/nodes/readfuncs.c | 14 ++++++++++----
2 files changed, 14 insertions(+), 8 deletions(-)
diff --git a/src/backend/nodes/outfuncs.c b/src/backend/nodes/outfuncs.c
index 3973c0e489..1a17eafd57 100644
--- a/src/backend/nodes/outfuncs.c
+++ b/src/backend/nodes/outfuncs.c
@@ -499,11 +499,11 @@ _outConst(StringInfo str, const Const *node, bool omitLocation)
WRITE_BOOL_FIELD(constisnull);
WRITE_LOCATION_FIELD(location);
- appendStringInfoString(str, " :constvalue ");
- if (node->constisnull)
- appendStringInfoString(str, "<>");
- else
+ if (!node->constisnull)
+ {
+ appendStringInfoString(str, " :constvalue ");
outDatum(str, node->constvalue, node->constlen, node->constbyval);
+ }
}
static void
diff --git a/src/backend/nodes/readfuncs.c b/src/backend/nodes/readfuncs.c
index 44ab140799..4cdbad9e7e 100644
--- a/src/backend/nodes/readfuncs.c
+++ b/src/backend/nodes/readfuncs.c
@@ -363,11 +363,17 @@ _readConst(void)
READ_BOOL_FIELD(constisnull);
READ_LOCATION_FIELD(location);
- token = pg_strtok(&length); /* skip :constvalue */
- if (local_node->constisnull)
- token = pg_strtok(&length); /* skip "<>" */
- else
+ if (pg_strtoken_next(":constvalue"))
+ {
+ token = pg_strtok(&length); /* skip :constvalue */
+ Assert(strncmp(token, ":constvalue", sizeof(":constvalue") - 1) == 0);
local_node->constvalue = readDatum(local_node->constbyval);
+ }
+ else
+ {
+ /* value was omitted */
+ Assert(local_node->constisnull);
+ }
READ_DONE();
}
--
2.40.1
[application/octet-stream] v4-0004-gen_node_support.pl-Add-a-TypMod-type-for-signall.patch (10.4K, ../../CAEze2WhR_99QHboeXU73u8tc_N4zSnkoyhpm8ZuRS6gh49ZGjw@mail.gmail.com/5-v4-0004-gen_node_support.pl-Add-a-TypMod-type-for-signall.patch)
download | inline diff:
From 72a05aba203c4421ef94587e847d2b6ba64968a7 Mon Sep 17 00:00:00 2001
From: Matthias van de Meent <[email protected]>
Date: Thu, 8 Feb 2024 17:37:39 +0100
Subject: [PATCH v4 4/7] gen_node_support.pl: Add a TypMod type for signalling
TypMod behaviour
Like Location, TypMod has its own default (-1). By using a type, we can
omit adding pg_node_attribute(default(-1)) markers to every typmod-valued
field, whilst still getting the benefits of a smaller size in serialization.
---
src/backend/nodes/gen_node_support.pl | 8 ++++++-
src/backend/nodes/outfuncs.c | 2 +-
src/backend/nodes/readfuncs.c | 2 +-
src/include/nodes/parsenodes.h | 4 ++--
src/include/nodes/pathnodes.h | 2 +-
src/include/nodes/primnodes.h | 33 ++++++++++++++-------------
6 files changed, 29 insertions(+), 22 deletions(-)
diff --git a/src/backend/nodes/gen_node_support.pl b/src/backend/nodes/gen_node_support.pl
index ec850c3484..fda8c3a494 100644
--- a/src/backend/nodes/gen_node_support.pl
+++ b/src/backend/nodes/gen_node_support.pl
@@ -135,7 +135,8 @@ my @nodetag_only;
# types that are copied by straight assignment
my @scalar_types = qw(
bits32 bool char double int int8 int16 int32 int64 long uint8 uint16 uint32 uint64
- AclMode AttrNumber Cardinality Cost Index Oid RelFileNumber Selectivity Size StrategyNumber SubTransactionId TimeLineID XLogRecPtr
+ AclMode AttrNumber Cardinality Cost Index Oid RelFileNumber Selectivity Size StrategyNumber
+ SubTransactionId TimeLineID XLogRecPtr TypMod
);
# collect enum types
@@ -1015,6 +1016,11 @@ _read${n}(void)
print $off "\tWRITE_LOCATION_FIELD($f);\n";
print $rff "\tREAD_LOCATION_FIELD($f);\n" unless $no_read;
}
+ elsif ($t eq 'TypMod')
+ {
+ print $off "\tWRITE_INT_FIELD_DEFAULT($f, -1);\n";
+ print $rff "\tREAD_INT_FIELD_DEFAULT($f, -1);\n" unless $no_read;
+ }
elsif ($t eq 'int'
|| $t eq 'int16'
|| $t eq 'int32'
diff --git a/src/backend/nodes/outfuncs.c b/src/backend/nodes/outfuncs.c
index eca3160104..3973c0e489 100644
--- a/src/backend/nodes/outfuncs.c
+++ b/src/backend/nodes/outfuncs.c
@@ -492,7 +492,7 @@ _outConst(StringInfo str, const Const *node, bool omitLocation)
WRITE_NODE_TYPE("CONST");
WRITE_OID_FIELD(consttype);
- WRITE_INT_FIELD(consttypmod);
+ WRITE_INT_FIELD_DEFAULT(consttypmod, -1);
WRITE_OID_FIELD(constcollid);
WRITE_INT_FIELD(constlen);
WRITE_BOOL_FIELD(constbyval);
diff --git a/src/backend/nodes/readfuncs.c b/src/backend/nodes/readfuncs.c
index 47f4ba2695..44ab140799 100644
--- a/src/backend/nodes/readfuncs.c
+++ b/src/backend/nodes/readfuncs.c
@@ -356,7 +356,7 @@ _readConst(void)
READ_LOCALS(Const);
READ_OID_FIELD(consttype);
- READ_INT_FIELD(consttypmod);
+ READ_INT_FIELD_DEFAULT(consttypmod, -1);
READ_OID_FIELD(constcollid);
READ_INT_FIELD(constlen);
READ_BOOL_FIELD(constbyval);
diff --git a/src/include/nodes/parsenodes.h b/src/include/nodes/parsenodes.h
index dd8aa30aaf..52d8504a9d 100644
--- a/src/include/nodes/parsenodes.h
+++ b/src/include/nodes/parsenodes.h
@@ -260,7 +260,7 @@ typedef struct TypeName
bool setof; /* is a set? */
bool pct_type; /* %TYPE specified? */
List *typmods; /* type modifier expression(s) */
- int32 typemod; /* prespecified type modifier */
+ TypMod typemod; /* prespecified type modifier */
List *arrayBounds; /* array bounds */
Location location; /* token location, or -1 if unknown */
} TypeName;
@@ -1608,7 +1608,7 @@ typedef struct CTECycleClause
Location location;
/* These fields are set during parse analysis: */
Oid cycle_mark_type; /* common type of _value and _default */
- int cycle_mark_typmod;
+ TypMod cycle_mark_typmod;
Oid cycle_mark_collation;
Oid cycle_mark_neop; /* <> operator for type */
} CTECycleClause;
diff --git a/src/include/nodes/pathnodes.h b/src/include/nodes/pathnodes.h
index 534692bee1..f7c2496a7f 100644
--- a/src/include/nodes/pathnodes.h
+++ b/src/include/nodes/pathnodes.h
@@ -3394,7 +3394,7 @@ typedef struct AggTransInfo
Oid aggtranstype;
/* Additional data about transtype */
- int32 aggtranstypmod;
+ TypMod aggtranstypmod;
int transtypeLen;
bool transtypeByVal;
diff --git a/src/include/nodes/primnodes.h b/src/include/nodes/primnodes.h
index 557be05657..94282497d7 100644
--- a/src/include/nodes/primnodes.h
+++ b/src/include/nodes/primnodes.h
@@ -29,8 +29,9 @@ typedef enum OverridingKind
OVERRIDING_SYSTEM_VALUE,
} OverridingKind;
-
+/* Type aliases for gen_node_support */
#define Location int
+#define TypMod int32
/* ----------------------------------------------------------------
* node definitions
@@ -250,7 +251,7 @@ typedef struct Var
/* pg_type OID for the type of this var */
Oid vartype pg_node_attr(query_jumble_ignore);
/* pg_attribute typmod value */
- int32 vartypmod pg_node_attr(query_jumble_ignore);
+ TypMod vartypmod pg_node_attr(query_jumble_ignore);
/* OID of collation, or InvalidOid if none */
Oid varcollid pg_node_attr(query_jumble_ignore);
@@ -299,7 +300,7 @@ typedef struct Const
/* pg_type OID of the constant's datatype */
Oid consttype;
/* typmod value, if any */
- int32 consttypmod pg_node_attr(query_jumble_ignore);
+ TypMod consttypmod pg_node_attr(query_jumble_ignore);
/* OID of collation, or InvalidOid if none */
Oid constcollid pg_node_attr(query_jumble_ignore);
/* typlen of the constant's datatype */
@@ -365,7 +366,7 @@ typedef struct Param
int paramid; /* numeric ID for parameter */
Oid paramtype; /* pg_type OID of parameter's datatype */
/* typmod value, if known */
- int32 paramtypmod pg_node_attr(query_jumble_ignore);
+ TypMod paramtypmod pg_node_attr(query_jumble_ignore);
/* OID of collation, or InvalidOid if none */
Oid paramcollid pg_node_attr(query_jumble_ignore);
/* token location, or -1 if unknown */
@@ -623,7 +624,7 @@ typedef struct SubscriptingRef
/* type of the SubscriptingRef's result */
Oid refrestype pg_node_attr(query_jumble_ignore);
/* typmod of the result */
- int32 reftypmod pg_node_attr(query_jumble_ignore);
+ TypMod reftypmod pg_node_attr(query_jumble_ignore);
/* collation of result, or InvalidOid if none */
Oid refcollid pg_node_attr(query_jumble_ignore);
/* expressions that evaluate to upper container indexes */
@@ -1009,7 +1010,7 @@ typedef struct SubPlan
char *plan_name; /* A name assigned during planning */
/* Extra data useful for determining subplan's output type: */
Oid firstColType; /* Type of first column of subplan result */
- int32 firstColTypmod; /* Typmod of first column of subplan result */
+ TypMod firstColTypmod; /* Typmod of first column of subplan result */
Oid firstColCollation; /* Collation of first column of subplan
* result */
/* Information about execution strategy: */
@@ -1067,7 +1068,7 @@ typedef struct FieldSelect
/* type of the field (result type of this node) */
Oid resulttype pg_node_attr(query_jumble_ignore);
/* output typmod (usually -1) */
- int32 resulttypmod pg_node_attr(query_jumble_ignore);
+ TypMod resulttypmod pg_node_attr(query_jumble_ignore);
/* OID of collation of the field */
Oid resultcollid pg_node_attr(query_jumble_ignore);
} FieldSelect;
@@ -1121,7 +1122,7 @@ typedef struct RelabelType
Expr *arg; /* input expression */
Oid resulttype; /* output type of coercion expression */
/* output typmod (usually -1) */
- int32 resulttypmod pg_node_attr(query_jumble_ignore);
+ TypMod resulttypmod pg_node_attr(query_jumble_ignore);
/* OID of collation, or InvalidOid if none */
Oid resultcollid pg_node_attr(query_jumble_ignore);
/* how to display this node */
@@ -1171,7 +1172,7 @@ typedef struct ArrayCoerceExpr
Expr *elemexpr; /* expression representing per-element work */
Oid resulttype; /* output type of coercion (an array type) */
/* output typmod (also element typmod) */
- int32 resulttypmod pg_node_attr(query_jumble_ignore);
+ TypMod resulttypmod pg_node_attr(query_jumble_ignore);
/* OID of collation, or InvalidOid if none */
Oid resultcollid pg_node_attr(query_jumble_ignore);
/* how to display this node */
@@ -1291,7 +1292,7 @@ typedef struct CaseTestExpr
Expr xpr;
Oid typeId; /* type for substituted value */
/* typemod for substituted value */
- int32 typeMod pg_node_attr(query_jumble_ignore);
+ TypMod typeMod pg_node_attr(query_jumble_ignore);
/* collation for the substituted value */
Oid collation pg_node_attr(query_jumble_ignore);
} CaseTestExpr;
@@ -1497,7 +1498,7 @@ typedef struct SQLValueFunction
* include this Oid in the query jumbling.
*/
Oid type pg_node_attr(query_jumble_ignore);
- int32 typmod;
+ TypMod typmod;
Location location; /* token location, or -1 if unknown */
} SQLValueFunction;
@@ -1549,7 +1550,7 @@ typedef struct XmlExpr
bool indent;
/* target type/typmod for XMLSERIALIZE */
Oid type pg_node_attr(query_jumble_ignore);
- int32 typmod pg_node_attr(query_jumble_ignore);
+ TypMod typmod pg_node_attr(query_jumble_ignore);
/* token location, or -1 if unknown */
Location location;
} XmlExpr;
@@ -1599,7 +1600,7 @@ typedef struct JsonReturning
NodeTag type;
JsonFormat *format; /* output JSON format */
Oid typid; /* target type Oid */
- int32 typmod; /* target type modifier */
+ TypMod typmod; /* target type modifier */
} JsonReturning;
/*
@@ -1762,7 +1763,7 @@ typedef struct CoerceToDomain
Expr *arg; /* input expression */
Oid resulttype; /* domain type ID (result type) */
/* output typmod (currently always -1) */
- int32 resulttypmod pg_node_attr(query_jumble_ignore);
+ TypMod resulttypmod pg_node_attr(query_jumble_ignore);
/* OID of collation, or InvalidOid if none */
Oid resultcollid pg_node_attr(query_jumble_ignore);
/* how to display this node */
@@ -1785,7 +1786,7 @@ typedef struct CoerceToDomainValue
/* type for substituted value */
Oid typeId;
/* typemod for substituted value */
- int32 typeMod pg_node_attr(query_jumble_ignore);
+ TypMod typeMod pg_node_attr(query_jumble_ignore);
/* collation for the substituted value */
Oid collation pg_node_attr(query_jumble_ignore);
/* token location, or -1 if unknown */
@@ -1805,7 +1806,7 @@ typedef struct SetToDefault
/* type for substituted value */
Oid typeId;
/* typemod for substituted value */
- int32 typeMod pg_node_attr(query_jumble_ignore);
+ TypMod typeMod pg_node_attr(query_jumble_ignore);
/* collation for the substituted value */
Oid collation pg_node_attr(query_jumble_ignore);
/* token location, or -1 if unknown */
--
2.40.1
[application/octet-stream] v4-0003-gen_node_support.pl-Mark-location-fields-as-type-.patch (26.4K, ../../CAEze2WhR_99QHboeXU73u8tc_N4zSnkoyhpm8ZuRS6gh49ZGjw@mail.gmail.com/6-v4-0003-gen_node_support.pl-Mark-location-fields-as-type-.patch)
download | inline diff:
From 4c0159875ed1633796f2e7d64624ba5857154bc8 Mon Sep 17 00:00:00 2001
From: Matthias van de Meent <[email protected]>
Date: Thu, 8 Feb 2024 17:15:22 +0100
Subject: [PATCH v4 3/7] gen_node_support.pl: Mark location fields as type
alias Location
Instead of the rather ugly type=int + name ~= location$, we now have a
special type for offset pointers or sizes that are only relevant when a
query text is included, which decreases the complexity required in
gen_node_support.pl for handling these values.
---
src/backend/nodes/gen_node_support.pl | 6 +-
src/include/nodes/parsenodes.h | 92 +++++++++++++--------------
src/include/nodes/primnodes.h | 72 +++++++++++----------
3 files changed, 86 insertions(+), 84 deletions(-)
diff --git a/src/backend/nodes/gen_node_support.pl b/src/backend/nodes/gen_node_support.pl
index 487f6f7728..ec850c3484 100644
--- a/src/backend/nodes/gen_node_support.pl
+++ b/src/backend/nodes/gen_node_support.pl
@@ -777,7 +777,7 @@ _equal${n}(const $n *a, const $n *b)
print $eff "\tCOMPARE_BITMAPSET_FIELD($f);\n"
unless $equal_ignore;
}
- elsif ($t eq 'int' && $f =~ 'location$')
+ elsif ($t eq 'Location')
{
print $cff "\tCOPY_LOCATION_FIELD($f);\n" unless $copy_ignore;
print $eff "\tCOMPARE_LOCATION_FIELD($f);\n" unless $equal_ignore;
@@ -1010,7 +1010,7 @@ _read${n}(void)
print $off "\tWRITE_BOOL_FIELD($f);\n";
print $rff "\tREAD_BOOL_FIELD($f);\n" unless $no_read;
}
- elsif ($t eq 'int' && $f =~ 'location$')
+ elsif ($t eq 'Location')
{
print $off "\tWRITE_LOCATION_FIELD($f);\n";
print $rff "\tREAD_LOCATION_FIELD($f);\n" unless $no_read;
@@ -1303,7 +1303,7 @@ _jumble${n}(JumbleState *jstate, Node *node)
print $jff "\tJUMBLE_NODE($f);\n"
unless $query_jumble_ignore;
}
- elsif ($t eq 'int' && $f =~ 'location$')
+ elsif ($t eq 'Location')
{
# Track the node's location only if directly requested.
if ($query_jumble_location)
diff --git a/src/include/nodes/parsenodes.h b/src/include/nodes/parsenodes.h
index 476d55dd24..dd8aa30aaf 100644
--- a/src/include/nodes/parsenodes.h
+++ b/src/include/nodes/parsenodes.h
@@ -225,9 +225,9 @@ typedef struct Query
* both be -1 meaning "unknown".
*/
/* start location, or -1 if unknown */
- int stmt_location;
+ Location stmt_location;
/* length in bytes; 0 means "rest of string" */
- int stmt_len pg_node_attr(query_jumble_ignore);
+ Location stmt_len pg_node_attr(query_jumble_ignore);
} Query;
@@ -262,7 +262,7 @@ typedef struct TypeName
List *typmods; /* type modifier expression(s) */
int32 typemod; /* prespecified type modifier */
List *arrayBounds; /* array bounds */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} TypeName;
/*
@@ -282,7 +282,7 @@ typedef struct ColumnRef
{
NodeTag type;
List *fields; /* field names (String nodes) or A_Star */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} ColumnRef;
/*
@@ -292,7 +292,7 @@ typedef struct ParamRef
{
NodeTag type;
int number; /* the number of the parameter */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} ParamRef;
/*
@@ -325,7 +325,7 @@ typedef struct A_Expr
List *name; /* possibly-qualified name of operator */
Node *lexpr; /* left argument, or NULL if none */
Node *rexpr; /* right argument, or NULL if none */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} A_Expr;
/*
@@ -351,7 +351,7 @@ typedef struct A_Const
NodeTag type;
union ValUnion val;
bool isnull; /* SQL NULL constant */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} A_Const;
/*
@@ -362,7 +362,7 @@ typedef struct TypeCast
NodeTag type;
Node *arg; /* the expression being casted */
TypeName *typeName; /* the target type */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} TypeCast;
/*
@@ -373,7 +373,7 @@ typedef struct CollateClause
NodeTag type;
Node *arg; /* input expression */
List *collname; /* possibly-qualified collation name */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} CollateClause;
/*
@@ -393,7 +393,7 @@ typedef struct RoleSpec
NodeTag type;
RoleSpecType roletype; /* Type of this rolespec */
char *rolename; /* filled only for ROLESPEC_CSTRING */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} RoleSpec;
/*
@@ -423,7 +423,7 @@ typedef struct FuncCall
bool agg_distinct; /* arguments were labeled DISTINCT */
bool func_variadic; /* last argument was labeled VARIADIC */
CoercionForm funcformat; /* how to display this node */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} FuncCall;
/*
@@ -480,7 +480,7 @@ typedef struct A_ArrayExpr
{
NodeTag type;
List *elements; /* array element expressions */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} A_ArrayExpr;
/*
@@ -507,7 +507,7 @@ typedef struct ResTarget
char *name; /* column name or NULL */
List *indirection; /* subscripts, field names, and '*', or NIL */
Node *val; /* the value expression to compute or assign */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} ResTarget;
/*
@@ -537,7 +537,7 @@ typedef struct SortBy
SortByDir sortby_dir; /* ASC/DESC/USING/default */
SortByNulls sortby_nulls; /* NULLS FIRST/LAST */
List *useOp; /* name of op to use, if SORTBY_USING */
- int location; /* operator location, or -1 if none/unknown */
+ Location location; /* operator location, or -1 if none/unknown */
} SortBy;
/*
@@ -558,7 +558,7 @@ typedef struct WindowDef
int frameOptions; /* frame_clause options, see below */
Node *startOffset; /* expression for starting bound, if any */
Node *endOffset; /* expression for ending bound, if any */
- int location; /* parse location, or -1 if none/unknown */
+ Location location; /* parse location, or -1 if none/unknown */
} WindowDef;
/*
@@ -648,7 +648,7 @@ typedef struct RangeTableFunc
List *namespaces; /* list of namespaces as ResTarget */
List *columns; /* list of RangeTableFuncCol */
Alias *alias; /* table alias & optional column aliases */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} RangeTableFunc;
/*
@@ -666,7 +666,7 @@ typedef struct RangeTableFuncCol
bool is_not_null; /* does it have NOT NULL? */
Node *colexpr; /* column filter expression */
Node *coldefexpr; /* column default value expression */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} RangeTableFuncCol;
/*
@@ -686,7 +686,7 @@ typedef struct RangeTableSample
List *method; /* sampling method name (possibly qualified) */
List *args; /* argument(s) for sampling method */
Node *repeatable; /* REPEATABLE expression, or NULL if none */
- int location; /* method name location, or -1 if unknown */
+ Location location; /* method name location, or -1 if unknown */
} RangeTableSample;
/*
@@ -729,7 +729,7 @@ typedef struct ColumnDef
Oid collOid; /* collation OID (InvalidOid if not set) */
List *constraints; /* other constraints on column */
List *fdwoptions; /* per-column FDW options */
- int location; /* parse location, or -1 if none/unknown */
+ Location location; /* parse location, or -1 if none/unknown */
} ColumnDef;
/*
@@ -803,7 +803,7 @@ typedef struct DefElem
Node *arg; /* typically Integer, Float, String, or
* TypeName */
DefElemAction defaction; /* unspecified action, or SET/ADD/DROP */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} DefElem;
/*
@@ -833,7 +833,7 @@ typedef struct XmlSerialize
Node *expr;
TypeName *typeName;
bool indent; /* [NO] INDENT */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} XmlSerialize;
/* Partitioning related definitions */
@@ -851,7 +851,7 @@ typedef struct PartitionElem
Node *expr; /* expression to partition on, or NULL */
List *collation; /* name of collation; NIL = default */
List *opclass; /* name of desired opclass; NIL = default */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} PartitionElem;
typedef enum PartitionStrategy
@@ -871,7 +871,7 @@ typedef struct PartitionSpec
NodeTag type;
PartitionStrategy strategy;
List *partParams; /* List of PartitionElems */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} PartitionSpec;
/*
@@ -898,7 +898,7 @@ struct PartitionBoundSpec
List *lowerdatums; /* List of PartitionRangeDatums */
List *upperdatums; /* List of PartitionRangeDatums */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
};
/*
@@ -921,7 +921,7 @@ typedef struct PartitionRangeDatum
Node *value; /* Const (or A_Const in raw tree), if kind is
* PARTITION_RANGE_DATUM_VALUE, else NULL */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} PartitionRangeDatum;
/*
@@ -1454,7 +1454,7 @@ typedef struct GroupingSet
NodeTag type;
GroupingSetKind kind pg_node_attr(query_jumble_ignore);
List *content;
- int location;
+ Location location;
} GroupingSet;
/*
@@ -1542,7 +1542,7 @@ typedef struct WithClause
NodeTag type;
List *ctes; /* list of CommonTableExprs */
bool recursive; /* true = WITH RECURSIVE */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} WithClause;
/*
@@ -1557,7 +1557,7 @@ typedef struct InferClause
List *indexElems; /* IndexElems to infer unique index */
Node *whereClause; /* qualification (partial-index predicate) */
char *conname; /* Constraint name, or NULL if unnamed */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} InferClause;
/*
@@ -1573,7 +1573,7 @@ typedef struct OnConflictClause
InferClause *infer; /* Optional index inference clause */
List *targetList; /* the target list (of ResTarget) */
Node *whereClause; /* qualifications */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} OnConflictClause;
/*
@@ -1594,7 +1594,7 @@ typedef struct CTESearchClause
List *search_col_list;
bool search_breadth_first;
char *search_seq_column;
- int location;
+ Location location;
} CTESearchClause;
typedef struct CTECycleClause
@@ -1605,7 +1605,7 @@ typedef struct CTECycleClause
Node *cycle_mark_value;
Node *cycle_mark_default;
char *cycle_path_column;
- int location;
+ Location location;
/* These fields are set during parse analysis: */
Oid cycle_mark_type; /* common type of _value and _default */
int cycle_mark_typmod;
@@ -1629,7 +1629,7 @@ typedef struct CommonTableExpr
Node *ctequery; /* the CTE's subquery */
CTESearchClause *search_clause pg_node_attr(query_jumble_ignore);
CTECycleClause *cycle_clause pg_node_attr(query_jumble_ignore);
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
/* These fields are set during parse analysis: */
/* is this CTE actually recursive? */
bool cterecursive pg_node_attr(query_jumble_ignore);
@@ -1725,7 +1725,7 @@ typedef struct JsonParseExpr
JsonValueExpr *expr; /* string expression */
JsonOutput *output; /* RETURNING clause, if specified */
bool unique_keys; /* WITH UNIQUE KEYS? */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} JsonParseExpr;
/*
@@ -1737,7 +1737,7 @@ typedef struct JsonScalarExpr
NodeTag type;
Expr *expr; /* scalar expression */
JsonOutput *output; /* RETURNING clause, if specified */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} JsonScalarExpr;
/*
@@ -1749,7 +1749,7 @@ typedef struct JsonSerializeExpr
NodeTag type;
JsonValueExpr *expr; /* json value expression */
JsonOutput *output; /* RETURNING clause, if specified */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} JsonSerializeExpr;
/*
@@ -1763,7 +1763,7 @@ typedef struct JsonObjectConstructor
JsonOutput *output; /* RETURNING clause, if specified */
bool absent_on_null; /* skip NULL values? */
bool unique; /* check key uniqueness? */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} JsonObjectConstructor;
/*
@@ -1776,7 +1776,7 @@ typedef struct JsonArrayConstructor
List *exprs; /* list of JsonValueExpr elements */
JsonOutput *output; /* RETURNING clause, if specified */
bool absent_on_null; /* skip NULL elements? */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} JsonArrayConstructor;
/*
@@ -1790,7 +1790,7 @@ typedef struct JsonArrayQueryConstructor
JsonOutput *output; /* RETURNING clause, if specified */
JsonFormat *format; /* FORMAT clause for subquery, if specified */
bool absent_on_null; /* skip NULL elements? */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} JsonArrayQueryConstructor;
/*
@@ -1805,7 +1805,7 @@ typedef struct JsonAggConstructor
Node *agg_filter; /* FILTER clause, if any */
List *agg_order; /* ORDER BY clause, if any */
struct WindowDef *over; /* OVER clause, if any */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} JsonAggConstructor;
/*
@@ -1859,8 +1859,8 @@ typedef struct RawStmt
NodeTag type;
Node *stmt; /* raw parse tree */
- int stmt_location; /* start location, or -1 if unknown */
- int stmt_len; /* length in bytes; 0 means "rest of string" */
+ Location stmt_location; /* start location, or -1 if unknown */
+ Location stmt_len; /* length in bytes; 0 means "rest of string" */
} RawStmt;
/*****************************************************************************
@@ -2067,7 +2067,7 @@ typedef struct PLAssignStmt
List *indirection; /* subscripts and field names, if any */
int nnames; /* number of names to use in ColumnRef */
SelectStmt *val; /* the PL/pgSQL expression to assign */
- int location; /* name's token location, or -1 if unknown */
+ Location location; /* name's token location, or -1 if unknown */
} PLAssignStmt;
@@ -2575,7 +2575,7 @@ typedef struct Constraint
char *conname; /* Constraint name, or NULL if unnamed */
bool deferrable; /* DEFERRABLE? */
bool initdeferred; /* INITIALLY DEFERRED? */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
/* Fields used for constraints with expressions (CHECK and DEFAULT): */
bool is_no_inherit; /* is constraint non-inheritable? */
@@ -3528,7 +3528,7 @@ typedef struct TransactionStmt
char *gid pg_node_attr(query_jumble_ignore);
bool chain; /* AND CHAIN option */
/* token location, or -1 if unknown */
- int location pg_node_attr(query_jumble_location);
+ Location location pg_node_attr(query_jumble_location);
} TransactionStmt;
/* ----------------------
@@ -3914,7 +3914,7 @@ typedef struct DeallocateStmt
/* true if DEALLOCATE ALL */
bool isall;
/* token location, or -1 if unknown */
- int location pg_node_attr(query_jumble_location);
+ Location location pg_node_attr(query_jumble_location);
} DeallocateStmt;
/*
@@ -4002,7 +4002,7 @@ typedef struct PublicationObjSpec
PublicationObjSpecType pubobjtype; /* type of this publication object */
char *name;
PublicationTable *pubtable;
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} PublicationObjSpec;
typedef struct CreatePublicationStmt
diff --git a/src/include/nodes/primnodes.h b/src/include/nodes/primnodes.h
index 4a154606d2..557be05657 100644
--- a/src/include/nodes/primnodes.h
+++ b/src/include/nodes/primnodes.h
@@ -30,6 +30,8 @@ typedef enum OverridingKind
} OverridingKind;
+#define Location int
+
/* ----------------------------------------------------------------
* node definitions
* ----------------------------------------------------------------
@@ -91,7 +93,7 @@ typedef struct RangeVar
Alias *alias;
/* token location, or -1 if unknown */
- int location;
+ Location location;
} RangeVar;
/*
@@ -128,7 +130,7 @@ typedef struct TableFunc
/* counts from 0; -1 if none specified */
int ordinalitycol pg_node_attr(query_jumble_ignore);
/* token location, or -1 if unknown */
- int location;
+ Location location;
} TableFunc;
/*
@@ -276,7 +278,7 @@ typedef struct Var
AttrNumber varattnosyn pg_node_attr(equal_ignore, query_jumble_ignore);
/* token location, or -1 if unknown */
- int location;
+ Location location;
} Var;
/*
@@ -318,7 +320,7 @@ typedef struct Const
* token location, or -1 if unknown. All constants are tracked as
* locations in query jumbling, to be marked as parameters.
*/
- int location pg_node_attr(query_jumble_location);
+ Location location pg_node_attr(query_jumble_location);
} Const;
/*
@@ -367,7 +369,7 @@ typedef struct Param
/* OID of collation, or InvalidOid if none */
Oid paramcollid pg_node_attr(query_jumble_ignore);
/* token location, or -1 if unknown */
- int location;
+ Location location;
} Param;
/*
@@ -490,7 +492,7 @@ typedef struct Aggref
int aggtransno pg_node_attr(query_jumble_ignore);
/* token location, or -1 if unknown */
- int location;
+ Location location;
} Aggref;
/*
@@ -537,7 +539,7 @@ typedef struct GroupingFunc
Index agglevelsup;
/* token location */
- int location;
+ Location location;
} GroupingFunc;
/*
@@ -568,7 +570,7 @@ typedef struct WindowFunc
/* is function a simple aggregate? */
bool winagg pg_node_attr(query_jumble_ignore);
/* token location, or -1 if unknown */
- int location;
+ Location location;
} WindowFunc;
/*
@@ -702,7 +704,7 @@ typedef struct FuncExpr
/* arguments to the function */
List *args;
/* token location, or -1 if unknown */
- int location;
+ Location location;
} FuncExpr;
/*
@@ -729,7 +731,7 @@ typedef struct NamedArgExpr
/* argument's number in positional notation */
int argnumber;
/* argument name location, or -1 if unknown */
- int location;
+ Location location;
} NamedArgExpr;
/*
@@ -771,7 +773,7 @@ typedef struct OpExpr
List *args;
/* token location, or -1 if unknown */
- int location;
+ Location location;
} OpExpr;
/*
@@ -851,7 +853,7 @@ typedef struct ScalarArrayOpExpr
List *args;
/* token location, or -1 if unknown */
- int location;
+ Location location;
} ScalarArrayOpExpr;
/*
@@ -873,7 +875,7 @@ typedef struct BoolExpr
Expr xpr;
BoolExprType boolop;
List *args; /* arguments to this expression */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} BoolExpr;
/*
@@ -950,7 +952,7 @@ typedef struct SubLink
List *operName pg_node_attr(query_jumble_ignore);
/* subselect as Query* or raw parsetree */
Node *subselect;
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} SubLink;
/*
@@ -1124,7 +1126,7 @@ typedef struct RelabelType
Oid resultcollid pg_node_attr(query_jumble_ignore);
/* how to display this node */
CoercionForm relabelformat pg_node_attr(query_jumble_ignore);
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} RelabelType;
/* ----------------
@@ -1146,7 +1148,7 @@ typedef struct CoerceViaIO
Oid resultcollid pg_node_attr(query_jumble_ignore);
/* how to display this node */
CoercionForm coerceformat pg_node_attr(query_jumble_ignore);
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} CoerceViaIO;
/* ----------------
@@ -1174,7 +1176,7 @@ typedef struct ArrayCoerceExpr
Oid resultcollid pg_node_attr(query_jumble_ignore);
/* how to display this node */
CoercionForm coerceformat pg_node_attr(query_jumble_ignore);
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} ArrayCoerceExpr;
/* ----------------
@@ -1198,7 +1200,7 @@ typedef struct ConvertRowtypeExpr
/* Like RowExpr, we deliberately omit a typmod and collation here */
/* how to display this node */
CoercionForm convertformat pg_node_attr(query_jumble_ignore);
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} ConvertRowtypeExpr;
/*----------
@@ -1213,7 +1215,7 @@ typedef struct CollateExpr
Expr xpr;
Expr *arg; /* input expression */
Oid collOid; /* collation's OID */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} CollateExpr;
/*----------
@@ -1248,7 +1250,7 @@ typedef struct CaseExpr
Expr *arg; /* implicit equality comparison argument */
List *args; /* the arguments (list of WHEN clauses) */
Expr *defresult; /* the default result (ELSE clause) */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} CaseExpr;
/*
@@ -1259,7 +1261,7 @@ typedef struct CaseWhen
Expr xpr;
Expr *expr; /* condition expression */
Expr *result; /* substitution result */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} CaseWhen;
/*
@@ -1316,7 +1318,7 @@ typedef struct ArrayExpr
/* true if elements are sub-arrays */
bool multidims pg_node_attr(query_jumble_ignore);
/* token location, or -1 if unknown */
- int location;
+ Location location;
} ArrayExpr;
/*
@@ -1367,7 +1369,7 @@ typedef struct RowExpr
/* list of String, or NIL */
List *colnames pg_node_attr(query_jumble_ignore);
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} RowExpr;
/*
@@ -1426,7 +1428,7 @@ typedef struct CoalesceExpr
/* the arguments */
List *args;
/* token location, or -1 if unknown */
- int location;
+ Location location;
} CoalesceExpr;
/*
@@ -1452,7 +1454,7 @@ typedef struct MinMaxExpr
/* the arguments */
List *args;
/* token location, or -1 if unknown */
- int location;
+ Location location;
} MinMaxExpr;
/*
@@ -1496,7 +1498,7 @@ typedef struct SQLValueFunction
*/
Oid type pg_node_attr(query_jumble_ignore);
int32 typmod;
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} SQLValueFunction;
/*
@@ -1549,7 +1551,7 @@ typedef struct XmlExpr
Oid type pg_node_attr(query_jumble_ignore);
int32 typmod pg_node_attr(query_jumble_ignore);
/* token location, or -1 if unknown */
- int location;
+ Location location;
} XmlExpr;
/*
@@ -1585,7 +1587,7 @@ typedef struct JsonFormat
NodeTag type;
JsonFormatType format_type; /* format type */
JsonEncoding encoding; /* JSON encoding */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} JsonFormat;
/*
@@ -1641,7 +1643,7 @@ typedef struct JsonConstructorExpr
JsonReturning *returning; /* RETURNING clause */
bool absent_on_null; /* ABSENT ON NULL? */
bool unique; /* WITH UNIQUE KEYS? (JSON_OBJECT[AGG] only) */
- int location;
+ Location location;
} JsonConstructorExpr;
/*
@@ -1667,7 +1669,7 @@ typedef struct JsonIsPredicate
JsonFormat *format; /* FORMAT clause, if specified */
JsonValueType item_type; /* JSON item type */
bool unique_keys; /* check key uniqueness? */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} JsonIsPredicate;
/* ----------------
@@ -1701,7 +1703,7 @@ typedef struct NullTest
NullTestType nulltesttype; /* IS NULL, IS NOT NULL */
/* T to perform field-by-field null checks */
bool argisrow pg_node_attr(query_jumble_ignore);
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} NullTest;
/*
@@ -1723,7 +1725,7 @@ typedef struct BooleanTest
Expr xpr;
Expr *arg; /* input expression */
BoolTestType booltesttype; /* test type */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} BooleanTest;
@@ -1765,7 +1767,7 @@ typedef struct CoerceToDomain
Oid resultcollid pg_node_attr(query_jumble_ignore);
/* how to display this node */
CoercionForm coercionformat pg_node_attr(query_jumble_ignore);
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} CoerceToDomain;
/*
@@ -1787,7 +1789,7 @@ typedef struct CoerceToDomainValue
/* collation for the substituted value */
Oid collation pg_node_attr(query_jumble_ignore);
/* token location, or -1 if unknown */
- int location;
+ Location location;
} CoerceToDomainValue;
/*
@@ -1807,7 +1809,7 @@ typedef struct SetToDefault
/* collation for the substituted value */
Oid collation pg_node_attr(query_jumble_ignore);
/* token location, or -1 if unknown */
- int location;
+ Location location;
} SetToDefault;
/*
--
2.40.1
[application/octet-stream] v4-0007-gen_node_support.pl-Optimize-serialization-of-fie.patch (9.2K, ../../CAEze2WhR_99QHboeXU73u8tc_N4zSnkoyhpm8ZuRS6gh49ZGjw@mail.gmail.com/7-v4-0007-gen_node_support.pl-Optimize-serialization-of-fie.patch)
download | inline diff:
From 022f9275c8b8a62832dfe94b5388baced84b0d8c Mon Sep 17 00:00:00 2001
From: Matthias van de Meent <[email protected]>
Date: Mon, 12 Feb 2024 17:12:02 +0100
Subject: [PATCH v4 7/7] gen_node_support.pl: Optimize serialization of fields
with copied values
The Var node's [syn] fields often contain the same data as their non-syn
counterparts. We invent a new pg_node_attr()ibute that represents this
relation, which allows us to omit these fields from serialization when they
are indeed copies of the original fields.
---
src/backend/nodes/gen_node_support.pl | 94 ++++++++++++++++++---------
src/backend/nodes/outfuncs.c | 3 +
src/backend/nodes/readfuncs.c | 3 +
src/include/nodes/primnodes.h | 4 +-
4 files changed, 70 insertions(+), 34 deletions(-)
diff --git a/src/backend/nodes/gen_node_support.pl b/src/backend/nodes/gen_node_support.pl
index fda8c3a494..66661a3881 100644
--- a/src/backend/nodes/gen_node_support.pl
+++ b/src/backend/nodes/gen_node_support.pl
@@ -363,6 +363,10 @@ foreach my $infile (@ARGV)
{
$manual_nodetag_number{$in_struct} = $1;
}
+ elsif ($attr =~ /^default_ref\(([\w\d."'-]+)\)$/)
+ {
+ # Unused at the node level
+ }
else
{
die
@@ -437,7 +441,7 @@ foreach my $infile (@ARGV)
}
# normal struct field
elsif ($line =~
- /^\s*(.+)\s*\b(\w+)(\[[\w\s+]+\])?\s*(?:pg_node_attr\(([\w(), ]*)\))?;/
+ /^\s*(.+)\s*\b(\w+)(\[[\w\s+]+\])?\s*(?:pg_node_attr\(([\w\d(), ."'-]*)\))?;/
)
{
if ($is_node_struct)
@@ -469,6 +473,7 @@ foreach my $infile (@ARGV)
if ( $attr !~ /^array_size\(\w+\)$/
&& $attr !~ /^copy_as\(\w+\)$/
&& $attr !~ /^read_as\(\w+\)$/
+ && $attr !~ /^default_ref\([\w\d."'-]+\)$/
&& !elem $attr,
qw(copy_as_scalar
equal_as_scalar
@@ -495,7 +500,7 @@ foreach my $infile (@ARGV)
}
# function pointer field
elsif ($line =~
- /^\s*([\w\s*]+)\s*\(\*(\w+)\)\s*\((.*)\)\s*(?:pg_node_attr\(([\w(), ]*)\))?;/
+ /^\s*([\w\s*]+)\s*\(\*(\w+)\)\s*\((.*)\)\s*(?:pg_node_attr\(([\w\d(), ."'-]*)\))?;/
)
{
if ($is_node_struct)
@@ -970,6 +975,15 @@ _read${n}(void)
my $array_size_field;
my $read_as_field;
my $read_write_ignore = 0;
+ # Type read/write macro suffix, "" for normal use, or "_DEFAULT" when
+ # value argument.
+ my $s = "";
+ # Default parameter to read/write macro. Includes comma separator so
+ # that MACRO_NAME$s($fieldname$defaultparam) is the full complete
+ # read/write expression for essentially all types.
+ # Note that this (currently) only works for scalar values.
+ my $d = "";
+
foreach my $a (@a)
{
if ($a =~ /^array_size\(([\w.]+)\)$/)
@@ -988,6 +1002,19 @@ _read${n}(void)
{
$read_write_ignore = 1;
}
+ elsif ($a =~ /^default_ref\(([\w\d+."'-]+)\)$/)
+ {
+ $s = "_DEFAULT";
+ $d = ", NODE_FIELD($1)";
+ }
+ }
+
+ if ($s eq "_DEFAULT")
+ {
+ die "custom defaults for non-scalar fields are not supported\n\tat $n.$f"
+ unless (elem $t, @scalar_types or elem $t, @enum_types);
+ die "custom defaults for Location fields are not supported\n\tat $n.$f"
+ if ($t eq "Location");
}
if ($read_write_ignore)
@@ -1008,8 +1035,8 @@ _read${n}(void)
# select instructions by field type
if ($t eq 'bool')
{
- print $off "\tWRITE_BOOL_FIELD($f);\n";
- print $rff "\tREAD_BOOL_FIELD($f);\n" unless $no_read;
+ print $off "\tWRITE_BOOL_FIELD$s($f$d);\n";
+ print $rff "\tREAD_BOOL_FIELD$s($f$d);\n" unless $no_read;
}
elsif ($t eq 'Location')
{
@@ -1018,8 +1045,11 @@ _read${n}(void)
}
elsif ($t eq 'TypMod')
{
- print $off "\tWRITE_INT_FIELD_DEFAULT($f, -1);\n";
- print $rff "\tREAD_INT_FIELD_DEFAULT($f, -1);\n" unless $no_read;
+ # The default value of a TypMod fields is -1, rather than 0
+ # for normal int fields.
+ $d = ", -1" if ($d eq "");
+ print $off "\tWRITE_INT_FIELD_DEFAULT($f$d);\n";
+ print $rff "\tREAD_INT_FIELD_DEFAULT($f$d);\n" unless $no_read;
}
elsif ($t eq 'int'
|| $t eq 'int16'
@@ -1027,8 +1057,8 @@ _read${n}(void)
|| $t eq 'AttrNumber'
|| $t eq 'StrategyNumber')
{
- print $off "\tWRITE_INT_FIELD($f);\n";
- print $rff "\tREAD_INT_FIELD($f);\n" unless $no_read;
+ print $off "\tWRITE_INT_FIELD$s($f$d);\n";
+ print $rff "\tREAD_INT_FIELD$s($f$d);\n" unless $no_read;
}
elsif ($t eq 'uint32'
|| $t eq 'bits32'
@@ -1036,56 +1066,56 @@ _read${n}(void)
|| $t eq 'Index'
|| $t eq 'SubTransactionId')
{
- print $off "\tWRITE_UINT_FIELD($f);\n";
- print $rff "\tREAD_UINT_FIELD($f);\n" unless $no_read;
+ print $off "\tWRITE_UINT_FIELD$s($f$d);\n";
+ print $rff "\tREAD_UINT_FIELD$s($f$d);\n" unless $no_read;
}
elsif ($t eq 'uint64'
|| $t eq 'AclMode')
{
- print $off "\tWRITE_UINT64_FIELD($f);\n";
- print $rff "\tREAD_UINT64_FIELD($f);\n" unless $no_read;
+ print $off "\tWRITE_UINT64_FIELD$s($f$d);\n";
+ print $rff "\tREAD_UINT64_FIELD$s($f$d);\n" unless $no_read;
}
elsif ($t eq 'Oid' || $t eq 'RelFileNumber')
{
- print $off "\tWRITE_OID_FIELD($f);\n";
- print $rff "\tREAD_OID_FIELD($f);\n" unless $no_read;
+ print $off "\tWRITE_OID_FIELD$s($f$d);\n";
+ print $rff "\tREAD_OID_FIELD$s($f$d);\n" unless $no_read;
}
elsif ($t eq 'long')
{
- print $off "\tWRITE_LONG_FIELD($f);\n";
- print $rff "\tREAD_LONG_FIELD($f);\n" unless $no_read;
+ print $off "\tWRITE_LONG_FIELD$s($f$d);\n";
+ print $rff "\tREAD_LONG_FIELD$s($f$d);\n" unless $no_read;
}
elsif ($t eq 'char')
{
- print $off "\tWRITE_CHAR_FIELD($f);\n";
- print $rff "\tREAD_CHAR_FIELD($f);\n" unless $no_read;
+ print $off "\tWRITE_CHAR_FIELD$s($f$d);\n";
+ print $rff "\tREAD_CHAR_FIELD$s($f$d);\n" unless $no_read;
}
elsif ($t eq 'double')
{
- print $off "\tWRITE_FLOAT_FIELD($f);\n";
- print $rff "\tREAD_FLOAT_FIELD($f);\n" unless $no_read;
+ print $off "\tWRITE_FLOAT_FIELD$s($f$d);\n";
+ print $rff "\tREAD_FLOAT_FIELD$s($f$d);\n" unless $no_read;
}
elsif ($t eq 'Cardinality')
{
- print $off "\tWRITE_FLOAT_FIELD($f);\n";
- print $rff "\tREAD_FLOAT_FIELD($f);\n" unless $no_read;
+ print $off "\tWRITE_FLOAT_FIELD$s($f$d);\n";
+ print $rff "\tREAD_FLOAT_FIELD$s($f$d);\n" unless $no_read;
}
elsif ($t eq 'Cost')
{
- print $off "\tWRITE_FLOAT_FIELD($f);\n";
- print $rff "\tREAD_FLOAT_FIELD($f);\n" unless $no_read;
+ print $off "\tWRITE_FLOAT_FIELD$s($f$d);\n";
+ print $rff "\tREAD_FLOAT_FIELD$s($f$d);\n" unless $no_read;
}
elsif ($t eq 'QualCost')
{
- print $off "\tWRITE_FLOAT_FIELD($f.startup);\n";
- print $off "\tWRITE_FLOAT_FIELD($f.per_tuple);\n";
- print $rff "\tREAD_FLOAT_FIELD($f.startup);\n" unless $no_read;
- print $rff "\tREAD_FLOAT_FIELD($f.per_tuple);\n" unless $no_read;
+ print $off "\tWRITE_FLOAT_FIELD$s($f.startup$d);\n";
+ print $off "\tWRITE_FLOAT_FIELD$s($f.per_tuple$d);\n";
+ print $rff "\tREAD_FLOAT_FIELD$s($f.startup$d);\n" unless $no_read;
+ print $rff "\tREAD_FLOAT_FIELD$s($f.per_tuple$d);\n" unless $no_read;
}
elsif ($t eq 'Selectivity')
{
- print $off "\tWRITE_FLOAT_FIELD($f);\n";
- print $rff "\tREAD_FLOAT_FIELD($f);\n" unless $no_read;
+ print $off "\tWRITE_FLOAT_FIELD$s($f$d);\n";
+ print $rff "\tREAD_FLOAT_FIELD$s($f$d);\n" unless $no_read;
}
elsif ($t eq 'char*')
{
@@ -1099,8 +1129,8 @@ _read${n}(void)
}
elsif (elem $t, @enum_types)
{
- print $off "\tWRITE_ENUM_FIELD($f, $t);\n";
- print $rff "\tREAD_ENUM_FIELD($f, $t);\n" unless $no_read;
+ print $off "\tWRITE_ENUM_FIELD$s($f, $t$d);\n";
+ print $rff "\tREAD_ENUM_FIELD$s($f, $t$d);\n" unless $no_read;
}
# arrays of scalar types
elsif ($t =~ /^(\w+)(\*|\[\w+\])$/ and elem $1, @scalar_types)
diff --git a/src/backend/nodes/outfuncs.c b/src/backend/nodes/outfuncs.c
index f96295b3e5..adbfc094a1 100644
--- a/src/backend/nodes/outfuncs.c
+++ b/src/backend/nodes/outfuncs.c
@@ -240,6 +240,9 @@ static void outDouble(StringInfo str, double d);
} \
} while (0)
+/* Accessing a node's field goes like this */
+#define NODE_FIELD(fldname) (node->fldname)
+
#define booltostr(x) ((x) ? "true" : "false")
diff --git a/src/backend/nodes/readfuncs.c b/src/backend/nodes/readfuncs.c
index eb87d17804..9adcc73102 100644
--- a/src/backend/nodes/readfuncs.c
+++ b/src/backend/nodes/readfuncs.c
@@ -258,6 +258,9 @@
#define READ_BOOL_ARRAY(fldname, len) \
READ_BOOL_ARRAY_DEFAULT(fldname, len, NULL)
+/* Accessing a node's field goes like this */
+#define NODE_FIELD(fldname) (local_node->fldname)
+
/* Routine exit */
#define READ_DONE() \
return local_node
diff --git a/src/include/nodes/primnodes.h b/src/include/nodes/primnodes.h
index 94282497d7..9dd60ad5c5 100644
--- a/src/include/nodes/primnodes.h
+++ b/src/include/nodes/primnodes.h
@@ -274,9 +274,9 @@ typedef struct Var
* their varno/varattno match.
*/
/* syntactic relation index (0 if unknown) */
- Index varnosyn pg_node_attr(equal_ignore, query_jumble_ignore);
+ Index varnosyn pg_node_attr(equal_ignore, query_jumble_ignore, default_ref(varno));
/* syntactic attribute number */
- AttrNumber varattnosyn pg_node_attr(equal_ignore, query_jumble_ignore);
+ AttrNumber varattnosyn pg_node_attr(equal_ignore, query_jumble_ignore, default_ref(varattno));
/* token location, or -1 if unknown */
Location location;
--
2.40.1
[application/octet-stream] v4-0006-nodeToString-Apply-RLE-on-Bitmapset-and-numeric-L.patch (7.9K, ../../CAEze2WhR_99QHboeXU73u8tc_N4zSnkoyhpm8ZuRS6gh49ZGjw@mail.gmail.com/8-v4-0006-nodeToString-Apply-RLE-on-Bitmapset-and-numeric-L.patch)
download | inline diff:
From d7daa505f0e76a88f455c8cc9a4cbe1297f4621f Mon Sep 17 00:00:00 2001
From: Matthias van de Meent <[email protected]>
Date: Sat, 10 Feb 2024 00:57:19 +0100
Subject: [PATCH v4 6/7] nodeToString: Apply RLE on Bitmapset and numeric List
types
This reduces the size of full serialized queries in pg_rewrite by several %,
reducing overhead in the system.
---
src/backend/nodes/outfuncs.c | 96 +++++++++++++++++++++++++++++++----
src/backend/nodes/read.c | 53 +++++++++++++++++--
src/backend/nodes/readfuncs.c | 16 +++++-
3 files changed, 149 insertions(+), 16 deletions(-)
diff --git a/src/backend/nodes/outfuncs.c b/src/backend/nodes/outfuncs.c
index 1a17eafd57..f96295b3e5 100644
--- a/src/backend/nodes/outfuncs.c
+++ b/src/backend/nodes/outfuncs.c
@@ -378,23 +378,64 @@ writeNodeArray(StringInfo str, const Node *const *arr, int len,
/*
* Print a List.
+ *
+ * Notes:
+ * NodeList is formatted as (<node1> <node2> ...)
+ *
+ * OidList is formatted as (o int +int int int ...), with +int indicating N
+ * successive identical values.
+ *
+ * IntList and XidList are formatted as (i/x int +int int int ...), with +int
+ * indicating N successively increasing values. (i 9 +3) is thus equivalent
+ * to (i 9 10 11 12).
*/
static void
_outList(StringInfo str, const List *node, bool omitLocation)
{
const ListCell *lc;
+ union LCSurrogate {
+ int32 i;
+ Oid o;
+ TransactionId x;
+ } previous = { .i = 0};
+ int run = 0;
+ int run_delta;
+ const char *fmt;
appendStringInfoChar(str, '(');
if (IsA(node, IntList))
+ {
appendStringInfoChar(str, 'i');
+ fmt = " %d";
+ run_delta = 1;
+ }
else if (IsA(node, OidList))
+ {
appendStringInfoChar(str, 'o');
+ fmt = " %u";
+ run_delta = 0;
+ }
else if (IsA(node, XidList))
+ {
appendStringInfoChar(str, 'x');
+ fmt = " %u";
+ run_delta = 1;
+ }
+ else if (IsA(node, List))
+ {
+ /* silence the compiler about uninitialized variables */
+ fmt = "";
+ run_delta = 0;
+ }
+ else
+ elog(ERROR, "unrecognized list node type: %d",
+ (int) node->type);
foreach(lc, node)
{
+ union LCSurrogate val = {.i = 0};
+
/*
* For the sake of backward compatibility, we emit a slightly
* different whitespace format for lists of nodes vs. other types of
@@ -405,26 +446,41 @@ _outList(StringInfo str, const List *node, bool omitLocation)
outNode(str, lfirst(lc), omitLocation);
if (lnext(node, lc))
appendStringInfoChar(str, ' ');
+ continue;
}
else if (IsA(node, IntList))
- appendStringInfo(str, " %d", lfirst_int(lc));
+ val.i = lfirst_int(lc);
else if (IsA(node, OidList))
- appendStringInfo(str, " %u", lfirst_oid(lc));
+ val.o = lfirst_oid(lc);
else if (IsA(node, XidList))
- appendStringInfo(str, " %u", lfirst_xid(lc));
+ val.x = lfirst_xid(lc);
+
+ if (val.i == previous.i + run_delta)
+ run += 1;
else
- elog(ERROR, "unrecognized list node type: %d",
- (int) node->type);
+ {
+ if (run > 0)
+ appendStringInfo(str, " +%d", run);
+ run = 0;
+ appendStringInfo(str, fmt, val.i);
+ }
+ previous = val;
}
- appendStringInfoChar(str, ')');
-}
+ if (run > 0)
+ appendStringInfo(str, " +%d", run);
+
+ appendStringInfoChar(str, ')');}
/*
* outBitmapset -
* converts a bitmap set of integers
*
- * Note: the output format is "(b int int ...)", similar to an integer List.
+ * Note: the output format is "(b int int +int ...)", similar to an
+ * integer List. Note that consecutive runs of incremental values are
+ * indicated by [... int +int ...], where the first int is the first set bit,
+ * and the int that's tagged with the '+' sign that follows is the number of
+ * subsequent set bits (resulting in a run of N+1 set bits).
*
* We export this function for use by extensions that define extensible nodes.
* That's somewhat historical, though, because calling outNode() will work.
@@ -432,13 +488,31 @@ _outList(StringInfo str, const List *node, bool omitLocation)
void
outBitmapset(StringInfo str, const Bitmapset *bms)
{
- int x;
+ int x = -1;
+ int prev = -2;
+ int run = 0;
appendStringInfoChar(str, '(');
appendStringInfoChar(str, 'b');
- x = -1;
+
while ((x = bms_next_member(bms, x)) >= 0)
- appendStringInfo(str, " %d", x);
+ {
+ if (x - prev == 1)
+ run++;
+ else
+ {
+ if (run > 0)
+ appendStringInfo(str, " +%d", run);
+
+ run = 0;
+ appendStringInfo(str, " %d", x);
+ }
+ prev = x;
+ }
+
+ if (run > 0)
+ appendStringInfo(str, " +%d", run);
+
appendStringInfoChar(str, ')');
}
diff --git a/src/backend/nodes/read.c b/src/backend/nodes/read.c
index a4cd3a8932..c3681e3615 100644
--- a/src/backend/nodes/read.c
+++ b/src/backend/nodes/read.c
@@ -402,6 +402,8 @@ nodeRead(const char *token, int tok_len)
elog(ERROR, "unterminated List structure");
if (tok_len == 1 && token[0] == 'i')
{
+ int prev = 0;
+
/* List of integers */
for (;;)
{
@@ -417,12 +419,23 @@ nodeRead(const char *token, int tok_len)
if (endptr != token + tok_len)
elog(ERROR, "unrecognized integer: \"%.*s\"",
tok_len, token);
- l = lappend_int(l, val);
+
+ if (token[0] == '+')
+ {
+ for (int i = 0; i < val; i++)
+ l = lappend_int(l, ++prev);
+ }
+ else
+ {
+ prev = val;
+ l = lappend_int(l, val);
+ }
}
result = (Node *) l;
}
else if (tok_len == 1 && token[0] == 'o')
{
+ Oid prev = 0;
/* List of OIDs */
for (;;)
{
@@ -438,12 +451,23 @@ nodeRead(const char *token, int tok_len)
if (endptr != token + tok_len)
elog(ERROR, "unrecognized OID: \"%.*s\"",
tok_len, token);
- l = lappend_oid(l, val);
+
+ if (token[0] == '+')
+ {
+ for (int i = 0; i < val; i++)
+ l = lappend_oid(l, prev);
+ }
+ else
+ {
+ prev = val;
+ l = lappend_oid(l, val);
+ }
}
result = (Node *) l;
}
else if (tok_len == 1 && token[0] == 'x')
{
+ TransactionId prev = 0;
/* List of TransactionIds */
for (;;)
{
@@ -459,7 +483,17 @@ nodeRead(const char *token, int tok_len)
if (endptr != token + tok_len)
elog(ERROR, "unrecognized Xid: \"%.*s\"",
tok_len, token);
- l = lappend_xid(l, val);
+
+ if (token[0] == '+')
+ {
+ for (int i = 0; i < val; i++)
+ l = lappend_xid(l, ++prev);
+ }
+ else
+ {
+ prev = val;
+ l = lappend_xid(l, val);
+ }
}
result = (Node *) l;
}
@@ -467,6 +501,7 @@ nodeRead(const char *token, int tok_len)
{
/* Bitmapset -- see also _readBitmapset() */
Bitmapset *bms = NULL;
+ int prev = -2;
for (;;)
{
@@ -482,7 +517,17 @@ nodeRead(const char *token, int tok_len)
if (endptr != token + tok_len)
elog(ERROR, "unrecognized integer: \"%.*s\"",
tok_len, token);
- bms = bms_add_member(bms, val);
+
+ if (token[0] == '+')
+ {
+ for (int i = 0; i < val; i++)
+ bms = bms_add_member(bms, ++prev);
+ }
+ else
+ {
+ bms = bms_add_member(bms, val);
+ prev = val;
+ }
}
result = (Node *) bms;
}
diff --git a/src/backend/nodes/readfuncs.c b/src/backend/nodes/readfuncs.c
index 4cdbad9e7e..eb87d17804 100644
--- a/src/backend/nodes/readfuncs.c
+++ b/src/backend/nodes/readfuncs.c
@@ -298,6 +298,7 @@ static Bitmapset *
_readBitmapset(void)
{
Bitmapset *result = NULL;
+ int prev = -2;
READ_TEMP_LOCALS();
@@ -326,7 +327,20 @@ _readBitmapset(void)
val = (int) strtol(token, &endptr, 10);
if (endptr != token + length)
elog(ERROR, "unrecognized integer: \"%.*s\"", length, token);
- result = bms_add_member(result, val);
+
+ if (token[0] == '+')
+ {
+ for (int i = 0; i < val; i++)
+ {
+ ++prev;
+ result = bms_add_member(result, prev);
+ }
+ }
+ else
+ {
+ result = bms_add_member(result, val);
+ prev = val;
+ }
}
return result;
--
2.40.1
^ permalink raw reply [nested|flat] 69+ messages in thread
* Re: Reducing output size of nodeToString
@ 2024-02-15 12:59 Peter Eisentraut <[email protected]>
parent: Matthias van de Meent <[email protected]>
0 siblings, 1 reply; 69+ messages in thread
From: Peter Eisentraut @ 2024-02-15 12:59 UTC (permalink / raw)
To: Matthias van de Meent <[email protected]>; +Cc: Robert Haas <[email protected]>; PostgreSQL Hackers <[email protected]>; Andres Freund <[email protected]>; Michel Pelletier <[email protected]>
Thanks, this patch set is a good way to incrementally work through these
changes.
I have looked at
v4-0001-pg_node_tree-Omit-serialization-of-fields-with-de.patch today.
Here are my thoughts:
I believe we had discussed offline to not omit enum fields with value 0
(WRITE_ENUM_FIELD). This is because the values of enum fields are
implementation artifacts, and this could be confusing for readers.
(This could be added as a squeeze-out-every-byte change later, but if
we're going to keep the format fit for human reading, I think we should
skip this.)
I have some concerns about the round-trippability of float values. If
we do, effectively, if (node->fldname != 0.0), then I think this would
also match negative zero, but when we read it back it, it would get
assigned positive zero. Maybe there are other edge cases like this.
Might be safer to not mess with this.
On the reading side, the macro nesting has gotten a bit out of hand. :)
We had talked earlier in the thread about the _DIRECT macros and you
said there were left over from something else you want to try, but I see
nothing else in this patch set uses this. I think this could all be
much simpler, like (omitting required punctuation)
#define READ_INT_FIELD(fldname, default)
if ((token = next_field(fldname, &length)))
local_node->fldname = atoi(token);
else
local_node->fldname = default;
where next_field() would
1. read the next token
2. if it is ":fldname", continue;
else rewind the read pointer and return NULL
3. read the next token and return that
Not only is this simpler, but it might also have better performance,
because we don't have separate pg_strtok_next() and pg_strtok() calls in
sequence.
^ permalink raw reply [nested|flat] 69+ messages in thread
* Re: Reducing output size of nodeToString
@ 2024-02-15 14:37 Matthias van de Meent <[email protected]>
parent: Peter Eisentraut <[email protected]>
0 siblings, 1 reply; 69+ messages in thread
From: Matthias van de Meent @ 2024-02-15 14:37 UTC (permalink / raw)
To: Peter Eisentraut <[email protected]>; +Cc: Robert Haas <[email protected]>; PostgreSQL Hackers <[email protected]>; Andres Freund <[email protected]>; Michel Pelletier <[email protected]>
On Thu, 15 Feb 2024 at 13:59, Peter Eisentraut <[email protected]> wrote:
>
> Thanks, this patch set is a good way to incrementally work through these
> changes.
>
> I have looked at
> v4-0001-pg_node_tree-Omit-serialization-of-fields-with-de.patch today.
> Here are my thoughts:
>
> I believe we had discussed offline to not omit enum fields with value 0
> (WRITE_ENUM_FIELD). This is because the values of enum fields are
> implementation artifacts, and this could be confusing for readers.
Thanks for reminding me, I didn't remember this when I worked on
updating the patchset. I'll update this soon.
> I have some concerns about the round-trippability of float values. If
> we do, effectively, if (node->fldname != 0.0), then I think this would
> also match negative zero, but when we read it back it, it would get
> assigned positive zero. Maybe there are other edge cases like this.
> Might be safer to not mess with this.
That's a good point. Would an additional check that the sign of the
field equals the default's sign be enough for this? As for other
cases, I'm not sure we currently want to support non-normal floats,
even if it is technically possible to do the round-trip in the current
format.
> On the reading side, the macro nesting has gotten a bit out of hand. :)
> We had talked earlier in the thread about the _DIRECT macros and you
> said there were left over from something else you want to try, but I see
> nothing else in this patch set uses this. I think this could all be
> much simpler, like (omitting required punctuation)
[...]
> Not only is this simpler, but it might also have better performance,
> because we don't have separate pg_strtok_next() and pg_strtok() calls in
> sequence.
Good points. I'll see what I can do here.
Kind regards,
Matthias van de Meent
Neon (https://neon.tech)
^ permalink raw reply [nested|flat] 69+ messages in thread
* Re: Reducing output size of nodeToString
@ 2024-02-19 13:19 Matthias van de Meent <[email protected]>
parent: Matthias van de Meent <[email protected]>
0 siblings, 1 reply; 69+ messages in thread
From: Matthias van de Meent @ 2024-02-19 13:19 UTC (permalink / raw)
To: Matthias van de Meent <[email protected]>; +Cc: Peter Eisentraut <[email protected]>; Robert Haas <[email protected]>; PostgreSQL Hackers <[email protected]>; Andres Freund <[email protected]>; Michel Pelletier <[email protected]>
On Thu, 15 Feb 2024 at 15:37, Matthias van de Meent
<[email protected]> wrote:
>
> On Thu, 15 Feb 2024 at 13:59, Peter Eisentraut <[email protected]> wrote:
> >
> > Thanks, this patch set is a good way to incrementally work through these
> > changes.
> >
> > I have looked at
> > v4-0001-pg_node_tree-Omit-serialization-of-fields-with-de.patch today.
> > Here are my thoughts:
> >
> > I believe we had discussed offline to not omit enum fields with value 0
> > (WRITE_ENUM_FIELD). This is because the values of enum fields are
> > implementation artifacts, and this could be confusing for readers.
>
> Thanks for reminding me, I didn't remember this when I worked on
> updating the patchset. I'll update this soon.
This has been split into patch 0008 in the set. A query on ev_action
shows that enum default-0-omission is effective on 1994 fields:
select match, count(*)
from pg_rewrite,
lateral (
select unnest(regexp_matches(ev_action, '(:\w+ 0)[^0-9]', 'g')) match
)
group by 1 order by 2 desc;
match | count
-----------------+-------
:funcformat 0 | 587
:rtekind 0 | 449
:limitOption 0 | 260
:querySource 0 | 260
:override 0 | 260
:jointype 0 | 156
:aggsplit 0 | 15
:subLinkType 0 | 5
:nulltesttype 0 | 2
> > On the reading side, the macro nesting has gotten a bit out of hand. :)
> > We had talked earlier in the thread about the _DIRECT macros and you
> > said there were left over from something else you want to try, but I see
> > nothing else in this patch set uses this. I think this could all be
> > much simpler, like (omitting required punctuation)
> [...]
> > Not only is this simpler, but it might also have better performance,
> > because we don't have separate pg_strtok_next() and pg_strtok() calls in
> > sequence.
>
> Good points. I'll see what I can do here.
Attached the updated version of the patch on top of 5497daf3, which
incorporates this last round of feedback. It moves the
default-0-omission for Enums to newly added 0008, and checks the sign
to deal with +0/-0 issues in float default checks.
See below for updated numbers.
Kind regards,
Matthias van de Meent
Neon (https://neon.tech)
New numbers:
select 'master' as "version"
, pg_database_size('template0') as "template0"
, pg_total_relation_size('pg_rewrite') as "rel_total"
, pg_relation_size('pg_rewrite', 'main') as "rel_main"
, sum(pg_column_size(ev_action)) as "toasted"
, sum(octet_length(ev_action)) as "raw"
from pg_rewrite;
version | template0 | rel_total | rel_main | toasted | raw
---------+-----------+-----------+----------+---------+---------
master | 7528975 | 770048 | 114688 | 574051 | 3002981
0001 | 7348751 | 630784 | 131072 | 448495 | 1972854
0002 | 7250447 | 589824 | 131072 | 412261 | 1866880
0003 | 7242255 | 581632 | 131072 | 410476 | 1864843
0004 | 7225871 | 565248 | 139264 | 393801 | 1678735
0005 | 7225871 | 565248 | 139264 | 393556 | 1675165
0006 | 7217679 | 557056 | 139264 | 379062 | 1654178
0007 | 7160335 | 491520 | 155648 | 322145 | 1363885
0008 | 7135759 | 475136 | 155648 | 311294 | 1337649
Attachments:
[application/octet-stream] v3-0004-gen_node_support.pl-Add-a-TypMod-type-for-signall.patch (10.4K, ../../CAEze2WhfRn0cdNer0Vkye_61BwAmMqM6D9_cJp8i6JmZ8U4wAA@mail.gmail.com/2-v3-0004-gen_node_support.pl-Add-a-TypMod-type-for-signall.patch)
download | inline diff:
From 2deead6a6749c6c3e300fa199f339efe821b1cb7 Mon Sep 17 00:00:00 2001
From: Matthias van de Meent <[email protected]>
Date: Thu, 8 Feb 2024 17:37:39 +0100
Subject: [PATCH v3 4/8] gen_node_support.pl: Add a TypMod type for signalling
TypMod behaviour
Like Location, TypMod has its own default (-1). By using a type, we can
omit adding pg_node_attribute(default(-1)) markers to every typmod-valued
field, whilst still getting the benefits of a smaller size in serialization.
---
src/backend/nodes/gen_node_support.pl | 8 ++++++-
src/backend/nodes/outfuncs.c | 2 +-
src/backend/nodes/readfuncs.c | 2 +-
src/include/nodes/parsenodes.h | 4 ++--
src/include/nodes/pathnodes.h | 2 +-
src/include/nodes/primnodes.h | 33 ++++++++++++++-------------
6 files changed, 29 insertions(+), 22 deletions(-)
diff --git a/src/backend/nodes/gen_node_support.pl b/src/backend/nodes/gen_node_support.pl
index ec850c3484..fda8c3a494 100644
--- a/src/backend/nodes/gen_node_support.pl
+++ b/src/backend/nodes/gen_node_support.pl
@@ -135,7 +135,8 @@ my @nodetag_only;
# types that are copied by straight assignment
my @scalar_types = qw(
bits32 bool char double int int8 int16 int32 int64 long uint8 uint16 uint32 uint64
- AclMode AttrNumber Cardinality Cost Index Oid RelFileNumber Selectivity Size StrategyNumber SubTransactionId TimeLineID XLogRecPtr
+ AclMode AttrNumber Cardinality Cost Index Oid RelFileNumber Selectivity Size StrategyNumber
+ SubTransactionId TimeLineID XLogRecPtr TypMod
);
# collect enum types
@@ -1015,6 +1016,11 @@ _read${n}(void)
print $off "\tWRITE_LOCATION_FIELD($f);\n";
print $rff "\tREAD_LOCATION_FIELD($f);\n" unless $no_read;
}
+ elsif ($t eq 'TypMod')
+ {
+ print $off "\tWRITE_INT_FIELD_DEFAULT($f, -1);\n";
+ print $rff "\tREAD_INT_FIELD_DEFAULT($f, -1);\n" unless $no_read;
+ }
elsif ($t eq 'int'
|| $t eq 'int16'
|| $t eq 'int32'
diff --git a/src/backend/nodes/outfuncs.c b/src/backend/nodes/outfuncs.c
index a9b5c4d754..ec22130a70 100644
--- a/src/backend/nodes/outfuncs.c
+++ b/src/backend/nodes/outfuncs.c
@@ -497,7 +497,7 @@ _outConst(StringInfo str, const Const *node, bool omitLocation)
WRITE_NODE_TYPE("CONST");
WRITE_OID_FIELD(consttype);
- WRITE_INT_FIELD(consttypmod);
+ WRITE_INT_FIELD_DEFAULT(consttypmod, -1);
WRITE_OID_FIELD(constcollid);
WRITE_INT_FIELD(constlen);
WRITE_BOOL_FIELD(constbyval);
diff --git a/src/backend/nodes/readfuncs.c b/src/backend/nodes/readfuncs.c
index 96ddf0f9e2..635f43fe2f 100644
--- a/src/backend/nodes/readfuncs.c
+++ b/src/backend/nodes/readfuncs.c
@@ -367,7 +367,7 @@ _readConst(void)
READ_LOCALS(Const);
READ_OID_FIELD(consttype);
- READ_INT_FIELD(consttypmod);
+ READ_INT_FIELD_DEFAULT(consttypmod, -1);
READ_OID_FIELD(constcollid);
READ_INT_FIELD(constlen);
READ_BOOL_FIELD(constbyval);
diff --git a/src/include/nodes/parsenodes.h b/src/include/nodes/parsenodes.h
index fc7003052d..3ed81af4b9 100644
--- a/src/include/nodes/parsenodes.h
+++ b/src/include/nodes/parsenodes.h
@@ -260,7 +260,7 @@ typedef struct TypeName
bool setof; /* is a set? */
bool pct_type; /* %TYPE specified? */
List *typmods; /* type modifier expression(s) */
- int32 typemod; /* prespecified type modifier */
+ TypMod typemod; /* prespecified type modifier */
List *arrayBounds; /* array bounds */
Location location; /* token location, or -1 if unknown */
} TypeName;
@@ -1607,7 +1607,7 @@ typedef struct CTECycleClause
Location location;
/* These fields are set during parse analysis: */
Oid cycle_mark_type; /* common type of _value and _default */
- int cycle_mark_typmod;
+ TypMod cycle_mark_typmod;
Oid cycle_mark_collation;
Oid cycle_mark_neop; /* <> operator for type */
} CTECycleClause;
diff --git a/src/include/nodes/pathnodes.h b/src/include/nodes/pathnodes.h
index 534692bee1..f7c2496a7f 100644
--- a/src/include/nodes/pathnodes.h
+++ b/src/include/nodes/pathnodes.h
@@ -3394,7 +3394,7 @@ typedef struct AggTransInfo
Oid aggtranstype;
/* Additional data about transtype */
- int32 aggtranstypmod;
+ TypMod aggtranstypmod;
int transtypeLen;
bool transtypeByVal;
diff --git a/src/include/nodes/primnodes.h b/src/include/nodes/primnodes.h
index 557be05657..94282497d7 100644
--- a/src/include/nodes/primnodes.h
+++ b/src/include/nodes/primnodes.h
@@ -29,8 +29,9 @@ typedef enum OverridingKind
OVERRIDING_SYSTEM_VALUE,
} OverridingKind;
-
+/* Type aliases for gen_node_support */
#define Location int
+#define TypMod int32
/* ----------------------------------------------------------------
* node definitions
@@ -250,7 +251,7 @@ typedef struct Var
/* pg_type OID for the type of this var */
Oid vartype pg_node_attr(query_jumble_ignore);
/* pg_attribute typmod value */
- int32 vartypmod pg_node_attr(query_jumble_ignore);
+ TypMod vartypmod pg_node_attr(query_jumble_ignore);
/* OID of collation, or InvalidOid if none */
Oid varcollid pg_node_attr(query_jumble_ignore);
@@ -299,7 +300,7 @@ typedef struct Const
/* pg_type OID of the constant's datatype */
Oid consttype;
/* typmod value, if any */
- int32 consttypmod pg_node_attr(query_jumble_ignore);
+ TypMod consttypmod pg_node_attr(query_jumble_ignore);
/* OID of collation, or InvalidOid if none */
Oid constcollid pg_node_attr(query_jumble_ignore);
/* typlen of the constant's datatype */
@@ -365,7 +366,7 @@ typedef struct Param
int paramid; /* numeric ID for parameter */
Oid paramtype; /* pg_type OID of parameter's datatype */
/* typmod value, if known */
- int32 paramtypmod pg_node_attr(query_jumble_ignore);
+ TypMod paramtypmod pg_node_attr(query_jumble_ignore);
/* OID of collation, or InvalidOid if none */
Oid paramcollid pg_node_attr(query_jumble_ignore);
/* token location, or -1 if unknown */
@@ -623,7 +624,7 @@ typedef struct SubscriptingRef
/* type of the SubscriptingRef's result */
Oid refrestype pg_node_attr(query_jumble_ignore);
/* typmod of the result */
- int32 reftypmod pg_node_attr(query_jumble_ignore);
+ TypMod reftypmod pg_node_attr(query_jumble_ignore);
/* collation of result, or InvalidOid if none */
Oid refcollid pg_node_attr(query_jumble_ignore);
/* expressions that evaluate to upper container indexes */
@@ -1009,7 +1010,7 @@ typedef struct SubPlan
char *plan_name; /* A name assigned during planning */
/* Extra data useful for determining subplan's output type: */
Oid firstColType; /* Type of first column of subplan result */
- int32 firstColTypmod; /* Typmod of first column of subplan result */
+ TypMod firstColTypmod; /* Typmod of first column of subplan result */
Oid firstColCollation; /* Collation of first column of subplan
* result */
/* Information about execution strategy: */
@@ -1067,7 +1068,7 @@ typedef struct FieldSelect
/* type of the field (result type of this node) */
Oid resulttype pg_node_attr(query_jumble_ignore);
/* output typmod (usually -1) */
- int32 resulttypmod pg_node_attr(query_jumble_ignore);
+ TypMod resulttypmod pg_node_attr(query_jumble_ignore);
/* OID of collation of the field */
Oid resultcollid pg_node_attr(query_jumble_ignore);
} FieldSelect;
@@ -1121,7 +1122,7 @@ typedef struct RelabelType
Expr *arg; /* input expression */
Oid resulttype; /* output type of coercion expression */
/* output typmod (usually -1) */
- int32 resulttypmod pg_node_attr(query_jumble_ignore);
+ TypMod resulttypmod pg_node_attr(query_jumble_ignore);
/* OID of collation, or InvalidOid if none */
Oid resultcollid pg_node_attr(query_jumble_ignore);
/* how to display this node */
@@ -1171,7 +1172,7 @@ typedef struct ArrayCoerceExpr
Expr *elemexpr; /* expression representing per-element work */
Oid resulttype; /* output type of coercion (an array type) */
/* output typmod (also element typmod) */
- int32 resulttypmod pg_node_attr(query_jumble_ignore);
+ TypMod resulttypmod pg_node_attr(query_jumble_ignore);
/* OID of collation, or InvalidOid if none */
Oid resultcollid pg_node_attr(query_jumble_ignore);
/* how to display this node */
@@ -1291,7 +1292,7 @@ typedef struct CaseTestExpr
Expr xpr;
Oid typeId; /* type for substituted value */
/* typemod for substituted value */
- int32 typeMod pg_node_attr(query_jumble_ignore);
+ TypMod typeMod pg_node_attr(query_jumble_ignore);
/* collation for the substituted value */
Oid collation pg_node_attr(query_jumble_ignore);
} CaseTestExpr;
@@ -1497,7 +1498,7 @@ typedef struct SQLValueFunction
* include this Oid in the query jumbling.
*/
Oid type pg_node_attr(query_jumble_ignore);
- int32 typmod;
+ TypMod typmod;
Location location; /* token location, or -1 if unknown */
} SQLValueFunction;
@@ -1549,7 +1550,7 @@ typedef struct XmlExpr
bool indent;
/* target type/typmod for XMLSERIALIZE */
Oid type pg_node_attr(query_jumble_ignore);
- int32 typmod pg_node_attr(query_jumble_ignore);
+ TypMod typmod pg_node_attr(query_jumble_ignore);
/* token location, or -1 if unknown */
Location location;
} XmlExpr;
@@ -1599,7 +1600,7 @@ typedef struct JsonReturning
NodeTag type;
JsonFormat *format; /* output JSON format */
Oid typid; /* target type Oid */
- int32 typmod; /* target type modifier */
+ TypMod typmod; /* target type modifier */
} JsonReturning;
/*
@@ -1762,7 +1763,7 @@ typedef struct CoerceToDomain
Expr *arg; /* input expression */
Oid resulttype; /* domain type ID (result type) */
/* output typmod (currently always -1) */
- int32 resulttypmod pg_node_attr(query_jumble_ignore);
+ TypMod resulttypmod pg_node_attr(query_jumble_ignore);
/* OID of collation, or InvalidOid if none */
Oid resultcollid pg_node_attr(query_jumble_ignore);
/* how to display this node */
@@ -1785,7 +1786,7 @@ typedef struct CoerceToDomainValue
/* type for substituted value */
Oid typeId;
/* typemod for substituted value */
- int32 typeMod pg_node_attr(query_jumble_ignore);
+ TypMod typeMod pg_node_attr(query_jumble_ignore);
/* collation for the substituted value */
Oid collation pg_node_attr(query_jumble_ignore);
/* token location, or -1 if unknown */
@@ -1805,7 +1806,7 @@ typedef struct SetToDefault
/* type for substituted value */
Oid typeId;
/* typemod for substituted value */
- int32 typeMod pg_node_attr(query_jumble_ignore);
+ TypMod typeMod pg_node_attr(query_jumble_ignore);
/* collation for the substituted value */
Oid collation pg_node_attr(query_jumble_ignore);
/* token location, or -1 if unknown */
--
2.40.1
[application/octet-stream] v3-0003-gen_node_support.pl-Mark-location-fields-as-type-.patch (26.4K, ../../CAEze2WhfRn0cdNer0Vkye_61BwAmMqM6D9_cJp8i6JmZ8U4wAA@mail.gmail.com/3-v3-0003-gen_node_support.pl-Mark-location-fields-as-type-.patch)
download | inline diff:
From a8225b71f280e5918b307d57938f06e9435ae2b3 Mon Sep 17 00:00:00 2001
From: Matthias van de Meent <[email protected]>
Date: Thu, 8 Feb 2024 17:15:22 +0100
Subject: [PATCH v3 3/8] gen_node_support.pl: Mark location fields as type
alias Location
Instead of the rather ugly type=int + name ~= location$, we now have a
special type for offset pointers or sizes that are only relevant when a
query text is included, which decreases the complexity required in
gen_node_support.pl for handling these values.
---
src/backend/nodes/gen_node_support.pl | 6 +-
src/include/nodes/parsenodes.h | 92 +++++++++++++--------------
src/include/nodes/primnodes.h | 72 +++++++++++----------
3 files changed, 86 insertions(+), 84 deletions(-)
diff --git a/src/backend/nodes/gen_node_support.pl b/src/backend/nodes/gen_node_support.pl
index 487f6f7728..ec850c3484 100644
--- a/src/backend/nodes/gen_node_support.pl
+++ b/src/backend/nodes/gen_node_support.pl
@@ -777,7 +777,7 @@ _equal${n}(const $n *a, const $n *b)
print $eff "\tCOMPARE_BITMAPSET_FIELD($f);\n"
unless $equal_ignore;
}
- elsif ($t eq 'int' && $f =~ 'location$')
+ elsif ($t eq 'Location')
{
print $cff "\tCOPY_LOCATION_FIELD($f);\n" unless $copy_ignore;
print $eff "\tCOMPARE_LOCATION_FIELD($f);\n" unless $equal_ignore;
@@ -1010,7 +1010,7 @@ _read${n}(void)
print $off "\tWRITE_BOOL_FIELD($f);\n";
print $rff "\tREAD_BOOL_FIELD($f);\n" unless $no_read;
}
- elsif ($t eq 'int' && $f =~ 'location$')
+ elsif ($t eq 'Location')
{
print $off "\tWRITE_LOCATION_FIELD($f);\n";
print $rff "\tREAD_LOCATION_FIELD($f);\n" unless $no_read;
@@ -1303,7 +1303,7 @@ _jumble${n}(JumbleState *jstate, Node *node)
print $jff "\tJUMBLE_NODE($f);\n"
unless $query_jumble_ignore;
}
- elsif ($t eq 'int' && $f =~ 'location$')
+ elsif ($t eq 'Location')
{
# Track the node's location only if directly requested.
if ($query_jumble_location)
diff --git a/src/include/nodes/parsenodes.h b/src/include/nodes/parsenodes.h
index d5b08ded44..fc7003052d 100644
--- a/src/include/nodes/parsenodes.h
+++ b/src/include/nodes/parsenodes.h
@@ -225,9 +225,9 @@ typedef struct Query
* both be -1 meaning "unknown".
*/
/* start location, or -1 if unknown */
- int stmt_location;
+ Location stmt_location;
/* length in bytes; 0 means "rest of string" */
- int stmt_len pg_node_attr(query_jumble_ignore);
+ Location stmt_len pg_node_attr(query_jumble_ignore);
} Query;
@@ -262,7 +262,7 @@ typedef struct TypeName
List *typmods; /* type modifier expression(s) */
int32 typemod; /* prespecified type modifier */
List *arrayBounds; /* array bounds */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} TypeName;
/*
@@ -282,7 +282,7 @@ typedef struct ColumnRef
{
NodeTag type;
List *fields; /* field names (String nodes) or A_Star */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} ColumnRef;
/*
@@ -292,7 +292,7 @@ typedef struct ParamRef
{
NodeTag type;
int number; /* the number of the parameter */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} ParamRef;
/*
@@ -325,7 +325,7 @@ typedef struct A_Expr
List *name; /* possibly-qualified name of operator */
Node *lexpr; /* left argument, or NULL if none */
Node *rexpr; /* right argument, or NULL if none */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} A_Expr;
/*
@@ -351,7 +351,7 @@ typedef struct A_Const
NodeTag type;
union ValUnion val;
bool isnull; /* SQL NULL constant */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} A_Const;
/*
@@ -362,7 +362,7 @@ typedef struct TypeCast
NodeTag type;
Node *arg; /* the expression being casted */
TypeName *typeName; /* the target type */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} TypeCast;
/*
@@ -373,7 +373,7 @@ typedef struct CollateClause
NodeTag type;
Node *arg; /* input expression */
List *collname; /* possibly-qualified collation name */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} CollateClause;
/*
@@ -393,7 +393,7 @@ typedef struct RoleSpec
NodeTag type;
RoleSpecType roletype; /* Type of this rolespec */
char *rolename; /* filled only for ROLESPEC_CSTRING */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} RoleSpec;
/*
@@ -423,7 +423,7 @@ typedef struct FuncCall
bool agg_distinct; /* arguments were labeled DISTINCT */
bool func_variadic; /* last argument was labeled VARIADIC */
CoercionForm funcformat; /* how to display this node */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} FuncCall;
/*
@@ -480,7 +480,7 @@ typedef struct A_ArrayExpr
{
NodeTag type;
List *elements; /* array element expressions */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} A_ArrayExpr;
/*
@@ -507,7 +507,7 @@ typedef struct ResTarget
char *name; /* column name or NULL */
List *indirection; /* subscripts, field names, and '*', or NIL */
Node *val; /* the value expression to compute or assign */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} ResTarget;
/*
@@ -537,7 +537,7 @@ typedef struct SortBy
SortByDir sortby_dir; /* ASC/DESC/USING/default */
SortByNulls sortby_nulls; /* NULLS FIRST/LAST */
List *useOp; /* name of op to use, if SORTBY_USING */
- int location; /* operator location, or -1 if none/unknown */
+ Location location; /* operator location, or -1 if none/unknown */
} SortBy;
/*
@@ -558,7 +558,7 @@ typedef struct WindowDef
int frameOptions; /* frame_clause options, see below */
Node *startOffset; /* expression for starting bound, if any */
Node *endOffset; /* expression for ending bound, if any */
- int location; /* parse location, or -1 if none/unknown */
+ Location location; /* parse location, or -1 if none/unknown */
} WindowDef;
/*
@@ -648,7 +648,7 @@ typedef struct RangeTableFunc
List *namespaces; /* list of namespaces as ResTarget */
List *columns; /* list of RangeTableFuncCol */
Alias *alias; /* table alias & optional column aliases */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} RangeTableFunc;
/*
@@ -666,7 +666,7 @@ typedef struct RangeTableFuncCol
bool is_not_null; /* does it have NOT NULL? */
Node *colexpr; /* column filter expression */
Node *coldefexpr; /* column default value expression */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} RangeTableFuncCol;
/*
@@ -686,7 +686,7 @@ typedef struct RangeTableSample
List *method; /* sampling method name (possibly qualified) */
List *args; /* argument(s) for sampling method */
Node *repeatable; /* REPEATABLE expression, or NULL if none */
- int location; /* method name location, or -1 if unknown */
+ Location location; /* method name location, or -1 if unknown */
} RangeTableSample;
/*
@@ -728,7 +728,7 @@ typedef struct ColumnDef
Oid collOid; /* collation OID (InvalidOid if not set) */
List *constraints; /* other constraints on column */
List *fdwoptions; /* per-column FDW options */
- int location; /* parse location, or -1 if none/unknown */
+ Location location; /* parse location, or -1 if none/unknown */
} ColumnDef;
/*
@@ -802,7 +802,7 @@ typedef struct DefElem
Node *arg; /* typically Integer, Float, String, or
* TypeName */
DefElemAction defaction; /* unspecified action, or SET/ADD/DROP */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} DefElem;
/*
@@ -832,7 +832,7 @@ typedef struct XmlSerialize
Node *expr;
TypeName *typeName;
bool indent; /* [NO] INDENT */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} XmlSerialize;
/* Partitioning related definitions */
@@ -850,7 +850,7 @@ typedef struct PartitionElem
Node *expr; /* expression to partition on, or NULL */
List *collation; /* name of collation; NIL = default */
List *opclass; /* name of desired opclass; NIL = default */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} PartitionElem;
typedef enum PartitionStrategy
@@ -870,7 +870,7 @@ typedef struct PartitionSpec
NodeTag type;
PartitionStrategy strategy;
List *partParams; /* List of PartitionElems */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} PartitionSpec;
/*
@@ -897,7 +897,7 @@ struct PartitionBoundSpec
List *lowerdatums; /* List of PartitionRangeDatums */
List *upperdatums; /* List of PartitionRangeDatums */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
};
/*
@@ -920,7 +920,7 @@ typedef struct PartitionRangeDatum
Node *value; /* Const (or A_Const in raw tree), if kind is
* PARTITION_RANGE_DATUM_VALUE, else NULL */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} PartitionRangeDatum;
/*
@@ -1453,7 +1453,7 @@ typedef struct GroupingSet
NodeTag type;
GroupingSetKind kind pg_node_attr(query_jumble_ignore);
List *content;
- int location;
+ Location location;
} GroupingSet;
/*
@@ -1541,7 +1541,7 @@ typedef struct WithClause
NodeTag type;
List *ctes; /* list of CommonTableExprs */
bool recursive; /* true = WITH RECURSIVE */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} WithClause;
/*
@@ -1556,7 +1556,7 @@ typedef struct InferClause
List *indexElems; /* IndexElems to infer unique index */
Node *whereClause; /* qualification (partial-index predicate) */
char *conname; /* Constraint name, or NULL if unnamed */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} InferClause;
/*
@@ -1572,7 +1572,7 @@ typedef struct OnConflictClause
InferClause *infer; /* Optional index inference clause */
List *targetList; /* the target list (of ResTarget) */
Node *whereClause; /* qualifications */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} OnConflictClause;
/*
@@ -1593,7 +1593,7 @@ typedef struct CTESearchClause
List *search_col_list;
bool search_breadth_first;
char *search_seq_column;
- int location;
+ Location location;
} CTESearchClause;
typedef struct CTECycleClause
@@ -1604,7 +1604,7 @@ typedef struct CTECycleClause
Node *cycle_mark_value;
Node *cycle_mark_default;
char *cycle_path_column;
- int location;
+ Location location;
/* These fields are set during parse analysis: */
Oid cycle_mark_type; /* common type of _value and _default */
int cycle_mark_typmod;
@@ -1628,7 +1628,7 @@ typedef struct CommonTableExpr
Node *ctequery; /* the CTE's subquery */
CTESearchClause *search_clause pg_node_attr(query_jumble_ignore);
CTECycleClause *cycle_clause pg_node_attr(query_jumble_ignore);
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
/* These fields are set during parse analysis: */
/* is this CTE actually recursive? */
bool cterecursive pg_node_attr(query_jumble_ignore);
@@ -1724,7 +1724,7 @@ typedef struct JsonParseExpr
JsonValueExpr *expr; /* string expression */
JsonOutput *output; /* RETURNING clause, if specified */
bool unique_keys; /* WITH UNIQUE KEYS? */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} JsonParseExpr;
/*
@@ -1736,7 +1736,7 @@ typedef struct JsonScalarExpr
NodeTag type;
Expr *expr; /* scalar expression */
JsonOutput *output; /* RETURNING clause, if specified */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} JsonScalarExpr;
/*
@@ -1748,7 +1748,7 @@ typedef struct JsonSerializeExpr
NodeTag type;
JsonValueExpr *expr; /* json value expression */
JsonOutput *output; /* RETURNING clause, if specified */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} JsonSerializeExpr;
/*
@@ -1762,7 +1762,7 @@ typedef struct JsonObjectConstructor
JsonOutput *output; /* RETURNING clause, if specified */
bool absent_on_null; /* skip NULL values? */
bool unique; /* check key uniqueness? */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} JsonObjectConstructor;
/*
@@ -1775,7 +1775,7 @@ typedef struct JsonArrayConstructor
List *exprs; /* list of JsonValueExpr elements */
JsonOutput *output; /* RETURNING clause, if specified */
bool absent_on_null; /* skip NULL elements? */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} JsonArrayConstructor;
/*
@@ -1789,7 +1789,7 @@ typedef struct JsonArrayQueryConstructor
JsonOutput *output; /* RETURNING clause, if specified */
JsonFormat *format; /* FORMAT clause for subquery, if specified */
bool absent_on_null; /* skip NULL elements? */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} JsonArrayQueryConstructor;
/*
@@ -1804,7 +1804,7 @@ typedef struct JsonAggConstructor
Node *agg_filter; /* FILTER clause, if any */
List *agg_order; /* ORDER BY clause, if any */
struct WindowDef *over; /* OVER clause, if any */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} JsonAggConstructor;
/*
@@ -1858,8 +1858,8 @@ typedef struct RawStmt
NodeTag type;
Node *stmt; /* raw parse tree */
- int stmt_location; /* start location, or -1 if unknown */
- int stmt_len; /* length in bytes; 0 means "rest of string" */
+ Location stmt_location; /* start location, or -1 if unknown */
+ Location stmt_len; /* length in bytes; 0 means "rest of string" */
} RawStmt;
/*****************************************************************************
@@ -2066,7 +2066,7 @@ typedef struct PLAssignStmt
List *indirection; /* subscripts and field names, if any */
int nnames; /* number of names to use in ColumnRef */
SelectStmt *val; /* the PL/pgSQL expression to assign */
- int location; /* name's token location, or -1 if unknown */
+ Location location; /* name's token location, or -1 if unknown */
} PLAssignStmt;
@@ -2574,7 +2574,7 @@ typedef struct Constraint
char *conname; /* Constraint name, or NULL if unnamed */
bool deferrable; /* DEFERRABLE? */
bool initdeferred; /* INITIALLY DEFERRED? */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
/* Fields used for constraints with expressions (CHECK and DEFAULT): */
bool is_no_inherit; /* is constraint non-inheritable? */
@@ -3527,7 +3527,7 @@ typedef struct TransactionStmt
char *gid pg_node_attr(query_jumble_ignore);
bool chain; /* AND CHAIN option */
/* token location, or -1 if unknown */
- int location pg_node_attr(query_jumble_location);
+ Location location pg_node_attr(query_jumble_location);
} TransactionStmt;
/* ----------------------
@@ -3913,7 +3913,7 @@ typedef struct DeallocateStmt
/* true if DEALLOCATE ALL */
bool isall;
/* token location, or -1 if unknown */
- int location pg_node_attr(query_jumble_location);
+ Location location pg_node_attr(query_jumble_location);
} DeallocateStmt;
/*
@@ -4001,7 +4001,7 @@ typedef struct PublicationObjSpec
PublicationObjSpecType pubobjtype; /* type of this publication object */
char *name;
PublicationTable *pubtable;
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} PublicationObjSpec;
typedef struct CreatePublicationStmt
diff --git a/src/include/nodes/primnodes.h b/src/include/nodes/primnodes.h
index 4a154606d2..557be05657 100644
--- a/src/include/nodes/primnodes.h
+++ b/src/include/nodes/primnodes.h
@@ -30,6 +30,8 @@ typedef enum OverridingKind
} OverridingKind;
+#define Location int
+
/* ----------------------------------------------------------------
* node definitions
* ----------------------------------------------------------------
@@ -91,7 +93,7 @@ typedef struct RangeVar
Alias *alias;
/* token location, or -1 if unknown */
- int location;
+ Location location;
} RangeVar;
/*
@@ -128,7 +130,7 @@ typedef struct TableFunc
/* counts from 0; -1 if none specified */
int ordinalitycol pg_node_attr(query_jumble_ignore);
/* token location, or -1 if unknown */
- int location;
+ Location location;
} TableFunc;
/*
@@ -276,7 +278,7 @@ typedef struct Var
AttrNumber varattnosyn pg_node_attr(equal_ignore, query_jumble_ignore);
/* token location, or -1 if unknown */
- int location;
+ Location location;
} Var;
/*
@@ -318,7 +320,7 @@ typedef struct Const
* token location, or -1 if unknown. All constants are tracked as
* locations in query jumbling, to be marked as parameters.
*/
- int location pg_node_attr(query_jumble_location);
+ Location location pg_node_attr(query_jumble_location);
} Const;
/*
@@ -367,7 +369,7 @@ typedef struct Param
/* OID of collation, or InvalidOid if none */
Oid paramcollid pg_node_attr(query_jumble_ignore);
/* token location, or -1 if unknown */
- int location;
+ Location location;
} Param;
/*
@@ -490,7 +492,7 @@ typedef struct Aggref
int aggtransno pg_node_attr(query_jumble_ignore);
/* token location, or -1 if unknown */
- int location;
+ Location location;
} Aggref;
/*
@@ -537,7 +539,7 @@ typedef struct GroupingFunc
Index agglevelsup;
/* token location */
- int location;
+ Location location;
} GroupingFunc;
/*
@@ -568,7 +570,7 @@ typedef struct WindowFunc
/* is function a simple aggregate? */
bool winagg pg_node_attr(query_jumble_ignore);
/* token location, or -1 if unknown */
- int location;
+ Location location;
} WindowFunc;
/*
@@ -702,7 +704,7 @@ typedef struct FuncExpr
/* arguments to the function */
List *args;
/* token location, or -1 if unknown */
- int location;
+ Location location;
} FuncExpr;
/*
@@ -729,7 +731,7 @@ typedef struct NamedArgExpr
/* argument's number in positional notation */
int argnumber;
/* argument name location, or -1 if unknown */
- int location;
+ Location location;
} NamedArgExpr;
/*
@@ -771,7 +773,7 @@ typedef struct OpExpr
List *args;
/* token location, or -1 if unknown */
- int location;
+ Location location;
} OpExpr;
/*
@@ -851,7 +853,7 @@ typedef struct ScalarArrayOpExpr
List *args;
/* token location, or -1 if unknown */
- int location;
+ Location location;
} ScalarArrayOpExpr;
/*
@@ -873,7 +875,7 @@ typedef struct BoolExpr
Expr xpr;
BoolExprType boolop;
List *args; /* arguments to this expression */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} BoolExpr;
/*
@@ -950,7 +952,7 @@ typedef struct SubLink
List *operName pg_node_attr(query_jumble_ignore);
/* subselect as Query* or raw parsetree */
Node *subselect;
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} SubLink;
/*
@@ -1124,7 +1126,7 @@ typedef struct RelabelType
Oid resultcollid pg_node_attr(query_jumble_ignore);
/* how to display this node */
CoercionForm relabelformat pg_node_attr(query_jumble_ignore);
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} RelabelType;
/* ----------------
@@ -1146,7 +1148,7 @@ typedef struct CoerceViaIO
Oid resultcollid pg_node_attr(query_jumble_ignore);
/* how to display this node */
CoercionForm coerceformat pg_node_attr(query_jumble_ignore);
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} CoerceViaIO;
/* ----------------
@@ -1174,7 +1176,7 @@ typedef struct ArrayCoerceExpr
Oid resultcollid pg_node_attr(query_jumble_ignore);
/* how to display this node */
CoercionForm coerceformat pg_node_attr(query_jumble_ignore);
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} ArrayCoerceExpr;
/* ----------------
@@ -1198,7 +1200,7 @@ typedef struct ConvertRowtypeExpr
/* Like RowExpr, we deliberately omit a typmod and collation here */
/* how to display this node */
CoercionForm convertformat pg_node_attr(query_jumble_ignore);
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} ConvertRowtypeExpr;
/*----------
@@ -1213,7 +1215,7 @@ typedef struct CollateExpr
Expr xpr;
Expr *arg; /* input expression */
Oid collOid; /* collation's OID */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} CollateExpr;
/*----------
@@ -1248,7 +1250,7 @@ typedef struct CaseExpr
Expr *arg; /* implicit equality comparison argument */
List *args; /* the arguments (list of WHEN clauses) */
Expr *defresult; /* the default result (ELSE clause) */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} CaseExpr;
/*
@@ -1259,7 +1261,7 @@ typedef struct CaseWhen
Expr xpr;
Expr *expr; /* condition expression */
Expr *result; /* substitution result */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} CaseWhen;
/*
@@ -1316,7 +1318,7 @@ typedef struct ArrayExpr
/* true if elements are sub-arrays */
bool multidims pg_node_attr(query_jumble_ignore);
/* token location, or -1 if unknown */
- int location;
+ Location location;
} ArrayExpr;
/*
@@ -1367,7 +1369,7 @@ typedef struct RowExpr
/* list of String, or NIL */
List *colnames pg_node_attr(query_jumble_ignore);
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} RowExpr;
/*
@@ -1426,7 +1428,7 @@ typedef struct CoalesceExpr
/* the arguments */
List *args;
/* token location, or -1 if unknown */
- int location;
+ Location location;
} CoalesceExpr;
/*
@@ -1452,7 +1454,7 @@ typedef struct MinMaxExpr
/* the arguments */
List *args;
/* token location, or -1 if unknown */
- int location;
+ Location location;
} MinMaxExpr;
/*
@@ -1496,7 +1498,7 @@ typedef struct SQLValueFunction
*/
Oid type pg_node_attr(query_jumble_ignore);
int32 typmod;
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} SQLValueFunction;
/*
@@ -1549,7 +1551,7 @@ typedef struct XmlExpr
Oid type pg_node_attr(query_jumble_ignore);
int32 typmod pg_node_attr(query_jumble_ignore);
/* token location, or -1 if unknown */
- int location;
+ Location location;
} XmlExpr;
/*
@@ -1585,7 +1587,7 @@ typedef struct JsonFormat
NodeTag type;
JsonFormatType format_type; /* format type */
JsonEncoding encoding; /* JSON encoding */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} JsonFormat;
/*
@@ -1641,7 +1643,7 @@ typedef struct JsonConstructorExpr
JsonReturning *returning; /* RETURNING clause */
bool absent_on_null; /* ABSENT ON NULL? */
bool unique; /* WITH UNIQUE KEYS? (JSON_OBJECT[AGG] only) */
- int location;
+ Location location;
} JsonConstructorExpr;
/*
@@ -1667,7 +1669,7 @@ typedef struct JsonIsPredicate
JsonFormat *format; /* FORMAT clause, if specified */
JsonValueType item_type; /* JSON item type */
bool unique_keys; /* check key uniqueness? */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} JsonIsPredicate;
/* ----------------
@@ -1701,7 +1703,7 @@ typedef struct NullTest
NullTestType nulltesttype; /* IS NULL, IS NOT NULL */
/* T to perform field-by-field null checks */
bool argisrow pg_node_attr(query_jumble_ignore);
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} NullTest;
/*
@@ -1723,7 +1725,7 @@ typedef struct BooleanTest
Expr xpr;
Expr *arg; /* input expression */
BoolTestType booltesttype; /* test type */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} BooleanTest;
@@ -1765,7 +1767,7 @@ typedef struct CoerceToDomain
Oid resultcollid pg_node_attr(query_jumble_ignore);
/* how to display this node */
CoercionForm coercionformat pg_node_attr(query_jumble_ignore);
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} CoerceToDomain;
/*
@@ -1787,7 +1789,7 @@ typedef struct CoerceToDomainValue
/* collation for the substituted value */
Oid collation pg_node_attr(query_jumble_ignore);
/* token location, or -1 if unknown */
- int location;
+ Location location;
} CoerceToDomainValue;
/*
@@ -1807,7 +1809,7 @@ typedef struct SetToDefault
/* collation for the substituted value */
Oid collation pg_node_attr(query_jumble_ignore);
/* token location, or -1 if unknown */
- int location;
+ Location location;
} SetToDefault;
/*
--
2.40.1
[application/octet-stream] v3-0001-incremental-backups-Add-new-items-to-glossary-mon.patch (3.1K, ../../CAEze2WhfRn0cdNer0Vkye_61BwAmMqM6D9_cJp8i6JmZ8U4wAA@mail.gmail.com/4-v3-0001-incremental-backups-Add-new-items-to-glossary-mon.patch)
download | inline diff:
From 12a2b5290f29e3897391cc7b7a2666198cb1e835 Mon Sep 17 00:00:00 2001
From: Matthias van de Meent <[email protected]>
Date: Mon, 15 Jan 2024 21:20:38 +0100
Subject: [PATCH v3] incremental backups: Add new items to glossary,
monitoring.sgml
The previous patches seem to have overlooked this.
---
doc/src/sgml/glossary.sgml | 35 +++++++++++++++++++++++++++++++++++
doc/src/sgml/monitoring.sgml | 3 ++-
2 files changed, 37 insertions(+), 1 deletion(-)
diff --git a/doc/src/sgml/glossary.sgml b/doc/src/sgml/glossary.sgml
index 5815fa4471..a0150e5c48 100644
--- a/doc/src/sgml/glossary.sgml
+++ b/doc/src/sgml/glossary.sgml
@@ -893,6 +893,27 @@
</para>
</glossdef>
</glossentry>
+ <glossentry id="glossary-incremental-backup">
+ <glossterm>Incremental backup</glossterm>
+ <glossdef>
+ <para>
+ A special <glossterm linkend="glossary-basebackup">base backup</glossterm>
+ that for some files may contain only those pages that were modified since
+ a previous backup, as opposed to the full contents of every file. Like
+ base backups, it is generated by the tool <xref linkend="app-pgbasebackup"/>.
+ </para>
+ <para>
+ To restore incremental backups the tool <xref linkend="app-pgcombinebackup"/>
+ is used, which combines incremental backups with a base backup and
+ <glossterm linkend="glossary-wal">WAL</glossterm> to restore a
+ <glossterm linkend="glossary-db-cluster">database cluster</glossterm> to
+ a consistent state.
+ </para>
+ <para>
+ For more information, see <xref linkend="backup-incremental-backup"/>.
+ </para>
+ </glossdef>
+ </glossentry>
<glossentry id="glossary-insert">
<glossterm>Insert</glossterm>
@@ -2157,6 +2178,20 @@
</glossdef>
</glossentry>
+ <glossentry id="glossary-wal-summarizer">
+ <glossterm>WAL summarizer (process)</glossterm>
+ <glossdef>
+ <para>
+ A special <glossterm linkend="glossary-backend">backend process</glossterm>
+ that summarizes WAL data for
+ <glossterm linkend="glossary-incremental-backup">incremental backups</glossterm>.
+ </para>
+ <para>
+ For more information, see <xref linkend="runtime-config-wal-summarization"/>.
+ </para>
+ </glossdef>
+ </glossentry>
+
<glossentry id="glossary-wal-writer">
<glossterm>WAL writer (process)</glossterm>
<glossdef>
diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml
index b804eb8b5e..6e74138a69 100644
--- a/doc/src/sgml/monitoring.sgml
+++ b/doc/src/sgml/monitoring.sgml
@@ -999,7 +999,8 @@ postgres 27093 0.0 0.0 30096 2752 ? Ss 11:34 0:00 postgres: ser
<literal>client backend</literal>, <literal>checkpointer</literal>,
<literal>archiver</literal>, <literal>standalone backend</literal>,
<literal>startup</literal>, <literal>walreceiver</literal>,
- <literal>walsender</literal> and <literal>walwriter</literal>.
+ <literal>walsender</literal>, <literal>walwriter</literal> and
+ <literal>walsummarizer</literal>.
In addition, background workers registered by extensions may have
additional types.
</para></entry>
--
2.40.1
[application/octet-stream] v3-0002-pg_node_tree-Don-t-store-query-text-locations-in-.patch (19.2K, ../../CAEze2WhfRn0cdNer0Vkye_61BwAmMqM6D9_cJp8i6JmZ8U4wAA@mail.gmail.com/5-v3-0002-pg_node_tree-Don-t-store-query-text-locations-in-.patch)
download | inline diff:
From 3d92084c00a4fabf2e8b3f2a05bc93a51610a199 Mon Sep 17 00:00:00 2001
From: Matthias van de Meent <[email protected]>
Date: Wed, 3 Jan 2024 01:39:42 +0100
Subject: [PATCH v3 2/8] pg_node_tree: Don't store query text locations in
pg_node_tree fields.
We don't store original query texts, so any lingering "location" value
can only be useful in forensic debugging. In normal operation, however,
a non-default value will show up as measurable overhead in
serialization, just omit serialization, saving several 10s of kBs.
---
src/backend/catalog/heap.c | 9 ++--
src/backend/catalog/index.c | 4 +-
src/backend/catalog/pg_attrdef.c | 6 ++-
src/backend/catalog/pg_proc.c | 10 +++-
src/backend/catalog/pg_publication.c | 6 ++-
src/backend/commands/policy.c | 11 +++-
src/backend/commands/statscmds.c | 2 +-
src/backend/commands/trigger.c | 3 +-
src/backend/commands/typecmds.c | 8 +--
src/backend/nodes/gen_node_support.pl | 4 +-
src/backend/nodes/outfuncs.c | 77 ++++++++++++++++-----------
src/backend/rewrite/rewriteDefine.c | 4 +-
src/include/nodes/nodes.h | 4 +-
13 files changed, 95 insertions(+), 53 deletions(-)
diff --git a/src/backend/catalog/heap.c b/src/backend/catalog/heap.c
index 252e106cad..b0e0e8191d 100644
--- a/src/backend/catalog/heap.c
+++ b/src/backend/catalog/heap.c
@@ -2063,9 +2063,9 @@ StoreRelCheck(Relation rel, const char *ccname, Node *expr,
Oid constrOid;
/*
- * Flatten expression to string form for storage.
+ * Flatten expression to string form for storage, without query refs.
*/
- ccbin = nodeToString(expr);
+ ccbin = nodeToStringNoQLocs(expr);
/*
* Find columns of rel that are used in expr
@@ -3679,7 +3679,7 @@ StorePartitionKey(Relation rel,
{
char *exprString;
- exprString = nodeToString(partexprs);
+ exprString = nodeToStringNoQLocs(partexprs);
partexprDatum = CStringGetTextDatum(exprString);
pfree(exprString);
}
@@ -3837,7 +3837,8 @@ StorePartitionBound(Relation rel, Relation parent, PartitionBoundSpec *bound)
memset(new_val, 0, sizeof(new_val));
memset(new_null, false, sizeof(new_null));
memset(new_repl, false, sizeof(new_repl));
- new_val[Anum_pg_class_relpartbound - 1] = CStringGetTextDatum(nodeToString(bound));
+ new_val[Anum_pg_class_relpartbound - 1] =
+ CStringGetTextDatum(nodeToStringNoQLocs(bound));
new_null[Anum_pg_class_relpartbound - 1] = false;
new_repl[Anum_pg_class_relpartbound - 1] = true;
newtuple = heap_modify_tuple(tuple, RelationGetDescr(classRel),
diff --git a/src/backend/catalog/index.c b/src/backend/catalog/index.c
index 4b88a9cb87..3d5f4e53d5 100644
--- a/src/backend/catalog/index.c
+++ b/src/backend/catalog/index.c
@@ -587,7 +587,7 @@ UpdateIndexRelation(Oid indexoid,
{
char *exprsString;
- exprsString = nodeToString(indexInfo->ii_Expressions);
+ exprsString = nodeToStringNoQLocs(indexInfo->ii_Expressions);
exprsDatum = CStringGetTextDatum(exprsString);
pfree(exprsString);
}
@@ -602,7 +602,7 @@ UpdateIndexRelation(Oid indexoid,
{
char *predString;
- predString = nodeToString(make_ands_explicit(indexInfo->ii_Predicate));
+ predString = nodeToStringNoQLocs(make_ands_explicit(indexInfo->ii_Predicate));
predDatum = CStringGetTextDatum(predString);
pfree(predString);
}
diff --git a/src/backend/catalog/pg_attrdef.c b/src/backend/catalog/pg_attrdef.c
index 003ae70b4d..a900c9bb28 100644
--- a/src/backend/catalog/pg_attrdef.c
+++ b/src/backend/catalog/pg_attrdef.c
@@ -23,6 +23,7 @@
#include "catalog/objectaccess.h"
#include "catalog/pg_attrdef.h"
#include "executor/executor.h"
+#include "nodes/nodeFuncs.h"
#include "optimizer/optimizer.h"
#include "utils/array.h"
#include "utils/builtins.h"
@@ -62,9 +63,10 @@ StoreAttrDefault(Relation rel, AttrNumber attnum,
adrel = table_open(AttrDefaultRelationId, RowExclusiveLock);
/*
- * Flatten expression to string form for storage.
+ * Flatten expression to string form for storage, without references to
+ * the original query string.
*/
- adbin = nodeToString(expr);
+ adbin = nodeToStringNoQLocs(expr);
/*
* Make the pg_attrdef entry.
diff --git a/src/backend/catalog/pg_proc.c b/src/backend/catalog/pg_proc.c
index b581d334d3..c5790a2224 100644
--- a/src/backend/catalog/pg_proc.c
+++ b/src/backend/catalog/pg_proc.c
@@ -331,7 +331,10 @@ ProcedureCreate(const char *procedureName,
else
nulls[Anum_pg_proc_proargnames - 1] = true;
if (parameterDefaults != NIL)
- values[Anum_pg_proc_proargdefaults - 1] = CStringGetTextDatum(nodeToString(parameterDefaults));
+ {
+ values[Anum_pg_proc_proargdefaults - 1] =
+ CStringGetTextDatum(nodeToStringNoQLocs(parameterDefaults));
+ }
else
nulls[Anum_pg_proc_proargdefaults - 1] = true;
if (trftypes != PointerGetDatum(NULL))
@@ -344,7 +347,10 @@ ProcedureCreate(const char *procedureName,
else
nulls[Anum_pg_proc_probin - 1] = true;
if (prosqlbody)
- values[Anum_pg_proc_prosqlbody - 1] = CStringGetTextDatum(nodeToString(prosqlbody));
+ {
+ values[Anum_pg_proc_prosqlbody - 1] =
+ CStringGetTextDatum(nodeToStringNoQLocs(prosqlbody));
+ }
else
nulls[Anum_pg_proc_prosqlbody - 1] = true;
if (proconfig != PointerGetDatum(NULL))
diff --git a/src/backend/catalog/pg_publication.c b/src/backend/catalog/pg_publication.c
index b98b0ce0ae..b201313430 100644
--- a/src/backend/catalog/pg_publication.c
+++ b/src/backend/catalog/pg_publication.c
@@ -36,6 +36,7 @@
#include "commands/publicationcmds.h"
#include "funcapi.h"
#include "miscadmin.h"
+#include "nodes/nodeFuncs.h"
#include "utils/array.h"
#include "utils/builtins.h"
#include "utils/catcache.h"
@@ -422,7 +423,10 @@ publication_add_relation(Oid pubid, PublicationRelInfo *pri,
/* Add qualifications, if available */
if (pri->whereClause != NULL)
- values[Anum_pg_publication_rel_prqual - 1] = CStringGetTextDatum(nodeToString(pri->whereClause));
+ {
+ values[Anum_pg_publication_rel_prqual - 1] =
+ CStringGetTextDatum(nodeToStringNoQLocs(pri->whereClause));
+ }
else
nulls[Anum_pg_publication_rel_prqual - 1] = true;
diff --git a/src/backend/commands/policy.c b/src/backend/commands/policy.c
index 596326e5ec..1e6842bf41 100644
--- a/src/backend/commands/policy.c
+++ b/src/backend/commands/policy.c
@@ -30,6 +30,7 @@
#include "commands/policy.h"
#include "miscadmin.h"
#include "nodes/makefuncs.h"
+#include "nodes/nodeFuncs.h"
#include "nodes/pg_list.h"
#include "parser/parse_clause.h"
#include "parser/parse_collate.h"
@@ -701,13 +702,19 @@ CreatePolicy(CreatePolicyStmt *stmt)
/* Add qual if present. */
if (qual)
- values[Anum_pg_policy_polqual - 1] = CStringGetTextDatum(nodeToString(qual));
+ {
+ values[Anum_pg_policy_polqual - 1] =
+ CStringGetTextDatum(nodeToStringNoQLocs(qual));
+ }
else
isnull[Anum_pg_policy_polqual - 1] = true;
/* Add WITH CHECK qual if present */
if (with_check_qual)
- values[Anum_pg_policy_polwithcheck - 1] = CStringGetTextDatum(nodeToString(with_check_qual));
+ {
+ values[Anum_pg_policy_polwithcheck - 1] =
+ CStringGetTextDatum(nodeToStringNoQLocs(with_check_qual));
+ }
else
isnull[Anum_pg_policy_polwithcheck - 1] = true;
diff --git a/src/backend/commands/statscmds.c b/src/backend/commands/statscmds.c
index b1a9c74bd6..58e8133f93 100644
--- a/src/backend/commands/statscmds.c
+++ b/src/backend/commands/statscmds.c
@@ -477,7 +477,7 @@ CreateStatistics(CreateStatsStmt *stmt)
{
char *exprsString;
- exprsString = nodeToString(stxexprs);
+ exprsString = nodeToStringNoQLocs(stxexprs);
exprsDatum = CStringGetTextDatum(exprsString);
pfree(exprsString);
}
diff --git a/src/backend/commands/trigger.c b/src/backend/commands/trigger.c
index c344ff0944..6a3dc13a67 100644
--- a/src/backend/commands/trigger.c
+++ b/src/backend/commands/trigger.c
@@ -39,6 +39,7 @@
#include "miscadmin.h"
#include "nodes/bitmapset.h"
#include "nodes/makefuncs.h"
+#include "nodes/nodeFuncs.h"
#include "optimizer/optimizer.h"
#include "parser/parse_clause.h"
#include "parser/parse_collate.h"
@@ -674,7 +675,7 @@ CreateTriggerFiringOn(CreateTrigStmt *stmt, const char *queryString,
/* we'll need the rtable for recordDependencyOnExpr */
whenRtable = pstate->p_rtable;
- qual = nodeToString(whenClause);
+ qual = nodeToStringNoQLocs(whenClause);
free_parsestate(pstate);
}
diff --git a/src/backend/commands/typecmds.c b/src/backend/commands/typecmds.c
index e0275e5fe9..3c0e7bc5db 100644
--- a/src/backend/commands/typecmds.c
+++ b/src/backend/commands/typecmds.c
@@ -58,6 +58,7 @@
#include "executor/executor.h"
#include "miscadmin.h"
#include "nodes/makefuncs.h"
+#include "nodes/nodeFuncs.h"
#include "optimizer/optimizer.h"
#include "parser/parse_coerce.h"
#include "parser/parse_collate.h"
@@ -924,7 +925,7 @@ DefineDomain(CreateDomainStmt *stmt)
defaultValue =
deparse_expression(defaultExpr,
NIL, false, false);
- defaultValueBin = nodeToString(defaultExpr);
+ defaultValueBin = nodeToStringNoQLocs(defaultExpr);
}
}
else
@@ -3506,9 +3507,10 @@ domainAddConstraint(Oid domainOid, Oid domainNamespace, Oid baseTypeOid,
errmsg("cannot use table references in domain check constraint")));
/*
- * Convert to string form for storage.
+ * Convert to string form for storage, without references to the original
+ * query text.
*/
- ccbin = nodeToString(expr);
+ ccbin = nodeToStringNoQLocs(expr);
/*
* Store the constraint in pg_constraint
diff --git a/src/backend/nodes/gen_node_support.pl b/src/backend/nodes/gen_node_support.pl
index 2f0a59bc87..487f6f7728 100644
--- a/src/backend/nodes/gen_node_support.pl
+++ b/src/backend/nodes/gen_node_support.pl
@@ -921,7 +921,7 @@ foreach my $n (@node_types)
my $N = uc $n;
print $ofs "\t\t\tcase T_${n}:\n"
- . "\t\t\t\t_out${n}(str, obj);\n"
+ . "\t\t\t\t_out${n}(str, obj, omitLocation);\n"
. "\t\t\t\tbreak;\n";
print $rfs "\tif (MATCH(\"$N\", "
@@ -933,7 +933,7 @@ foreach my $n (@node_types)
print $off "
static void
-_out${n}(StringInfo str, const $n *node)
+_out${n}(StringInfo str, const $n *node, bool omitLocation)
{
\tWRITE_NODE_TYPE(\"$N\");
diff --git a/src/backend/nodes/outfuncs.c b/src/backend/nodes/outfuncs.c
index 25b0a21f79..a9b5c4d754 100644
--- a/src/backend/nodes/outfuncs.c
+++ b/src/backend/nodes/outfuncs.c
@@ -159,7 +159,7 @@ static void outDouble(StringInfo str, double d);
/* Write a parse location field (actually same as INT case) */
#define WRITE_LOCATION_FIELD(fldname) \
do { \
- if (node->fldname != -1) \
+ if (node->fldname != -1 && !omitLocation) \
appendStringInfo(str, " :" CppAsString(fldname) " %d", \
node->fldname); \
} while (0)
@@ -170,7 +170,7 @@ static void outDouble(StringInfo str, double d);
if (node->fldname != NULL) \
{ \
appendStringInfoString(str, " :" CppAsString(fldname) " "); \
- outNode(str, node->fldname); \
+ outNode(str, node->fldname, omitLocation); \
} \
} while (0)
@@ -190,7 +190,8 @@ static void outDouble(StringInfo str, double d);
if (node->fldname != NULL) \
{ \
appendStringInfoString(str, " :" CppAsString(fldname) " "); \
- writeNodeArray(str, (const Node * const *) node->fldname, len); \
+ writeNodeArray(str, (const Node * const *) node->fldname, len, \
+ omitLocation); \
} \
} while (0)
@@ -363,7 +364,8 @@ WRITE_SCALAR_ARRAY(writeBoolCols, bool, " %s", booltostr)
* quite use appendStringInfo() in the loop.
*/
static void
-writeNodeArray(StringInfo str, const Node *const *arr, int len)
+writeNodeArray(StringInfo str, const Node *const *arr, int len,
+ bool omitLocation)
{
if (arr != NULL)
{
@@ -371,7 +373,7 @@ writeNodeArray(StringInfo str, const Node *const *arr, int len)
for (int i = 0; i < len; i++)
{
appendStringInfoChar(str, ' ');
- outNode(str, arr[i]);
+ outNode(str, arr[i], omitLocation);
}
appendStringInfoChar(str, ')');
}
@@ -383,7 +385,7 @@ writeNodeArray(StringInfo str, const Node *const *arr, int len)
* Print a List.
*/
static void
-_outList(StringInfo str, const List *node)
+_outList(StringInfo str, const List *node, bool omitLocation)
{
const ListCell *lc;
@@ -405,7 +407,7 @@ _outList(StringInfo str, const List *node)
*/
if (IsA(node, List))
{
- outNode(str, lfirst(lc));
+ outNode(str, lfirst(lc), omitLocation);
if (lnext(node, lc))
appendStringInfoChar(str, ' ');
}
@@ -490,7 +492,7 @@ outDatum(StringInfo str, Datum value, int typlen, bool typbyval)
*/
static void
-_outConst(StringInfo str, const Const *node)
+_outConst(StringInfo str, const Const *node, bool omitLocation)
{
WRITE_NODE_TYPE("CONST");
@@ -510,7 +512,7 @@ _outConst(StringInfo str, const Const *node)
}
static void
-_outBoolExpr(StringInfo str, const BoolExpr *node)
+_outBoolExpr(StringInfo str, const BoolExpr *node, bool omitLocation)
{
char *opstr = NULL;
@@ -537,7 +539,8 @@ _outBoolExpr(StringInfo str, const BoolExpr *node)
}
static void
-_outForeignKeyOptInfo(StringInfo str, const ForeignKeyOptInfo *node)
+_outForeignKeyOptInfo(StringInfo str, const ForeignKeyOptInfo *node,
+ bool omitLocation)
{
int i;
@@ -563,7 +566,8 @@ _outForeignKeyOptInfo(StringInfo str, const ForeignKeyOptInfo *node)
}
static void
-_outEquivalenceClass(StringInfo str, const EquivalenceClass *node)
+_outEquivalenceClass(StringInfo str, const EquivalenceClass *node,
+ bool omitLocation)
{
/*
* To simplify reading, we just chase up to the topmost merged EC and
@@ -589,7 +593,8 @@ _outEquivalenceClass(StringInfo str, const EquivalenceClass *node)
}
static void
-_outExtensibleNode(StringInfo str, const ExtensibleNode *node)
+_outExtensibleNode(StringInfo str, const ExtensibleNode *node,
+ bool omitLocation)
{
const ExtensibleNodeMethods *methods;
@@ -604,7 +609,8 @@ _outExtensibleNode(StringInfo str, const ExtensibleNode *node)
}
static void
-_outRangeTblEntry(StringInfo str, const RangeTblEntry *node)
+_outRangeTblEntry(StringInfo str, const RangeTblEntry *node,
+ bool omitLocation)
{
WRITE_NODE_TYPE("RANGETBLENTRY");
@@ -684,7 +690,7 @@ _outRangeTblEntry(StringInfo str, const RangeTblEntry *node)
}
static void
-_outA_Expr(StringInfo str, const A_Expr *node)
+_outA_Expr(StringInfo str, const A_Expr *node, bool omitLocation)
{
WRITE_NODE_TYPE("A_EXPR");
@@ -756,13 +762,13 @@ _outA_Expr(StringInfo str, const A_Expr *node)
}
static void
-_outInteger(StringInfo str, const Integer *node)
+_outInteger(StringInfo str, const Integer *node, bool omitLocation)
{
appendStringInfo(str, "%d", node->ival);
}
static void
-_outFloat(StringInfo str, const Float *node)
+_outFloat(StringInfo str, const Float *node, bool omitLocation)
{
/*
* We assume the value is a valid numeric literal and so does not need
@@ -772,13 +778,13 @@ _outFloat(StringInfo str, const Float *node)
}
static void
-_outBoolean(StringInfo str, const Boolean *node)
+_outBoolean(StringInfo str, const Boolean *node, bool omitLocation)
{
appendStringInfoString(str, node->boolval ? "true" : "false");
}
static void
-_outString(StringInfo str, const String *node)
+_outString(StringInfo str, const String *node, bool omitLocation)
{
/*
* We use outToken to provide escaping of the string's content, but we
@@ -792,7 +798,7 @@ _outString(StringInfo str, const String *node)
}
static void
-_outBitString(StringInfo str, const BitString *node)
+_outBitString(StringInfo str, const BitString *node, bool omitLocation)
{
/*
* The lexer will always produce a string starting with 'b' or 'x'. There
@@ -804,7 +810,7 @@ _outBitString(StringInfo str, const BitString *node)
}
static void
-_outA_Const(StringInfo str, const A_Const *node)
+_outA_Const(StringInfo str, const A_Const *node, bool omitLocation)
{
WRITE_NODE_TYPE("A_CONST");
@@ -813,13 +819,13 @@ _outA_Const(StringInfo str, const A_Const *node)
else
{
appendStringInfoString(str, " :val ");
- outNode(str, &node->val);
+ outNode(str, &node->val, omitLocation);
}
WRITE_LOCATION_FIELD(location);
}
static void
-_outConstraint(StringInfo str, const Constraint *node)
+_outConstraint(StringInfo str, const Constraint *node, bool omitLocation)
{
WRITE_NODE_TYPE("CONSTRAINT");
@@ -952,7 +958,7 @@ _outConstraint(StringInfo str, const Constraint *node)
* converts a Node into ascii string and append it to 'str'
*/
void
-outNode(StringInfo str, const void *obj)
+outNode(StringInfo str, const void *obj, bool omitLocation)
{
/* Guard against stack overflow due to overly complex expressions */
check_stack_depth();
@@ -961,18 +967,18 @@ outNode(StringInfo str, const void *obj)
appendStringInfoString(str, "<>");
else if (IsA(obj, List) || IsA(obj, IntList) || IsA(obj, OidList) ||
IsA(obj, XidList))
- _outList(str, obj);
+ _outList(str, obj, omitLocation);
/* nodeRead does not want to see { } around these! */
else if (IsA(obj, Integer))
- _outInteger(str, (Integer *) obj);
+ _outInteger(str, (Integer *) obj, omitLocation);
else if (IsA(obj, Float))
- _outFloat(str, (Float *) obj);
+ _outFloat(str, (Float *) obj, omitLocation);
else if (IsA(obj, Boolean))
- _outBoolean(str, (Boolean *) obj);
+ _outBoolean(str, (Boolean *) obj, omitLocation);
else if (IsA(obj, String))
- _outString(str, (String *) obj);
+ _outString(str, (String *) obj, omitLocation);
else if (IsA(obj, BitString))
- _outBitString(str, (BitString *) obj);
+ _outBitString(str, (BitString *) obj, omitLocation);
else if (IsA(obj, Bitmapset))
outBitmapset(str, (Bitmapset *) obj);
else
@@ -1007,7 +1013,18 @@ nodeToString(const void *obj)
/* see stringinfo.h for an explanation of this maneuver */
initStringInfo(&str);
- outNode(&str, obj);
+ outNode(&str, obj, false);
+ return str.data;
+}
+
+char *
+nodeToStringNoQLocs(const void *obj)
+{
+ StringInfoData str;
+
+ /* see stringinfo.h for an explanation of this maneuver */
+ initStringInfo(&str);
+ outNode(&str, obj, true);
return str.data;
}
diff --git a/src/backend/rewrite/rewriteDefine.c b/src/backend/rewrite/rewriteDefine.c
index b449244a53..6302cd1472 100644
--- a/src/backend/rewrite/rewriteDefine.c
+++ b/src/backend/rewrite/rewriteDefine.c
@@ -64,8 +64,8 @@ InsertRule(const char *rulname,
List *action,
bool replace)
{
- char *evqual = nodeToString(event_qual);
- char *actiontree = nodeToString((Node *) action);
+ char *evqual = nodeToStringNoQLocs(event_qual);
+ char *actiontree = nodeToStringNoQLocs((Node *) action);
Datum values[Natts_pg_rewrite];
bool nulls[Natts_pg_rewrite] = {0};
NameData rname;
diff --git a/src/include/nodes/nodes.h b/src/include/nodes/nodes.h
index 2969dd831b..f7adb5e767 100644
--- a/src/include/nodes/nodes.h
+++ b/src/include/nodes/nodes.h
@@ -188,13 +188,15 @@ castNodeImpl(NodeTag type, void *ptr)
struct Bitmapset; /* not to include bitmapset.h here */
struct StringInfoData; /* not to include stringinfo.h here */
-extern void outNode(struct StringInfoData *str, const void *obj);
+extern void outNode(struct StringInfoData *str, const void *obj,
+ bool omitLocation);
extern void outToken(struct StringInfoData *str, const char *s);
extern void outBitmapset(struct StringInfoData *str,
const struct Bitmapset *bms);
extern void outDatum(struct StringInfoData *str, uintptr_t value,
int typlen, bool typbyval);
extern char *nodeToString(const void *obj);
+extern char *nodeToStringNoQLocs(const void *obj);
extern char *bmsToString(const struct Bitmapset *bms);
/*
--
2.40.1
[application/octet-stream] v3-0001-pg_node_tree-Omit-serialization-of-fields-with-de.patch (22.8K, ../../CAEze2WhfRn0cdNer0Vkye_61BwAmMqM6D9_cJp8i6JmZ8U4wAA@mail.gmail.com/6-v3-0001-pg_node_tree-Omit-serialization-of-fields-with-de.patch)
download | inline diff:
From c74218f4301d617bb64b821d263c9c67edbfbc1b Mon Sep 17 00:00:00 2001
From: Matthias van de Meent <[email protected]>
Date: Tue, 2 Jan 2024 23:55:02 +0100
Subject: [PATCH v3 1/8] pg_node_tree: Omit serialization of fields with
default values.
Often, the values in nodes are their default values. By not serializing
those fields and inserting the defaults during deserialization, we
reduce the size of pg_node_tree attributes seen in e.g. pg_rewrite by a
significant factor.
Note: Enum fields are excluded from this for now, as it would hinder
debugging.
In passing, we fix a test that had a strict dependency on the
serialization of pg_node_tree; we now do the checks in a more generic
manner, making it more stable and ensuring its stability in future work.
---
src/backend/nodes/outfuncs.c | 180 +++++++++++++++----
src/backend/nodes/read.c | 58 ++++++
src/backend/nodes/readfuncs.c | 210 ++++++++++++++++------
src/include/nodes/readfuncs.h | 1 +
src/test/regress/expected/rowsecurity.out | 5 +-
src/test/regress/sql/rowsecurity.sql | 5 +-
6 files changed, 372 insertions(+), 87 deletions(-)
diff --git a/src/backend/nodes/outfuncs.c b/src/backend/nodes/outfuncs.c
index 25171864db..25b0a21f79 100644
--- a/src/backend/nodes/outfuncs.c
+++ b/src/backend/nodes/outfuncs.c
@@ -15,6 +15,7 @@
#include "postgres.h"
#include <ctype.h>
+#include <math.h>
#include "access/attnum.h"
#include "common/shortest_dec.h"
@@ -41,94 +42,207 @@ static void outDouble(StringInfo str, double d);
appendStringInfoString(str, nodelabel)
/* Write an integer field (anything written as ":fldname %d") */
+#define WRITE_INT_FIELD_DEFAULT(fldname, default) \
+ do { \
+ if (node->fldname != default) \
+ appendStringInfo(str, " :" CppAsString(fldname) " %d", \
+ node->fldname); \
+ } while (0)
#define WRITE_INT_FIELD(fldname) \
- appendStringInfo(str, " :" CppAsString(fldname) " %d", node->fldname)
+ WRITE_INT_FIELD_DEFAULT(fldname, 0)
/* Write an unsigned integer field (anything written as ":fldname %u") */
+#define WRITE_UINT_FIELD_DEFAULT(fldname, default) \
+ do { \
+ if (node->fldname != default) \
+ appendStringInfo(str, " :" CppAsString(fldname) " %u", \
+ node->fldname); \
+ } while (0)
#define WRITE_UINT_FIELD(fldname) \
- appendStringInfo(str, " :" CppAsString(fldname) " %u", node->fldname)
+ WRITE_UINT_FIELD_DEFAULT(fldname, 0)
/* Write an unsigned integer field (anything written with UINT64_FORMAT) */
+#define WRITE_UINT64_FIELD_DEFAULT(fldname, default) \
+ do { \
+ if (node->fldname != default) \
+ appendStringInfo(str, " :" CppAsString(fldname) " " UINT64_FORMAT, \
+ node->fldname); \
+ } while (0)
#define WRITE_UINT64_FIELD(fldname) \
- appendStringInfo(str, " :" CppAsString(fldname) " " UINT64_FORMAT, \
- node->fldname)
+ WRITE_UINT64_FIELD_DEFAULT(fldname, 0)
/* Write an OID field (don't hard-wire assumption that OID is same as uint) */
+#define WRITE_OID_FIELD_DEFAULT(fldname, default) \
+ do { \
+ if (node->fldname != default) \
+ appendStringInfo(str, " :" CppAsString(fldname) " %u", \
+ node->fldname); \
+ } while (0)
#define WRITE_OID_FIELD(fldname) \
- appendStringInfo(str, " :" CppAsString(fldname) " %u", node->fldname)
+ WRITE_OID_FIELD_DEFAULT(fldname, 0)
/* Write a long-integer field */
+#define WRITE_LONG_FIELD_DEFAULT(fldname, default) \
+ do { \
+ if (node->fldname != default) \
+ appendStringInfo(str, " :" CppAsString(fldname) " %ld", \
+ node->fldname); \
+ } while (0)
#define WRITE_LONG_FIELD(fldname) \
- appendStringInfo(str, " :" CppAsString(fldname) " %ld", node->fldname)
+ WRITE_LONG_FIELD_DEFAULT(fldname, 0)
+
/* Write a char field (ie, one ascii character) */
+#define WRITE_CHAR_FIELD_DEFAULT(fldname, default) \
+ do { \
+ if (node->fldname != default) \
+ { \
+ appendStringInfoString(str, " :" CppAsString(fldname) " "); \
+ outChar(str, node->fldname); \
+ } \
+ } while (0)
#define WRITE_CHAR_FIELD(fldname) \
- (appendStringInfo(str, " :" CppAsString(fldname) " "), \
- outChar(str, node->fldname))
+ WRITE_CHAR_FIELD_DEFAULT(fldname, '\0')
/* Write an enumerated-type field as an integer code */
+#define WRITE_ENUM_FIELD_DEFAULT(fldname, enumtype, default) \
+ do { \
+ if (node->fldname != default) \
+ appendStringInfo(str, " :" CppAsString(fldname) " %d", \
+ (int) node->fldname); \
+ } while (0)
#define WRITE_ENUM_FIELD(fldname, enumtype) \
- appendStringInfo(str, " :" CppAsString(fldname) " %d", \
- (int) node->fldname)
+ do { \
+ appendStringInfo(str, " :" CppAsString(fldname) " %d", \
+ (int) node->fldname); \
+ } while (0)
/* Write a float field (actually, they're double) */
+#define WRITE_FLOAT_FIELD_DEFAULT(fldname, default) \
+ do { \
+ if (node->fldname != default || \
+ signbit(node->fldname) != signbit(default)) \
+ { \
+ appendStringInfoString(str, " :" CppAsString(fldname) " "); \
+ outDouble(str, node->fldname); \
+ } \
+ } while (0)
#define WRITE_FLOAT_FIELD(fldname) \
- (appendStringInfo(str, " :" CppAsString(fldname) " "), \
- outDouble(str, node->fldname))
+ WRITE_FLOAT_FIELD_DEFAULT(fldname, 0.0)
/* Write a boolean field */
+#define WRITE_BOOL_FIELD_DEFAULT(fldname, default) \
+ do { \
+ if (node->fldname != default) \
+ appendStringInfo(str, " :" CppAsString(fldname) " %s", \
+ booltostr(node->fldname)); \
+ } while (0)
#define WRITE_BOOL_FIELD(fldname) \
- appendStringInfo(str, " :" CppAsString(fldname) " %s", \
- booltostr(node->fldname))
+ WRITE_BOOL_FIELD_DEFAULT(fldname, false)
+
+/*
+ * Non-null defaults of by-ref types are exceedingly rare (if not generally
+ * nonexistent), so we don't (yet) have a specialized macro for non-NULL
+ * defaults omission.
+ */
/* Write a character-string (possibly NULL) field */
#define WRITE_STRING_FIELD(fldname) \
- (appendStringInfoString(str, " :" CppAsString(fldname) " "), \
- outToken(str, node->fldname))
+ do { \
+ if (node->fldname != NULL) \
+ { \
+ appendStringInfoString(str, " :" CppAsString(fldname) " "); \
+ outToken(str, node->fldname); \
+ } \
+ } while (0)
/* Write a parse location field (actually same as INT case) */
#define WRITE_LOCATION_FIELD(fldname) \
- appendStringInfo(str, " :" CppAsString(fldname) " %d", node->fldname)
+ do { \
+ if (node->fldname != -1) \
+ appendStringInfo(str, " :" CppAsString(fldname) " %d", \
+ node->fldname); \
+ } while (0)
/* Write a Node field */
#define WRITE_NODE_FIELD(fldname) \
- (appendStringInfoString(str, " :" CppAsString(fldname) " "), \
- outNode(str, node->fldname))
+ do { \
+ if (node->fldname != NULL) \
+ { \
+ appendStringInfoString(str, " :" CppAsString(fldname) " "); \
+ outNode(str, node->fldname); \
+ } \
+ } while (0)
/* Write a bitmapset field */
#define WRITE_BITMAPSET_FIELD(fldname) \
- (appendStringInfoString(str, " :" CppAsString(fldname) " "), \
- outBitmapset(str, node->fldname))
+ do { \
+ if (node->fldname != NULL) \
+ { \
+ appendStringInfoString(str, " :" CppAsString(fldname) " "); \
+ outBitmapset(str, node->fldname); \
+ } \
+ } while (0)
/* Write a variable-length array (not a List) of Node pointers */
#define WRITE_NODE_ARRAY(fldname, len) \
- (appendStringInfoString(str, " :" CppAsString(fldname) " "), \
- writeNodeArray(str, (const Node * const *) node->fldname, len))
+ do { \
+ if (node->fldname != NULL) \
+ { \
+ appendStringInfoString(str, " :" CppAsString(fldname) " "); \
+ writeNodeArray(str, (const Node * const *) node->fldname, len); \
+ } \
+ } while (0)
/* Write a variable-length array of AttrNumber */
#define WRITE_ATTRNUMBER_ARRAY(fldname, len) \
- (appendStringInfoString(str, " :" CppAsString(fldname) " "), \
- writeAttrNumberCols(str, node->fldname, len))
+ do { \
+ if (node->fldname != NULL) \
+ { \
+ appendStringInfoString(str, " :" CppAsString(fldname) " "); \
+ writeAttrNumberCols(str, node->fldname, len); \
+ } \
+ } while (0)
/* Write a variable-length array of Oid */
#define WRITE_OID_ARRAY(fldname, len) \
- (appendStringInfoString(str, " :" CppAsString(fldname) " "), \
- writeOidCols(str, node->fldname, len))
+ do { \
+ if (node->fldname != NULL) \
+ { \
+ appendStringInfoString(str, " :" CppAsString(fldname) " "); \
+ writeOidCols(str, node->fldname, len); \
+ } \
+ } while (0)
/* Write a variable-length array of Index */
#define WRITE_INDEX_ARRAY(fldname, len) \
- (appendStringInfoString(str, " :" CppAsString(fldname) " "), \
- writeIndexCols(str, node->fldname, len))
+ do { \
+ if (node->fldname != NULL) \
+ { \
+ appendStringInfoString(str, " :" CppAsString(fldname) " "); \
+ writeIndexCols(str, node->fldname, len); \
+ } \
+ } while (0)
/* Write a variable-length array of int */
#define WRITE_INT_ARRAY(fldname, len) \
- (appendStringInfoString(str, " :" CppAsString(fldname) " "), \
- writeIntCols(str, node->fldname, len))
+ do { \
+ if (node->fldname != NULL) \
+ { \
+ appendStringInfoString(str, " :" CppAsString(fldname) " "); \
+ writeIntCols(str, node->fldname, len); \
+ } \
+ } while (0)
/* Write a variable-length array of bool */
#define WRITE_BOOL_ARRAY(fldname, len) \
- (appendStringInfoString(str, " :" CppAsString(fldname) " "), \
- writeBoolCols(str, node->fldname, len))
+ do { \
+ if (node->fldname != NULL) \
+ { \
+ appendStringInfoString(str, " :" CppAsString(fldname) " "); \
+ writeBoolCols(str, node->fldname, len); \
+ } \
+ } while (0)
#define booltostr(x) ((x) ? "true" : "false")
diff --git a/src/backend/nodes/read.c b/src/backend/nodes/read.c
index 4eb42445c5..cffd913ba6 100644
--- a/src/backend/nodes/read.c
+++ b/src/backend/nodes/read.c
@@ -205,6 +205,64 @@ pg_strtok(int *length)
return ret_str;
}
+/*
+ * Check if the next token is the expected field name, and if so,
+ * consume and return it.
+ *
+ * It handles similar to pg_strtok, except that it is special-cased for user-
+ * provided field names, rather than arbitrary tokens.
+ */
+const char *
+pg_strtok_fieldname(const char *fldname, int *length)
+{
+ const char *local_str; /* working pointer to string */
+ int expect_len;
+ char next_char;
+
+ /* skip global state to the next token */
+ while (*pg_strtok_ptr == ' ' ||
+ *pg_strtok_ptr == '\n' ||
+ *pg_strtok_ptr == '\t')
+ pg_strtok_ptr++;
+
+ local_str = pg_strtok_ptr;
+
+ if (*local_str != ':')
+ return false; /* not a field name */
+
+ expect_len = strlen(fldname);
+
+ Assert(expect_len > 0);
+
+ /* check if the next few bytes match the token */
+ if (strncmp(local_str, fldname, expect_len) != 0)
+ return false;
+
+ next_char = local_str[expect_len];
+
+ /*
+ * Check that the token was actually terminated at the end of the
+ * expected token with a character that is a separate token.
+ * Otherwise, we'd get positive matches for mathing the token of "is"
+ * against a local_str of "isn't", which is clearly wrong.
+ */
+ if (next_char == '\0' ||
+ next_char == ' ' ||
+ next_char == '\n' ||
+ next_char == '\t' ||
+ next_char == '(' ||
+ next_char == ')' ||
+ next_char == '{' ||
+ next_char == '}')
+ {
+ pg_strtok_ptr = local_str + expect_len;
+ *length = expect_len;
+
+ return local_str;
+ }
+ return NULL;
+}
+
/*
* debackslash -
* create a palloc'd string holding the given token.
diff --git a/src/backend/nodes/readfuncs.c b/src/backend/nodes/readfuncs.c
index cfb552fde7..96ddf0f9e2 100644
--- a/src/backend/nodes/readfuncs.c
+++ b/src/backend/nodes/readfuncs.c
@@ -57,111 +57,217 @@
READ_TEMP_LOCALS()
/* Read an integer field (anything written as ":fldname %d") */
+#define READ_INT_FIELD_DEFAULT(fldname, default_value) \
+ do { \
+ if ((token = pg_strtok_fieldname(":" CppAsString(fldname), &length))) \
+ { \
+ token = pg_strtok(&length); /* get field value */ \
+ local_node->fldname = atoi(token); \
+ } \
+ else \
+ local_node->fldname = default_value; \
+ } while (0)
#define READ_INT_FIELD(fldname) \
- token = pg_strtok(&length); /* skip :fldname */ \
- token = pg_strtok(&length); /* get field value */ \
- local_node->fldname = atoi(token)
+ READ_INT_FIELD_DEFAULT(fldname, 0)
/* Read an unsigned integer field (anything written as ":fldname %u") */
+#define READ_UINT_FIELD_DEFAULT(fldname, default_value) \
+ do { \
+ if ((token = pg_strtok_fieldname(":" CppAsString(fldname), &length))) \
+ { \
+ token = pg_strtok(&length); /* get field value */ \
+ local_node->fldname = atoui(token); \
+ } \
+ else \
+ local_node->fldname = default_value; \
+ } while (0)
#define READ_UINT_FIELD(fldname) \
- token = pg_strtok(&length); /* skip :fldname */ \
- token = pg_strtok(&length); /* get field value */ \
- local_node->fldname = atoui(token)
+ READ_UINT_FIELD_DEFAULT(fldname, 0)
/* Read an unsigned integer field (anything written using UINT64_FORMAT) */
+#define READ_UINT64_FIELD_DEFAULT(fldname, default_value) \
+ do { \
+ if ((token = pg_strtok_fieldname(":" CppAsString(fldname), &length))) \
+ { \
+ token = pg_strtok(&length); /* get field value */ \
+ local_node->fldname = strtou64(token, NULL, 10); \
+ } \
+ else \
+ local_node->fldname = default_value; \
+ } while (0)
#define READ_UINT64_FIELD(fldname) \
- token = pg_strtok(&length); /* skip :fldname */ \
- token = pg_strtok(&length); /* get field value */ \
- local_node->fldname = strtou64(token, NULL, 10)
+ READ_UINT64_FIELD_DEFAULT(fldname, 0)
/* Read a long integer field (anything written as ":fldname %ld") */
+#define READ_LONG_FIELD_DEFAULT(fldname, default_value) \
+ do { \
+ if ((token = pg_strtok_fieldname(":" CppAsString(fldname), &length))) \
+ { \
+ token = pg_strtok(&length); /* get field value */ \
+ local_node->fldname = atol(token); \
+ } \
+ else \
+ local_node->fldname = default_value; \
+ } while (0)
#define READ_LONG_FIELD(fldname) \
- token = pg_strtok(&length); /* skip :fldname */ \
- token = pg_strtok(&length); /* get field value */ \
- local_node->fldname = atol(token)
+ READ_LONG_FIELD_DEFAULT(fldname, 0)
/* Read an OID field (don't hard-wire assumption that OID is same as uint) */
+#define READ_OID_FIELD_DEFAULT(fldname, default_value) \
+ do { \
+ if ((token = pg_strtok_fieldname(":" CppAsString(fldname), &length))) \
+ { \
+ token = pg_strtok(&length); /* get field value */ \
+ local_node->fldname = atooid(token); \
+ } \
+ else \
+ local_node->fldname = default_value; \
+ } while (0)
#define READ_OID_FIELD(fldname) \
- token = pg_strtok(&length); /* skip :fldname */ \
- token = pg_strtok(&length); /* get field value */ \
- local_node->fldname = atooid(token)
+ READ_OID_FIELD_DEFAULT(fldname, 0)
/* Read a char field (ie, one ascii character) */
+#define READ_CHAR_FIELD_DEFAULT(fldname, default_value) \
+ do { \
+ if ((token = pg_strtok_fieldname(":" CppAsString(fldname), &length))) \
+ { \
+ token = pg_strtok(&length); /* get field value */ \
+ /* avoid overhead of calling debackslash() for one char */ \
+ local_node->fldname = (length == 0) ? '\0' : (token[0] == '\\' ? token[1] : token[0]); \
+ } \
+ else \
+ local_node->fldname = default_value; \
+ } while (0)
#define READ_CHAR_FIELD(fldname) \
- token = pg_strtok(&length); /* skip :fldname */ \
- token = pg_strtok(&length); /* get field value */ \
- /* avoid overhead of calling debackslash() for one char */ \
- local_node->fldname = (length == 0) ? '\0' : (token[0] == '\\' ? token[1] : token[0])
+ READ_CHAR_FIELD_DEFAULT(fldname, '\0')
/* Read an enumerated-type field that was written as an integer code */
#define READ_ENUM_FIELD(fldname, enumtype) \
- token = pg_strtok(&length); /* skip :fldname */ \
- token = pg_strtok(&length); /* get field value */ \
- local_node->fldname = (enumtype) atoi(token)
+ do { \
+ token = pg_strtok_fieldname(":" CppAsString(fldname), &length); \
+ Assert(token != NULL); \
+ token = pg_strtok(&length); /* get field value */ \
+ local_node->fldname = (enumtype) atoi(token); \
+ } while (0)
/* Read a float field */
+#define READ_FLOAT_FIELD_DEFAULT(fldname, default_value) \
+ do { \
+ if ((token = pg_strtok_fieldname(":" CppAsString(fldname), &length))) \
+ { \
+ token = pg_strtok(&length); /* get field value */ \
+ local_node->fldname = atof(token); \
+ } \
+ else \
+ local_node->fldname = default_value; \
+ } while (0)
#define READ_FLOAT_FIELD(fldname) \
- token = pg_strtok(&length); /* skip :fldname */ \
- token = pg_strtok(&length); /* get field value */ \
- local_node->fldname = atof(token)
+ READ_FLOAT_FIELD_DEFAULT(fldname, 0.0)
/* Read a boolean field */
+#define READ_BOOL_FIELD_DEFAULT(fldname, default_value) \
+ do { \
+ if ((token = pg_strtok_fieldname(":" CppAsString(fldname), &length))) \
+ { \
+ token = pg_strtok(&length); /* get field value */ \
+ local_node->fldname = strtobool(token); \
+ } \
+ else \
+ local_node->fldname = default_value; \
+ } while (0)
#define READ_BOOL_FIELD(fldname) \
- token = pg_strtok(&length); /* skip :fldname */ \
- token = pg_strtok(&length); /* get field value */ \
- local_node->fldname = strtobool(token)
+ READ_BOOL_FIELD_DEFAULT(fldname, false)
/* Read a character-string field */
+/* see WRITE_STRING_FIELD in outfuncs.c */
#define READ_STRING_FIELD(fldname) \
- token = pg_strtok(&length); /* skip :fldname */ \
- token = pg_strtok(&length); /* get field value */ \
- local_node->fldname = nullable_string(token, length)
+ do { \
+ if ((token = pg_strtok_fieldname(":" CppAsString(fldname), &length))) \
+ { \
+ token = pg_strtok(&length); /* get field value */ \
+ local_node->fldname = nullable_string(token, length); \
+ } \
+ else \
+ local_node->fldname = NULL; \
+ } while (0)
/* Read a parse location field (and possibly throw away the value) */
#ifdef WRITE_READ_PARSE_PLAN_TREES
#define READ_LOCATION_FIELD(fldname) \
- token = pg_strtok(&length); /* skip :fldname */ \
- token = pg_strtok(&length); /* get field value */ \
- local_node->fldname = restore_location_fields ? atoi(token) : -1
+ do { \
+ if ((token = pg_strtok_fieldname(":" CppAsString(fldname), &length))) \
+ { \
+ token = pg_strtok(&length); /* get field value */ \
+ local_node->fldname = restore_location_fields ? atoi(token) : -1; \
+ } \
+ local_node->fldname = -1; \
+ } while (0)
#else
#define READ_LOCATION_FIELD(fldname) \
- token = pg_strtok(&length); /* skip :fldname */ \
- token = pg_strtok(&length); /* get field value */ \
- (void) token; /* in case not used elsewhere */ \
- local_node->fldname = -1 /* set field to "unknown" */
+ do { \
+ if ((token = pg_strtok_fieldname(":" CppAsString(fldname), &length))) \
+ { \
+ token = pg_strtok(&length); /* the field value is useless */ \
+ (void) token; \
+ } \
+ local_node->fldname = -1; \
+ } while (0)
#endif
+
/* Read a Node field */
#define READ_NODE_FIELD(fldname) \
- token = pg_strtok(&length); /* skip :fldname */ \
- (void) token; /* in case not used elsewhere */ \
- local_node->fldname = nodeRead(NULL, 0)
+ do { \
+ if ((token = pg_strtok_fieldname(":" CppAsString(fldname), &length))) \
+ local_node->fldname = nodeRead(NULL, 0); \
+ else \
+ local_node->fldname = NULL; \
+ } while (0)
/* Read a bitmapset field */
#define READ_BITMAPSET_FIELD(fldname) \
- token = pg_strtok(&length); /* skip :fldname */ \
- (void) token; /* in case not used elsewhere */ \
- local_node->fldname = _readBitmapset()
+ do { \
+ if ((token = pg_strtok_fieldname(":" CppAsString(fldname), &length))) \
+ local_node->fldname = _readBitmapset(); \
+ else \
+ local_node->fldname = NULL; \
+ } while (0)
/* Read an attribute number array */
#define READ_ATTRNUMBER_ARRAY(fldname, len) \
- token = pg_strtok(&length); /* skip :fldname */ \
- local_node->fldname = readAttrNumberCols(len)
+ do { \
+ if ((token = pg_strtok_fieldname(":" CppAsString(fldname), &length))) \
+ local_node->fldname = readAttrNumberCols(len); \
+ else \
+ local_node->fldname = NULL; \
+ } while (0)
/* Read an oid array */
#define READ_OID_ARRAY(fldname, len) \
- token = pg_strtok(&length); /* skip :fldname */ \
- local_node->fldname = readOidCols(len)
+ do { \
+ if ((token = pg_strtok_fieldname(":" CppAsString(fldname), &length))) \
+ local_node->fldname = readOidCols(len); \
+ else \
+ local_node->fldname = NULL; \
+ } while (0)
/* Read an int array */
#define READ_INT_ARRAY(fldname, len) \
- token = pg_strtok(&length); /* skip :fldname */ \
- local_node->fldname = readIntCols(len)
+ do { \
+ if ((token = pg_strtok_fieldname(":" CppAsString(fldname), &length))) \
+ local_node->fldname = readIntCols(len); \
+ else \
+ local_node->fldname = NULL; \
+ } while (0)
/* Read a bool array */
#define READ_BOOL_ARRAY(fldname, len) \
- token = pg_strtok(&length); /* skip :fldname */ \
- local_node->fldname = readBoolCols(len)
+ do { \
+ if ((token = pg_strtok_fieldname(":" CppAsString(fldname), &length))) \
+ local_node->fldname = readBoolCols(len); \
+ else \
+ local_node->fldname = NULL; \
+ } while (0)
/* Routine exit */
#define READ_DONE() \
diff --git a/src/include/nodes/readfuncs.h b/src/include/nodes/readfuncs.h
index 8466038ed0..0579c62973 100644
--- a/src/include/nodes/readfuncs.h
+++ b/src/include/nodes/readfuncs.h
@@ -27,6 +27,7 @@ extern PGDLLIMPORT bool restore_location_fields;
* prototypes for functions in read.c (the lisp token parser)
*/
extern const char *pg_strtok(int *length);
+extern const char *pg_strtok_fieldname(const char *fldname, int *length);
extern char *debackslash(const char *token, int length);
extern void *nodeRead(const char *token, int tok_len);
diff --git a/src/test/regress/expected/rowsecurity.out b/src/test/regress/expected/rowsecurity.out
index 6988128aa4..a69aa40f82 100644
--- a/src/test/regress/expected/rowsecurity.out
+++ b/src/test/regress/expected/rowsecurity.out
@@ -3937,7 +3937,10 @@ CREATE TABLE coll_t (c) AS VALUES ('bar'::text);
CREATE POLICY coll_p ON coll_t USING (c < ('foo'::text COLLATE "C"));
ALTER TABLE coll_t ENABLE ROW LEVEL SECURITY;
GRANT SELECT ON coll_t TO regress_rls_alice;
-SELECT (string_to_array(polqual, ':'))[7] AS inputcollid FROM pg_policy WHERE polrelid = 'coll_t'::regclass;
+SELECT split AS inputcollid
+FROM pg_policy,
+ lateral unnest(string_to_array(polqual, ':')) as split
+WHERE polrelid = 'coll_t'::regclass and split LIKE '%inputcollid%';
inputcollid
------------------
inputcollid 950
diff --git a/src/test/regress/sql/rowsecurity.sql b/src/test/regress/sql/rowsecurity.sql
index dec7340538..cb7a4c776b 100644
--- a/src/test/regress/sql/rowsecurity.sql
+++ b/src/test/regress/sql/rowsecurity.sql
@@ -1732,7 +1732,10 @@ CREATE TABLE coll_t (c) AS VALUES ('bar'::text);
CREATE POLICY coll_p ON coll_t USING (c < ('foo'::text COLLATE "C"));
ALTER TABLE coll_t ENABLE ROW LEVEL SECURITY;
GRANT SELECT ON coll_t TO regress_rls_alice;
-SELECT (string_to_array(polqual, ':'))[7] AS inputcollid FROM pg_policy WHERE polrelid = 'coll_t'::regclass;
+SELECT split AS inputcollid
+FROM pg_policy,
+ lateral unnest(string_to_array(polqual, ':')) as split
+WHERE polrelid = 'coll_t'::regclass and split LIKE '%inputcollid%';
SET SESSION AUTHORIZATION regress_rls_alice;
SELECT * FROM coll_t;
ROLLBACK;
--
2.40.1
[application/octet-stream] v3-0005-nodeToString-omit-serializing-NULL-datums-in-Cons.patch (1.8K, ../../CAEze2WhfRn0cdNer0Vkye_61BwAmMqM6D9_cJp8i6JmZ8U4wAA@mail.gmail.com/7-v3-0005-nodeToString-omit-serializing-NULL-datums-in-Cons.patch)
download | inline diff:
From 7fb024eb8f80940bd6985bd4ade73d3e8ef8afea Mon Sep 17 00:00:00 2001
From: Matthias van de Meent <[email protected]>
Date: Thu, 8 Feb 2024 20:24:22 +0100
Subject: [PATCH v3 5/8] nodeToString: omit serializing NULL datums in Const
nodes
This saves some bytes in certain cases, and aligns its serialization conditions
with other field's serialization conditions.
---
src/backend/nodes/outfuncs.c | 8 ++++----
src/backend/nodes/readfuncs.c | 10 ++++++----
2 files changed, 10 insertions(+), 8 deletions(-)
diff --git a/src/backend/nodes/outfuncs.c b/src/backend/nodes/outfuncs.c
index ec22130a70..6c6fec8059 100644
--- a/src/backend/nodes/outfuncs.c
+++ b/src/backend/nodes/outfuncs.c
@@ -504,11 +504,11 @@ _outConst(StringInfo str, const Const *node, bool omitLocation)
WRITE_BOOL_FIELD(constisnull);
WRITE_LOCATION_FIELD(location);
- appendStringInfoString(str, " :constvalue ");
- if (node->constisnull)
- appendStringInfoString(str, "<>");
- else
+ if (!node->constisnull)
+ {
+ appendStringInfoString(str, " :constvalue ");
outDatum(str, node->constvalue, node->constlen, node->constbyval);
+ }
}
static void
diff --git a/src/backend/nodes/readfuncs.c b/src/backend/nodes/readfuncs.c
index 635f43fe2f..c5763c0256 100644
--- a/src/backend/nodes/readfuncs.c
+++ b/src/backend/nodes/readfuncs.c
@@ -374,11 +374,13 @@ _readConst(void)
READ_BOOL_FIELD(constisnull);
READ_LOCATION_FIELD(location);
- token = pg_strtok(&length); /* skip :constvalue */
- if (local_node->constisnull)
- token = pg_strtok(&length); /* skip "<>" */
- else
+ if ((token = pg_strtok_fieldname(":constvalue", &length)))
local_node->constvalue = readDatum(local_node->constbyval);
+ else
+ {
+ /* value was omitted */
+ Assert(local_node->constisnull);
+ }
READ_DONE();
}
--
2.40.1
[application/octet-stream] v3-0007-gen_node_support.pl-Optimize-serialization-of-fie.patch (9.2K, ../../CAEze2WhfRn0cdNer0Vkye_61BwAmMqM6D9_cJp8i6JmZ8U4wAA@mail.gmail.com/8-v3-0007-gen_node_support.pl-Optimize-serialization-of-fie.patch)
download | inline diff:
From 1ec4351d1d1284c5bb0e6a0f33c2447994968981 Mon Sep 17 00:00:00 2001
From: Matthias van de Meent <[email protected]>
Date: Mon, 12 Feb 2024 17:12:02 +0100
Subject: [PATCH v3 7/8] gen_node_support.pl: Optimize serialization of fields
with copied values
The Var node's [syn] fields often contain the same data as their non-syn
counterparts. We invent a new pg_node_attr()ibute that represents this
relation, which allows us to omit these fields from serialization when they
are indeed copies of the original fields.
---
src/backend/nodes/gen_node_support.pl | 94 ++++++++++++++++++---------
src/backend/nodes/outfuncs.c | 3 +
src/backend/nodes/readfuncs.c | 3 +
src/include/nodes/primnodes.h | 4 +-
4 files changed, 70 insertions(+), 34 deletions(-)
diff --git a/src/backend/nodes/gen_node_support.pl b/src/backend/nodes/gen_node_support.pl
index fda8c3a494..66661a3881 100644
--- a/src/backend/nodes/gen_node_support.pl
+++ b/src/backend/nodes/gen_node_support.pl
@@ -363,6 +363,10 @@ foreach my $infile (@ARGV)
{
$manual_nodetag_number{$in_struct} = $1;
}
+ elsif ($attr =~ /^default_ref\(([\w\d."'-]+)\)$/)
+ {
+ # Unused at the node level
+ }
else
{
die
@@ -437,7 +441,7 @@ foreach my $infile (@ARGV)
}
# normal struct field
elsif ($line =~
- /^\s*(.+)\s*\b(\w+)(\[[\w\s+]+\])?\s*(?:pg_node_attr\(([\w(), ]*)\))?;/
+ /^\s*(.+)\s*\b(\w+)(\[[\w\s+]+\])?\s*(?:pg_node_attr\(([\w\d(), ."'-]*)\))?;/
)
{
if ($is_node_struct)
@@ -469,6 +473,7 @@ foreach my $infile (@ARGV)
if ( $attr !~ /^array_size\(\w+\)$/
&& $attr !~ /^copy_as\(\w+\)$/
&& $attr !~ /^read_as\(\w+\)$/
+ && $attr !~ /^default_ref\([\w\d."'-]+\)$/
&& !elem $attr,
qw(copy_as_scalar
equal_as_scalar
@@ -495,7 +500,7 @@ foreach my $infile (@ARGV)
}
# function pointer field
elsif ($line =~
- /^\s*([\w\s*]+)\s*\(\*(\w+)\)\s*\((.*)\)\s*(?:pg_node_attr\(([\w(), ]*)\))?;/
+ /^\s*([\w\s*]+)\s*\(\*(\w+)\)\s*\((.*)\)\s*(?:pg_node_attr\(([\w\d(), ."'-]*)\))?;/
)
{
if ($is_node_struct)
@@ -970,6 +975,15 @@ _read${n}(void)
my $array_size_field;
my $read_as_field;
my $read_write_ignore = 0;
+ # Type read/write macro suffix, "" for normal use, or "_DEFAULT" when
+ # value argument.
+ my $s = "";
+ # Default parameter to read/write macro. Includes comma separator so
+ # that MACRO_NAME$s($fieldname$defaultparam) is the full complete
+ # read/write expression for essentially all types.
+ # Note that this (currently) only works for scalar values.
+ my $d = "";
+
foreach my $a (@a)
{
if ($a =~ /^array_size\(([\w.]+)\)$/)
@@ -988,6 +1002,19 @@ _read${n}(void)
{
$read_write_ignore = 1;
}
+ elsif ($a =~ /^default_ref\(([\w\d+."'-]+)\)$/)
+ {
+ $s = "_DEFAULT";
+ $d = ", NODE_FIELD($1)";
+ }
+ }
+
+ if ($s eq "_DEFAULT")
+ {
+ die "custom defaults for non-scalar fields are not supported\n\tat $n.$f"
+ unless (elem $t, @scalar_types or elem $t, @enum_types);
+ die "custom defaults for Location fields are not supported\n\tat $n.$f"
+ if ($t eq "Location");
}
if ($read_write_ignore)
@@ -1008,8 +1035,8 @@ _read${n}(void)
# select instructions by field type
if ($t eq 'bool')
{
- print $off "\tWRITE_BOOL_FIELD($f);\n";
- print $rff "\tREAD_BOOL_FIELD($f);\n" unless $no_read;
+ print $off "\tWRITE_BOOL_FIELD$s($f$d);\n";
+ print $rff "\tREAD_BOOL_FIELD$s($f$d);\n" unless $no_read;
}
elsif ($t eq 'Location')
{
@@ -1018,8 +1045,11 @@ _read${n}(void)
}
elsif ($t eq 'TypMod')
{
- print $off "\tWRITE_INT_FIELD_DEFAULT($f, -1);\n";
- print $rff "\tREAD_INT_FIELD_DEFAULT($f, -1);\n" unless $no_read;
+ # The default value of a TypMod fields is -1, rather than 0
+ # for normal int fields.
+ $d = ", -1" if ($d eq "");
+ print $off "\tWRITE_INT_FIELD_DEFAULT($f$d);\n";
+ print $rff "\tREAD_INT_FIELD_DEFAULT($f$d);\n" unless $no_read;
}
elsif ($t eq 'int'
|| $t eq 'int16'
@@ -1027,8 +1057,8 @@ _read${n}(void)
|| $t eq 'AttrNumber'
|| $t eq 'StrategyNumber')
{
- print $off "\tWRITE_INT_FIELD($f);\n";
- print $rff "\tREAD_INT_FIELD($f);\n" unless $no_read;
+ print $off "\tWRITE_INT_FIELD$s($f$d);\n";
+ print $rff "\tREAD_INT_FIELD$s($f$d);\n" unless $no_read;
}
elsif ($t eq 'uint32'
|| $t eq 'bits32'
@@ -1036,56 +1066,56 @@ _read${n}(void)
|| $t eq 'Index'
|| $t eq 'SubTransactionId')
{
- print $off "\tWRITE_UINT_FIELD($f);\n";
- print $rff "\tREAD_UINT_FIELD($f);\n" unless $no_read;
+ print $off "\tWRITE_UINT_FIELD$s($f$d);\n";
+ print $rff "\tREAD_UINT_FIELD$s($f$d);\n" unless $no_read;
}
elsif ($t eq 'uint64'
|| $t eq 'AclMode')
{
- print $off "\tWRITE_UINT64_FIELD($f);\n";
- print $rff "\tREAD_UINT64_FIELD($f);\n" unless $no_read;
+ print $off "\tWRITE_UINT64_FIELD$s($f$d);\n";
+ print $rff "\tREAD_UINT64_FIELD$s($f$d);\n" unless $no_read;
}
elsif ($t eq 'Oid' || $t eq 'RelFileNumber')
{
- print $off "\tWRITE_OID_FIELD($f);\n";
- print $rff "\tREAD_OID_FIELD($f);\n" unless $no_read;
+ print $off "\tWRITE_OID_FIELD$s($f$d);\n";
+ print $rff "\tREAD_OID_FIELD$s($f$d);\n" unless $no_read;
}
elsif ($t eq 'long')
{
- print $off "\tWRITE_LONG_FIELD($f);\n";
- print $rff "\tREAD_LONG_FIELD($f);\n" unless $no_read;
+ print $off "\tWRITE_LONG_FIELD$s($f$d);\n";
+ print $rff "\tREAD_LONG_FIELD$s($f$d);\n" unless $no_read;
}
elsif ($t eq 'char')
{
- print $off "\tWRITE_CHAR_FIELD($f);\n";
- print $rff "\tREAD_CHAR_FIELD($f);\n" unless $no_read;
+ print $off "\tWRITE_CHAR_FIELD$s($f$d);\n";
+ print $rff "\tREAD_CHAR_FIELD$s($f$d);\n" unless $no_read;
}
elsif ($t eq 'double')
{
- print $off "\tWRITE_FLOAT_FIELD($f);\n";
- print $rff "\tREAD_FLOAT_FIELD($f);\n" unless $no_read;
+ print $off "\tWRITE_FLOAT_FIELD$s($f$d);\n";
+ print $rff "\tREAD_FLOAT_FIELD$s($f$d);\n" unless $no_read;
}
elsif ($t eq 'Cardinality')
{
- print $off "\tWRITE_FLOAT_FIELD($f);\n";
- print $rff "\tREAD_FLOAT_FIELD($f);\n" unless $no_read;
+ print $off "\tWRITE_FLOAT_FIELD$s($f$d);\n";
+ print $rff "\tREAD_FLOAT_FIELD$s($f$d);\n" unless $no_read;
}
elsif ($t eq 'Cost')
{
- print $off "\tWRITE_FLOAT_FIELD($f);\n";
- print $rff "\tREAD_FLOAT_FIELD($f);\n" unless $no_read;
+ print $off "\tWRITE_FLOAT_FIELD$s($f$d);\n";
+ print $rff "\tREAD_FLOAT_FIELD$s($f$d);\n" unless $no_read;
}
elsif ($t eq 'QualCost')
{
- print $off "\tWRITE_FLOAT_FIELD($f.startup);\n";
- print $off "\tWRITE_FLOAT_FIELD($f.per_tuple);\n";
- print $rff "\tREAD_FLOAT_FIELD($f.startup);\n" unless $no_read;
- print $rff "\tREAD_FLOAT_FIELD($f.per_tuple);\n" unless $no_read;
+ print $off "\tWRITE_FLOAT_FIELD$s($f.startup$d);\n";
+ print $off "\tWRITE_FLOAT_FIELD$s($f.per_tuple$d);\n";
+ print $rff "\tREAD_FLOAT_FIELD$s($f.startup$d);\n" unless $no_read;
+ print $rff "\tREAD_FLOAT_FIELD$s($f.per_tuple$d);\n" unless $no_read;
}
elsif ($t eq 'Selectivity')
{
- print $off "\tWRITE_FLOAT_FIELD($f);\n";
- print $rff "\tREAD_FLOAT_FIELD($f);\n" unless $no_read;
+ print $off "\tWRITE_FLOAT_FIELD$s($f$d);\n";
+ print $rff "\tREAD_FLOAT_FIELD$s($f$d);\n" unless $no_read;
}
elsif ($t eq 'char*')
{
@@ -1099,8 +1129,8 @@ _read${n}(void)
}
elsif (elem $t, @enum_types)
{
- print $off "\tWRITE_ENUM_FIELD($f, $t);\n";
- print $rff "\tREAD_ENUM_FIELD($f, $t);\n" unless $no_read;
+ print $off "\tWRITE_ENUM_FIELD$s($f, $t$d);\n";
+ print $rff "\tREAD_ENUM_FIELD$s($f, $t$d);\n" unless $no_read;
}
# arrays of scalar types
elsif ($t =~ /^(\w+)(\*|\[\w+\])$/ and elem $1, @scalar_types)
diff --git a/src/backend/nodes/outfuncs.c b/src/backend/nodes/outfuncs.c
index 131122f239..5d88d252d8 100644
--- a/src/backend/nodes/outfuncs.c
+++ b/src/backend/nodes/outfuncs.c
@@ -245,6 +245,9 @@ static void outDouble(StringInfo str, double d);
} \
} while (0)
+/* Accessing a node's field goes like this */
+#define NODE_FIELD(fldname) (node->fldname)
+
#define booltostr(x) ((x) ? "true" : "false")
diff --git a/src/backend/nodes/readfuncs.c b/src/backend/nodes/readfuncs.c
index f872f27ae0..49285fa86d 100644
--- a/src/backend/nodes/readfuncs.c
+++ b/src/backend/nodes/readfuncs.c
@@ -269,6 +269,9 @@
local_node->fldname = NULL; \
} while (0)
+/* Accessing a node's field goes like this */
+#define NODE_FIELD(fldname) (local_node->fldname)
+
/* Routine exit */
#define READ_DONE() \
return local_node
diff --git a/src/include/nodes/primnodes.h b/src/include/nodes/primnodes.h
index 94282497d7..9dd60ad5c5 100644
--- a/src/include/nodes/primnodes.h
+++ b/src/include/nodes/primnodes.h
@@ -274,9 +274,9 @@ typedef struct Var
* their varno/varattno match.
*/
/* syntactic relation index (0 if unknown) */
- Index varnosyn pg_node_attr(equal_ignore, query_jumble_ignore);
+ Index varnosyn pg_node_attr(equal_ignore, query_jumble_ignore, default_ref(varno));
/* syntactic attribute number */
- AttrNumber varattnosyn pg_node_attr(equal_ignore, query_jumble_ignore);
+ AttrNumber varattnosyn pg_node_attr(equal_ignore, query_jumble_ignore, default_ref(varattno));
/* token location, or -1 if unknown */
Location location;
--
2.40.1
[application/octet-stream] v3-0008-nodeToString-omit-serializing-0s-in-enum-typed-fi.patch (2.1K, ../../CAEze2WhfRn0cdNer0Vkye_61BwAmMqM6D9_cJp8i6JmZ8U4wAA@mail.gmail.com/9-v3-0008-nodeToString-omit-serializing-0s-in-enum-typed-fi.patch)
download | inline diff:
From 2d7810d6de60eca741ad832dea8521c14d60951d Mon Sep 17 00:00:00 2001
From: Matthias van de Meent <[email protected]>
Date: Mon, 19 Feb 2024 13:53:00 +0100
Subject: [PATCH v3 8/8] nodeToString: omit serializing 0s in enum-typed
fields.
---
src/backend/nodes/outfuncs.c | 5 +----
src/backend/nodes/readfuncs.c | 16 +++++++++++-----
2 files changed, 12 insertions(+), 9 deletions(-)
diff --git a/src/backend/nodes/outfuncs.c b/src/backend/nodes/outfuncs.c
index 5d88d252d8..22dd2d6083 100644
--- a/src/backend/nodes/outfuncs.c
+++ b/src/backend/nodes/outfuncs.c
@@ -112,10 +112,7 @@ static void outDouble(StringInfo str, double d);
(int) node->fldname); \
} while (0)
#define WRITE_ENUM_FIELD(fldname, enumtype) \
- do { \
- appendStringInfo(str, " :" CppAsString(fldname) " %d", \
- (int) node->fldname); \
- } while (0)
+ WRITE_ENUM_FIELD_DEFAULT(fldname, enumtype, 0)
/* Write a float field (actually, they're double) */
#define WRITE_FLOAT_FIELD_DEFAULT(fldname, default) \
diff --git a/src/backend/nodes/readfuncs.c b/src/backend/nodes/readfuncs.c
index 49285fa86d..3559e3cb06 100644
--- a/src/backend/nodes/readfuncs.c
+++ b/src/backend/nodes/readfuncs.c
@@ -142,13 +142,19 @@
READ_CHAR_FIELD_DEFAULT(fldname, '\0')
/* Read an enumerated-type field that was written as an integer code */
-#define READ_ENUM_FIELD(fldname, enumtype) \
+
+#define READ_ENUM_FIELD_DEFAULT(fldname, enumtype, default) \
do { \
- token = pg_strtok_fieldname(":" CppAsString(fldname), &length); \
- Assert(token != NULL); \
- token = pg_strtok(&length); /* get field value */ \
- local_node->fldname = (enumtype) atoi(token); \
+ if ((token = pg_strtok_fieldname(":" CppAsString(fldname), &length))) \
+ { \
+ token = pg_strtok(&length); /* get field value */ \
+ local_node->fldname = (enumtype) atoi(token); \
+ } \
+ else \
+ local_node->fldname = (enumtype) default; \
} while (0)
+#define READ_ENUM_FIELD(fldname, enumtype) \
+ READ_ENUM_FIELD_DEFAULT(fldname, enumtype, 0)
/* Read a float field */
#define READ_FLOAT_FIELD_DEFAULT(fldname, default_value) \
--
2.40.1
[application/octet-stream] v3-0006-nodeToString-Apply-RLE-on-Bitmapset-and-numeric-L.patch (7.9K, ../../CAEze2WhfRn0cdNer0Vkye_61BwAmMqM6D9_cJp8i6JmZ8U4wAA@mail.gmail.com/10-v3-0006-nodeToString-Apply-RLE-on-Bitmapset-and-numeric-L.patch)
download | inline diff:
From b04c0dd796a6fe19d3bffe39675fd54e149b80fe Mon Sep 17 00:00:00 2001
From: Matthias van de Meent <[email protected]>
Date: Sat, 10 Feb 2024 00:57:19 +0100
Subject: [PATCH v3 6/8] nodeToString: Apply RLE on Bitmapset and numeric List
types
This reduces the size of full serialized queries in pg_rewrite by several %,
reducing overhead in the system.
---
src/backend/nodes/outfuncs.c | 96 +++++++++++++++++++++++++++++++----
src/backend/nodes/read.c | 53 +++++++++++++++++--
src/backend/nodes/readfuncs.c | 16 +++++-
3 files changed, 149 insertions(+), 16 deletions(-)
diff --git a/src/backend/nodes/outfuncs.c b/src/backend/nodes/outfuncs.c
index 6c6fec8059..131122f239 100644
--- a/src/backend/nodes/outfuncs.c
+++ b/src/backend/nodes/outfuncs.c
@@ -383,23 +383,64 @@ writeNodeArray(StringInfo str, const Node *const *arr, int len,
/*
* Print a List.
+ *
+ * Notes:
+ * NodeList is formatted as (<node1> <node2> ...)
+ *
+ * OidList is formatted as (o int +int int int ...), with +int indicating N
+ * successive identical values.
+ *
+ * IntList and XidList are formatted as (i/x int +int int int ...), with +int
+ * indicating N successively increasing values. (i 9 +3) is thus equivalent
+ * to (i 9 10 11 12).
*/
static void
_outList(StringInfo str, const List *node, bool omitLocation)
{
const ListCell *lc;
+ union LCSurrogate {
+ int32 i;
+ Oid o;
+ TransactionId x;
+ } previous = { .i = 0};
+ int run = 0;
+ int run_delta;
+ const char *fmt;
appendStringInfoChar(str, '(');
if (IsA(node, IntList))
+ {
appendStringInfoChar(str, 'i');
+ fmt = " %d";
+ run_delta = 1;
+ }
else if (IsA(node, OidList))
+ {
appendStringInfoChar(str, 'o');
+ fmt = " %u";
+ run_delta = 0;
+ }
else if (IsA(node, XidList))
+ {
appendStringInfoChar(str, 'x');
+ fmt = " %u";
+ run_delta = 1;
+ }
+ else if (IsA(node, List))
+ {
+ /* silence the compiler about uninitialized variables */
+ fmt = "";
+ run_delta = 0;
+ }
+ else
+ elog(ERROR, "unrecognized list node type: %d",
+ (int) node->type);
foreach(lc, node)
{
+ union LCSurrogate val = {.i = 0};
+
/*
* For the sake of backward compatibility, we emit a slightly
* different whitespace format for lists of nodes vs. other types of
@@ -410,26 +451,41 @@ _outList(StringInfo str, const List *node, bool omitLocation)
outNode(str, lfirst(lc), omitLocation);
if (lnext(node, lc))
appendStringInfoChar(str, ' ');
+ continue;
}
else if (IsA(node, IntList))
- appendStringInfo(str, " %d", lfirst_int(lc));
+ val.i = lfirst_int(lc);
else if (IsA(node, OidList))
- appendStringInfo(str, " %u", lfirst_oid(lc));
+ val.o = lfirst_oid(lc);
else if (IsA(node, XidList))
- appendStringInfo(str, " %u", lfirst_xid(lc));
+ val.x = lfirst_xid(lc);
+
+ if (val.i == previous.i + run_delta)
+ run += 1;
else
- elog(ERROR, "unrecognized list node type: %d",
- (int) node->type);
+ {
+ if (run > 0)
+ appendStringInfo(str, " +%d", run);
+ run = 0;
+ appendStringInfo(str, fmt, val.i);
+ }
+ previous = val;
}
- appendStringInfoChar(str, ')');
-}
+ if (run > 0)
+ appendStringInfo(str, " +%d", run);
+
+ appendStringInfoChar(str, ')');}
/*
* outBitmapset -
* converts a bitmap set of integers
*
- * Note: the output format is "(b int int ...)", similar to an integer List.
+ * Note: the output format is "(b int int +int ...)", similar to an
+ * integer List. Note that consecutive runs of incremental values are
+ * indicated by [... int +int ...], where the first int is the first set bit,
+ * and the int that's tagged with the '+' sign that follows is the number of
+ * subsequent set bits (resulting in a run of N+1 set bits).
*
* We export this function for use by extensions that define extensible nodes.
* That's somewhat historical, though, because calling outNode() will work.
@@ -437,13 +493,31 @@ _outList(StringInfo str, const List *node, bool omitLocation)
void
outBitmapset(StringInfo str, const Bitmapset *bms)
{
- int x;
+ int x = -1;
+ int prev = -2;
+ int run = 0;
appendStringInfoChar(str, '(');
appendStringInfoChar(str, 'b');
- x = -1;
+
while ((x = bms_next_member(bms, x)) >= 0)
- appendStringInfo(str, " %d", x);
+ {
+ if (x - prev == 1)
+ run++;
+ else
+ {
+ if (run > 0)
+ appendStringInfo(str, " +%d", run);
+
+ run = 0;
+ appendStringInfo(str, " %d", x);
+ }
+ prev = x;
+ }
+
+ if (run > 0)
+ appendStringInfo(str, " +%d", run);
+
appendStringInfoChar(str, ')');
}
diff --git a/src/backend/nodes/read.c b/src/backend/nodes/read.c
index cffd913ba6..7456626e82 100644
--- a/src/backend/nodes/read.c
+++ b/src/backend/nodes/read.c
@@ -415,6 +415,8 @@ nodeRead(const char *token, int tok_len)
elog(ERROR, "unterminated List structure");
if (tok_len == 1 && token[0] == 'i')
{
+ int prev = 0;
+
/* List of integers */
for (;;)
{
@@ -430,12 +432,23 @@ nodeRead(const char *token, int tok_len)
if (endptr != token + tok_len)
elog(ERROR, "unrecognized integer: \"%.*s\"",
tok_len, token);
- l = lappend_int(l, val);
+
+ if (token[0] == '+')
+ {
+ for (int i = 0; i < val; i++)
+ l = lappend_int(l, ++prev);
+ }
+ else
+ {
+ prev = val;
+ l = lappend_int(l, val);
+ }
}
result = (Node *) l;
}
else if (tok_len == 1 && token[0] == 'o')
{
+ Oid prev = 0;
/* List of OIDs */
for (;;)
{
@@ -451,12 +464,23 @@ nodeRead(const char *token, int tok_len)
if (endptr != token + tok_len)
elog(ERROR, "unrecognized OID: \"%.*s\"",
tok_len, token);
- l = lappend_oid(l, val);
+
+ if (token[0] == '+')
+ {
+ for (int i = 0; i < val; i++)
+ l = lappend_oid(l, prev);
+ }
+ else
+ {
+ prev = val;
+ l = lappend_oid(l, val);
+ }
}
result = (Node *) l;
}
else if (tok_len == 1 && token[0] == 'x')
{
+ TransactionId prev = 0;
/* List of TransactionIds */
for (;;)
{
@@ -472,7 +496,17 @@ nodeRead(const char *token, int tok_len)
if (endptr != token + tok_len)
elog(ERROR, "unrecognized Xid: \"%.*s\"",
tok_len, token);
- l = lappend_xid(l, val);
+
+ if (token[0] == '+')
+ {
+ for (int i = 0; i < val; i++)
+ l = lappend_xid(l, ++prev);
+ }
+ else
+ {
+ prev = val;
+ l = lappend_xid(l, val);
+ }
}
result = (Node *) l;
}
@@ -480,6 +514,7 @@ nodeRead(const char *token, int tok_len)
{
/* Bitmapset -- see also _readBitmapset() */
Bitmapset *bms = NULL;
+ int prev = -2;
for (;;)
{
@@ -495,7 +530,17 @@ nodeRead(const char *token, int tok_len)
if (endptr != token + tok_len)
elog(ERROR, "unrecognized integer: \"%.*s\"",
tok_len, token);
- bms = bms_add_member(bms, val);
+
+ if (token[0] == '+')
+ {
+ for (int i = 0; i < val; i++)
+ bms = bms_add_member(bms, ++prev);
+ }
+ else
+ {
+ bms = bms_add_member(bms, val);
+ prev = val;
+ }
}
result = (Node *) bms;
}
diff --git a/src/backend/nodes/readfuncs.c b/src/backend/nodes/readfuncs.c
index c5763c0256..f872f27ae0 100644
--- a/src/backend/nodes/readfuncs.c
+++ b/src/backend/nodes/readfuncs.c
@@ -309,6 +309,7 @@ static Bitmapset *
_readBitmapset(void)
{
Bitmapset *result = NULL;
+ int prev = -2;
READ_TEMP_LOCALS();
@@ -337,7 +338,20 @@ _readBitmapset(void)
val = (int) strtol(token, &endptr, 10);
if (endptr != token + length)
elog(ERROR, "unrecognized integer: \"%.*s\"", length, token);
- result = bms_add_member(result, val);
+
+ if (token[0] == '+')
+ {
+ for (int i = 0; i < val; i++)
+ {
+ ++prev;
+ result = bms_add_member(result, prev);
+ }
+ }
+ else
+ {
+ result = bms_add_member(result, val);
+ prev = val;
+ }
}
return result;
--
2.40.1
^ permalink raw reply [nested|flat] 69+ messages in thread
* Re: Reducing output size of nodeToString
@ 2024-02-22 12:37 Matthias van de Meent <[email protected]>
parent: Matthias van de Meent <[email protected]>
0 siblings, 1 reply; 69+ messages in thread
From: Matthias van de Meent @ 2024-02-22 12:37 UTC (permalink / raw)
To: Peter Eisentraut <[email protected]>; +Cc: Robert Haas <[email protected]>; PostgreSQL Hackers <[email protected]>; Andres Freund <[email protected]>; Michel Pelletier <[email protected]>
On Mon, 19 Feb 2024 at 14:19, Matthias van de Meent
<[email protected]> wrote:
> Attached the updated version of the patch on top of 5497daf3, which
> incorporates this last round of feedback.
Now attached rebased on top of 93db6cbd to fix conflicts with fbc93b8b
and an issue in the previous patchset: I attached one too many v3-0001
from a previous patch I worked on.
Kind regards,
Matthias van de Meent
Neon (https://neon.tech)
Attachments:
[application/octet-stream] v6-0003-gen_node_support.pl-Mark-location-fields-as-type-.patch (26.2K, ../../CAEze2Wj3KOaXDD0+Viv8u5yeg1tNFQZ3it-pdY267A_v57voyA@mail.gmail.com/2-v6-0003-gen_node_support.pl-Mark-location-fields-as-type-.patch)
download | inline diff:
From db3da7d24e0ffebf3bbed2e05e9794927ec73418 Mon Sep 17 00:00:00 2001
From: Matthias van de Meent <[email protected]>
Date: Thu, 8 Feb 2024 17:15:22 +0100
Subject: [PATCH v6 3/8] gen_node_support.pl: Mark location fields as type
alias Location
Instead of the rather ugly type=int + name ~= location$, we now have a
special type for offset pointers or sizes that are only relevant when a
query text is included, which decreases the complexity required in
gen_node_support.pl for handling these values.
---
src/backend/nodes/gen_node_support.pl | 6 +-
src/include/nodes/parsenodes.h | 92 +++++++++++++--------------
src/include/nodes/primnodes.h | 72 +++++++++++----------
3 files changed, 86 insertions(+), 84 deletions(-)
diff --git a/src/backend/nodes/gen_node_support.pl b/src/backend/nodes/gen_node_support.pl
index 487f6f7728..ec850c3484 100644
--- a/src/backend/nodes/gen_node_support.pl
+++ b/src/backend/nodes/gen_node_support.pl
@@ -777,7 +777,7 @@ _equal${n}(const $n *a, const $n *b)
print $eff "\tCOMPARE_BITMAPSET_FIELD($f);\n"
unless $equal_ignore;
}
- elsif ($t eq 'int' && $f =~ 'location$')
+ elsif ($t eq 'Location')
{
print $cff "\tCOPY_LOCATION_FIELD($f);\n" unless $copy_ignore;
print $eff "\tCOMPARE_LOCATION_FIELD($f);\n" unless $equal_ignore;
@@ -1010,7 +1010,7 @@ _read${n}(void)
print $off "\tWRITE_BOOL_FIELD($f);\n";
print $rff "\tREAD_BOOL_FIELD($f);\n" unless $no_read;
}
- elsif ($t eq 'int' && $f =~ 'location$')
+ elsif ($t eq 'Location')
{
print $off "\tWRITE_LOCATION_FIELD($f);\n";
print $rff "\tREAD_LOCATION_FIELD($f);\n" unless $no_read;
@@ -1303,7 +1303,7 @@ _jumble${n}(JumbleState *jstate, Node *node)
print $jff "\tJUMBLE_NODE($f);\n"
unless $query_jumble_ignore;
}
- elsif ($t eq 'int' && $f =~ 'location$')
+ elsif ($t eq 'Location')
{
# Track the node's location only if directly requested.
if ($query_jumble_location)
diff --git a/src/include/nodes/parsenodes.h b/src/include/nodes/parsenodes.h
index baa6a97c7e..37e4c53ebe 100644
--- a/src/include/nodes/parsenodes.h
+++ b/src/include/nodes/parsenodes.h
@@ -225,9 +225,9 @@ typedef struct Query
* both be -1 meaning "unknown".
*/
/* start location, or -1 if unknown */
- int stmt_location;
+ Location stmt_location;
/* length in bytes; 0 means "rest of string" */
- int stmt_len pg_node_attr(query_jumble_ignore);
+ Location stmt_len pg_node_attr(query_jumble_ignore);
} Query;
@@ -262,7 +262,7 @@ typedef struct TypeName
List *typmods; /* type modifier expression(s) */
int32 typemod; /* prespecified type modifier */
List *arrayBounds; /* array bounds */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} TypeName;
/*
@@ -282,7 +282,7 @@ typedef struct ColumnRef
{
NodeTag type;
List *fields; /* field names (String nodes) or A_Star */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} ColumnRef;
/*
@@ -292,7 +292,7 @@ typedef struct ParamRef
{
NodeTag type;
int number; /* the number of the parameter */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} ParamRef;
/*
@@ -325,7 +325,7 @@ typedef struct A_Expr
List *name; /* possibly-qualified name of operator */
Node *lexpr; /* left argument, or NULL if none */
Node *rexpr; /* right argument, or NULL if none */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} A_Expr;
/*
@@ -351,7 +351,7 @@ typedef struct A_Const
NodeTag type;
union ValUnion val;
bool isnull; /* SQL NULL constant */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} A_Const;
/*
@@ -362,7 +362,7 @@ typedef struct TypeCast
NodeTag type;
Node *arg; /* the expression being casted */
TypeName *typeName; /* the target type */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} TypeCast;
/*
@@ -373,7 +373,7 @@ typedef struct CollateClause
NodeTag type;
Node *arg; /* input expression */
List *collname; /* possibly-qualified collation name */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} CollateClause;
/*
@@ -393,7 +393,7 @@ typedef struct RoleSpec
NodeTag type;
RoleSpecType roletype; /* Type of this rolespec */
char *rolename; /* filled only for ROLESPEC_CSTRING */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} RoleSpec;
/*
@@ -423,7 +423,7 @@ typedef struct FuncCall
bool agg_distinct; /* arguments were labeled DISTINCT */
bool func_variadic; /* last argument was labeled VARIADIC */
CoercionForm funcformat; /* how to display this node */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} FuncCall;
/*
@@ -480,7 +480,7 @@ typedef struct A_ArrayExpr
{
NodeTag type;
List *elements; /* array element expressions */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} A_ArrayExpr;
/*
@@ -507,7 +507,7 @@ typedef struct ResTarget
char *name; /* column name or NULL */
List *indirection; /* subscripts, field names, and '*', or NIL */
Node *val; /* the value expression to compute or assign */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} ResTarget;
/*
@@ -537,7 +537,7 @@ typedef struct SortBy
SortByDir sortby_dir; /* ASC/DESC/USING/default */
SortByNulls sortby_nulls; /* NULLS FIRST/LAST */
List *useOp; /* name of op to use, if SORTBY_USING */
- int location; /* operator location, or -1 if none/unknown */
+ Location location; /* operator location, or -1 if none/unknown */
} SortBy;
/*
@@ -558,7 +558,7 @@ typedef struct WindowDef
int frameOptions; /* frame_clause options, see below */
Node *startOffset; /* expression for starting bound, if any */
Node *endOffset; /* expression for ending bound, if any */
- int location; /* parse location, or -1 if none/unknown */
+ Location location; /* parse location, or -1 if none/unknown */
} WindowDef;
/*
@@ -648,7 +648,7 @@ typedef struct RangeTableFunc
List *namespaces; /* list of namespaces as ResTarget */
List *columns; /* list of RangeTableFuncCol */
Alias *alias; /* table alias & optional column aliases */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} RangeTableFunc;
/*
@@ -666,7 +666,7 @@ typedef struct RangeTableFuncCol
bool is_not_null; /* does it have NOT NULL? */
Node *colexpr; /* column filter expression */
Node *coldefexpr; /* column default value expression */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} RangeTableFuncCol;
/*
@@ -686,7 +686,7 @@ typedef struct RangeTableSample
List *method; /* sampling method name (possibly qualified) */
List *args; /* argument(s) for sampling method */
Node *repeatable; /* REPEATABLE expression, or NULL if none */
- int location; /* method name location, or -1 if unknown */
+ Location location; /* method name location, or -1 if unknown */
} RangeTableSample;
/*
@@ -729,7 +729,7 @@ typedef struct ColumnDef
Oid collOid; /* collation OID (InvalidOid if not set) */
List *constraints; /* other constraints on column */
List *fdwoptions; /* per-column FDW options */
- int location; /* parse location, or -1 if none/unknown */
+ Location location; /* parse location, or -1 if none/unknown */
} ColumnDef;
/*
@@ -803,7 +803,7 @@ typedef struct DefElem
Node *arg; /* typically Integer, Float, String, or
* TypeName */
DefElemAction defaction; /* unspecified action, or SET/ADD/DROP */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} DefElem;
/*
@@ -833,7 +833,7 @@ typedef struct XmlSerialize
Node *expr;
TypeName *typeName;
bool indent; /* [NO] INDENT */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} XmlSerialize;
/* Partitioning related definitions */
@@ -851,7 +851,7 @@ typedef struct PartitionElem
Node *expr; /* expression to partition on, or NULL */
List *collation; /* name of collation; NIL = default */
List *opclass; /* name of desired opclass; NIL = default */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} PartitionElem;
typedef enum PartitionStrategy
@@ -871,7 +871,7 @@ typedef struct PartitionSpec
NodeTag type;
PartitionStrategy strategy;
List *partParams; /* List of PartitionElems */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} PartitionSpec;
/*
@@ -898,7 +898,7 @@ struct PartitionBoundSpec
List *lowerdatums; /* List of PartitionRangeDatums */
List *upperdatums; /* List of PartitionRangeDatums */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
};
/*
@@ -921,7 +921,7 @@ typedef struct PartitionRangeDatum
Node *value; /* Const (or A_Const in raw tree), if kind is
* PARTITION_RANGE_DATUM_VALUE, else NULL */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} PartitionRangeDatum;
/*
@@ -1454,7 +1454,7 @@ typedef struct GroupingSet
NodeTag type;
GroupingSetKind kind pg_node_attr(query_jumble_ignore);
List *content;
- int location;
+ Location location;
} GroupingSet;
/*
@@ -1542,7 +1542,7 @@ typedef struct WithClause
NodeTag type;
List *ctes; /* list of CommonTableExprs */
bool recursive; /* true = WITH RECURSIVE */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} WithClause;
/*
@@ -1557,7 +1557,7 @@ typedef struct InferClause
List *indexElems; /* IndexElems to infer unique index */
Node *whereClause; /* qualification (partial-index predicate) */
char *conname; /* Constraint name, or NULL if unnamed */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} InferClause;
/*
@@ -1573,7 +1573,7 @@ typedef struct OnConflictClause
InferClause *infer; /* Optional index inference clause */
List *targetList; /* the target list (of ResTarget) */
Node *whereClause; /* qualifications */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} OnConflictClause;
/*
@@ -1594,7 +1594,7 @@ typedef struct CTESearchClause
List *search_col_list;
bool search_breadth_first;
char *search_seq_column;
- int location;
+ Location location;
} CTESearchClause;
typedef struct CTECycleClause
@@ -1605,7 +1605,7 @@ typedef struct CTECycleClause
Node *cycle_mark_value;
Node *cycle_mark_default;
char *cycle_path_column;
- int location;
+ Location location;
/* These fields are set during parse analysis: */
Oid cycle_mark_type; /* common type of _value and _default */
int cycle_mark_typmod;
@@ -1629,7 +1629,7 @@ typedef struct CommonTableExpr
Node *ctequery; /* the CTE's subquery */
CTESearchClause *search_clause pg_node_attr(query_jumble_ignore);
CTECycleClause *cycle_clause pg_node_attr(query_jumble_ignore);
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
/* These fields are set during parse analysis: */
/* is this CTE actually recursive? */
bool cterecursive pg_node_attr(query_jumble_ignore);
@@ -1725,7 +1725,7 @@ typedef struct JsonParseExpr
JsonValueExpr *expr; /* string expression */
JsonOutput *output; /* RETURNING clause, if specified */
bool unique_keys; /* WITH UNIQUE KEYS? */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} JsonParseExpr;
/*
@@ -1737,7 +1737,7 @@ typedef struct JsonScalarExpr
NodeTag type;
Expr *expr; /* scalar expression */
JsonOutput *output; /* RETURNING clause, if specified */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} JsonScalarExpr;
/*
@@ -1749,7 +1749,7 @@ typedef struct JsonSerializeExpr
NodeTag type;
JsonValueExpr *expr; /* json value expression */
JsonOutput *output; /* RETURNING clause, if specified */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} JsonSerializeExpr;
/*
@@ -1763,7 +1763,7 @@ typedef struct JsonObjectConstructor
JsonOutput *output; /* RETURNING clause, if specified */
bool absent_on_null; /* skip NULL values? */
bool unique; /* check key uniqueness? */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} JsonObjectConstructor;
/*
@@ -1776,7 +1776,7 @@ typedef struct JsonArrayConstructor
List *exprs; /* list of JsonValueExpr elements */
JsonOutput *output; /* RETURNING clause, if specified */
bool absent_on_null; /* skip NULL elements? */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} JsonArrayConstructor;
/*
@@ -1790,7 +1790,7 @@ typedef struct JsonArrayQueryConstructor
JsonOutput *output; /* RETURNING clause, if specified */
JsonFormat *format; /* FORMAT clause for subquery, if specified */
bool absent_on_null; /* skip NULL elements? */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} JsonArrayQueryConstructor;
/*
@@ -1805,7 +1805,7 @@ typedef struct JsonAggConstructor
Node *agg_filter; /* FILTER clause, if any */
List *agg_order; /* ORDER BY clause, if any */
struct WindowDef *over; /* OVER clause, if any */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} JsonAggConstructor;
/*
@@ -1859,8 +1859,8 @@ typedef struct RawStmt
NodeTag type;
Node *stmt; /* raw parse tree */
- int stmt_location; /* start location, or -1 if unknown */
- int stmt_len; /* length in bytes; 0 means "rest of string" */
+ Location stmt_location; /* start location, or -1 if unknown */
+ Location stmt_len; /* length in bytes; 0 means "rest of string" */
} RawStmt;
/*****************************************************************************
@@ -2067,7 +2067,7 @@ typedef struct PLAssignStmt
List *indirection; /* subscripts and field names, if any */
int nnames; /* number of names to use in ColumnRef */
SelectStmt *val; /* the PL/pgSQL expression to assign */
- int location; /* name's token location, or -1 if unknown */
+ Location location; /* name's token location, or -1 if unknown */
} PLAssignStmt;
@@ -2609,7 +2609,7 @@ typedef struct Constraint
Oid old_pktable_oid; /* pg_constraint.confrelid of my former
* self */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} Constraint;
/* ----------------------
@@ -3516,7 +3516,7 @@ typedef struct TransactionStmt
char *gid pg_node_attr(query_jumble_ignore);
bool chain; /* AND CHAIN option */
/* token location, or -1 if unknown */
- int location pg_node_attr(query_jumble_location);
+ Location location pg_node_attr(query_jumble_location);
} TransactionStmt;
/* ----------------------
@@ -3902,7 +3902,7 @@ typedef struct DeallocateStmt
/* true if DEALLOCATE ALL */
bool isall;
/* token location, or -1 if unknown */
- int location pg_node_attr(query_jumble_location);
+ Location location pg_node_attr(query_jumble_location);
} DeallocateStmt;
/*
@@ -3990,7 +3990,7 @@ typedef struct PublicationObjSpec
PublicationObjSpecType pubobjtype; /* type of this publication object */
char *name;
PublicationTable *pubtable;
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} PublicationObjSpec;
typedef struct CreatePublicationStmt
diff --git a/src/include/nodes/primnodes.h b/src/include/nodes/primnodes.h
index 4a154606d2..557be05657 100644
--- a/src/include/nodes/primnodes.h
+++ b/src/include/nodes/primnodes.h
@@ -30,6 +30,8 @@ typedef enum OverridingKind
} OverridingKind;
+#define Location int
+
/* ----------------------------------------------------------------
* node definitions
* ----------------------------------------------------------------
@@ -91,7 +93,7 @@ typedef struct RangeVar
Alias *alias;
/* token location, or -1 if unknown */
- int location;
+ Location location;
} RangeVar;
/*
@@ -128,7 +130,7 @@ typedef struct TableFunc
/* counts from 0; -1 if none specified */
int ordinalitycol pg_node_attr(query_jumble_ignore);
/* token location, or -1 if unknown */
- int location;
+ Location location;
} TableFunc;
/*
@@ -276,7 +278,7 @@ typedef struct Var
AttrNumber varattnosyn pg_node_attr(equal_ignore, query_jumble_ignore);
/* token location, or -1 if unknown */
- int location;
+ Location location;
} Var;
/*
@@ -318,7 +320,7 @@ typedef struct Const
* token location, or -1 if unknown. All constants are tracked as
* locations in query jumbling, to be marked as parameters.
*/
- int location pg_node_attr(query_jumble_location);
+ Location location pg_node_attr(query_jumble_location);
} Const;
/*
@@ -367,7 +369,7 @@ typedef struct Param
/* OID of collation, or InvalidOid if none */
Oid paramcollid pg_node_attr(query_jumble_ignore);
/* token location, or -1 if unknown */
- int location;
+ Location location;
} Param;
/*
@@ -490,7 +492,7 @@ typedef struct Aggref
int aggtransno pg_node_attr(query_jumble_ignore);
/* token location, or -1 if unknown */
- int location;
+ Location location;
} Aggref;
/*
@@ -537,7 +539,7 @@ typedef struct GroupingFunc
Index agglevelsup;
/* token location */
- int location;
+ Location location;
} GroupingFunc;
/*
@@ -568,7 +570,7 @@ typedef struct WindowFunc
/* is function a simple aggregate? */
bool winagg pg_node_attr(query_jumble_ignore);
/* token location, or -1 if unknown */
- int location;
+ Location location;
} WindowFunc;
/*
@@ -702,7 +704,7 @@ typedef struct FuncExpr
/* arguments to the function */
List *args;
/* token location, or -1 if unknown */
- int location;
+ Location location;
} FuncExpr;
/*
@@ -729,7 +731,7 @@ typedef struct NamedArgExpr
/* argument's number in positional notation */
int argnumber;
/* argument name location, or -1 if unknown */
- int location;
+ Location location;
} NamedArgExpr;
/*
@@ -771,7 +773,7 @@ typedef struct OpExpr
List *args;
/* token location, or -1 if unknown */
- int location;
+ Location location;
} OpExpr;
/*
@@ -851,7 +853,7 @@ typedef struct ScalarArrayOpExpr
List *args;
/* token location, or -1 if unknown */
- int location;
+ Location location;
} ScalarArrayOpExpr;
/*
@@ -873,7 +875,7 @@ typedef struct BoolExpr
Expr xpr;
BoolExprType boolop;
List *args; /* arguments to this expression */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} BoolExpr;
/*
@@ -950,7 +952,7 @@ typedef struct SubLink
List *operName pg_node_attr(query_jumble_ignore);
/* subselect as Query* or raw parsetree */
Node *subselect;
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} SubLink;
/*
@@ -1124,7 +1126,7 @@ typedef struct RelabelType
Oid resultcollid pg_node_attr(query_jumble_ignore);
/* how to display this node */
CoercionForm relabelformat pg_node_attr(query_jumble_ignore);
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} RelabelType;
/* ----------------
@@ -1146,7 +1148,7 @@ typedef struct CoerceViaIO
Oid resultcollid pg_node_attr(query_jumble_ignore);
/* how to display this node */
CoercionForm coerceformat pg_node_attr(query_jumble_ignore);
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} CoerceViaIO;
/* ----------------
@@ -1174,7 +1176,7 @@ typedef struct ArrayCoerceExpr
Oid resultcollid pg_node_attr(query_jumble_ignore);
/* how to display this node */
CoercionForm coerceformat pg_node_attr(query_jumble_ignore);
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} ArrayCoerceExpr;
/* ----------------
@@ -1198,7 +1200,7 @@ typedef struct ConvertRowtypeExpr
/* Like RowExpr, we deliberately omit a typmod and collation here */
/* how to display this node */
CoercionForm convertformat pg_node_attr(query_jumble_ignore);
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} ConvertRowtypeExpr;
/*----------
@@ -1213,7 +1215,7 @@ typedef struct CollateExpr
Expr xpr;
Expr *arg; /* input expression */
Oid collOid; /* collation's OID */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} CollateExpr;
/*----------
@@ -1248,7 +1250,7 @@ typedef struct CaseExpr
Expr *arg; /* implicit equality comparison argument */
List *args; /* the arguments (list of WHEN clauses) */
Expr *defresult; /* the default result (ELSE clause) */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} CaseExpr;
/*
@@ -1259,7 +1261,7 @@ typedef struct CaseWhen
Expr xpr;
Expr *expr; /* condition expression */
Expr *result; /* substitution result */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} CaseWhen;
/*
@@ -1316,7 +1318,7 @@ typedef struct ArrayExpr
/* true if elements are sub-arrays */
bool multidims pg_node_attr(query_jumble_ignore);
/* token location, or -1 if unknown */
- int location;
+ Location location;
} ArrayExpr;
/*
@@ -1367,7 +1369,7 @@ typedef struct RowExpr
/* list of String, or NIL */
List *colnames pg_node_attr(query_jumble_ignore);
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} RowExpr;
/*
@@ -1426,7 +1428,7 @@ typedef struct CoalesceExpr
/* the arguments */
List *args;
/* token location, or -1 if unknown */
- int location;
+ Location location;
} CoalesceExpr;
/*
@@ -1452,7 +1454,7 @@ typedef struct MinMaxExpr
/* the arguments */
List *args;
/* token location, or -1 if unknown */
- int location;
+ Location location;
} MinMaxExpr;
/*
@@ -1496,7 +1498,7 @@ typedef struct SQLValueFunction
*/
Oid type pg_node_attr(query_jumble_ignore);
int32 typmod;
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} SQLValueFunction;
/*
@@ -1549,7 +1551,7 @@ typedef struct XmlExpr
Oid type pg_node_attr(query_jumble_ignore);
int32 typmod pg_node_attr(query_jumble_ignore);
/* token location, or -1 if unknown */
- int location;
+ Location location;
} XmlExpr;
/*
@@ -1585,7 +1587,7 @@ typedef struct JsonFormat
NodeTag type;
JsonFormatType format_type; /* format type */
JsonEncoding encoding; /* JSON encoding */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} JsonFormat;
/*
@@ -1641,7 +1643,7 @@ typedef struct JsonConstructorExpr
JsonReturning *returning; /* RETURNING clause */
bool absent_on_null; /* ABSENT ON NULL? */
bool unique; /* WITH UNIQUE KEYS? (JSON_OBJECT[AGG] only) */
- int location;
+ Location location;
} JsonConstructorExpr;
/*
@@ -1667,7 +1669,7 @@ typedef struct JsonIsPredicate
JsonFormat *format; /* FORMAT clause, if specified */
JsonValueType item_type; /* JSON item type */
bool unique_keys; /* check key uniqueness? */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} JsonIsPredicate;
/* ----------------
@@ -1701,7 +1703,7 @@ typedef struct NullTest
NullTestType nulltesttype; /* IS NULL, IS NOT NULL */
/* T to perform field-by-field null checks */
bool argisrow pg_node_attr(query_jumble_ignore);
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} NullTest;
/*
@@ -1723,7 +1725,7 @@ typedef struct BooleanTest
Expr xpr;
Expr *arg; /* input expression */
BoolTestType booltesttype; /* test type */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} BooleanTest;
@@ -1765,7 +1767,7 @@ typedef struct CoerceToDomain
Oid resultcollid pg_node_attr(query_jumble_ignore);
/* how to display this node */
CoercionForm coercionformat pg_node_attr(query_jumble_ignore);
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} CoerceToDomain;
/*
@@ -1787,7 +1789,7 @@ typedef struct CoerceToDomainValue
/* collation for the substituted value */
Oid collation pg_node_attr(query_jumble_ignore);
/* token location, or -1 if unknown */
- int location;
+ Location location;
} CoerceToDomainValue;
/*
@@ -1807,7 +1809,7 @@ typedef struct SetToDefault
/* collation for the substituted value */
Oid collation pg_node_attr(query_jumble_ignore);
/* token location, or -1 if unknown */
- int location;
+ Location location;
} SetToDefault;
/*
--
2.40.1
[application/octet-stream] v6-0002-pg_node_tree-Don-t-store-query-text-locations-in-.patch (19.0K, ../../CAEze2Wj3KOaXDD0+Viv8u5yeg1tNFQZ3it-pdY267A_v57voyA@mail.gmail.com/3-v6-0002-pg_node_tree-Don-t-store-query-text-locations-in-.patch)
download | inline diff:
From 9e54a1034a5126c52b605e13289d6e9e73b25892 Mon Sep 17 00:00:00 2001
From: Matthias van de Meent <[email protected]>
Date: Wed, 3 Jan 2024 01:39:42 +0100
Subject: [PATCH v6 2/8] pg_node_tree: Don't store query text locations in
pg_node_tree fields.
We don't store original query texts, so any lingering "location" value
can only be useful in forensic debugging. In normal operation, however,
a non-default value will show up as measurable overhead in
serialization, just omit serialization, saving several 10s of kBs.
---
src/backend/catalog/heap.c | 9 ++--
src/backend/catalog/index.c | 4 +-
src/backend/catalog/pg_attrdef.c | 6 ++-
src/backend/catalog/pg_proc.c | 10 +++-
src/backend/catalog/pg_publication.c | 6 ++-
src/backend/commands/policy.c | 11 +++-
src/backend/commands/statscmds.c | 2 +-
src/backend/commands/trigger.c | 3 +-
src/backend/commands/typecmds.c | 8 +--
src/backend/nodes/gen_node_support.pl | 4 +-
src/backend/nodes/outfuncs.c | 75 ++++++++++++++++-----------
src/backend/rewrite/rewriteDefine.c | 4 +-
src/include/nodes/nodes.h | 4 +-
13 files changed, 94 insertions(+), 52 deletions(-)
diff --git a/src/backend/catalog/heap.c b/src/backend/catalog/heap.c
index 348943e36c..9d2fd6fae8 100644
--- a/src/backend/catalog/heap.c
+++ b/src/backend/catalog/heap.c
@@ -2064,9 +2064,9 @@ StoreRelCheck(Relation rel, const char *ccname, Node *expr,
Oid constrOid;
/*
- * Flatten expression to string form for storage.
+ * Flatten expression to string form for storage, without query refs.
*/
- ccbin = nodeToString(expr);
+ ccbin = nodeToStringNoQLocs(expr);
/*
* Find columns of rel that are used in expr
@@ -3676,7 +3676,7 @@ StorePartitionKey(Relation rel,
{
char *exprString;
- exprString = nodeToString(partexprs);
+ exprString = nodeToStringNoQLocs(partexprs);
partexprDatum = CStringGetTextDatum(exprString);
pfree(exprString);
}
@@ -3834,7 +3834,8 @@ StorePartitionBound(Relation rel, Relation parent, PartitionBoundSpec *bound)
memset(new_val, 0, sizeof(new_val));
memset(new_null, false, sizeof(new_null));
memset(new_repl, false, sizeof(new_repl));
- new_val[Anum_pg_class_relpartbound - 1] = CStringGetTextDatum(nodeToString(bound));
+ new_val[Anum_pg_class_relpartbound - 1] =
+ CStringGetTextDatum(nodeToStringNoQLocs(bound));
new_null[Anum_pg_class_relpartbound - 1] = false;
new_repl[Anum_pg_class_relpartbound - 1] = true;
newtuple = heap_modify_tuple(tuple, RelationGetDescr(classRel),
diff --git a/src/backend/catalog/index.c b/src/backend/catalog/index.c
index 4b88a9cb87..3d5f4e53d5 100644
--- a/src/backend/catalog/index.c
+++ b/src/backend/catalog/index.c
@@ -587,7 +587,7 @@ UpdateIndexRelation(Oid indexoid,
{
char *exprsString;
- exprsString = nodeToString(indexInfo->ii_Expressions);
+ exprsString = nodeToStringNoQLocs(indexInfo->ii_Expressions);
exprsDatum = CStringGetTextDatum(exprsString);
pfree(exprsString);
}
@@ -602,7 +602,7 @@ UpdateIndexRelation(Oid indexoid,
{
char *predString;
- predString = nodeToString(make_ands_explicit(indexInfo->ii_Predicate));
+ predString = nodeToStringNoQLocs(make_ands_explicit(indexInfo->ii_Predicate));
predDatum = CStringGetTextDatum(predString);
pfree(predString);
}
diff --git a/src/backend/catalog/pg_attrdef.c b/src/backend/catalog/pg_attrdef.c
index 003ae70b4d..a900c9bb28 100644
--- a/src/backend/catalog/pg_attrdef.c
+++ b/src/backend/catalog/pg_attrdef.c
@@ -23,6 +23,7 @@
#include "catalog/objectaccess.h"
#include "catalog/pg_attrdef.h"
#include "executor/executor.h"
+#include "nodes/nodeFuncs.h"
#include "optimizer/optimizer.h"
#include "utils/array.h"
#include "utils/builtins.h"
@@ -62,9 +63,10 @@ StoreAttrDefault(Relation rel, AttrNumber attnum,
adrel = table_open(AttrDefaultRelationId, RowExclusiveLock);
/*
- * Flatten expression to string form for storage.
+ * Flatten expression to string form for storage, without references to
+ * the original query string.
*/
- adbin = nodeToString(expr);
+ adbin = nodeToStringNoQLocs(expr);
/*
* Make the pg_attrdef entry.
diff --git a/src/backend/catalog/pg_proc.c b/src/backend/catalog/pg_proc.c
index b581d334d3..c5790a2224 100644
--- a/src/backend/catalog/pg_proc.c
+++ b/src/backend/catalog/pg_proc.c
@@ -331,7 +331,10 @@ ProcedureCreate(const char *procedureName,
else
nulls[Anum_pg_proc_proargnames - 1] = true;
if (parameterDefaults != NIL)
- values[Anum_pg_proc_proargdefaults - 1] = CStringGetTextDatum(nodeToString(parameterDefaults));
+ {
+ values[Anum_pg_proc_proargdefaults - 1] =
+ CStringGetTextDatum(nodeToStringNoQLocs(parameterDefaults));
+ }
else
nulls[Anum_pg_proc_proargdefaults - 1] = true;
if (trftypes != PointerGetDatum(NULL))
@@ -344,7 +347,10 @@ ProcedureCreate(const char *procedureName,
else
nulls[Anum_pg_proc_probin - 1] = true;
if (prosqlbody)
- values[Anum_pg_proc_prosqlbody - 1] = CStringGetTextDatum(nodeToString(prosqlbody));
+ {
+ values[Anum_pg_proc_prosqlbody - 1] =
+ CStringGetTextDatum(nodeToStringNoQLocs(prosqlbody));
+ }
else
nulls[Anum_pg_proc_prosqlbody - 1] = true;
if (proconfig != PointerGetDatum(NULL))
diff --git a/src/backend/catalog/pg_publication.c b/src/backend/catalog/pg_publication.c
index b98b0ce0ae..b201313430 100644
--- a/src/backend/catalog/pg_publication.c
+++ b/src/backend/catalog/pg_publication.c
@@ -36,6 +36,7 @@
#include "commands/publicationcmds.h"
#include "funcapi.h"
#include "miscadmin.h"
+#include "nodes/nodeFuncs.h"
#include "utils/array.h"
#include "utils/builtins.h"
#include "utils/catcache.h"
@@ -422,7 +423,10 @@ publication_add_relation(Oid pubid, PublicationRelInfo *pri,
/* Add qualifications, if available */
if (pri->whereClause != NULL)
- values[Anum_pg_publication_rel_prqual - 1] = CStringGetTextDatum(nodeToString(pri->whereClause));
+ {
+ values[Anum_pg_publication_rel_prqual - 1] =
+ CStringGetTextDatum(nodeToStringNoQLocs(pri->whereClause));
+ }
else
nulls[Anum_pg_publication_rel_prqual - 1] = true;
diff --git a/src/backend/commands/policy.c b/src/backend/commands/policy.c
index 596326e5ec..1e6842bf41 100644
--- a/src/backend/commands/policy.c
+++ b/src/backend/commands/policy.c
@@ -30,6 +30,7 @@
#include "commands/policy.h"
#include "miscadmin.h"
#include "nodes/makefuncs.h"
+#include "nodes/nodeFuncs.h"
#include "nodes/pg_list.h"
#include "parser/parse_clause.h"
#include "parser/parse_collate.h"
@@ -701,13 +702,19 @@ CreatePolicy(CreatePolicyStmt *stmt)
/* Add qual if present. */
if (qual)
- values[Anum_pg_policy_polqual - 1] = CStringGetTextDatum(nodeToString(qual));
+ {
+ values[Anum_pg_policy_polqual - 1] =
+ CStringGetTextDatum(nodeToStringNoQLocs(qual));
+ }
else
isnull[Anum_pg_policy_polqual - 1] = true;
/* Add WITH CHECK qual if present */
if (with_check_qual)
- values[Anum_pg_policy_polwithcheck - 1] = CStringGetTextDatum(nodeToString(with_check_qual));
+ {
+ values[Anum_pg_policy_polwithcheck - 1] =
+ CStringGetTextDatum(nodeToStringNoQLocs(with_check_qual));
+ }
else
isnull[Anum_pg_policy_polwithcheck - 1] = true;
diff --git a/src/backend/commands/statscmds.c b/src/backend/commands/statscmds.c
index b1a9c74bd6..58e8133f93 100644
--- a/src/backend/commands/statscmds.c
+++ b/src/backend/commands/statscmds.c
@@ -477,7 +477,7 @@ CreateStatistics(CreateStatsStmt *stmt)
{
char *exprsString;
- exprsString = nodeToString(stxexprs);
+ exprsString = nodeToStringNoQLocs(stxexprs);
exprsDatum = CStringGetTextDatum(exprsString);
pfree(exprsString);
}
diff --git a/src/backend/commands/trigger.c b/src/backend/commands/trigger.c
index c344ff0944..6a3dc13a67 100644
--- a/src/backend/commands/trigger.c
+++ b/src/backend/commands/trigger.c
@@ -39,6 +39,7 @@
#include "miscadmin.h"
#include "nodes/bitmapset.h"
#include "nodes/makefuncs.h"
+#include "nodes/nodeFuncs.h"
#include "optimizer/optimizer.h"
#include "parser/parse_clause.h"
#include "parser/parse_collate.h"
@@ -674,7 +675,7 @@ CreateTriggerFiringOn(CreateTrigStmt *stmt, const char *queryString,
/* we'll need the rtable for recordDependencyOnExpr */
whenRtable = pstate->p_rtable;
- qual = nodeToString(whenClause);
+ qual = nodeToStringNoQLocs(whenClause);
free_parsestate(pstate);
}
diff --git a/src/backend/commands/typecmds.c b/src/backend/commands/typecmds.c
index e0275e5fe9..3c0e7bc5db 100644
--- a/src/backend/commands/typecmds.c
+++ b/src/backend/commands/typecmds.c
@@ -58,6 +58,7 @@
#include "executor/executor.h"
#include "miscadmin.h"
#include "nodes/makefuncs.h"
+#include "nodes/nodeFuncs.h"
#include "optimizer/optimizer.h"
#include "parser/parse_coerce.h"
#include "parser/parse_collate.h"
@@ -924,7 +925,7 @@ DefineDomain(CreateDomainStmt *stmt)
defaultValue =
deparse_expression(defaultExpr,
NIL, false, false);
- defaultValueBin = nodeToString(defaultExpr);
+ defaultValueBin = nodeToStringNoQLocs(defaultExpr);
}
}
else
@@ -3506,9 +3507,10 @@ domainAddConstraint(Oid domainOid, Oid domainNamespace, Oid baseTypeOid,
errmsg("cannot use table references in domain check constraint")));
/*
- * Convert to string form for storage.
+ * Convert to string form for storage, without references to the original
+ * query text.
*/
- ccbin = nodeToString(expr);
+ ccbin = nodeToStringNoQLocs(expr);
/*
* Store the constraint in pg_constraint
diff --git a/src/backend/nodes/gen_node_support.pl b/src/backend/nodes/gen_node_support.pl
index 2f0a59bc87..487f6f7728 100644
--- a/src/backend/nodes/gen_node_support.pl
+++ b/src/backend/nodes/gen_node_support.pl
@@ -921,7 +921,7 @@ foreach my $n (@node_types)
my $N = uc $n;
print $ofs "\t\t\tcase T_${n}:\n"
- . "\t\t\t\t_out${n}(str, obj);\n"
+ . "\t\t\t\t_out${n}(str, obj, omitLocation);\n"
. "\t\t\t\tbreak;\n";
print $rfs "\tif (MATCH(\"$N\", "
@@ -933,7 +933,7 @@ foreach my $n (@node_types)
print $off "
static void
-_out${n}(StringInfo str, const $n *node)
+_out${n}(StringInfo str, const $n *node, bool omitLocation)
{
\tWRITE_NODE_TYPE(\"$N\");
diff --git a/src/backend/nodes/outfuncs.c b/src/backend/nodes/outfuncs.c
index 8ad81be1cd..c4fd16dc37 100644
--- a/src/backend/nodes/outfuncs.c
+++ b/src/backend/nodes/outfuncs.c
@@ -159,7 +159,7 @@ static void outDouble(StringInfo str, double d);
/* Write a parse location field (actually same as INT case) */
#define WRITE_LOCATION_FIELD(fldname) \
do { \
- if (node->fldname != -1) \
+ if (node->fldname != -1 && !omitLocation) \
appendStringInfo(str, " :" CppAsString(fldname) " %d", \
node->fldname); \
} while (0)
@@ -170,7 +170,7 @@ static void outDouble(StringInfo str, double d);
if (node->fldname != NULL) \
{ \
appendStringInfoString(str, " :" CppAsString(fldname) " "); \
- outNode(str, node->fldname); \
+ outNode(str, node->fldname, omitLocation); \
} \
} while (0)
@@ -190,7 +190,8 @@ static void outDouble(StringInfo str, double d);
if (node->fldname != NULL) \
{ \
appendStringInfoString(str, " :" CppAsString(fldname) " "); \
- writeNodeArray(str, (const Node * const *) node->fldname, len); \
+ writeNodeArray(str, (const Node * const *) node->fldname, len, \
+ omitLocation); \
} \
} while (0)
@@ -363,7 +364,8 @@ WRITE_SCALAR_ARRAY(writeBoolCols, bool, " %s", booltostr)
* quite use appendStringInfo() in the loop.
*/
static void
-writeNodeArray(StringInfo str, const Node *const *arr, int len)
+writeNodeArray(StringInfo str, const Node *const *arr, int len,
+ bool omitLocation)
{
if (arr != NULL)
{
@@ -371,7 +373,7 @@ writeNodeArray(StringInfo str, const Node *const *arr, int len)
for (int i = 0; i < len; i++)
{
appendStringInfoChar(str, ' ');
- outNode(str, arr[i]);
+ outNode(str, arr[i], omitLocation);
}
appendStringInfoChar(str, ')');
}
@@ -383,7 +385,7 @@ writeNodeArray(StringInfo str, const Node *const *arr, int len)
* Print a List.
*/
static void
-_outList(StringInfo str, const List *node)
+_outList(StringInfo str, const List *node, bool omitLocation)
{
const ListCell *lc;
@@ -405,7 +407,7 @@ _outList(StringInfo str, const List *node)
*/
if (IsA(node, List))
{
- outNode(str, lfirst(lc));
+ outNode(str, lfirst(lc), omitLocation);
if (lnext(node, lc))
appendStringInfoChar(str, ' ');
}
@@ -490,7 +492,7 @@ outDatum(StringInfo str, Datum value, int typlen, bool typbyval)
*/
static void
-_outConst(StringInfo str, const Const *node)
+_outConst(StringInfo str, const Const *node, bool omitLocation)
{
WRITE_NODE_TYPE("CONST");
@@ -510,7 +512,7 @@ _outConst(StringInfo str, const Const *node)
}
static void
-_outBoolExpr(StringInfo str, const BoolExpr *node)
+_outBoolExpr(StringInfo str, const BoolExpr *node, bool omitLocation)
{
char *opstr = NULL;
@@ -537,7 +539,8 @@ _outBoolExpr(StringInfo str, const BoolExpr *node)
}
static void
-_outForeignKeyOptInfo(StringInfo str, const ForeignKeyOptInfo *node)
+_outForeignKeyOptInfo(StringInfo str, const ForeignKeyOptInfo *node,
+ bool omitLocation)
{
int i;
@@ -563,7 +566,8 @@ _outForeignKeyOptInfo(StringInfo str, const ForeignKeyOptInfo *node)
}
static void
-_outEquivalenceClass(StringInfo str, const EquivalenceClass *node)
+_outEquivalenceClass(StringInfo str, const EquivalenceClass *node,
+ bool omitLocation)
{
/*
* To simplify reading, we just chase up to the topmost merged EC and
@@ -589,7 +593,8 @@ _outEquivalenceClass(StringInfo str, const EquivalenceClass *node)
}
static void
-_outExtensibleNode(StringInfo str, const ExtensibleNode *node)
+_outExtensibleNode(StringInfo str, const ExtensibleNode *node,
+ bool omitLocation)
{
const ExtensibleNodeMethods *methods;
@@ -604,7 +609,8 @@ _outExtensibleNode(StringInfo str, const ExtensibleNode *node)
}
static void
-_outRangeTblEntry(StringInfo str, const RangeTblEntry *node)
+_outRangeTblEntry(StringInfo str, const RangeTblEntry *node,
+ bool omitLocation)
{
WRITE_NODE_TYPE("RANGETBLENTRY");
@@ -684,7 +690,7 @@ _outRangeTblEntry(StringInfo str, const RangeTblEntry *node)
}
static void
-_outA_Expr(StringInfo str, const A_Expr *node)
+_outA_Expr(StringInfo str, const A_Expr *node, bool omitLocation)
{
WRITE_NODE_TYPE("A_EXPR");
@@ -756,13 +762,13 @@ _outA_Expr(StringInfo str, const A_Expr *node)
}
static void
-_outInteger(StringInfo str, const Integer *node)
+_outInteger(StringInfo str, const Integer *node, bool omitLocation)
{
appendStringInfo(str, "%d", node->ival);
}
static void
-_outFloat(StringInfo str, const Float *node)
+_outFloat(StringInfo str, const Float *node, bool omitLocation)
{
/*
* We assume the value is a valid numeric literal and so does not need
@@ -772,13 +778,13 @@ _outFloat(StringInfo str, const Float *node)
}
static void
-_outBoolean(StringInfo str, const Boolean *node)
+_outBoolean(StringInfo str, const Boolean *node, bool omitLocation)
{
appendStringInfoString(str, node->boolval ? "true" : "false");
}
static void
-_outString(StringInfo str, const String *node)
+_outString(StringInfo str, const String *node, bool omitLocation)
{
/*
* We use outToken to provide escaping of the string's content, but we
@@ -792,7 +798,7 @@ _outString(StringInfo str, const String *node)
}
static void
-_outBitString(StringInfo str, const BitString *node)
+_outBitString(StringInfo str, const BitString *node, bool omitLocation)
{
/*
* The lexer will always produce a string starting with 'b' or 'x'. There
@@ -804,7 +810,7 @@ _outBitString(StringInfo str, const BitString *node)
}
static void
-_outA_Const(StringInfo str, const A_Const *node)
+_outA_Const(StringInfo str, const A_Const *node, bool omitLocation)
{
WRITE_NODE_TYPE("A_CONST");
@@ -813,7 +819,7 @@ _outA_Const(StringInfo str, const A_Const *node)
else
{
appendStringInfoString(str, " :val ");
- outNode(str, &node->val);
+ outNode(str, &node->val, omitLocation);
}
WRITE_LOCATION_FIELD(location);
}
@@ -824,7 +830,7 @@ _outA_Const(StringInfo str, const A_Const *node)
* converts a Node into ascii string and append it to 'str'
*/
void
-outNode(StringInfo str, const void *obj)
+outNode(StringInfo str, const void *obj, bool omitLocation)
{
/* Guard against stack overflow due to overly complex expressions */
check_stack_depth();
@@ -833,18 +839,18 @@ outNode(StringInfo str, const void *obj)
appendStringInfoString(str, "<>");
else if (IsA(obj, List) || IsA(obj, IntList) || IsA(obj, OidList) ||
IsA(obj, XidList))
- _outList(str, obj);
+ _outList(str, obj, omitLocation);
/* nodeRead does not want to see { } around these! */
else if (IsA(obj, Integer))
- _outInteger(str, (Integer *) obj);
+ _outInteger(str, (Integer *) obj, omitLocation);
else if (IsA(obj, Float))
- _outFloat(str, (Float *) obj);
+ _outFloat(str, (Float *) obj, omitLocation);
else if (IsA(obj, Boolean))
- _outBoolean(str, (Boolean *) obj);
+ _outBoolean(str, (Boolean *) obj, omitLocation);
else if (IsA(obj, String))
- _outString(str, (String *) obj);
+ _outString(str, (String *) obj, omitLocation);
else if (IsA(obj, BitString))
- _outBitString(str, (BitString *) obj);
+ _outBitString(str, (BitString *) obj, omitLocation);
else if (IsA(obj, Bitmapset))
outBitmapset(str, (Bitmapset *) obj);
else
@@ -879,7 +885,18 @@ nodeToString(const void *obj)
/* see stringinfo.h for an explanation of this maneuver */
initStringInfo(&str);
- outNode(&str, obj);
+ outNode(&str, obj, false);
+ return str.data;
+}
+
+char *
+nodeToStringNoQLocs(const void *obj)
+{
+ StringInfoData str;
+
+ /* see stringinfo.h for an explanation of this maneuver */
+ initStringInfo(&str);
+ outNode(&str, obj, true);
return str.data;
}
diff --git a/src/backend/rewrite/rewriteDefine.c b/src/backend/rewrite/rewriteDefine.c
index b449244a53..6302cd1472 100644
--- a/src/backend/rewrite/rewriteDefine.c
+++ b/src/backend/rewrite/rewriteDefine.c
@@ -64,8 +64,8 @@ InsertRule(const char *rulname,
List *action,
bool replace)
{
- char *evqual = nodeToString(event_qual);
- char *actiontree = nodeToString((Node *) action);
+ char *evqual = nodeToStringNoQLocs(event_qual);
+ char *actiontree = nodeToStringNoQLocs((Node *) action);
Datum values[Natts_pg_rewrite];
bool nulls[Natts_pg_rewrite] = {0};
NameData rname;
diff --git a/src/include/nodes/nodes.h b/src/include/nodes/nodes.h
index 2969dd831b..f7adb5e767 100644
--- a/src/include/nodes/nodes.h
+++ b/src/include/nodes/nodes.h
@@ -188,13 +188,15 @@ castNodeImpl(NodeTag type, void *ptr)
struct Bitmapset; /* not to include bitmapset.h here */
struct StringInfoData; /* not to include stringinfo.h here */
-extern void outNode(struct StringInfoData *str, const void *obj);
+extern void outNode(struct StringInfoData *str, const void *obj,
+ bool omitLocation);
extern void outToken(struct StringInfoData *str, const char *s);
extern void outBitmapset(struct StringInfoData *str,
const struct Bitmapset *bms);
extern void outDatum(struct StringInfoData *str, uintptr_t value,
int typlen, bool typbyval);
extern char *nodeToString(const void *obj);
+extern char *nodeToStringNoQLocs(const void *obj);
extern char *bmsToString(const struct Bitmapset *bms);
/*
--
2.40.1
[application/octet-stream] v6-0005-nodeToString-omit-serializing-NULL-datums-in-Cons.patch (1.8K, ../../CAEze2Wj3KOaXDD0+Viv8u5yeg1tNFQZ3it-pdY267A_v57voyA@mail.gmail.com/4-v6-0005-nodeToString-omit-serializing-NULL-datums-in-Cons.patch)
download | inline diff:
From 3158be824bfddfcfa7d32f641016fe678695773d Mon Sep 17 00:00:00 2001
From: Matthias van de Meent <[email protected]>
Date: Thu, 8 Feb 2024 20:24:22 +0100
Subject: [PATCH v6 5/8] nodeToString: omit serializing NULL datums in Const
nodes
This saves some bytes in certain cases, and aligns its serialization conditions
with other field's serialization conditions.
---
src/backend/nodes/outfuncs.c | 8 ++++----
src/backend/nodes/readfuncs.c | 10 ++++++----
2 files changed, 10 insertions(+), 8 deletions(-)
diff --git a/src/backend/nodes/outfuncs.c b/src/backend/nodes/outfuncs.c
index d1395b1940..bc57b7add6 100644
--- a/src/backend/nodes/outfuncs.c
+++ b/src/backend/nodes/outfuncs.c
@@ -504,11 +504,11 @@ _outConst(StringInfo str, const Const *node, bool omitLocation)
WRITE_BOOL_FIELD(constisnull);
WRITE_LOCATION_FIELD(location);
- appendStringInfoString(str, " :constvalue ");
- if (node->constisnull)
- appendStringInfoString(str, "<>");
- else
+ if (!node->constisnull)
+ {
+ appendStringInfoString(str, " :constvalue ");
outDatum(str, node->constvalue, node->constlen, node->constbyval);
+ }
}
static void
diff --git a/src/backend/nodes/readfuncs.c b/src/backend/nodes/readfuncs.c
index 84df1c2868..bcdc39ba18 100644
--- a/src/backend/nodes/readfuncs.c
+++ b/src/backend/nodes/readfuncs.c
@@ -374,11 +374,13 @@ _readConst(void)
READ_BOOL_FIELD(constisnull);
READ_LOCATION_FIELD(location);
- token = pg_strtok(&length); /* skip :constvalue */
- if (local_node->constisnull)
- token = pg_strtok(&length); /* skip "<>" */
- else
+ if ((token = pg_strtok_fieldname(":constvalue", &length)))
local_node->constvalue = readDatum(local_node->constbyval);
+ else
+ {
+ /* value was omitted */
+ Assert(local_node->constisnull);
+ }
READ_DONE();
}
--
2.40.1
[application/octet-stream] v6-0001-pg_node_tree-Omit-serialization-of-fields-with-de.patch (22.8K, ../../CAEze2Wj3KOaXDD0+Viv8u5yeg1tNFQZ3it-pdY267A_v57voyA@mail.gmail.com/5-v6-0001-pg_node_tree-Omit-serialization-of-fields-with-de.patch)
download | inline diff:
From 934716646b5f999ba8e9f67a627fd461f37874b1 Mon Sep 17 00:00:00 2001
From: Matthias van de Meent <[email protected]>
Date: Tue, 2 Jan 2024 23:55:02 +0100
Subject: [PATCH v6 1/8] pg_node_tree: Omit serialization of fields with
default values.
Often, the values in nodes are their default values. By not serializing
those fields and inserting the defaults during deserialization, we
reduce the size of pg_node_tree attributes seen in e.g. pg_rewrite by a
significant factor.
Note: Enum fields are excluded from this for now, as it would hinder
debugging.
In passing, we fix a test that had a strict dependency on the
serialization of pg_node_tree; we now do the checks in a more generic
manner, making it more stable and ensuring its stability in future work.
---
src/backend/nodes/outfuncs.c | 180 +++++++++++++++----
src/backend/nodes/read.c | 58 ++++++
src/backend/nodes/readfuncs.c | 210 ++++++++++++++++------
src/include/nodes/readfuncs.h | 1 +
src/test/regress/expected/rowsecurity.out | 5 +-
src/test/regress/sql/rowsecurity.sql | 5 +-
6 files changed, 372 insertions(+), 87 deletions(-)
diff --git a/src/backend/nodes/outfuncs.c b/src/backend/nodes/outfuncs.c
index 2c30bba212..8ad81be1cd 100644
--- a/src/backend/nodes/outfuncs.c
+++ b/src/backend/nodes/outfuncs.c
@@ -15,6 +15,7 @@
#include "postgres.h"
#include <ctype.h>
+#include <math.h>
#include "access/attnum.h"
#include "common/shortest_dec.h"
@@ -41,94 +42,207 @@ static void outDouble(StringInfo str, double d);
appendStringInfoString(str, nodelabel)
/* Write an integer field (anything written as ":fldname %d") */
+#define WRITE_INT_FIELD_DEFAULT(fldname, default) \
+ do { \
+ if (node->fldname != default) \
+ appendStringInfo(str, " :" CppAsString(fldname) " %d", \
+ node->fldname); \
+ } while (0)
#define WRITE_INT_FIELD(fldname) \
- appendStringInfo(str, " :" CppAsString(fldname) " %d", node->fldname)
+ WRITE_INT_FIELD_DEFAULT(fldname, 0)
/* Write an unsigned integer field (anything written as ":fldname %u") */
+#define WRITE_UINT_FIELD_DEFAULT(fldname, default) \
+ do { \
+ if (node->fldname != default) \
+ appendStringInfo(str, " :" CppAsString(fldname) " %u", \
+ node->fldname); \
+ } while (0)
#define WRITE_UINT_FIELD(fldname) \
- appendStringInfo(str, " :" CppAsString(fldname) " %u", node->fldname)
+ WRITE_UINT_FIELD_DEFAULT(fldname, 0)
/* Write an unsigned integer field (anything written with UINT64_FORMAT) */
+#define WRITE_UINT64_FIELD_DEFAULT(fldname, default) \
+ do { \
+ if (node->fldname != default) \
+ appendStringInfo(str, " :" CppAsString(fldname) " " UINT64_FORMAT, \
+ node->fldname); \
+ } while (0)
#define WRITE_UINT64_FIELD(fldname) \
- appendStringInfo(str, " :" CppAsString(fldname) " " UINT64_FORMAT, \
- node->fldname)
+ WRITE_UINT64_FIELD_DEFAULT(fldname, 0)
/* Write an OID field (don't hard-wire assumption that OID is same as uint) */
+#define WRITE_OID_FIELD_DEFAULT(fldname, default) \
+ do { \
+ if (node->fldname != default) \
+ appendStringInfo(str, " :" CppAsString(fldname) " %u", \
+ node->fldname); \
+ } while (0)
#define WRITE_OID_FIELD(fldname) \
- appendStringInfo(str, " :" CppAsString(fldname) " %u", node->fldname)
+ WRITE_OID_FIELD_DEFAULT(fldname, 0)
/* Write a long-integer field */
+#define WRITE_LONG_FIELD_DEFAULT(fldname, default) \
+ do { \
+ if (node->fldname != default) \
+ appendStringInfo(str, " :" CppAsString(fldname) " %ld", \
+ node->fldname); \
+ } while (0)
#define WRITE_LONG_FIELD(fldname) \
- appendStringInfo(str, " :" CppAsString(fldname) " %ld", node->fldname)
+ WRITE_LONG_FIELD_DEFAULT(fldname, 0)
+
/* Write a char field (ie, one ascii character) */
+#define WRITE_CHAR_FIELD_DEFAULT(fldname, default) \
+ do { \
+ if (node->fldname != default) \
+ { \
+ appendStringInfoString(str, " :" CppAsString(fldname) " "); \
+ outChar(str, node->fldname); \
+ } \
+ } while (0)
#define WRITE_CHAR_FIELD(fldname) \
- (appendStringInfo(str, " :" CppAsString(fldname) " "), \
- outChar(str, node->fldname))
+ WRITE_CHAR_FIELD_DEFAULT(fldname, '\0')
/* Write an enumerated-type field as an integer code */
+#define WRITE_ENUM_FIELD_DEFAULT(fldname, enumtype, default) \
+ do { \
+ if (node->fldname != default) \
+ appendStringInfo(str, " :" CppAsString(fldname) " %d", \
+ (int) node->fldname); \
+ } while (0)
#define WRITE_ENUM_FIELD(fldname, enumtype) \
- appendStringInfo(str, " :" CppAsString(fldname) " %d", \
- (int) node->fldname)
+ do { \
+ appendStringInfo(str, " :" CppAsString(fldname) " %d", \
+ (int) node->fldname); \
+ } while (0)
/* Write a float field (actually, they're double) */
+#define WRITE_FLOAT_FIELD_DEFAULT(fldname, default) \
+ do { \
+ if (node->fldname != default || \
+ signbit(node->fldname) != signbit(default)) \
+ { \
+ appendStringInfoString(str, " :" CppAsString(fldname) " "); \
+ outDouble(str, node->fldname); \
+ } \
+ } while (0)
#define WRITE_FLOAT_FIELD(fldname) \
- (appendStringInfo(str, " :" CppAsString(fldname) " "), \
- outDouble(str, node->fldname))
+ WRITE_FLOAT_FIELD_DEFAULT(fldname, 0.0)
/* Write a boolean field */
+#define WRITE_BOOL_FIELD_DEFAULT(fldname, default) \
+ do { \
+ if (node->fldname != default) \
+ appendStringInfo(str, " :" CppAsString(fldname) " %s", \
+ booltostr(node->fldname)); \
+ } while (0)
#define WRITE_BOOL_FIELD(fldname) \
- appendStringInfo(str, " :" CppAsString(fldname) " %s", \
- booltostr(node->fldname))
+ WRITE_BOOL_FIELD_DEFAULT(fldname, false)
+
+/*
+ * Non-null defaults of by-ref types are exceedingly rare (if not generally
+ * nonexistent), so we don't (yet) have a specialized macro for non-NULL
+ * defaults omission.
+ */
/* Write a character-string (possibly NULL) field */
#define WRITE_STRING_FIELD(fldname) \
- (appendStringInfoString(str, " :" CppAsString(fldname) " "), \
- outToken(str, node->fldname))
+ do { \
+ if (node->fldname != NULL) \
+ { \
+ appendStringInfoString(str, " :" CppAsString(fldname) " "); \
+ outToken(str, node->fldname); \
+ } \
+ } while (0)
/* Write a parse location field (actually same as INT case) */
#define WRITE_LOCATION_FIELD(fldname) \
- appendStringInfo(str, " :" CppAsString(fldname) " %d", node->fldname)
+ do { \
+ if (node->fldname != -1) \
+ appendStringInfo(str, " :" CppAsString(fldname) " %d", \
+ node->fldname); \
+ } while (0)
/* Write a Node field */
#define WRITE_NODE_FIELD(fldname) \
- (appendStringInfoString(str, " :" CppAsString(fldname) " "), \
- outNode(str, node->fldname))
+ do { \
+ if (node->fldname != NULL) \
+ { \
+ appendStringInfoString(str, " :" CppAsString(fldname) " "); \
+ outNode(str, node->fldname); \
+ } \
+ } while (0)
/* Write a bitmapset field */
#define WRITE_BITMAPSET_FIELD(fldname) \
- (appendStringInfoString(str, " :" CppAsString(fldname) " "), \
- outBitmapset(str, node->fldname))
+ do { \
+ if (node->fldname != NULL) \
+ { \
+ appendStringInfoString(str, " :" CppAsString(fldname) " "); \
+ outBitmapset(str, node->fldname); \
+ } \
+ } while (0)
/* Write a variable-length array (not a List) of Node pointers */
#define WRITE_NODE_ARRAY(fldname, len) \
- (appendStringInfoString(str, " :" CppAsString(fldname) " "), \
- writeNodeArray(str, (const Node * const *) node->fldname, len))
+ do { \
+ if (node->fldname != NULL) \
+ { \
+ appendStringInfoString(str, " :" CppAsString(fldname) " "); \
+ writeNodeArray(str, (const Node * const *) node->fldname, len); \
+ } \
+ } while (0)
/* Write a variable-length array of AttrNumber */
#define WRITE_ATTRNUMBER_ARRAY(fldname, len) \
- (appendStringInfoString(str, " :" CppAsString(fldname) " "), \
- writeAttrNumberCols(str, node->fldname, len))
+ do { \
+ if (node->fldname != NULL) \
+ { \
+ appendStringInfoString(str, " :" CppAsString(fldname) " "); \
+ writeAttrNumberCols(str, node->fldname, len); \
+ } \
+ } while (0)
/* Write a variable-length array of Oid */
#define WRITE_OID_ARRAY(fldname, len) \
- (appendStringInfoString(str, " :" CppAsString(fldname) " "), \
- writeOidCols(str, node->fldname, len))
+ do { \
+ if (node->fldname != NULL) \
+ { \
+ appendStringInfoString(str, " :" CppAsString(fldname) " "); \
+ writeOidCols(str, node->fldname, len); \
+ } \
+ } while (0)
/* Write a variable-length array of Index */
#define WRITE_INDEX_ARRAY(fldname, len) \
- (appendStringInfoString(str, " :" CppAsString(fldname) " "), \
- writeIndexCols(str, node->fldname, len))
+ do { \
+ if (node->fldname != NULL) \
+ { \
+ appendStringInfoString(str, " :" CppAsString(fldname) " "); \
+ writeIndexCols(str, node->fldname, len); \
+ } \
+ } while (0)
/* Write a variable-length array of int */
#define WRITE_INT_ARRAY(fldname, len) \
- (appendStringInfoString(str, " :" CppAsString(fldname) " "), \
- writeIntCols(str, node->fldname, len))
+ do { \
+ if (node->fldname != NULL) \
+ { \
+ appendStringInfoString(str, " :" CppAsString(fldname) " "); \
+ writeIntCols(str, node->fldname, len); \
+ } \
+ } while (0)
/* Write a variable-length array of bool */
#define WRITE_BOOL_ARRAY(fldname, len) \
- (appendStringInfoString(str, " :" CppAsString(fldname) " "), \
- writeBoolCols(str, node->fldname, len))
+ do { \
+ if (node->fldname != NULL) \
+ { \
+ appendStringInfoString(str, " :" CppAsString(fldname) " "); \
+ writeBoolCols(str, node->fldname, len); \
+ } \
+ } while (0)
#define booltostr(x) ((x) ? "true" : "false")
diff --git a/src/backend/nodes/read.c b/src/backend/nodes/read.c
index 4eb42445c5..cffd913ba6 100644
--- a/src/backend/nodes/read.c
+++ b/src/backend/nodes/read.c
@@ -205,6 +205,64 @@ pg_strtok(int *length)
return ret_str;
}
+/*
+ * Check if the next token is the expected field name, and if so,
+ * consume and return it.
+ *
+ * It handles similar to pg_strtok, except that it is special-cased for user-
+ * provided field names, rather than arbitrary tokens.
+ */
+const char *
+pg_strtok_fieldname(const char *fldname, int *length)
+{
+ const char *local_str; /* working pointer to string */
+ int expect_len;
+ char next_char;
+
+ /* skip global state to the next token */
+ while (*pg_strtok_ptr == ' ' ||
+ *pg_strtok_ptr == '\n' ||
+ *pg_strtok_ptr == '\t')
+ pg_strtok_ptr++;
+
+ local_str = pg_strtok_ptr;
+
+ if (*local_str != ':')
+ return false; /* not a field name */
+
+ expect_len = strlen(fldname);
+
+ Assert(expect_len > 0);
+
+ /* check if the next few bytes match the token */
+ if (strncmp(local_str, fldname, expect_len) != 0)
+ return false;
+
+ next_char = local_str[expect_len];
+
+ /*
+ * Check that the token was actually terminated at the end of the
+ * expected token with a character that is a separate token.
+ * Otherwise, we'd get positive matches for mathing the token of "is"
+ * against a local_str of "isn't", which is clearly wrong.
+ */
+ if (next_char == '\0' ||
+ next_char == ' ' ||
+ next_char == '\n' ||
+ next_char == '\t' ||
+ next_char == '(' ||
+ next_char == ')' ||
+ next_char == '{' ||
+ next_char == '}')
+ {
+ pg_strtok_ptr = local_str + expect_len;
+ *length = expect_len;
+
+ return local_str;
+ }
+ return NULL;
+}
+
/*
* debackslash -
* create a palloc'd string holding the given token.
diff --git a/src/backend/nodes/readfuncs.c b/src/backend/nodes/readfuncs.c
index b1e2f2b440..0c415edb31 100644
--- a/src/backend/nodes/readfuncs.c
+++ b/src/backend/nodes/readfuncs.c
@@ -57,111 +57,217 @@
READ_TEMP_LOCALS()
/* Read an integer field (anything written as ":fldname %d") */
+#define READ_INT_FIELD_DEFAULT(fldname, default_value) \
+ do { \
+ if ((token = pg_strtok_fieldname(":" CppAsString(fldname), &length))) \
+ { \
+ token = pg_strtok(&length); /* get field value */ \
+ local_node->fldname = atoi(token); \
+ } \
+ else \
+ local_node->fldname = default_value; \
+ } while (0)
#define READ_INT_FIELD(fldname) \
- token = pg_strtok(&length); /* skip :fldname */ \
- token = pg_strtok(&length); /* get field value */ \
- local_node->fldname = atoi(token)
+ READ_INT_FIELD_DEFAULT(fldname, 0)
/* Read an unsigned integer field (anything written as ":fldname %u") */
+#define READ_UINT_FIELD_DEFAULT(fldname, default_value) \
+ do { \
+ if ((token = pg_strtok_fieldname(":" CppAsString(fldname), &length))) \
+ { \
+ token = pg_strtok(&length); /* get field value */ \
+ local_node->fldname = atoui(token); \
+ } \
+ else \
+ local_node->fldname = default_value; \
+ } while (0)
#define READ_UINT_FIELD(fldname) \
- token = pg_strtok(&length); /* skip :fldname */ \
- token = pg_strtok(&length); /* get field value */ \
- local_node->fldname = atoui(token)
+ READ_UINT_FIELD_DEFAULT(fldname, 0)
/* Read an unsigned integer field (anything written using UINT64_FORMAT) */
+#define READ_UINT64_FIELD_DEFAULT(fldname, default_value) \
+ do { \
+ if ((token = pg_strtok_fieldname(":" CppAsString(fldname), &length))) \
+ { \
+ token = pg_strtok(&length); /* get field value */ \
+ local_node->fldname = strtou64(token, NULL, 10); \
+ } \
+ else \
+ local_node->fldname = default_value; \
+ } while (0)
#define READ_UINT64_FIELD(fldname) \
- token = pg_strtok(&length); /* skip :fldname */ \
- token = pg_strtok(&length); /* get field value */ \
- local_node->fldname = strtou64(token, NULL, 10)
+ READ_UINT64_FIELD_DEFAULT(fldname, 0)
/* Read a long integer field (anything written as ":fldname %ld") */
+#define READ_LONG_FIELD_DEFAULT(fldname, default_value) \
+ do { \
+ if ((token = pg_strtok_fieldname(":" CppAsString(fldname), &length))) \
+ { \
+ token = pg_strtok(&length); /* get field value */ \
+ local_node->fldname = atol(token); \
+ } \
+ else \
+ local_node->fldname = default_value; \
+ } while (0)
#define READ_LONG_FIELD(fldname) \
- token = pg_strtok(&length); /* skip :fldname */ \
- token = pg_strtok(&length); /* get field value */ \
- local_node->fldname = atol(token)
+ READ_LONG_FIELD_DEFAULT(fldname, 0)
/* Read an OID field (don't hard-wire assumption that OID is same as uint) */
+#define READ_OID_FIELD_DEFAULT(fldname, default_value) \
+ do { \
+ if ((token = pg_strtok_fieldname(":" CppAsString(fldname), &length))) \
+ { \
+ token = pg_strtok(&length); /* get field value */ \
+ local_node->fldname = atooid(token); \
+ } \
+ else \
+ local_node->fldname = default_value; \
+ } while (0)
#define READ_OID_FIELD(fldname) \
- token = pg_strtok(&length); /* skip :fldname */ \
- token = pg_strtok(&length); /* get field value */ \
- local_node->fldname = atooid(token)
+ READ_OID_FIELD_DEFAULT(fldname, 0)
/* Read a char field (ie, one ascii character) */
+#define READ_CHAR_FIELD_DEFAULT(fldname, default_value) \
+ do { \
+ if ((token = pg_strtok_fieldname(":" CppAsString(fldname), &length))) \
+ { \
+ token = pg_strtok(&length); /* get field value */ \
+ /* avoid overhead of calling debackslash() for one char */ \
+ local_node->fldname = (length == 0) ? '\0' : (token[0] == '\\' ? token[1] : token[0]); \
+ } \
+ else \
+ local_node->fldname = default_value; \
+ } while (0)
#define READ_CHAR_FIELD(fldname) \
- token = pg_strtok(&length); /* skip :fldname */ \
- token = pg_strtok(&length); /* get field value */ \
- /* avoid overhead of calling debackslash() for one char */ \
- local_node->fldname = (length == 0) ? '\0' : (token[0] == '\\' ? token[1] : token[0])
+ READ_CHAR_FIELD_DEFAULT(fldname, '\0')
/* Read an enumerated-type field that was written as an integer code */
#define READ_ENUM_FIELD(fldname, enumtype) \
- token = pg_strtok(&length); /* skip :fldname */ \
- token = pg_strtok(&length); /* get field value */ \
- local_node->fldname = (enumtype) atoi(token)
+ do { \
+ token = pg_strtok_fieldname(":" CppAsString(fldname), &length); \
+ Assert(token != NULL); \
+ token = pg_strtok(&length); /* get field value */ \
+ local_node->fldname = (enumtype) atoi(token); \
+ } while (0)
/* Read a float field */
+#define READ_FLOAT_FIELD_DEFAULT(fldname, default_value) \
+ do { \
+ if ((token = pg_strtok_fieldname(":" CppAsString(fldname), &length))) \
+ { \
+ token = pg_strtok(&length); /* get field value */ \
+ local_node->fldname = atof(token); \
+ } \
+ else \
+ local_node->fldname = default_value; \
+ } while (0)
#define READ_FLOAT_FIELD(fldname) \
- token = pg_strtok(&length); /* skip :fldname */ \
- token = pg_strtok(&length); /* get field value */ \
- local_node->fldname = atof(token)
+ READ_FLOAT_FIELD_DEFAULT(fldname, 0.0)
/* Read a boolean field */
+#define READ_BOOL_FIELD_DEFAULT(fldname, default_value) \
+ do { \
+ if ((token = pg_strtok_fieldname(":" CppAsString(fldname), &length))) \
+ { \
+ token = pg_strtok(&length); /* get field value */ \
+ local_node->fldname = strtobool(token); \
+ } \
+ else \
+ local_node->fldname = default_value; \
+ } while (0)
#define READ_BOOL_FIELD(fldname) \
- token = pg_strtok(&length); /* skip :fldname */ \
- token = pg_strtok(&length); /* get field value */ \
- local_node->fldname = strtobool(token)
+ READ_BOOL_FIELD_DEFAULT(fldname, false)
/* Read a character-string field */
+/* see WRITE_STRING_FIELD in outfuncs.c */
#define READ_STRING_FIELD(fldname) \
- token = pg_strtok(&length); /* skip :fldname */ \
- token = pg_strtok(&length); /* get field value */ \
- local_node->fldname = nullable_string(token, length)
+ do { \
+ if ((token = pg_strtok_fieldname(":" CppAsString(fldname), &length))) \
+ { \
+ token = pg_strtok(&length); /* get field value */ \
+ local_node->fldname = nullable_string(token, length); \
+ } \
+ else \
+ local_node->fldname = NULL; \
+ } while (0)
/* Read a parse location field (and possibly throw away the value) */
#ifdef WRITE_READ_PARSE_PLAN_TREES
#define READ_LOCATION_FIELD(fldname) \
- token = pg_strtok(&length); /* skip :fldname */ \
- token = pg_strtok(&length); /* get field value */ \
- local_node->fldname = restore_location_fields ? atoi(token) : -1
+ do { \
+ if ((token = pg_strtok_fieldname(":" CppAsString(fldname), &length))) \
+ { \
+ token = pg_strtok(&length); /* get field value */ \
+ local_node->fldname = restore_location_fields ? atoi(token) : -1; \
+ } \
+ local_node->fldname = -1; \
+ } while (0)
#else
#define READ_LOCATION_FIELD(fldname) \
- token = pg_strtok(&length); /* skip :fldname */ \
- token = pg_strtok(&length); /* get field value */ \
- (void) token; /* in case not used elsewhere */ \
- local_node->fldname = -1 /* set field to "unknown" */
+ do { \
+ if ((token = pg_strtok_fieldname(":" CppAsString(fldname), &length))) \
+ { \
+ token = pg_strtok(&length); /* the field value is useless */ \
+ (void) token; \
+ } \
+ local_node->fldname = -1; \
+ } while (0)
#endif
+
/* Read a Node field */
#define READ_NODE_FIELD(fldname) \
- token = pg_strtok(&length); /* skip :fldname */ \
- (void) token; /* in case not used elsewhere */ \
- local_node->fldname = nodeRead(NULL, 0)
+ do { \
+ if ((token = pg_strtok_fieldname(":" CppAsString(fldname), &length))) \
+ local_node->fldname = nodeRead(NULL, 0); \
+ else \
+ local_node->fldname = NULL; \
+ } while (0)
/* Read a bitmapset field */
#define READ_BITMAPSET_FIELD(fldname) \
- token = pg_strtok(&length); /* skip :fldname */ \
- (void) token; /* in case not used elsewhere */ \
- local_node->fldname = _readBitmapset()
+ do { \
+ if ((token = pg_strtok_fieldname(":" CppAsString(fldname), &length))) \
+ local_node->fldname = _readBitmapset(); \
+ else \
+ local_node->fldname = NULL; \
+ } while (0)
/* Read an attribute number array */
#define READ_ATTRNUMBER_ARRAY(fldname, len) \
- token = pg_strtok(&length); /* skip :fldname */ \
- local_node->fldname = readAttrNumberCols(len)
+ do { \
+ if ((token = pg_strtok_fieldname(":" CppAsString(fldname), &length))) \
+ local_node->fldname = readAttrNumberCols(len); \
+ else \
+ local_node->fldname = NULL; \
+ } while (0)
/* Read an oid array */
#define READ_OID_ARRAY(fldname, len) \
- token = pg_strtok(&length); /* skip :fldname */ \
- local_node->fldname = readOidCols(len)
+ do { \
+ if ((token = pg_strtok_fieldname(":" CppAsString(fldname), &length))) \
+ local_node->fldname = readOidCols(len); \
+ else \
+ local_node->fldname = NULL; \
+ } while (0)
/* Read an int array */
#define READ_INT_ARRAY(fldname, len) \
- token = pg_strtok(&length); /* skip :fldname */ \
- local_node->fldname = readIntCols(len)
+ do { \
+ if ((token = pg_strtok_fieldname(":" CppAsString(fldname), &length))) \
+ local_node->fldname = readIntCols(len); \
+ else \
+ local_node->fldname = NULL; \
+ } while (0)
/* Read a bool array */
#define READ_BOOL_ARRAY(fldname, len) \
- token = pg_strtok(&length); /* skip :fldname */ \
- local_node->fldname = readBoolCols(len)
+ do { \
+ if ((token = pg_strtok_fieldname(":" CppAsString(fldname), &length))) \
+ local_node->fldname = readBoolCols(len); \
+ else \
+ local_node->fldname = NULL; \
+ } while (0)
/* Routine exit */
#define READ_DONE() \
diff --git a/src/include/nodes/readfuncs.h b/src/include/nodes/readfuncs.h
index 8466038ed0..0579c62973 100644
--- a/src/include/nodes/readfuncs.h
+++ b/src/include/nodes/readfuncs.h
@@ -27,6 +27,7 @@ extern PGDLLIMPORT bool restore_location_fields;
* prototypes for functions in read.c (the lisp token parser)
*/
extern const char *pg_strtok(int *length);
+extern const char *pg_strtok_fieldname(const char *fldname, int *length);
extern char *debackslash(const char *token, int length);
extern void *nodeRead(const char *token, int tok_len);
diff --git a/src/test/regress/expected/rowsecurity.out b/src/test/regress/expected/rowsecurity.out
index 6988128aa4..a69aa40f82 100644
--- a/src/test/regress/expected/rowsecurity.out
+++ b/src/test/regress/expected/rowsecurity.out
@@ -3937,7 +3937,10 @@ CREATE TABLE coll_t (c) AS VALUES ('bar'::text);
CREATE POLICY coll_p ON coll_t USING (c < ('foo'::text COLLATE "C"));
ALTER TABLE coll_t ENABLE ROW LEVEL SECURITY;
GRANT SELECT ON coll_t TO regress_rls_alice;
-SELECT (string_to_array(polqual, ':'))[7] AS inputcollid FROM pg_policy WHERE polrelid = 'coll_t'::regclass;
+SELECT split AS inputcollid
+FROM pg_policy,
+ lateral unnest(string_to_array(polqual, ':')) as split
+WHERE polrelid = 'coll_t'::regclass and split LIKE '%inputcollid%';
inputcollid
------------------
inputcollid 950
diff --git a/src/test/regress/sql/rowsecurity.sql b/src/test/regress/sql/rowsecurity.sql
index dec7340538..cb7a4c776b 100644
--- a/src/test/regress/sql/rowsecurity.sql
+++ b/src/test/regress/sql/rowsecurity.sql
@@ -1732,7 +1732,10 @@ CREATE TABLE coll_t (c) AS VALUES ('bar'::text);
CREATE POLICY coll_p ON coll_t USING (c < ('foo'::text COLLATE "C"));
ALTER TABLE coll_t ENABLE ROW LEVEL SECURITY;
GRANT SELECT ON coll_t TO regress_rls_alice;
-SELECT (string_to_array(polqual, ':'))[7] AS inputcollid FROM pg_policy WHERE polrelid = 'coll_t'::regclass;
+SELECT split AS inputcollid
+FROM pg_policy,
+ lateral unnest(string_to_array(polqual, ':')) as split
+WHERE polrelid = 'coll_t'::regclass and split LIKE '%inputcollid%';
SET SESSION AUTHORIZATION regress_rls_alice;
SELECT * FROM coll_t;
ROLLBACK;
--
2.40.1
[application/octet-stream] v6-0008-nodeToString-omit-serializing-0s-in-enum-typed-fi.patch (2.1K, ../../CAEze2Wj3KOaXDD0+Viv8u5yeg1tNFQZ3it-pdY267A_v57voyA@mail.gmail.com/6-v6-0008-nodeToString-omit-serializing-0s-in-enum-typed-fi.patch)
download | inline diff:
From 7beb1186f3b58b0f76dae05cfb08a4ce48694518 Mon Sep 17 00:00:00 2001
From: Matthias van de Meent <[email protected]>
Date: Mon, 19 Feb 2024 13:53:00 +0100
Subject: [PATCH v6 8/8] nodeToString: omit serializing 0s in enum-typed
fields.
---
src/backend/nodes/outfuncs.c | 5 +----
src/backend/nodes/readfuncs.c | 16 +++++++++++-----
2 files changed, 12 insertions(+), 9 deletions(-)
diff --git a/src/backend/nodes/outfuncs.c b/src/backend/nodes/outfuncs.c
index e64f3c5975..0269daf484 100644
--- a/src/backend/nodes/outfuncs.c
+++ b/src/backend/nodes/outfuncs.c
@@ -112,10 +112,7 @@ static void outDouble(StringInfo str, double d);
(int) node->fldname); \
} while (0)
#define WRITE_ENUM_FIELD(fldname, enumtype) \
- do { \
- appendStringInfo(str, " :" CppAsString(fldname) " %d", \
- (int) node->fldname); \
- } while (0)
+ WRITE_ENUM_FIELD_DEFAULT(fldname, enumtype, 0)
/* Write a float field (actually, they're double) */
#define WRITE_FLOAT_FIELD_DEFAULT(fldname, default) \
diff --git a/src/backend/nodes/readfuncs.c b/src/backend/nodes/readfuncs.c
index 5e082a2c37..c401edc686 100644
--- a/src/backend/nodes/readfuncs.c
+++ b/src/backend/nodes/readfuncs.c
@@ -142,13 +142,19 @@
READ_CHAR_FIELD_DEFAULT(fldname, '\0')
/* Read an enumerated-type field that was written as an integer code */
-#define READ_ENUM_FIELD(fldname, enumtype) \
+
+#define READ_ENUM_FIELD_DEFAULT(fldname, enumtype, default) \
do { \
- token = pg_strtok_fieldname(":" CppAsString(fldname), &length); \
- Assert(token != NULL); \
- token = pg_strtok(&length); /* get field value */ \
- local_node->fldname = (enumtype) atoi(token); \
+ if ((token = pg_strtok_fieldname(":" CppAsString(fldname), &length))) \
+ { \
+ token = pg_strtok(&length); /* get field value */ \
+ local_node->fldname = (enumtype) atoi(token); \
+ } \
+ else \
+ local_node->fldname = (enumtype) default; \
} while (0)
+#define READ_ENUM_FIELD(fldname, enumtype) \
+ READ_ENUM_FIELD_DEFAULT(fldname, enumtype, 0)
/* Read a float field */
#define READ_FLOAT_FIELD_DEFAULT(fldname, default_value) \
--
2.40.1
[application/octet-stream] v6-0007-gen_node_support.pl-Optimize-serialization-of-fie.patch (9.2K, ../../CAEze2Wj3KOaXDD0+Viv8u5yeg1tNFQZ3it-pdY267A_v57voyA@mail.gmail.com/7-v6-0007-gen_node_support.pl-Optimize-serialization-of-fie.patch)
download | inline diff:
From 089305ddbc433177e68cc173a396fc7392e4c9ed Mon Sep 17 00:00:00 2001
From: Matthias van de Meent <[email protected]>
Date: Mon, 12 Feb 2024 17:12:02 +0100
Subject: [PATCH v6 7/8] gen_node_support.pl: Optimize serialization of fields
with copied values
The Var node's [syn] fields often contain the same data as their non-syn
counterparts. We invent a new pg_node_attr()ibute that represents this
relation, which allows us to omit these fields from serialization when they
are indeed copies of the original fields.
---
src/backend/nodes/gen_node_support.pl | 94 ++++++++++++++++++---------
src/backend/nodes/outfuncs.c | 3 +
src/backend/nodes/readfuncs.c | 3 +
src/include/nodes/primnodes.h | 4 +-
4 files changed, 70 insertions(+), 34 deletions(-)
diff --git a/src/backend/nodes/gen_node_support.pl b/src/backend/nodes/gen_node_support.pl
index fda8c3a494..66661a3881 100644
--- a/src/backend/nodes/gen_node_support.pl
+++ b/src/backend/nodes/gen_node_support.pl
@@ -363,6 +363,10 @@ foreach my $infile (@ARGV)
{
$manual_nodetag_number{$in_struct} = $1;
}
+ elsif ($attr =~ /^default_ref\(([\w\d."'-]+)\)$/)
+ {
+ # Unused at the node level
+ }
else
{
die
@@ -437,7 +441,7 @@ foreach my $infile (@ARGV)
}
# normal struct field
elsif ($line =~
- /^\s*(.+)\s*\b(\w+)(\[[\w\s+]+\])?\s*(?:pg_node_attr\(([\w(), ]*)\))?;/
+ /^\s*(.+)\s*\b(\w+)(\[[\w\s+]+\])?\s*(?:pg_node_attr\(([\w\d(), ."'-]*)\))?;/
)
{
if ($is_node_struct)
@@ -469,6 +473,7 @@ foreach my $infile (@ARGV)
if ( $attr !~ /^array_size\(\w+\)$/
&& $attr !~ /^copy_as\(\w+\)$/
&& $attr !~ /^read_as\(\w+\)$/
+ && $attr !~ /^default_ref\([\w\d."'-]+\)$/
&& !elem $attr,
qw(copy_as_scalar
equal_as_scalar
@@ -495,7 +500,7 @@ foreach my $infile (@ARGV)
}
# function pointer field
elsif ($line =~
- /^\s*([\w\s*]+)\s*\(\*(\w+)\)\s*\((.*)\)\s*(?:pg_node_attr\(([\w(), ]*)\))?;/
+ /^\s*([\w\s*]+)\s*\(\*(\w+)\)\s*\((.*)\)\s*(?:pg_node_attr\(([\w\d(), ."'-]*)\))?;/
)
{
if ($is_node_struct)
@@ -970,6 +975,15 @@ _read${n}(void)
my $array_size_field;
my $read_as_field;
my $read_write_ignore = 0;
+ # Type read/write macro suffix, "" for normal use, or "_DEFAULT" when
+ # value argument.
+ my $s = "";
+ # Default parameter to read/write macro. Includes comma separator so
+ # that MACRO_NAME$s($fieldname$defaultparam) is the full complete
+ # read/write expression for essentially all types.
+ # Note that this (currently) only works for scalar values.
+ my $d = "";
+
foreach my $a (@a)
{
if ($a =~ /^array_size\(([\w.]+)\)$/)
@@ -988,6 +1002,19 @@ _read${n}(void)
{
$read_write_ignore = 1;
}
+ elsif ($a =~ /^default_ref\(([\w\d+."'-]+)\)$/)
+ {
+ $s = "_DEFAULT";
+ $d = ", NODE_FIELD($1)";
+ }
+ }
+
+ if ($s eq "_DEFAULT")
+ {
+ die "custom defaults for non-scalar fields are not supported\n\tat $n.$f"
+ unless (elem $t, @scalar_types or elem $t, @enum_types);
+ die "custom defaults for Location fields are not supported\n\tat $n.$f"
+ if ($t eq "Location");
}
if ($read_write_ignore)
@@ -1008,8 +1035,8 @@ _read${n}(void)
# select instructions by field type
if ($t eq 'bool')
{
- print $off "\tWRITE_BOOL_FIELD($f);\n";
- print $rff "\tREAD_BOOL_FIELD($f);\n" unless $no_read;
+ print $off "\tWRITE_BOOL_FIELD$s($f$d);\n";
+ print $rff "\tREAD_BOOL_FIELD$s($f$d);\n" unless $no_read;
}
elsif ($t eq 'Location')
{
@@ -1018,8 +1045,11 @@ _read${n}(void)
}
elsif ($t eq 'TypMod')
{
- print $off "\tWRITE_INT_FIELD_DEFAULT($f, -1);\n";
- print $rff "\tREAD_INT_FIELD_DEFAULT($f, -1);\n" unless $no_read;
+ # The default value of a TypMod fields is -1, rather than 0
+ # for normal int fields.
+ $d = ", -1" if ($d eq "");
+ print $off "\tWRITE_INT_FIELD_DEFAULT($f$d);\n";
+ print $rff "\tREAD_INT_FIELD_DEFAULT($f$d);\n" unless $no_read;
}
elsif ($t eq 'int'
|| $t eq 'int16'
@@ -1027,8 +1057,8 @@ _read${n}(void)
|| $t eq 'AttrNumber'
|| $t eq 'StrategyNumber')
{
- print $off "\tWRITE_INT_FIELD($f);\n";
- print $rff "\tREAD_INT_FIELD($f);\n" unless $no_read;
+ print $off "\tWRITE_INT_FIELD$s($f$d);\n";
+ print $rff "\tREAD_INT_FIELD$s($f$d);\n" unless $no_read;
}
elsif ($t eq 'uint32'
|| $t eq 'bits32'
@@ -1036,56 +1066,56 @@ _read${n}(void)
|| $t eq 'Index'
|| $t eq 'SubTransactionId')
{
- print $off "\tWRITE_UINT_FIELD($f);\n";
- print $rff "\tREAD_UINT_FIELD($f);\n" unless $no_read;
+ print $off "\tWRITE_UINT_FIELD$s($f$d);\n";
+ print $rff "\tREAD_UINT_FIELD$s($f$d);\n" unless $no_read;
}
elsif ($t eq 'uint64'
|| $t eq 'AclMode')
{
- print $off "\tWRITE_UINT64_FIELD($f);\n";
- print $rff "\tREAD_UINT64_FIELD($f);\n" unless $no_read;
+ print $off "\tWRITE_UINT64_FIELD$s($f$d);\n";
+ print $rff "\tREAD_UINT64_FIELD$s($f$d);\n" unless $no_read;
}
elsif ($t eq 'Oid' || $t eq 'RelFileNumber')
{
- print $off "\tWRITE_OID_FIELD($f);\n";
- print $rff "\tREAD_OID_FIELD($f);\n" unless $no_read;
+ print $off "\tWRITE_OID_FIELD$s($f$d);\n";
+ print $rff "\tREAD_OID_FIELD$s($f$d);\n" unless $no_read;
}
elsif ($t eq 'long')
{
- print $off "\tWRITE_LONG_FIELD($f);\n";
- print $rff "\tREAD_LONG_FIELD($f);\n" unless $no_read;
+ print $off "\tWRITE_LONG_FIELD$s($f$d);\n";
+ print $rff "\tREAD_LONG_FIELD$s($f$d);\n" unless $no_read;
}
elsif ($t eq 'char')
{
- print $off "\tWRITE_CHAR_FIELD($f);\n";
- print $rff "\tREAD_CHAR_FIELD($f);\n" unless $no_read;
+ print $off "\tWRITE_CHAR_FIELD$s($f$d);\n";
+ print $rff "\tREAD_CHAR_FIELD$s($f$d);\n" unless $no_read;
}
elsif ($t eq 'double')
{
- print $off "\tWRITE_FLOAT_FIELD($f);\n";
- print $rff "\tREAD_FLOAT_FIELD($f);\n" unless $no_read;
+ print $off "\tWRITE_FLOAT_FIELD$s($f$d);\n";
+ print $rff "\tREAD_FLOAT_FIELD$s($f$d);\n" unless $no_read;
}
elsif ($t eq 'Cardinality')
{
- print $off "\tWRITE_FLOAT_FIELD($f);\n";
- print $rff "\tREAD_FLOAT_FIELD($f);\n" unless $no_read;
+ print $off "\tWRITE_FLOAT_FIELD$s($f$d);\n";
+ print $rff "\tREAD_FLOAT_FIELD$s($f$d);\n" unless $no_read;
}
elsif ($t eq 'Cost')
{
- print $off "\tWRITE_FLOAT_FIELD($f);\n";
- print $rff "\tREAD_FLOAT_FIELD($f);\n" unless $no_read;
+ print $off "\tWRITE_FLOAT_FIELD$s($f$d);\n";
+ print $rff "\tREAD_FLOAT_FIELD$s($f$d);\n" unless $no_read;
}
elsif ($t eq 'QualCost')
{
- print $off "\tWRITE_FLOAT_FIELD($f.startup);\n";
- print $off "\tWRITE_FLOAT_FIELD($f.per_tuple);\n";
- print $rff "\tREAD_FLOAT_FIELD($f.startup);\n" unless $no_read;
- print $rff "\tREAD_FLOAT_FIELD($f.per_tuple);\n" unless $no_read;
+ print $off "\tWRITE_FLOAT_FIELD$s($f.startup$d);\n";
+ print $off "\tWRITE_FLOAT_FIELD$s($f.per_tuple$d);\n";
+ print $rff "\tREAD_FLOAT_FIELD$s($f.startup$d);\n" unless $no_read;
+ print $rff "\tREAD_FLOAT_FIELD$s($f.per_tuple$d);\n" unless $no_read;
}
elsif ($t eq 'Selectivity')
{
- print $off "\tWRITE_FLOAT_FIELD($f);\n";
- print $rff "\tREAD_FLOAT_FIELD($f);\n" unless $no_read;
+ print $off "\tWRITE_FLOAT_FIELD$s($f$d);\n";
+ print $rff "\tREAD_FLOAT_FIELD$s($f$d);\n" unless $no_read;
}
elsif ($t eq 'char*')
{
@@ -1099,8 +1129,8 @@ _read${n}(void)
}
elsif (elem $t, @enum_types)
{
- print $off "\tWRITE_ENUM_FIELD($f, $t);\n";
- print $rff "\tREAD_ENUM_FIELD($f, $t);\n" unless $no_read;
+ print $off "\tWRITE_ENUM_FIELD$s($f, $t$d);\n";
+ print $rff "\tREAD_ENUM_FIELD$s($f, $t$d);\n" unless $no_read;
}
# arrays of scalar types
elsif ($t =~ /^(\w+)(\*|\[\w+\])$/ and elem $1, @scalar_types)
diff --git a/src/backend/nodes/outfuncs.c b/src/backend/nodes/outfuncs.c
index cfafdaa033..e64f3c5975 100644
--- a/src/backend/nodes/outfuncs.c
+++ b/src/backend/nodes/outfuncs.c
@@ -245,6 +245,9 @@ static void outDouble(StringInfo str, double d);
} \
} while (0)
+/* Accessing a node's field goes like this */
+#define NODE_FIELD(fldname) (node->fldname)
+
#define booltostr(x) ((x) ? "true" : "false")
diff --git a/src/backend/nodes/readfuncs.c b/src/backend/nodes/readfuncs.c
index d77dd30f34..5e082a2c37 100644
--- a/src/backend/nodes/readfuncs.c
+++ b/src/backend/nodes/readfuncs.c
@@ -269,6 +269,9 @@
local_node->fldname = NULL; \
} while (0)
+/* Accessing a node's field goes like this */
+#define NODE_FIELD(fldname) (local_node->fldname)
+
/* Routine exit */
#define READ_DONE() \
return local_node
diff --git a/src/include/nodes/primnodes.h b/src/include/nodes/primnodes.h
index 94282497d7..9dd60ad5c5 100644
--- a/src/include/nodes/primnodes.h
+++ b/src/include/nodes/primnodes.h
@@ -274,9 +274,9 @@ typedef struct Var
* their varno/varattno match.
*/
/* syntactic relation index (0 if unknown) */
- Index varnosyn pg_node_attr(equal_ignore, query_jumble_ignore);
+ Index varnosyn pg_node_attr(equal_ignore, query_jumble_ignore, default_ref(varno));
/* syntactic attribute number */
- AttrNumber varattnosyn pg_node_attr(equal_ignore, query_jumble_ignore);
+ AttrNumber varattnosyn pg_node_attr(equal_ignore, query_jumble_ignore, default_ref(varattno));
/* token location, or -1 if unknown */
Location location;
--
2.40.1
[application/octet-stream] v6-0006-nodeToString-Apply-RLE-on-Bitmapset-and-numeric-L.patch (7.9K, ../../CAEze2Wj3KOaXDD0+Viv8u5yeg1tNFQZ3it-pdY267A_v57voyA@mail.gmail.com/8-v6-0006-nodeToString-Apply-RLE-on-Bitmapset-and-numeric-L.patch)
download | inline diff:
From 3e08dbfd214a66125d44648563875004c792708b Mon Sep 17 00:00:00 2001
From: Matthias van de Meent <[email protected]>
Date: Sat, 10 Feb 2024 00:57:19 +0100
Subject: [PATCH v6 6/8] nodeToString: Apply RLE on Bitmapset and numeric List
types
This reduces the size of full serialized queries in pg_rewrite by several %,
reducing overhead in the system.
---
src/backend/nodes/outfuncs.c | 96 +++++++++++++++++++++++++++++++----
src/backend/nodes/read.c | 53 +++++++++++++++++--
src/backend/nodes/readfuncs.c | 16 +++++-
3 files changed, 149 insertions(+), 16 deletions(-)
diff --git a/src/backend/nodes/outfuncs.c b/src/backend/nodes/outfuncs.c
index bc57b7add6..cfafdaa033 100644
--- a/src/backend/nodes/outfuncs.c
+++ b/src/backend/nodes/outfuncs.c
@@ -383,23 +383,64 @@ writeNodeArray(StringInfo str, const Node *const *arr, int len,
/*
* Print a List.
+ *
+ * Notes:
+ * NodeList is formatted as (<node1> <node2> ...)
+ *
+ * OidList is formatted as (o int +int int int ...), with +int indicating N
+ * successive identical values.
+ *
+ * IntList and XidList are formatted as (i/x int +int int int ...), with +int
+ * indicating N successively increasing values. (i 9 +3) is thus equivalent
+ * to (i 9 10 11 12).
*/
static void
_outList(StringInfo str, const List *node, bool omitLocation)
{
const ListCell *lc;
+ union LCSurrogate {
+ int32 i;
+ Oid o;
+ TransactionId x;
+ } previous = { .i = 0};
+ int run = 0;
+ int run_delta;
+ const char *fmt;
appendStringInfoChar(str, '(');
if (IsA(node, IntList))
+ {
appendStringInfoChar(str, 'i');
+ fmt = " %d";
+ run_delta = 1;
+ }
else if (IsA(node, OidList))
+ {
appendStringInfoChar(str, 'o');
+ fmt = " %u";
+ run_delta = 0;
+ }
else if (IsA(node, XidList))
+ {
appendStringInfoChar(str, 'x');
+ fmt = " %u";
+ run_delta = 1;
+ }
+ else if (IsA(node, List))
+ {
+ /* silence the compiler about uninitialized variables */
+ fmt = "";
+ run_delta = 0;
+ }
+ else
+ elog(ERROR, "unrecognized list node type: %d",
+ (int) node->type);
foreach(lc, node)
{
+ union LCSurrogate val = {.i = 0};
+
/*
* For the sake of backward compatibility, we emit a slightly
* different whitespace format for lists of nodes vs. other types of
@@ -410,26 +451,41 @@ _outList(StringInfo str, const List *node, bool omitLocation)
outNode(str, lfirst(lc), omitLocation);
if (lnext(node, lc))
appendStringInfoChar(str, ' ');
+ continue;
}
else if (IsA(node, IntList))
- appendStringInfo(str, " %d", lfirst_int(lc));
+ val.i = lfirst_int(lc);
else if (IsA(node, OidList))
- appendStringInfo(str, " %u", lfirst_oid(lc));
+ val.o = lfirst_oid(lc);
else if (IsA(node, XidList))
- appendStringInfo(str, " %u", lfirst_xid(lc));
+ val.x = lfirst_xid(lc);
+
+ if (val.i == previous.i + run_delta)
+ run += 1;
else
- elog(ERROR, "unrecognized list node type: %d",
- (int) node->type);
+ {
+ if (run > 0)
+ appendStringInfo(str, " +%d", run);
+ run = 0;
+ appendStringInfo(str, fmt, val.i);
+ }
+ previous = val;
}
- appendStringInfoChar(str, ')');
-}
+ if (run > 0)
+ appendStringInfo(str, " +%d", run);
+
+ appendStringInfoChar(str, ')');}
/*
* outBitmapset -
* converts a bitmap set of integers
*
- * Note: the output format is "(b int int ...)", similar to an integer List.
+ * Note: the output format is "(b int int +int ...)", similar to an
+ * integer List. Note that consecutive runs of incremental values are
+ * indicated by [... int +int ...], where the first int is the first set bit,
+ * and the int that's tagged with the '+' sign that follows is the number of
+ * subsequent set bits (resulting in a run of N+1 set bits).
*
* We export this function for use by extensions that define extensible nodes.
* That's somewhat historical, though, because calling outNode() will work.
@@ -437,13 +493,31 @@ _outList(StringInfo str, const List *node, bool omitLocation)
void
outBitmapset(StringInfo str, const Bitmapset *bms)
{
- int x;
+ int x = -1;
+ int prev = -2;
+ int run = 0;
appendStringInfoChar(str, '(');
appendStringInfoChar(str, 'b');
- x = -1;
+
while ((x = bms_next_member(bms, x)) >= 0)
- appendStringInfo(str, " %d", x);
+ {
+ if (x - prev == 1)
+ run++;
+ else
+ {
+ if (run > 0)
+ appendStringInfo(str, " +%d", run);
+
+ run = 0;
+ appendStringInfo(str, " %d", x);
+ }
+ prev = x;
+ }
+
+ if (run > 0)
+ appendStringInfo(str, " +%d", run);
+
appendStringInfoChar(str, ')');
}
diff --git a/src/backend/nodes/read.c b/src/backend/nodes/read.c
index cffd913ba6..7456626e82 100644
--- a/src/backend/nodes/read.c
+++ b/src/backend/nodes/read.c
@@ -415,6 +415,8 @@ nodeRead(const char *token, int tok_len)
elog(ERROR, "unterminated List structure");
if (tok_len == 1 && token[0] == 'i')
{
+ int prev = 0;
+
/* List of integers */
for (;;)
{
@@ -430,12 +432,23 @@ nodeRead(const char *token, int tok_len)
if (endptr != token + tok_len)
elog(ERROR, "unrecognized integer: \"%.*s\"",
tok_len, token);
- l = lappend_int(l, val);
+
+ if (token[0] == '+')
+ {
+ for (int i = 0; i < val; i++)
+ l = lappend_int(l, ++prev);
+ }
+ else
+ {
+ prev = val;
+ l = lappend_int(l, val);
+ }
}
result = (Node *) l;
}
else if (tok_len == 1 && token[0] == 'o')
{
+ Oid prev = 0;
/* List of OIDs */
for (;;)
{
@@ -451,12 +464,23 @@ nodeRead(const char *token, int tok_len)
if (endptr != token + tok_len)
elog(ERROR, "unrecognized OID: \"%.*s\"",
tok_len, token);
- l = lappend_oid(l, val);
+
+ if (token[0] == '+')
+ {
+ for (int i = 0; i < val; i++)
+ l = lappend_oid(l, prev);
+ }
+ else
+ {
+ prev = val;
+ l = lappend_oid(l, val);
+ }
}
result = (Node *) l;
}
else if (tok_len == 1 && token[0] == 'x')
{
+ TransactionId prev = 0;
/* List of TransactionIds */
for (;;)
{
@@ -472,7 +496,17 @@ nodeRead(const char *token, int tok_len)
if (endptr != token + tok_len)
elog(ERROR, "unrecognized Xid: \"%.*s\"",
tok_len, token);
- l = lappend_xid(l, val);
+
+ if (token[0] == '+')
+ {
+ for (int i = 0; i < val; i++)
+ l = lappend_xid(l, ++prev);
+ }
+ else
+ {
+ prev = val;
+ l = lappend_xid(l, val);
+ }
}
result = (Node *) l;
}
@@ -480,6 +514,7 @@ nodeRead(const char *token, int tok_len)
{
/* Bitmapset -- see also _readBitmapset() */
Bitmapset *bms = NULL;
+ int prev = -2;
for (;;)
{
@@ -495,7 +530,17 @@ nodeRead(const char *token, int tok_len)
if (endptr != token + tok_len)
elog(ERROR, "unrecognized integer: \"%.*s\"",
tok_len, token);
- bms = bms_add_member(bms, val);
+
+ if (token[0] == '+')
+ {
+ for (int i = 0; i < val; i++)
+ bms = bms_add_member(bms, ++prev);
+ }
+ else
+ {
+ bms = bms_add_member(bms, val);
+ prev = val;
+ }
}
result = (Node *) bms;
}
diff --git a/src/backend/nodes/readfuncs.c b/src/backend/nodes/readfuncs.c
index bcdc39ba18..d77dd30f34 100644
--- a/src/backend/nodes/readfuncs.c
+++ b/src/backend/nodes/readfuncs.c
@@ -309,6 +309,7 @@ static Bitmapset *
_readBitmapset(void)
{
Bitmapset *result = NULL;
+ int prev = -2;
READ_TEMP_LOCALS();
@@ -337,7 +338,20 @@ _readBitmapset(void)
val = (int) strtol(token, &endptr, 10);
if (endptr != token + length)
elog(ERROR, "unrecognized integer: \"%.*s\"", length, token);
- result = bms_add_member(result, val);
+
+ if (token[0] == '+')
+ {
+ for (int i = 0; i < val; i++)
+ {
+ ++prev;
+ result = bms_add_member(result, prev);
+ }
+ }
+ else
+ {
+ result = bms_add_member(result, val);
+ prev = val;
+ }
}
return result;
--
2.40.1
[application/octet-stream] v6-0004-gen_node_support.pl-Add-a-TypMod-type-for-signall.patch (10.4K, ../../CAEze2Wj3KOaXDD0+Viv8u5yeg1tNFQZ3it-pdY267A_v57voyA@mail.gmail.com/9-v6-0004-gen_node_support.pl-Add-a-TypMod-type-for-signall.patch)
download | inline diff:
From ac7a06d3f72204f2c9ce9a73c9b5c0fa0a4481bd Mon Sep 17 00:00:00 2001
From: Matthias van de Meent <[email protected]>
Date: Thu, 8 Feb 2024 17:37:39 +0100
Subject: [PATCH v6 4/8] gen_node_support.pl: Add a TypMod type for signalling
TypMod behaviour
Like Location, TypMod has its own default (-1). By using a type, we can
omit adding pg_node_attribute(default(-1)) markers to every typmod-valued
field, whilst still getting the benefits of a smaller size in serialization.
---
src/backend/nodes/gen_node_support.pl | 8 ++++++-
src/backend/nodes/outfuncs.c | 2 +-
src/backend/nodes/readfuncs.c | 2 +-
src/include/nodes/parsenodes.h | 4 ++--
src/include/nodes/pathnodes.h | 2 +-
src/include/nodes/primnodes.h | 33 ++++++++++++++-------------
6 files changed, 29 insertions(+), 22 deletions(-)
diff --git a/src/backend/nodes/gen_node_support.pl b/src/backend/nodes/gen_node_support.pl
index ec850c3484..fda8c3a494 100644
--- a/src/backend/nodes/gen_node_support.pl
+++ b/src/backend/nodes/gen_node_support.pl
@@ -135,7 +135,8 @@ my @nodetag_only;
# types that are copied by straight assignment
my @scalar_types = qw(
bits32 bool char double int int8 int16 int32 int64 long uint8 uint16 uint32 uint64
- AclMode AttrNumber Cardinality Cost Index Oid RelFileNumber Selectivity Size StrategyNumber SubTransactionId TimeLineID XLogRecPtr
+ AclMode AttrNumber Cardinality Cost Index Oid RelFileNumber Selectivity Size StrategyNumber
+ SubTransactionId TimeLineID XLogRecPtr TypMod
);
# collect enum types
@@ -1015,6 +1016,11 @@ _read${n}(void)
print $off "\tWRITE_LOCATION_FIELD($f);\n";
print $rff "\tREAD_LOCATION_FIELD($f);\n" unless $no_read;
}
+ elsif ($t eq 'TypMod')
+ {
+ print $off "\tWRITE_INT_FIELD_DEFAULT($f, -1);\n";
+ print $rff "\tREAD_INT_FIELD_DEFAULT($f, -1);\n" unless $no_read;
+ }
elsif ($t eq 'int'
|| $t eq 'int16'
|| $t eq 'int32'
diff --git a/src/backend/nodes/outfuncs.c b/src/backend/nodes/outfuncs.c
index c4fd16dc37..d1395b1940 100644
--- a/src/backend/nodes/outfuncs.c
+++ b/src/backend/nodes/outfuncs.c
@@ -497,7 +497,7 @@ _outConst(StringInfo str, const Const *node, bool omitLocation)
WRITE_NODE_TYPE("CONST");
WRITE_OID_FIELD(consttype);
- WRITE_INT_FIELD(consttypmod);
+ WRITE_INT_FIELD_DEFAULT(consttypmod, -1);
WRITE_OID_FIELD(constcollid);
WRITE_INT_FIELD(constlen);
WRITE_BOOL_FIELD(constbyval);
diff --git a/src/backend/nodes/readfuncs.c b/src/backend/nodes/readfuncs.c
index 0c415edb31..84df1c2868 100644
--- a/src/backend/nodes/readfuncs.c
+++ b/src/backend/nodes/readfuncs.c
@@ -367,7 +367,7 @@ _readConst(void)
READ_LOCALS(Const);
READ_OID_FIELD(consttype);
- READ_INT_FIELD(consttypmod);
+ READ_INT_FIELD_DEFAULT(consttypmod, -1);
READ_OID_FIELD(constcollid);
READ_INT_FIELD(constlen);
READ_BOOL_FIELD(constbyval);
diff --git a/src/include/nodes/parsenodes.h b/src/include/nodes/parsenodes.h
index 37e4c53ebe..d4490e1dcc 100644
--- a/src/include/nodes/parsenodes.h
+++ b/src/include/nodes/parsenodes.h
@@ -260,7 +260,7 @@ typedef struct TypeName
bool setof; /* is a set? */
bool pct_type; /* %TYPE specified? */
List *typmods; /* type modifier expression(s) */
- int32 typemod; /* prespecified type modifier */
+ TypMod typemod; /* prespecified type modifier */
List *arrayBounds; /* array bounds */
Location location; /* token location, or -1 if unknown */
} TypeName;
@@ -1608,7 +1608,7 @@ typedef struct CTECycleClause
Location location;
/* These fields are set during parse analysis: */
Oid cycle_mark_type; /* common type of _value and _default */
- int cycle_mark_typmod;
+ TypMod cycle_mark_typmod;
Oid cycle_mark_collation;
Oid cycle_mark_neop; /* <> operator for type */
} CTECycleClause;
diff --git a/src/include/nodes/pathnodes.h b/src/include/nodes/pathnodes.h
index 534692bee1..f7c2496a7f 100644
--- a/src/include/nodes/pathnodes.h
+++ b/src/include/nodes/pathnodes.h
@@ -3394,7 +3394,7 @@ typedef struct AggTransInfo
Oid aggtranstype;
/* Additional data about transtype */
- int32 aggtranstypmod;
+ TypMod aggtranstypmod;
int transtypeLen;
bool transtypeByVal;
diff --git a/src/include/nodes/primnodes.h b/src/include/nodes/primnodes.h
index 557be05657..94282497d7 100644
--- a/src/include/nodes/primnodes.h
+++ b/src/include/nodes/primnodes.h
@@ -29,8 +29,9 @@ typedef enum OverridingKind
OVERRIDING_SYSTEM_VALUE,
} OverridingKind;
-
+/* Type aliases for gen_node_support */
#define Location int
+#define TypMod int32
/* ----------------------------------------------------------------
* node definitions
@@ -250,7 +251,7 @@ typedef struct Var
/* pg_type OID for the type of this var */
Oid vartype pg_node_attr(query_jumble_ignore);
/* pg_attribute typmod value */
- int32 vartypmod pg_node_attr(query_jumble_ignore);
+ TypMod vartypmod pg_node_attr(query_jumble_ignore);
/* OID of collation, or InvalidOid if none */
Oid varcollid pg_node_attr(query_jumble_ignore);
@@ -299,7 +300,7 @@ typedef struct Const
/* pg_type OID of the constant's datatype */
Oid consttype;
/* typmod value, if any */
- int32 consttypmod pg_node_attr(query_jumble_ignore);
+ TypMod consttypmod pg_node_attr(query_jumble_ignore);
/* OID of collation, or InvalidOid if none */
Oid constcollid pg_node_attr(query_jumble_ignore);
/* typlen of the constant's datatype */
@@ -365,7 +366,7 @@ typedef struct Param
int paramid; /* numeric ID for parameter */
Oid paramtype; /* pg_type OID of parameter's datatype */
/* typmod value, if known */
- int32 paramtypmod pg_node_attr(query_jumble_ignore);
+ TypMod paramtypmod pg_node_attr(query_jumble_ignore);
/* OID of collation, or InvalidOid if none */
Oid paramcollid pg_node_attr(query_jumble_ignore);
/* token location, or -1 if unknown */
@@ -623,7 +624,7 @@ typedef struct SubscriptingRef
/* type of the SubscriptingRef's result */
Oid refrestype pg_node_attr(query_jumble_ignore);
/* typmod of the result */
- int32 reftypmod pg_node_attr(query_jumble_ignore);
+ TypMod reftypmod pg_node_attr(query_jumble_ignore);
/* collation of result, or InvalidOid if none */
Oid refcollid pg_node_attr(query_jumble_ignore);
/* expressions that evaluate to upper container indexes */
@@ -1009,7 +1010,7 @@ typedef struct SubPlan
char *plan_name; /* A name assigned during planning */
/* Extra data useful for determining subplan's output type: */
Oid firstColType; /* Type of first column of subplan result */
- int32 firstColTypmod; /* Typmod of first column of subplan result */
+ TypMod firstColTypmod; /* Typmod of first column of subplan result */
Oid firstColCollation; /* Collation of first column of subplan
* result */
/* Information about execution strategy: */
@@ -1067,7 +1068,7 @@ typedef struct FieldSelect
/* type of the field (result type of this node) */
Oid resulttype pg_node_attr(query_jumble_ignore);
/* output typmod (usually -1) */
- int32 resulttypmod pg_node_attr(query_jumble_ignore);
+ TypMod resulttypmod pg_node_attr(query_jumble_ignore);
/* OID of collation of the field */
Oid resultcollid pg_node_attr(query_jumble_ignore);
} FieldSelect;
@@ -1121,7 +1122,7 @@ typedef struct RelabelType
Expr *arg; /* input expression */
Oid resulttype; /* output type of coercion expression */
/* output typmod (usually -1) */
- int32 resulttypmod pg_node_attr(query_jumble_ignore);
+ TypMod resulttypmod pg_node_attr(query_jumble_ignore);
/* OID of collation, or InvalidOid if none */
Oid resultcollid pg_node_attr(query_jumble_ignore);
/* how to display this node */
@@ -1171,7 +1172,7 @@ typedef struct ArrayCoerceExpr
Expr *elemexpr; /* expression representing per-element work */
Oid resulttype; /* output type of coercion (an array type) */
/* output typmod (also element typmod) */
- int32 resulttypmod pg_node_attr(query_jumble_ignore);
+ TypMod resulttypmod pg_node_attr(query_jumble_ignore);
/* OID of collation, or InvalidOid if none */
Oid resultcollid pg_node_attr(query_jumble_ignore);
/* how to display this node */
@@ -1291,7 +1292,7 @@ typedef struct CaseTestExpr
Expr xpr;
Oid typeId; /* type for substituted value */
/* typemod for substituted value */
- int32 typeMod pg_node_attr(query_jumble_ignore);
+ TypMod typeMod pg_node_attr(query_jumble_ignore);
/* collation for the substituted value */
Oid collation pg_node_attr(query_jumble_ignore);
} CaseTestExpr;
@@ -1497,7 +1498,7 @@ typedef struct SQLValueFunction
* include this Oid in the query jumbling.
*/
Oid type pg_node_attr(query_jumble_ignore);
- int32 typmod;
+ TypMod typmod;
Location location; /* token location, or -1 if unknown */
} SQLValueFunction;
@@ -1549,7 +1550,7 @@ typedef struct XmlExpr
bool indent;
/* target type/typmod for XMLSERIALIZE */
Oid type pg_node_attr(query_jumble_ignore);
- int32 typmod pg_node_attr(query_jumble_ignore);
+ TypMod typmod pg_node_attr(query_jumble_ignore);
/* token location, or -1 if unknown */
Location location;
} XmlExpr;
@@ -1599,7 +1600,7 @@ typedef struct JsonReturning
NodeTag type;
JsonFormat *format; /* output JSON format */
Oid typid; /* target type Oid */
- int32 typmod; /* target type modifier */
+ TypMod typmod; /* target type modifier */
} JsonReturning;
/*
@@ -1762,7 +1763,7 @@ typedef struct CoerceToDomain
Expr *arg; /* input expression */
Oid resulttype; /* domain type ID (result type) */
/* output typmod (currently always -1) */
- int32 resulttypmod pg_node_attr(query_jumble_ignore);
+ TypMod resulttypmod pg_node_attr(query_jumble_ignore);
/* OID of collation, or InvalidOid if none */
Oid resultcollid pg_node_attr(query_jumble_ignore);
/* how to display this node */
@@ -1785,7 +1786,7 @@ typedef struct CoerceToDomainValue
/* type for substituted value */
Oid typeId;
/* typemod for substituted value */
- int32 typeMod pg_node_attr(query_jumble_ignore);
+ TypMod typeMod pg_node_attr(query_jumble_ignore);
/* collation for the substituted value */
Oid collation pg_node_attr(query_jumble_ignore);
/* token location, or -1 if unknown */
@@ -1805,7 +1806,7 @@ typedef struct SetToDefault
/* type for substituted value */
Oid typeId;
/* typemod for substituted value */
- int32 typeMod pg_node_attr(query_jumble_ignore);
+ TypMod typeMod pg_node_attr(query_jumble_ignore);
/* collation for the substituted value */
Oid collation pg_node_attr(query_jumble_ignore);
/* token location, or -1 if unknown */
--
2.40.1
^ permalink raw reply [nested|flat] 69+ messages in thread
* Re: Reducing output size of nodeToString
@ 2024-02-22 15:07 Matthias van de Meent <[email protected]>
parent: Matthias van de Meent <[email protected]>
0 siblings, 1 reply; 69+ messages in thread
From: Matthias van de Meent @ 2024-02-22 15:07 UTC (permalink / raw)
To: Peter Eisentraut <[email protected]>; +Cc: Robert Haas <[email protected]>; PostgreSQL Hackers <[email protected]>; Andres Freund <[email protected]>; Michel Pelletier <[email protected]>
On Thu, 22 Feb 2024 at 13:37, Matthias van de Meent
<[email protected]> wrote:
>
> On Mon, 19 Feb 2024 at 14:19, Matthias van de Meent
> <[email protected]> wrote:
> > Attached the updated version of the patch on top of 5497daf3, which
> > incorporates this last round of feedback.
>
> Now attached rebased on top of 93db6cbd to fix conflicts with fbc93b8b
> and an issue in the previous patchset: I attached one too many v3-0001
> from a previous patch I worked on.
... and now with a fix for not overwriting newly deserialized location
attributes with -1, which breaks test output for
READ_WRITE_PARSE_PLAN_TREES installations. Again, no other significant
changes since the patch of last Monday.
Sorry for the noise,
-Matthias
Attachments:
[application/octet-stream] v7-0004-gen_node_support.pl-Add-a-TypMod-type-for-signall.patch (10.4K, ../../CAEze2WgKVdmMLAGga8Gfh7DsvuJPxiqPvYH8fS0Px8f3w5g64g@mail.gmail.com/2-v7-0004-gen_node_support.pl-Add-a-TypMod-type-for-signall.patch)
download | inline diff:
From fb6fde0c39a8eec4ea0cd1fd49c08bde77ea7f13 Mon Sep 17 00:00:00 2001
From: Matthias van de Meent <[email protected]>
Date: Thu, 8 Feb 2024 17:37:39 +0100
Subject: [PATCH v7 4/8] gen_node_support.pl: Add a TypMod type for signalling
TypMod behaviour
Like Location, TypMod has its own default (-1). By using a type, we can
omit adding pg_node_attribute(default(-1)) markers to every typmod-valued
field, whilst still getting the benefits of a smaller size in serialization.
---
src/backend/nodes/gen_node_support.pl | 8 ++++++-
src/backend/nodes/outfuncs.c | 2 +-
src/backend/nodes/readfuncs.c | 2 +-
src/include/nodes/parsenodes.h | 4 ++--
src/include/nodes/pathnodes.h | 2 +-
src/include/nodes/primnodes.h | 33 ++++++++++++++-------------
6 files changed, 29 insertions(+), 22 deletions(-)
diff --git a/src/backend/nodes/gen_node_support.pl b/src/backend/nodes/gen_node_support.pl
index ec850c3484..fda8c3a494 100644
--- a/src/backend/nodes/gen_node_support.pl
+++ b/src/backend/nodes/gen_node_support.pl
@@ -135,7 +135,8 @@ my @nodetag_only;
# types that are copied by straight assignment
my @scalar_types = qw(
bits32 bool char double int int8 int16 int32 int64 long uint8 uint16 uint32 uint64
- AclMode AttrNumber Cardinality Cost Index Oid RelFileNumber Selectivity Size StrategyNumber SubTransactionId TimeLineID XLogRecPtr
+ AclMode AttrNumber Cardinality Cost Index Oid RelFileNumber Selectivity Size StrategyNumber
+ SubTransactionId TimeLineID XLogRecPtr TypMod
);
# collect enum types
@@ -1015,6 +1016,11 @@ _read${n}(void)
print $off "\tWRITE_LOCATION_FIELD($f);\n";
print $rff "\tREAD_LOCATION_FIELD($f);\n" unless $no_read;
}
+ elsif ($t eq 'TypMod')
+ {
+ print $off "\tWRITE_INT_FIELD_DEFAULT($f, -1);\n";
+ print $rff "\tREAD_INT_FIELD_DEFAULT($f, -1);\n" unless $no_read;
+ }
elsif ($t eq 'int'
|| $t eq 'int16'
|| $t eq 'int32'
diff --git a/src/backend/nodes/outfuncs.c b/src/backend/nodes/outfuncs.c
index c4fd16dc37..d1395b1940 100644
--- a/src/backend/nodes/outfuncs.c
+++ b/src/backend/nodes/outfuncs.c
@@ -497,7 +497,7 @@ _outConst(StringInfo str, const Const *node, bool omitLocation)
WRITE_NODE_TYPE("CONST");
WRITE_OID_FIELD(consttype);
- WRITE_INT_FIELD(consttypmod);
+ WRITE_INT_FIELD_DEFAULT(consttypmod, -1);
WRITE_OID_FIELD(constcollid);
WRITE_INT_FIELD(constlen);
WRITE_BOOL_FIELD(constbyval);
diff --git a/src/backend/nodes/readfuncs.c b/src/backend/nodes/readfuncs.c
index e61bdbaef4..5453f53128 100644
--- a/src/backend/nodes/readfuncs.c
+++ b/src/backend/nodes/readfuncs.c
@@ -368,7 +368,7 @@ _readConst(void)
READ_LOCALS(Const);
READ_OID_FIELD(consttype);
- READ_INT_FIELD(consttypmod);
+ READ_INT_FIELD_DEFAULT(consttypmod, -1);
READ_OID_FIELD(constcollid);
READ_INT_FIELD(constlen);
READ_BOOL_FIELD(constbyval);
diff --git a/src/include/nodes/parsenodes.h b/src/include/nodes/parsenodes.h
index 37e4c53ebe..d4490e1dcc 100644
--- a/src/include/nodes/parsenodes.h
+++ b/src/include/nodes/parsenodes.h
@@ -260,7 +260,7 @@ typedef struct TypeName
bool setof; /* is a set? */
bool pct_type; /* %TYPE specified? */
List *typmods; /* type modifier expression(s) */
- int32 typemod; /* prespecified type modifier */
+ TypMod typemod; /* prespecified type modifier */
List *arrayBounds; /* array bounds */
Location location; /* token location, or -1 if unknown */
} TypeName;
@@ -1608,7 +1608,7 @@ typedef struct CTECycleClause
Location location;
/* These fields are set during parse analysis: */
Oid cycle_mark_type; /* common type of _value and _default */
- int cycle_mark_typmod;
+ TypMod cycle_mark_typmod;
Oid cycle_mark_collation;
Oid cycle_mark_neop; /* <> operator for type */
} CTECycleClause;
diff --git a/src/include/nodes/pathnodes.h b/src/include/nodes/pathnodes.h
index 534692bee1..f7c2496a7f 100644
--- a/src/include/nodes/pathnodes.h
+++ b/src/include/nodes/pathnodes.h
@@ -3394,7 +3394,7 @@ typedef struct AggTransInfo
Oid aggtranstype;
/* Additional data about transtype */
- int32 aggtranstypmod;
+ TypMod aggtranstypmod;
int transtypeLen;
bool transtypeByVal;
diff --git a/src/include/nodes/primnodes.h b/src/include/nodes/primnodes.h
index 557be05657..94282497d7 100644
--- a/src/include/nodes/primnodes.h
+++ b/src/include/nodes/primnodes.h
@@ -29,8 +29,9 @@ typedef enum OverridingKind
OVERRIDING_SYSTEM_VALUE,
} OverridingKind;
-
+/* Type aliases for gen_node_support */
#define Location int
+#define TypMod int32
/* ----------------------------------------------------------------
* node definitions
@@ -250,7 +251,7 @@ typedef struct Var
/* pg_type OID for the type of this var */
Oid vartype pg_node_attr(query_jumble_ignore);
/* pg_attribute typmod value */
- int32 vartypmod pg_node_attr(query_jumble_ignore);
+ TypMod vartypmod pg_node_attr(query_jumble_ignore);
/* OID of collation, or InvalidOid if none */
Oid varcollid pg_node_attr(query_jumble_ignore);
@@ -299,7 +300,7 @@ typedef struct Const
/* pg_type OID of the constant's datatype */
Oid consttype;
/* typmod value, if any */
- int32 consttypmod pg_node_attr(query_jumble_ignore);
+ TypMod consttypmod pg_node_attr(query_jumble_ignore);
/* OID of collation, or InvalidOid if none */
Oid constcollid pg_node_attr(query_jumble_ignore);
/* typlen of the constant's datatype */
@@ -365,7 +366,7 @@ typedef struct Param
int paramid; /* numeric ID for parameter */
Oid paramtype; /* pg_type OID of parameter's datatype */
/* typmod value, if known */
- int32 paramtypmod pg_node_attr(query_jumble_ignore);
+ TypMod paramtypmod pg_node_attr(query_jumble_ignore);
/* OID of collation, or InvalidOid if none */
Oid paramcollid pg_node_attr(query_jumble_ignore);
/* token location, or -1 if unknown */
@@ -623,7 +624,7 @@ typedef struct SubscriptingRef
/* type of the SubscriptingRef's result */
Oid refrestype pg_node_attr(query_jumble_ignore);
/* typmod of the result */
- int32 reftypmod pg_node_attr(query_jumble_ignore);
+ TypMod reftypmod pg_node_attr(query_jumble_ignore);
/* collation of result, or InvalidOid if none */
Oid refcollid pg_node_attr(query_jumble_ignore);
/* expressions that evaluate to upper container indexes */
@@ -1009,7 +1010,7 @@ typedef struct SubPlan
char *plan_name; /* A name assigned during planning */
/* Extra data useful for determining subplan's output type: */
Oid firstColType; /* Type of first column of subplan result */
- int32 firstColTypmod; /* Typmod of first column of subplan result */
+ TypMod firstColTypmod; /* Typmod of first column of subplan result */
Oid firstColCollation; /* Collation of first column of subplan
* result */
/* Information about execution strategy: */
@@ -1067,7 +1068,7 @@ typedef struct FieldSelect
/* type of the field (result type of this node) */
Oid resulttype pg_node_attr(query_jumble_ignore);
/* output typmod (usually -1) */
- int32 resulttypmod pg_node_attr(query_jumble_ignore);
+ TypMod resulttypmod pg_node_attr(query_jumble_ignore);
/* OID of collation of the field */
Oid resultcollid pg_node_attr(query_jumble_ignore);
} FieldSelect;
@@ -1121,7 +1122,7 @@ typedef struct RelabelType
Expr *arg; /* input expression */
Oid resulttype; /* output type of coercion expression */
/* output typmod (usually -1) */
- int32 resulttypmod pg_node_attr(query_jumble_ignore);
+ TypMod resulttypmod pg_node_attr(query_jumble_ignore);
/* OID of collation, or InvalidOid if none */
Oid resultcollid pg_node_attr(query_jumble_ignore);
/* how to display this node */
@@ -1171,7 +1172,7 @@ typedef struct ArrayCoerceExpr
Expr *elemexpr; /* expression representing per-element work */
Oid resulttype; /* output type of coercion (an array type) */
/* output typmod (also element typmod) */
- int32 resulttypmod pg_node_attr(query_jumble_ignore);
+ TypMod resulttypmod pg_node_attr(query_jumble_ignore);
/* OID of collation, or InvalidOid if none */
Oid resultcollid pg_node_attr(query_jumble_ignore);
/* how to display this node */
@@ -1291,7 +1292,7 @@ typedef struct CaseTestExpr
Expr xpr;
Oid typeId; /* type for substituted value */
/* typemod for substituted value */
- int32 typeMod pg_node_attr(query_jumble_ignore);
+ TypMod typeMod pg_node_attr(query_jumble_ignore);
/* collation for the substituted value */
Oid collation pg_node_attr(query_jumble_ignore);
} CaseTestExpr;
@@ -1497,7 +1498,7 @@ typedef struct SQLValueFunction
* include this Oid in the query jumbling.
*/
Oid type pg_node_attr(query_jumble_ignore);
- int32 typmod;
+ TypMod typmod;
Location location; /* token location, or -1 if unknown */
} SQLValueFunction;
@@ -1549,7 +1550,7 @@ typedef struct XmlExpr
bool indent;
/* target type/typmod for XMLSERIALIZE */
Oid type pg_node_attr(query_jumble_ignore);
- int32 typmod pg_node_attr(query_jumble_ignore);
+ TypMod typmod pg_node_attr(query_jumble_ignore);
/* token location, or -1 if unknown */
Location location;
} XmlExpr;
@@ -1599,7 +1600,7 @@ typedef struct JsonReturning
NodeTag type;
JsonFormat *format; /* output JSON format */
Oid typid; /* target type Oid */
- int32 typmod; /* target type modifier */
+ TypMod typmod; /* target type modifier */
} JsonReturning;
/*
@@ -1762,7 +1763,7 @@ typedef struct CoerceToDomain
Expr *arg; /* input expression */
Oid resulttype; /* domain type ID (result type) */
/* output typmod (currently always -1) */
- int32 resulttypmod pg_node_attr(query_jumble_ignore);
+ TypMod resulttypmod pg_node_attr(query_jumble_ignore);
/* OID of collation, or InvalidOid if none */
Oid resultcollid pg_node_attr(query_jumble_ignore);
/* how to display this node */
@@ -1785,7 +1786,7 @@ typedef struct CoerceToDomainValue
/* type for substituted value */
Oid typeId;
/* typemod for substituted value */
- int32 typeMod pg_node_attr(query_jumble_ignore);
+ TypMod typeMod pg_node_attr(query_jumble_ignore);
/* collation for the substituted value */
Oid collation pg_node_attr(query_jumble_ignore);
/* token location, or -1 if unknown */
@@ -1805,7 +1806,7 @@ typedef struct SetToDefault
/* type for substituted value */
Oid typeId;
/* typemod for substituted value */
- int32 typeMod pg_node_attr(query_jumble_ignore);
+ TypMod typeMod pg_node_attr(query_jumble_ignore);
/* collation for the substituted value */
Oid collation pg_node_attr(query_jumble_ignore);
/* token location, or -1 if unknown */
--
2.40.1
[application/octet-stream] v7-0003-gen_node_support.pl-Mark-location-fields-as-type-.patch (26.2K, ../../CAEze2WgKVdmMLAGga8Gfh7DsvuJPxiqPvYH8fS0Px8f3w5g64g@mail.gmail.com/3-v7-0003-gen_node_support.pl-Mark-location-fields-as-type-.patch)
download | inline diff:
From fa3e4d157867935dc1c22a1132e9a4c3254acb26 Mon Sep 17 00:00:00 2001
From: Matthias van de Meent <[email protected]>
Date: Thu, 8 Feb 2024 17:15:22 +0100
Subject: [PATCH v7 3/8] gen_node_support.pl: Mark location fields as type
alias Location
Instead of the rather ugly type=int + name ~= location$, we now have a
special type for offset pointers or sizes that are only relevant when a
query text is included, which decreases the complexity required in
gen_node_support.pl for handling these values.
---
src/backend/nodes/gen_node_support.pl | 6 +-
src/include/nodes/parsenodes.h | 92 +++++++++++++--------------
src/include/nodes/primnodes.h | 72 +++++++++++----------
3 files changed, 86 insertions(+), 84 deletions(-)
diff --git a/src/backend/nodes/gen_node_support.pl b/src/backend/nodes/gen_node_support.pl
index 487f6f7728..ec850c3484 100644
--- a/src/backend/nodes/gen_node_support.pl
+++ b/src/backend/nodes/gen_node_support.pl
@@ -777,7 +777,7 @@ _equal${n}(const $n *a, const $n *b)
print $eff "\tCOMPARE_BITMAPSET_FIELD($f);\n"
unless $equal_ignore;
}
- elsif ($t eq 'int' && $f =~ 'location$')
+ elsif ($t eq 'Location')
{
print $cff "\tCOPY_LOCATION_FIELD($f);\n" unless $copy_ignore;
print $eff "\tCOMPARE_LOCATION_FIELD($f);\n" unless $equal_ignore;
@@ -1010,7 +1010,7 @@ _read${n}(void)
print $off "\tWRITE_BOOL_FIELD($f);\n";
print $rff "\tREAD_BOOL_FIELD($f);\n" unless $no_read;
}
- elsif ($t eq 'int' && $f =~ 'location$')
+ elsif ($t eq 'Location')
{
print $off "\tWRITE_LOCATION_FIELD($f);\n";
print $rff "\tREAD_LOCATION_FIELD($f);\n" unless $no_read;
@@ -1303,7 +1303,7 @@ _jumble${n}(JumbleState *jstate, Node *node)
print $jff "\tJUMBLE_NODE($f);\n"
unless $query_jumble_ignore;
}
- elsif ($t eq 'int' && $f =~ 'location$')
+ elsif ($t eq 'Location')
{
# Track the node's location only if directly requested.
if ($query_jumble_location)
diff --git a/src/include/nodes/parsenodes.h b/src/include/nodes/parsenodes.h
index baa6a97c7e..37e4c53ebe 100644
--- a/src/include/nodes/parsenodes.h
+++ b/src/include/nodes/parsenodes.h
@@ -225,9 +225,9 @@ typedef struct Query
* both be -1 meaning "unknown".
*/
/* start location, or -1 if unknown */
- int stmt_location;
+ Location stmt_location;
/* length in bytes; 0 means "rest of string" */
- int stmt_len pg_node_attr(query_jumble_ignore);
+ Location stmt_len pg_node_attr(query_jumble_ignore);
} Query;
@@ -262,7 +262,7 @@ typedef struct TypeName
List *typmods; /* type modifier expression(s) */
int32 typemod; /* prespecified type modifier */
List *arrayBounds; /* array bounds */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} TypeName;
/*
@@ -282,7 +282,7 @@ typedef struct ColumnRef
{
NodeTag type;
List *fields; /* field names (String nodes) or A_Star */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} ColumnRef;
/*
@@ -292,7 +292,7 @@ typedef struct ParamRef
{
NodeTag type;
int number; /* the number of the parameter */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} ParamRef;
/*
@@ -325,7 +325,7 @@ typedef struct A_Expr
List *name; /* possibly-qualified name of operator */
Node *lexpr; /* left argument, or NULL if none */
Node *rexpr; /* right argument, or NULL if none */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} A_Expr;
/*
@@ -351,7 +351,7 @@ typedef struct A_Const
NodeTag type;
union ValUnion val;
bool isnull; /* SQL NULL constant */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} A_Const;
/*
@@ -362,7 +362,7 @@ typedef struct TypeCast
NodeTag type;
Node *arg; /* the expression being casted */
TypeName *typeName; /* the target type */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} TypeCast;
/*
@@ -373,7 +373,7 @@ typedef struct CollateClause
NodeTag type;
Node *arg; /* input expression */
List *collname; /* possibly-qualified collation name */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} CollateClause;
/*
@@ -393,7 +393,7 @@ typedef struct RoleSpec
NodeTag type;
RoleSpecType roletype; /* Type of this rolespec */
char *rolename; /* filled only for ROLESPEC_CSTRING */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} RoleSpec;
/*
@@ -423,7 +423,7 @@ typedef struct FuncCall
bool agg_distinct; /* arguments were labeled DISTINCT */
bool func_variadic; /* last argument was labeled VARIADIC */
CoercionForm funcformat; /* how to display this node */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} FuncCall;
/*
@@ -480,7 +480,7 @@ typedef struct A_ArrayExpr
{
NodeTag type;
List *elements; /* array element expressions */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} A_ArrayExpr;
/*
@@ -507,7 +507,7 @@ typedef struct ResTarget
char *name; /* column name or NULL */
List *indirection; /* subscripts, field names, and '*', or NIL */
Node *val; /* the value expression to compute or assign */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} ResTarget;
/*
@@ -537,7 +537,7 @@ typedef struct SortBy
SortByDir sortby_dir; /* ASC/DESC/USING/default */
SortByNulls sortby_nulls; /* NULLS FIRST/LAST */
List *useOp; /* name of op to use, if SORTBY_USING */
- int location; /* operator location, or -1 if none/unknown */
+ Location location; /* operator location, or -1 if none/unknown */
} SortBy;
/*
@@ -558,7 +558,7 @@ typedef struct WindowDef
int frameOptions; /* frame_clause options, see below */
Node *startOffset; /* expression for starting bound, if any */
Node *endOffset; /* expression for ending bound, if any */
- int location; /* parse location, or -1 if none/unknown */
+ Location location; /* parse location, or -1 if none/unknown */
} WindowDef;
/*
@@ -648,7 +648,7 @@ typedef struct RangeTableFunc
List *namespaces; /* list of namespaces as ResTarget */
List *columns; /* list of RangeTableFuncCol */
Alias *alias; /* table alias & optional column aliases */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} RangeTableFunc;
/*
@@ -666,7 +666,7 @@ typedef struct RangeTableFuncCol
bool is_not_null; /* does it have NOT NULL? */
Node *colexpr; /* column filter expression */
Node *coldefexpr; /* column default value expression */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} RangeTableFuncCol;
/*
@@ -686,7 +686,7 @@ typedef struct RangeTableSample
List *method; /* sampling method name (possibly qualified) */
List *args; /* argument(s) for sampling method */
Node *repeatable; /* REPEATABLE expression, or NULL if none */
- int location; /* method name location, or -1 if unknown */
+ Location location; /* method name location, or -1 if unknown */
} RangeTableSample;
/*
@@ -729,7 +729,7 @@ typedef struct ColumnDef
Oid collOid; /* collation OID (InvalidOid if not set) */
List *constraints; /* other constraints on column */
List *fdwoptions; /* per-column FDW options */
- int location; /* parse location, or -1 if none/unknown */
+ Location location; /* parse location, or -1 if none/unknown */
} ColumnDef;
/*
@@ -803,7 +803,7 @@ typedef struct DefElem
Node *arg; /* typically Integer, Float, String, or
* TypeName */
DefElemAction defaction; /* unspecified action, or SET/ADD/DROP */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} DefElem;
/*
@@ -833,7 +833,7 @@ typedef struct XmlSerialize
Node *expr;
TypeName *typeName;
bool indent; /* [NO] INDENT */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} XmlSerialize;
/* Partitioning related definitions */
@@ -851,7 +851,7 @@ typedef struct PartitionElem
Node *expr; /* expression to partition on, or NULL */
List *collation; /* name of collation; NIL = default */
List *opclass; /* name of desired opclass; NIL = default */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} PartitionElem;
typedef enum PartitionStrategy
@@ -871,7 +871,7 @@ typedef struct PartitionSpec
NodeTag type;
PartitionStrategy strategy;
List *partParams; /* List of PartitionElems */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} PartitionSpec;
/*
@@ -898,7 +898,7 @@ struct PartitionBoundSpec
List *lowerdatums; /* List of PartitionRangeDatums */
List *upperdatums; /* List of PartitionRangeDatums */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
};
/*
@@ -921,7 +921,7 @@ typedef struct PartitionRangeDatum
Node *value; /* Const (or A_Const in raw tree), if kind is
* PARTITION_RANGE_DATUM_VALUE, else NULL */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} PartitionRangeDatum;
/*
@@ -1454,7 +1454,7 @@ typedef struct GroupingSet
NodeTag type;
GroupingSetKind kind pg_node_attr(query_jumble_ignore);
List *content;
- int location;
+ Location location;
} GroupingSet;
/*
@@ -1542,7 +1542,7 @@ typedef struct WithClause
NodeTag type;
List *ctes; /* list of CommonTableExprs */
bool recursive; /* true = WITH RECURSIVE */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} WithClause;
/*
@@ -1557,7 +1557,7 @@ typedef struct InferClause
List *indexElems; /* IndexElems to infer unique index */
Node *whereClause; /* qualification (partial-index predicate) */
char *conname; /* Constraint name, or NULL if unnamed */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} InferClause;
/*
@@ -1573,7 +1573,7 @@ typedef struct OnConflictClause
InferClause *infer; /* Optional index inference clause */
List *targetList; /* the target list (of ResTarget) */
Node *whereClause; /* qualifications */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} OnConflictClause;
/*
@@ -1594,7 +1594,7 @@ typedef struct CTESearchClause
List *search_col_list;
bool search_breadth_first;
char *search_seq_column;
- int location;
+ Location location;
} CTESearchClause;
typedef struct CTECycleClause
@@ -1605,7 +1605,7 @@ typedef struct CTECycleClause
Node *cycle_mark_value;
Node *cycle_mark_default;
char *cycle_path_column;
- int location;
+ Location location;
/* These fields are set during parse analysis: */
Oid cycle_mark_type; /* common type of _value and _default */
int cycle_mark_typmod;
@@ -1629,7 +1629,7 @@ typedef struct CommonTableExpr
Node *ctequery; /* the CTE's subquery */
CTESearchClause *search_clause pg_node_attr(query_jumble_ignore);
CTECycleClause *cycle_clause pg_node_attr(query_jumble_ignore);
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
/* These fields are set during parse analysis: */
/* is this CTE actually recursive? */
bool cterecursive pg_node_attr(query_jumble_ignore);
@@ -1725,7 +1725,7 @@ typedef struct JsonParseExpr
JsonValueExpr *expr; /* string expression */
JsonOutput *output; /* RETURNING clause, if specified */
bool unique_keys; /* WITH UNIQUE KEYS? */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} JsonParseExpr;
/*
@@ -1737,7 +1737,7 @@ typedef struct JsonScalarExpr
NodeTag type;
Expr *expr; /* scalar expression */
JsonOutput *output; /* RETURNING clause, if specified */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} JsonScalarExpr;
/*
@@ -1749,7 +1749,7 @@ typedef struct JsonSerializeExpr
NodeTag type;
JsonValueExpr *expr; /* json value expression */
JsonOutput *output; /* RETURNING clause, if specified */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} JsonSerializeExpr;
/*
@@ -1763,7 +1763,7 @@ typedef struct JsonObjectConstructor
JsonOutput *output; /* RETURNING clause, if specified */
bool absent_on_null; /* skip NULL values? */
bool unique; /* check key uniqueness? */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} JsonObjectConstructor;
/*
@@ -1776,7 +1776,7 @@ typedef struct JsonArrayConstructor
List *exprs; /* list of JsonValueExpr elements */
JsonOutput *output; /* RETURNING clause, if specified */
bool absent_on_null; /* skip NULL elements? */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} JsonArrayConstructor;
/*
@@ -1790,7 +1790,7 @@ typedef struct JsonArrayQueryConstructor
JsonOutput *output; /* RETURNING clause, if specified */
JsonFormat *format; /* FORMAT clause for subquery, if specified */
bool absent_on_null; /* skip NULL elements? */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} JsonArrayQueryConstructor;
/*
@@ -1805,7 +1805,7 @@ typedef struct JsonAggConstructor
Node *agg_filter; /* FILTER clause, if any */
List *agg_order; /* ORDER BY clause, if any */
struct WindowDef *over; /* OVER clause, if any */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} JsonAggConstructor;
/*
@@ -1859,8 +1859,8 @@ typedef struct RawStmt
NodeTag type;
Node *stmt; /* raw parse tree */
- int stmt_location; /* start location, or -1 if unknown */
- int stmt_len; /* length in bytes; 0 means "rest of string" */
+ Location stmt_location; /* start location, or -1 if unknown */
+ Location stmt_len; /* length in bytes; 0 means "rest of string" */
} RawStmt;
/*****************************************************************************
@@ -2067,7 +2067,7 @@ typedef struct PLAssignStmt
List *indirection; /* subscripts and field names, if any */
int nnames; /* number of names to use in ColumnRef */
SelectStmt *val; /* the PL/pgSQL expression to assign */
- int location; /* name's token location, or -1 if unknown */
+ Location location; /* name's token location, or -1 if unknown */
} PLAssignStmt;
@@ -2609,7 +2609,7 @@ typedef struct Constraint
Oid old_pktable_oid; /* pg_constraint.confrelid of my former
* self */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} Constraint;
/* ----------------------
@@ -3516,7 +3516,7 @@ typedef struct TransactionStmt
char *gid pg_node_attr(query_jumble_ignore);
bool chain; /* AND CHAIN option */
/* token location, or -1 if unknown */
- int location pg_node_attr(query_jumble_location);
+ Location location pg_node_attr(query_jumble_location);
} TransactionStmt;
/* ----------------------
@@ -3902,7 +3902,7 @@ typedef struct DeallocateStmt
/* true if DEALLOCATE ALL */
bool isall;
/* token location, or -1 if unknown */
- int location pg_node_attr(query_jumble_location);
+ Location location pg_node_attr(query_jumble_location);
} DeallocateStmt;
/*
@@ -3990,7 +3990,7 @@ typedef struct PublicationObjSpec
PublicationObjSpecType pubobjtype; /* type of this publication object */
char *name;
PublicationTable *pubtable;
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} PublicationObjSpec;
typedef struct CreatePublicationStmt
diff --git a/src/include/nodes/primnodes.h b/src/include/nodes/primnodes.h
index 4a154606d2..557be05657 100644
--- a/src/include/nodes/primnodes.h
+++ b/src/include/nodes/primnodes.h
@@ -30,6 +30,8 @@ typedef enum OverridingKind
} OverridingKind;
+#define Location int
+
/* ----------------------------------------------------------------
* node definitions
* ----------------------------------------------------------------
@@ -91,7 +93,7 @@ typedef struct RangeVar
Alias *alias;
/* token location, or -1 if unknown */
- int location;
+ Location location;
} RangeVar;
/*
@@ -128,7 +130,7 @@ typedef struct TableFunc
/* counts from 0; -1 if none specified */
int ordinalitycol pg_node_attr(query_jumble_ignore);
/* token location, or -1 if unknown */
- int location;
+ Location location;
} TableFunc;
/*
@@ -276,7 +278,7 @@ typedef struct Var
AttrNumber varattnosyn pg_node_attr(equal_ignore, query_jumble_ignore);
/* token location, or -1 if unknown */
- int location;
+ Location location;
} Var;
/*
@@ -318,7 +320,7 @@ typedef struct Const
* token location, or -1 if unknown. All constants are tracked as
* locations in query jumbling, to be marked as parameters.
*/
- int location pg_node_attr(query_jumble_location);
+ Location location pg_node_attr(query_jumble_location);
} Const;
/*
@@ -367,7 +369,7 @@ typedef struct Param
/* OID of collation, or InvalidOid if none */
Oid paramcollid pg_node_attr(query_jumble_ignore);
/* token location, or -1 if unknown */
- int location;
+ Location location;
} Param;
/*
@@ -490,7 +492,7 @@ typedef struct Aggref
int aggtransno pg_node_attr(query_jumble_ignore);
/* token location, or -1 if unknown */
- int location;
+ Location location;
} Aggref;
/*
@@ -537,7 +539,7 @@ typedef struct GroupingFunc
Index agglevelsup;
/* token location */
- int location;
+ Location location;
} GroupingFunc;
/*
@@ -568,7 +570,7 @@ typedef struct WindowFunc
/* is function a simple aggregate? */
bool winagg pg_node_attr(query_jumble_ignore);
/* token location, or -1 if unknown */
- int location;
+ Location location;
} WindowFunc;
/*
@@ -702,7 +704,7 @@ typedef struct FuncExpr
/* arguments to the function */
List *args;
/* token location, or -1 if unknown */
- int location;
+ Location location;
} FuncExpr;
/*
@@ -729,7 +731,7 @@ typedef struct NamedArgExpr
/* argument's number in positional notation */
int argnumber;
/* argument name location, or -1 if unknown */
- int location;
+ Location location;
} NamedArgExpr;
/*
@@ -771,7 +773,7 @@ typedef struct OpExpr
List *args;
/* token location, or -1 if unknown */
- int location;
+ Location location;
} OpExpr;
/*
@@ -851,7 +853,7 @@ typedef struct ScalarArrayOpExpr
List *args;
/* token location, or -1 if unknown */
- int location;
+ Location location;
} ScalarArrayOpExpr;
/*
@@ -873,7 +875,7 @@ typedef struct BoolExpr
Expr xpr;
BoolExprType boolop;
List *args; /* arguments to this expression */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} BoolExpr;
/*
@@ -950,7 +952,7 @@ typedef struct SubLink
List *operName pg_node_attr(query_jumble_ignore);
/* subselect as Query* or raw parsetree */
Node *subselect;
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} SubLink;
/*
@@ -1124,7 +1126,7 @@ typedef struct RelabelType
Oid resultcollid pg_node_attr(query_jumble_ignore);
/* how to display this node */
CoercionForm relabelformat pg_node_attr(query_jumble_ignore);
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} RelabelType;
/* ----------------
@@ -1146,7 +1148,7 @@ typedef struct CoerceViaIO
Oid resultcollid pg_node_attr(query_jumble_ignore);
/* how to display this node */
CoercionForm coerceformat pg_node_attr(query_jumble_ignore);
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} CoerceViaIO;
/* ----------------
@@ -1174,7 +1176,7 @@ typedef struct ArrayCoerceExpr
Oid resultcollid pg_node_attr(query_jumble_ignore);
/* how to display this node */
CoercionForm coerceformat pg_node_attr(query_jumble_ignore);
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} ArrayCoerceExpr;
/* ----------------
@@ -1198,7 +1200,7 @@ typedef struct ConvertRowtypeExpr
/* Like RowExpr, we deliberately omit a typmod and collation here */
/* how to display this node */
CoercionForm convertformat pg_node_attr(query_jumble_ignore);
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} ConvertRowtypeExpr;
/*----------
@@ -1213,7 +1215,7 @@ typedef struct CollateExpr
Expr xpr;
Expr *arg; /* input expression */
Oid collOid; /* collation's OID */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} CollateExpr;
/*----------
@@ -1248,7 +1250,7 @@ typedef struct CaseExpr
Expr *arg; /* implicit equality comparison argument */
List *args; /* the arguments (list of WHEN clauses) */
Expr *defresult; /* the default result (ELSE clause) */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} CaseExpr;
/*
@@ -1259,7 +1261,7 @@ typedef struct CaseWhen
Expr xpr;
Expr *expr; /* condition expression */
Expr *result; /* substitution result */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} CaseWhen;
/*
@@ -1316,7 +1318,7 @@ typedef struct ArrayExpr
/* true if elements are sub-arrays */
bool multidims pg_node_attr(query_jumble_ignore);
/* token location, or -1 if unknown */
- int location;
+ Location location;
} ArrayExpr;
/*
@@ -1367,7 +1369,7 @@ typedef struct RowExpr
/* list of String, or NIL */
List *colnames pg_node_attr(query_jumble_ignore);
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} RowExpr;
/*
@@ -1426,7 +1428,7 @@ typedef struct CoalesceExpr
/* the arguments */
List *args;
/* token location, or -1 if unknown */
- int location;
+ Location location;
} CoalesceExpr;
/*
@@ -1452,7 +1454,7 @@ typedef struct MinMaxExpr
/* the arguments */
List *args;
/* token location, or -1 if unknown */
- int location;
+ Location location;
} MinMaxExpr;
/*
@@ -1496,7 +1498,7 @@ typedef struct SQLValueFunction
*/
Oid type pg_node_attr(query_jumble_ignore);
int32 typmod;
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} SQLValueFunction;
/*
@@ -1549,7 +1551,7 @@ typedef struct XmlExpr
Oid type pg_node_attr(query_jumble_ignore);
int32 typmod pg_node_attr(query_jumble_ignore);
/* token location, or -1 if unknown */
- int location;
+ Location location;
} XmlExpr;
/*
@@ -1585,7 +1587,7 @@ typedef struct JsonFormat
NodeTag type;
JsonFormatType format_type; /* format type */
JsonEncoding encoding; /* JSON encoding */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} JsonFormat;
/*
@@ -1641,7 +1643,7 @@ typedef struct JsonConstructorExpr
JsonReturning *returning; /* RETURNING clause */
bool absent_on_null; /* ABSENT ON NULL? */
bool unique; /* WITH UNIQUE KEYS? (JSON_OBJECT[AGG] only) */
- int location;
+ Location location;
} JsonConstructorExpr;
/*
@@ -1667,7 +1669,7 @@ typedef struct JsonIsPredicate
JsonFormat *format; /* FORMAT clause, if specified */
JsonValueType item_type; /* JSON item type */
bool unique_keys; /* check key uniqueness? */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} JsonIsPredicate;
/* ----------------
@@ -1701,7 +1703,7 @@ typedef struct NullTest
NullTestType nulltesttype; /* IS NULL, IS NOT NULL */
/* T to perform field-by-field null checks */
bool argisrow pg_node_attr(query_jumble_ignore);
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} NullTest;
/*
@@ -1723,7 +1725,7 @@ typedef struct BooleanTest
Expr xpr;
Expr *arg; /* input expression */
BoolTestType booltesttype; /* test type */
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} BooleanTest;
@@ -1765,7 +1767,7 @@ typedef struct CoerceToDomain
Oid resultcollid pg_node_attr(query_jumble_ignore);
/* how to display this node */
CoercionForm coercionformat pg_node_attr(query_jumble_ignore);
- int location; /* token location, or -1 if unknown */
+ Location location; /* token location, or -1 if unknown */
} CoerceToDomain;
/*
@@ -1787,7 +1789,7 @@ typedef struct CoerceToDomainValue
/* collation for the substituted value */
Oid collation pg_node_attr(query_jumble_ignore);
/* token location, or -1 if unknown */
- int location;
+ Location location;
} CoerceToDomainValue;
/*
@@ -1807,7 +1809,7 @@ typedef struct SetToDefault
/* collation for the substituted value */
Oid collation pg_node_attr(query_jumble_ignore);
/* token location, or -1 if unknown */
- int location;
+ Location location;
} SetToDefault;
/*
--
2.40.1
[application/octet-stream] v7-0002-pg_node_tree-Don-t-store-query-text-locations-in-.patch (19.0K, ../../CAEze2WgKVdmMLAGga8Gfh7DsvuJPxiqPvYH8fS0Px8f3w5g64g@mail.gmail.com/4-v7-0002-pg_node_tree-Don-t-store-query-text-locations-in-.patch)
download | inline diff:
From be5400d8c1772357b35d7ec2dd7c8a7a7eee9972 Mon Sep 17 00:00:00 2001
From: Matthias van de Meent <[email protected]>
Date: Wed, 3 Jan 2024 01:39:42 +0100
Subject: [PATCH v7 2/8] pg_node_tree: Don't store query text locations in
pg_node_tree fields.
We don't store original query texts, so any lingering "location" value
can only be useful in forensic debugging. In normal operation, however,
a non-default value will show up as measurable overhead in
serialization, just omit serialization, saving several 10s of kBs.
---
src/backend/catalog/heap.c | 9 ++--
src/backend/catalog/index.c | 4 +-
src/backend/catalog/pg_attrdef.c | 6 ++-
src/backend/catalog/pg_proc.c | 10 +++-
src/backend/catalog/pg_publication.c | 6 ++-
src/backend/commands/policy.c | 11 +++-
src/backend/commands/statscmds.c | 2 +-
src/backend/commands/trigger.c | 3 +-
src/backend/commands/typecmds.c | 8 +--
src/backend/nodes/gen_node_support.pl | 4 +-
src/backend/nodes/outfuncs.c | 75 ++++++++++++++++-----------
src/backend/rewrite/rewriteDefine.c | 4 +-
src/include/nodes/nodes.h | 4 +-
13 files changed, 94 insertions(+), 52 deletions(-)
diff --git a/src/backend/catalog/heap.c b/src/backend/catalog/heap.c
index 348943e36c..9d2fd6fae8 100644
--- a/src/backend/catalog/heap.c
+++ b/src/backend/catalog/heap.c
@@ -2064,9 +2064,9 @@ StoreRelCheck(Relation rel, const char *ccname, Node *expr,
Oid constrOid;
/*
- * Flatten expression to string form for storage.
+ * Flatten expression to string form for storage, without query refs.
*/
- ccbin = nodeToString(expr);
+ ccbin = nodeToStringNoQLocs(expr);
/*
* Find columns of rel that are used in expr
@@ -3676,7 +3676,7 @@ StorePartitionKey(Relation rel,
{
char *exprString;
- exprString = nodeToString(partexprs);
+ exprString = nodeToStringNoQLocs(partexprs);
partexprDatum = CStringGetTextDatum(exprString);
pfree(exprString);
}
@@ -3834,7 +3834,8 @@ StorePartitionBound(Relation rel, Relation parent, PartitionBoundSpec *bound)
memset(new_val, 0, sizeof(new_val));
memset(new_null, false, sizeof(new_null));
memset(new_repl, false, sizeof(new_repl));
- new_val[Anum_pg_class_relpartbound - 1] = CStringGetTextDatum(nodeToString(bound));
+ new_val[Anum_pg_class_relpartbound - 1] =
+ CStringGetTextDatum(nodeToStringNoQLocs(bound));
new_null[Anum_pg_class_relpartbound - 1] = false;
new_repl[Anum_pg_class_relpartbound - 1] = true;
newtuple = heap_modify_tuple(tuple, RelationGetDescr(classRel),
diff --git a/src/backend/catalog/index.c b/src/backend/catalog/index.c
index 4b88a9cb87..3d5f4e53d5 100644
--- a/src/backend/catalog/index.c
+++ b/src/backend/catalog/index.c
@@ -587,7 +587,7 @@ UpdateIndexRelation(Oid indexoid,
{
char *exprsString;
- exprsString = nodeToString(indexInfo->ii_Expressions);
+ exprsString = nodeToStringNoQLocs(indexInfo->ii_Expressions);
exprsDatum = CStringGetTextDatum(exprsString);
pfree(exprsString);
}
@@ -602,7 +602,7 @@ UpdateIndexRelation(Oid indexoid,
{
char *predString;
- predString = nodeToString(make_ands_explicit(indexInfo->ii_Predicate));
+ predString = nodeToStringNoQLocs(make_ands_explicit(indexInfo->ii_Predicate));
predDatum = CStringGetTextDatum(predString);
pfree(predString);
}
diff --git a/src/backend/catalog/pg_attrdef.c b/src/backend/catalog/pg_attrdef.c
index 003ae70b4d..a900c9bb28 100644
--- a/src/backend/catalog/pg_attrdef.c
+++ b/src/backend/catalog/pg_attrdef.c
@@ -23,6 +23,7 @@
#include "catalog/objectaccess.h"
#include "catalog/pg_attrdef.h"
#include "executor/executor.h"
+#include "nodes/nodeFuncs.h"
#include "optimizer/optimizer.h"
#include "utils/array.h"
#include "utils/builtins.h"
@@ -62,9 +63,10 @@ StoreAttrDefault(Relation rel, AttrNumber attnum,
adrel = table_open(AttrDefaultRelationId, RowExclusiveLock);
/*
- * Flatten expression to string form for storage.
+ * Flatten expression to string form for storage, without references to
+ * the original query string.
*/
- adbin = nodeToString(expr);
+ adbin = nodeToStringNoQLocs(expr);
/*
* Make the pg_attrdef entry.
diff --git a/src/backend/catalog/pg_proc.c b/src/backend/catalog/pg_proc.c
index b581d334d3..c5790a2224 100644
--- a/src/backend/catalog/pg_proc.c
+++ b/src/backend/catalog/pg_proc.c
@@ -331,7 +331,10 @@ ProcedureCreate(const char *procedureName,
else
nulls[Anum_pg_proc_proargnames - 1] = true;
if (parameterDefaults != NIL)
- values[Anum_pg_proc_proargdefaults - 1] = CStringGetTextDatum(nodeToString(parameterDefaults));
+ {
+ values[Anum_pg_proc_proargdefaults - 1] =
+ CStringGetTextDatum(nodeToStringNoQLocs(parameterDefaults));
+ }
else
nulls[Anum_pg_proc_proargdefaults - 1] = true;
if (trftypes != PointerGetDatum(NULL))
@@ -344,7 +347,10 @@ ProcedureCreate(const char *procedureName,
else
nulls[Anum_pg_proc_probin - 1] = true;
if (prosqlbody)
- values[Anum_pg_proc_prosqlbody - 1] = CStringGetTextDatum(nodeToString(prosqlbody));
+ {
+ values[Anum_pg_proc_prosqlbody - 1] =
+ CStringGetTextDatum(nodeToStringNoQLocs(prosqlbody));
+ }
else
nulls[Anum_pg_proc_prosqlbody - 1] = true;
if (proconfig != PointerGetDatum(NULL))
diff --git a/src/backend/catalog/pg_publication.c b/src/backend/catalog/pg_publication.c
index b98b0ce0ae..b201313430 100644
--- a/src/backend/catalog/pg_publication.c
+++ b/src/backend/catalog/pg_publication.c
@@ -36,6 +36,7 @@
#include "commands/publicationcmds.h"
#include "funcapi.h"
#include "miscadmin.h"
+#include "nodes/nodeFuncs.h"
#include "utils/array.h"
#include "utils/builtins.h"
#include "utils/catcache.h"
@@ -422,7 +423,10 @@ publication_add_relation(Oid pubid, PublicationRelInfo *pri,
/* Add qualifications, if available */
if (pri->whereClause != NULL)
- values[Anum_pg_publication_rel_prqual - 1] = CStringGetTextDatum(nodeToString(pri->whereClause));
+ {
+ values[Anum_pg_publication_rel_prqual - 1] =
+ CStringGetTextDatum(nodeToStringNoQLocs(pri->whereClause));
+ }
else
nulls[Anum_pg_publication_rel_prqual - 1] = true;
diff --git a/src/backend/commands/policy.c b/src/backend/commands/policy.c
index 596326e5ec..1e6842bf41 100644
--- a/src/backend/commands/policy.c
+++ b/src/backend/commands/policy.c
@@ -30,6 +30,7 @@
#include "commands/policy.h"
#include "miscadmin.h"
#include "nodes/makefuncs.h"
+#include "nodes/nodeFuncs.h"
#include "nodes/pg_list.h"
#include "parser/parse_clause.h"
#include "parser/parse_collate.h"
@@ -701,13 +702,19 @@ CreatePolicy(CreatePolicyStmt *stmt)
/* Add qual if present. */
if (qual)
- values[Anum_pg_policy_polqual - 1] = CStringGetTextDatum(nodeToString(qual));
+ {
+ values[Anum_pg_policy_polqual - 1] =
+ CStringGetTextDatum(nodeToStringNoQLocs(qual));
+ }
else
isnull[Anum_pg_policy_polqual - 1] = true;
/* Add WITH CHECK qual if present */
if (with_check_qual)
- values[Anum_pg_policy_polwithcheck - 1] = CStringGetTextDatum(nodeToString(with_check_qual));
+ {
+ values[Anum_pg_policy_polwithcheck - 1] =
+ CStringGetTextDatum(nodeToStringNoQLocs(with_check_qual));
+ }
else
isnull[Anum_pg_policy_polwithcheck - 1] = true;
diff --git a/src/backend/commands/statscmds.c b/src/backend/commands/statscmds.c
index b1a9c74bd6..58e8133f93 100644
--- a/src/backend/commands/statscmds.c
+++ b/src/backend/commands/statscmds.c
@@ -477,7 +477,7 @@ CreateStatistics(CreateStatsStmt *stmt)
{
char *exprsString;
- exprsString = nodeToString(stxexprs);
+ exprsString = nodeToStringNoQLocs(stxexprs);
exprsDatum = CStringGetTextDatum(exprsString);
pfree(exprsString);
}
diff --git a/src/backend/commands/trigger.c b/src/backend/commands/trigger.c
index c344ff0944..6a3dc13a67 100644
--- a/src/backend/commands/trigger.c
+++ b/src/backend/commands/trigger.c
@@ -39,6 +39,7 @@
#include "miscadmin.h"
#include "nodes/bitmapset.h"
#include "nodes/makefuncs.h"
+#include "nodes/nodeFuncs.h"
#include "optimizer/optimizer.h"
#include "parser/parse_clause.h"
#include "parser/parse_collate.h"
@@ -674,7 +675,7 @@ CreateTriggerFiringOn(CreateTrigStmt *stmt, const char *queryString,
/* we'll need the rtable for recordDependencyOnExpr */
whenRtable = pstate->p_rtable;
- qual = nodeToString(whenClause);
+ qual = nodeToStringNoQLocs(whenClause);
free_parsestate(pstate);
}
diff --git a/src/backend/commands/typecmds.c b/src/backend/commands/typecmds.c
index e0275e5fe9..3c0e7bc5db 100644
--- a/src/backend/commands/typecmds.c
+++ b/src/backend/commands/typecmds.c
@@ -58,6 +58,7 @@
#include "executor/executor.h"
#include "miscadmin.h"
#include "nodes/makefuncs.h"
+#include "nodes/nodeFuncs.h"
#include "optimizer/optimizer.h"
#include "parser/parse_coerce.h"
#include "parser/parse_collate.h"
@@ -924,7 +925,7 @@ DefineDomain(CreateDomainStmt *stmt)
defaultValue =
deparse_expression(defaultExpr,
NIL, false, false);
- defaultValueBin = nodeToString(defaultExpr);
+ defaultValueBin = nodeToStringNoQLocs(defaultExpr);
}
}
else
@@ -3506,9 +3507,10 @@ domainAddConstraint(Oid domainOid, Oid domainNamespace, Oid baseTypeOid,
errmsg("cannot use table references in domain check constraint")));
/*
- * Convert to string form for storage.
+ * Convert to string form for storage, without references to the original
+ * query text.
*/
- ccbin = nodeToString(expr);
+ ccbin = nodeToStringNoQLocs(expr);
/*
* Store the constraint in pg_constraint
diff --git a/src/backend/nodes/gen_node_support.pl b/src/backend/nodes/gen_node_support.pl
index 2f0a59bc87..487f6f7728 100644
--- a/src/backend/nodes/gen_node_support.pl
+++ b/src/backend/nodes/gen_node_support.pl
@@ -921,7 +921,7 @@ foreach my $n (@node_types)
my $N = uc $n;
print $ofs "\t\t\tcase T_${n}:\n"
- . "\t\t\t\t_out${n}(str, obj);\n"
+ . "\t\t\t\t_out${n}(str, obj, omitLocation);\n"
. "\t\t\t\tbreak;\n";
print $rfs "\tif (MATCH(\"$N\", "
@@ -933,7 +933,7 @@ foreach my $n (@node_types)
print $off "
static void
-_out${n}(StringInfo str, const $n *node)
+_out${n}(StringInfo str, const $n *node, bool omitLocation)
{
\tWRITE_NODE_TYPE(\"$N\");
diff --git a/src/backend/nodes/outfuncs.c b/src/backend/nodes/outfuncs.c
index 8ad81be1cd..c4fd16dc37 100644
--- a/src/backend/nodes/outfuncs.c
+++ b/src/backend/nodes/outfuncs.c
@@ -159,7 +159,7 @@ static void outDouble(StringInfo str, double d);
/* Write a parse location field (actually same as INT case) */
#define WRITE_LOCATION_FIELD(fldname) \
do { \
- if (node->fldname != -1) \
+ if (node->fldname != -1 && !omitLocation) \
appendStringInfo(str, " :" CppAsString(fldname) " %d", \
node->fldname); \
} while (0)
@@ -170,7 +170,7 @@ static void outDouble(StringInfo str, double d);
if (node->fldname != NULL) \
{ \
appendStringInfoString(str, " :" CppAsString(fldname) " "); \
- outNode(str, node->fldname); \
+ outNode(str, node->fldname, omitLocation); \
} \
} while (0)
@@ -190,7 +190,8 @@ static void outDouble(StringInfo str, double d);
if (node->fldname != NULL) \
{ \
appendStringInfoString(str, " :" CppAsString(fldname) " "); \
- writeNodeArray(str, (const Node * const *) node->fldname, len); \
+ writeNodeArray(str, (const Node * const *) node->fldname, len, \
+ omitLocation); \
} \
} while (0)
@@ -363,7 +364,8 @@ WRITE_SCALAR_ARRAY(writeBoolCols, bool, " %s", booltostr)
* quite use appendStringInfo() in the loop.
*/
static void
-writeNodeArray(StringInfo str, const Node *const *arr, int len)
+writeNodeArray(StringInfo str, const Node *const *arr, int len,
+ bool omitLocation)
{
if (arr != NULL)
{
@@ -371,7 +373,7 @@ writeNodeArray(StringInfo str, const Node *const *arr, int len)
for (int i = 0; i < len; i++)
{
appendStringInfoChar(str, ' ');
- outNode(str, arr[i]);
+ outNode(str, arr[i], omitLocation);
}
appendStringInfoChar(str, ')');
}
@@ -383,7 +385,7 @@ writeNodeArray(StringInfo str, const Node *const *arr, int len)
* Print a List.
*/
static void
-_outList(StringInfo str, const List *node)
+_outList(StringInfo str, const List *node, bool omitLocation)
{
const ListCell *lc;
@@ -405,7 +407,7 @@ _outList(StringInfo str, const List *node)
*/
if (IsA(node, List))
{
- outNode(str, lfirst(lc));
+ outNode(str, lfirst(lc), omitLocation);
if (lnext(node, lc))
appendStringInfoChar(str, ' ');
}
@@ -490,7 +492,7 @@ outDatum(StringInfo str, Datum value, int typlen, bool typbyval)
*/
static void
-_outConst(StringInfo str, const Const *node)
+_outConst(StringInfo str, const Const *node, bool omitLocation)
{
WRITE_NODE_TYPE("CONST");
@@ -510,7 +512,7 @@ _outConst(StringInfo str, const Const *node)
}
static void
-_outBoolExpr(StringInfo str, const BoolExpr *node)
+_outBoolExpr(StringInfo str, const BoolExpr *node, bool omitLocation)
{
char *opstr = NULL;
@@ -537,7 +539,8 @@ _outBoolExpr(StringInfo str, const BoolExpr *node)
}
static void
-_outForeignKeyOptInfo(StringInfo str, const ForeignKeyOptInfo *node)
+_outForeignKeyOptInfo(StringInfo str, const ForeignKeyOptInfo *node,
+ bool omitLocation)
{
int i;
@@ -563,7 +566,8 @@ _outForeignKeyOptInfo(StringInfo str, const ForeignKeyOptInfo *node)
}
static void
-_outEquivalenceClass(StringInfo str, const EquivalenceClass *node)
+_outEquivalenceClass(StringInfo str, const EquivalenceClass *node,
+ bool omitLocation)
{
/*
* To simplify reading, we just chase up to the topmost merged EC and
@@ -589,7 +593,8 @@ _outEquivalenceClass(StringInfo str, const EquivalenceClass *node)
}
static void
-_outExtensibleNode(StringInfo str, const ExtensibleNode *node)
+_outExtensibleNode(StringInfo str, const ExtensibleNode *node,
+ bool omitLocation)
{
const ExtensibleNodeMethods *methods;
@@ -604,7 +609,8 @@ _outExtensibleNode(StringInfo str, const ExtensibleNode *node)
}
static void
-_outRangeTblEntry(StringInfo str, const RangeTblEntry *node)
+_outRangeTblEntry(StringInfo str, const RangeTblEntry *node,
+ bool omitLocation)
{
WRITE_NODE_TYPE("RANGETBLENTRY");
@@ -684,7 +690,7 @@ _outRangeTblEntry(StringInfo str, const RangeTblEntry *node)
}
static void
-_outA_Expr(StringInfo str, const A_Expr *node)
+_outA_Expr(StringInfo str, const A_Expr *node, bool omitLocation)
{
WRITE_NODE_TYPE("A_EXPR");
@@ -756,13 +762,13 @@ _outA_Expr(StringInfo str, const A_Expr *node)
}
static void
-_outInteger(StringInfo str, const Integer *node)
+_outInteger(StringInfo str, const Integer *node, bool omitLocation)
{
appendStringInfo(str, "%d", node->ival);
}
static void
-_outFloat(StringInfo str, const Float *node)
+_outFloat(StringInfo str, const Float *node, bool omitLocation)
{
/*
* We assume the value is a valid numeric literal and so does not need
@@ -772,13 +778,13 @@ _outFloat(StringInfo str, const Float *node)
}
static void
-_outBoolean(StringInfo str, const Boolean *node)
+_outBoolean(StringInfo str, const Boolean *node, bool omitLocation)
{
appendStringInfoString(str, node->boolval ? "true" : "false");
}
static void
-_outString(StringInfo str, const String *node)
+_outString(StringInfo str, const String *node, bool omitLocation)
{
/*
* We use outToken to provide escaping of the string's content, but we
@@ -792,7 +798,7 @@ _outString(StringInfo str, const String *node)
}
static void
-_outBitString(StringInfo str, const BitString *node)
+_outBitString(StringInfo str, const BitString *node, bool omitLocation)
{
/*
* The lexer will always produce a string starting with 'b' or 'x'. There
@@ -804,7 +810,7 @@ _outBitString(StringInfo str, const BitString *node)
}
static void
-_outA_Const(StringInfo str, const A_Const *node)
+_outA_Const(StringInfo str, const A_Const *node, bool omitLocation)
{
WRITE_NODE_TYPE("A_CONST");
@@ -813,7 +819,7 @@ _outA_Const(StringInfo str, const A_Const *node)
else
{
appendStringInfoString(str, " :val ");
- outNode(str, &node->val);
+ outNode(str, &node->val, omitLocation);
}
WRITE_LOCATION_FIELD(location);
}
@@ -824,7 +830,7 @@ _outA_Const(StringInfo str, const A_Const *node)
* converts a Node into ascii string and append it to 'str'
*/
void
-outNode(StringInfo str, const void *obj)
+outNode(StringInfo str, const void *obj, bool omitLocation)
{
/* Guard against stack overflow due to overly complex expressions */
check_stack_depth();
@@ -833,18 +839,18 @@ outNode(StringInfo str, const void *obj)
appendStringInfoString(str, "<>");
else if (IsA(obj, List) || IsA(obj, IntList) || IsA(obj, OidList) ||
IsA(obj, XidList))
- _outList(str, obj);
+ _outList(str, obj, omitLocation);
/* nodeRead does not want to see { } around these! */
else if (IsA(obj, Integer))
- _outInteger(str, (Integer *) obj);
+ _outInteger(str, (Integer *) obj, omitLocation);
else if (IsA(obj, Float))
- _outFloat(str, (Float *) obj);
+ _outFloat(str, (Float *) obj, omitLocation);
else if (IsA(obj, Boolean))
- _outBoolean(str, (Boolean *) obj);
+ _outBoolean(str, (Boolean *) obj, omitLocation);
else if (IsA(obj, String))
- _outString(str, (String *) obj);
+ _outString(str, (String *) obj, omitLocation);
else if (IsA(obj, BitString))
- _outBitString(str, (BitString *) obj);
+ _outBitString(str, (BitString *) obj, omitLocation);
else if (IsA(obj, Bitmapset))
outBitmapset(str, (Bitmapset *) obj);
else
@@ -879,7 +885,18 @@ nodeToString(const void *obj)
/* see stringinfo.h for an explanation of this maneuver */
initStringInfo(&str);
- outNode(&str, obj);
+ outNode(&str, obj, false);
+ return str.data;
+}
+
+char *
+nodeToStringNoQLocs(const void *obj)
+{
+ StringInfoData str;
+
+ /* see stringinfo.h for an explanation of this maneuver */
+ initStringInfo(&str);
+ outNode(&str, obj, true);
return str.data;
}
diff --git a/src/backend/rewrite/rewriteDefine.c b/src/backend/rewrite/rewriteDefine.c
index b449244a53..6302cd1472 100644
--- a/src/backend/rewrite/rewriteDefine.c
+++ b/src/backend/rewrite/rewriteDefine.c
@@ -64,8 +64,8 @@ InsertRule(const char *rulname,
List *action,
bool replace)
{
- char *evqual = nodeToString(event_qual);
- char *actiontree = nodeToString((Node *) action);
+ char *evqual = nodeToStringNoQLocs(event_qual);
+ char *actiontree = nodeToStringNoQLocs((Node *) action);
Datum values[Natts_pg_rewrite];
bool nulls[Natts_pg_rewrite] = {0};
NameData rname;
diff --git a/src/include/nodes/nodes.h b/src/include/nodes/nodes.h
index 2969dd831b..f7adb5e767 100644
--- a/src/include/nodes/nodes.h
+++ b/src/include/nodes/nodes.h
@@ -188,13 +188,15 @@ castNodeImpl(NodeTag type, void *ptr)
struct Bitmapset; /* not to include bitmapset.h here */
struct StringInfoData; /* not to include stringinfo.h here */
-extern void outNode(struct StringInfoData *str, const void *obj);
+extern void outNode(struct StringInfoData *str, const void *obj,
+ bool omitLocation);
extern void outToken(struct StringInfoData *str, const char *s);
extern void outBitmapset(struct StringInfoData *str,
const struct Bitmapset *bms);
extern void outDatum(struct StringInfoData *str, uintptr_t value,
int typlen, bool typbyval);
extern char *nodeToString(const void *obj);
+extern char *nodeToStringNoQLocs(const void *obj);
extern char *bmsToString(const struct Bitmapset *bms);
/*
--
2.40.1
[application/octet-stream] v7-0001-pg_node_tree-Omit-serialization-of-fields-with-de.patch (22.8K, ../../CAEze2WgKVdmMLAGga8Gfh7DsvuJPxiqPvYH8fS0Px8f3w5g64g@mail.gmail.com/5-v7-0001-pg_node_tree-Omit-serialization-of-fields-with-de.patch)
download | inline diff:
From fae34af93b14949ac672e756aa606e4fba8bd66c Mon Sep 17 00:00:00 2001
From: Matthias van de Meent <[email protected]>
Date: Tue, 2 Jan 2024 23:55:02 +0100
Subject: [PATCH v7 1/8] pg_node_tree: Omit serialization of fields with
default values.
Often, the values in nodes are their default values. By not serializing
those fields and inserting the defaults during deserialization, we
reduce the size of pg_node_tree attributes seen in e.g. pg_rewrite by a
significant factor.
Note: Enum fields are excluded from this for now, as it would hinder
debugging.
In passing, we fix a test that had a strict dependency on the
serialization of pg_node_tree; we now do the checks in a more generic
manner, making it more stable and ensuring its stability in future work.
---
src/backend/nodes/outfuncs.c | 180 ++++++++++++++----
src/backend/nodes/read.c | 58 ++++++
src/backend/nodes/readfuncs.c | 211 ++++++++++++++++------
src/include/nodes/readfuncs.h | 1 +
src/test/regress/expected/rowsecurity.out | 5 +-
src/test/regress/sql/rowsecurity.sql | 5 +-
6 files changed, 373 insertions(+), 87 deletions(-)
diff --git a/src/backend/nodes/outfuncs.c b/src/backend/nodes/outfuncs.c
index 2c30bba212..8ad81be1cd 100644
--- a/src/backend/nodes/outfuncs.c
+++ b/src/backend/nodes/outfuncs.c
@@ -15,6 +15,7 @@
#include "postgres.h"
#include <ctype.h>
+#include <math.h>
#include "access/attnum.h"
#include "common/shortest_dec.h"
@@ -41,94 +42,207 @@ static void outDouble(StringInfo str, double d);
appendStringInfoString(str, nodelabel)
/* Write an integer field (anything written as ":fldname %d") */
+#define WRITE_INT_FIELD_DEFAULT(fldname, default) \
+ do { \
+ if (node->fldname != default) \
+ appendStringInfo(str, " :" CppAsString(fldname) " %d", \
+ node->fldname); \
+ } while (0)
#define WRITE_INT_FIELD(fldname) \
- appendStringInfo(str, " :" CppAsString(fldname) " %d", node->fldname)
+ WRITE_INT_FIELD_DEFAULT(fldname, 0)
/* Write an unsigned integer field (anything written as ":fldname %u") */
+#define WRITE_UINT_FIELD_DEFAULT(fldname, default) \
+ do { \
+ if (node->fldname != default) \
+ appendStringInfo(str, " :" CppAsString(fldname) " %u", \
+ node->fldname); \
+ } while (0)
#define WRITE_UINT_FIELD(fldname) \
- appendStringInfo(str, " :" CppAsString(fldname) " %u", node->fldname)
+ WRITE_UINT_FIELD_DEFAULT(fldname, 0)
/* Write an unsigned integer field (anything written with UINT64_FORMAT) */
+#define WRITE_UINT64_FIELD_DEFAULT(fldname, default) \
+ do { \
+ if (node->fldname != default) \
+ appendStringInfo(str, " :" CppAsString(fldname) " " UINT64_FORMAT, \
+ node->fldname); \
+ } while (0)
#define WRITE_UINT64_FIELD(fldname) \
- appendStringInfo(str, " :" CppAsString(fldname) " " UINT64_FORMAT, \
- node->fldname)
+ WRITE_UINT64_FIELD_DEFAULT(fldname, 0)
/* Write an OID field (don't hard-wire assumption that OID is same as uint) */
+#define WRITE_OID_FIELD_DEFAULT(fldname, default) \
+ do { \
+ if (node->fldname != default) \
+ appendStringInfo(str, " :" CppAsString(fldname) " %u", \
+ node->fldname); \
+ } while (0)
#define WRITE_OID_FIELD(fldname) \
- appendStringInfo(str, " :" CppAsString(fldname) " %u", node->fldname)
+ WRITE_OID_FIELD_DEFAULT(fldname, 0)
/* Write a long-integer field */
+#define WRITE_LONG_FIELD_DEFAULT(fldname, default) \
+ do { \
+ if (node->fldname != default) \
+ appendStringInfo(str, " :" CppAsString(fldname) " %ld", \
+ node->fldname); \
+ } while (0)
#define WRITE_LONG_FIELD(fldname) \
- appendStringInfo(str, " :" CppAsString(fldname) " %ld", node->fldname)
+ WRITE_LONG_FIELD_DEFAULT(fldname, 0)
+
/* Write a char field (ie, one ascii character) */
+#define WRITE_CHAR_FIELD_DEFAULT(fldname, default) \
+ do { \
+ if (node->fldname != default) \
+ { \
+ appendStringInfoString(str, " :" CppAsString(fldname) " "); \
+ outChar(str, node->fldname); \
+ } \
+ } while (0)
#define WRITE_CHAR_FIELD(fldname) \
- (appendStringInfo(str, " :" CppAsString(fldname) " "), \
- outChar(str, node->fldname))
+ WRITE_CHAR_FIELD_DEFAULT(fldname, '\0')
/* Write an enumerated-type field as an integer code */
+#define WRITE_ENUM_FIELD_DEFAULT(fldname, enumtype, default) \
+ do { \
+ if (node->fldname != default) \
+ appendStringInfo(str, " :" CppAsString(fldname) " %d", \
+ (int) node->fldname); \
+ } while (0)
#define WRITE_ENUM_FIELD(fldname, enumtype) \
- appendStringInfo(str, " :" CppAsString(fldname) " %d", \
- (int) node->fldname)
+ do { \
+ appendStringInfo(str, " :" CppAsString(fldname) " %d", \
+ (int) node->fldname); \
+ } while (0)
/* Write a float field (actually, they're double) */
+#define WRITE_FLOAT_FIELD_DEFAULT(fldname, default) \
+ do { \
+ if (node->fldname != default || \
+ signbit(node->fldname) != signbit(default)) \
+ { \
+ appendStringInfoString(str, " :" CppAsString(fldname) " "); \
+ outDouble(str, node->fldname); \
+ } \
+ } while (0)
#define WRITE_FLOAT_FIELD(fldname) \
- (appendStringInfo(str, " :" CppAsString(fldname) " "), \
- outDouble(str, node->fldname))
+ WRITE_FLOAT_FIELD_DEFAULT(fldname, 0.0)
/* Write a boolean field */
+#define WRITE_BOOL_FIELD_DEFAULT(fldname, default) \
+ do { \
+ if (node->fldname != default) \
+ appendStringInfo(str, " :" CppAsString(fldname) " %s", \
+ booltostr(node->fldname)); \
+ } while (0)
#define WRITE_BOOL_FIELD(fldname) \
- appendStringInfo(str, " :" CppAsString(fldname) " %s", \
- booltostr(node->fldname))
+ WRITE_BOOL_FIELD_DEFAULT(fldname, false)
+
+/*
+ * Non-null defaults of by-ref types are exceedingly rare (if not generally
+ * nonexistent), so we don't (yet) have a specialized macro for non-NULL
+ * defaults omission.
+ */
/* Write a character-string (possibly NULL) field */
#define WRITE_STRING_FIELD(fldname) \
- (appendStringInfoString(str, " :" CppAsString(fldname) " "), \
- outToken(str, node->fldname))
+ do { \
+ if (node->fldname != NULL) \
+ { \
+ appendStringInfoString(str, " :" CppAsString(fldname) " "); \
+ outToken(str, node->fldname); \
+ } \
+ } while (0)
/* Write a parse location field (actually same as INT case) */
#define WRITE_LOCATION_FIELD(fldname) \
- appendStringInfo(str, " :" CppAsString(fldname) " %d", node->fldname)
+ do { \
+ if (node->fldname != -1) \
+ appendStringInfo(str, " :" CppAsString(fldname) " %d", \
+ node->fldname); \
+ } while (0)
/* Write a Node field */
#define WRITE_NODE_FIELD(fldname) \
- (appendStringInfoString(str, " :" CppAsString(fldname) " "), \
- outNode(str, node->fldname))
+ do { \
+ if (node->fldname != NULL) \
+ { \
+ appendStringInfoString(str, " :" CppAsString(fldname) " "); \
+ outNode(str, node->fldname); \
+ } \
+ } while (0)
/* Write a bitmapset field */
#define WRITE_BITMAPSET_FIELD(fldname) \
- (appendStringInfoString(str, " :" CppAsString(fldname) " "), \
- outBitmapset(str, node->fldname))
+ do { \
+ if (node->fldname != NULL) \
+ { \
+ appendStringInfoString(str, " :" CppAsString(fldname) " "); \
+ outBitmapset(str, node->fldname); \
+ } \
+ } while (0)
/* Write a variable-length array (not a List) of Node pointers */
#define WRITE_NODE_ARRAY(fldname, len) \
- (appendStringInfoString(str, " :" CppAsString(fldname) " "), \
- writeNodeArray(str, (const Node * const *) node->fldname, len))
+ do { \
+ if (node->fldname != NULL) \
+ { \
+ appendStringInfoString(str, " :" CppAsString(fldname) " "); \
+ writeNodeArray(str, (const Node * const *) node->fldname, len); \
+ } \
+ } while (0)
/* Write a variable-length array of AttrNumber */
#define WRITE_ATTRNUMBER_ARRAY(fldname, len) \
- (appendStringInfoString(str, " :" CppAsString(fldname) " "), \
- writeAttrNumberCols(str, node->fldname, len))
+ do { \
+ if (node->fldname != NULL) \
+ { \
+ appendStringInfoString(str, " :" CppAsString(fldname) " "); \
+ writeAttrNumberCols(str, node->fldname, len); \
+ } \
+ } while (0)
/* Write a variable-length array of Oid */
#define WRITE_OID_ARRAY(fldname, len) \
- (appendStringInfoString(str, " :" CppAsString(fldname) " "), \
- writeOidCols(str, node->fldname, len))
+ do { \
+ if (node->fldname != NULL) \
+ { \
+ appendStringInfoString(str, " :" CppAsString(fldname) " "); \
+ writeOidCols(str, node->fldname, len); \
+ } \
+ } while (0)
/* Write a variable-length array of Index */
#define WRITE_INDEX_ARRAY(fldname, len) \
- (appendStringInfoString(str, " :" CppAsString(fldname) " "), \
- writeIndexCols(str, node->fldname, len))
+ do { \
+ if (node->fldname != NULL) \
+ { \
+ appendStringInfoString(str, " :" CppAsString(fldname) " "); \
+ writeIndexCols(str, node->fldname, len); \
+ } \
+ } while (0)
/* Write a variable-length array of int */
#define WRITE_INT_ARRAY(fldname, len) \
- (appendStringInfoString(str, " :" CppAsString(fldname) " "), \
- writeIntCols(str, node->fldname, len))
+ do { \
+ if (node->fldname != NULL) \
+ { \
+ appendStringInfoString(str, " :" CppAsString(fldname) " "); \
+ writeIntCols(str, node->fldname, len); \
+ } \
+ } while (0)
/* Write a variable-length array of bool */
#define WRITE_BOOL_ARRAY(fldname, len) \
- (appendStringInfoString(str, " :" CppAsString(fldname) " "), \
- writeBoolCols(str, node->fldname, len))
+ do { \
+ if (node->fldname != NULL) \
+ { \
+ appendStringInfoString(str, " :" CppAsString(fldname) " "); \
+ writeBoolCols(str, node->fldname, len); \
+ } \
+ } while (0)
#define booltostr(x) ((x) ? "true" : "false")
diff --git a/src/backend/nodes/read.c b/src/backend/nodes/read.c
index 4eb42445c5..cffd913ba6 100644
--- a/src/backend/nodes/read.c
+++ b/src/backend/nodes/read.c
@@ -205,6 +205,64 @@ pg_strtok(int *length)
return ret_str;
}
+/*
+ * Check if the next token is the expected field name, and if so,
+ * consume and return it.
+ *
+ * It handles similar to pg_strtok, except that it is special-cased for user-
+ * provided field names, rather than arbitrary tokens.
+ */
+const char *
+pg_strtok_fieldname(const char *fldname, int *length)
+{
+ const char *local_str; /* working pointer to string */
+ int expect_len;
+ char next_char;
+
+ /* skip global state to the next token */
+ while (*pg_strtok_ptr == ' ' ||
+ *pg_strtok_ptr == '\n' ||
+ *pg_strtok_ptr == '\t')
+ pg_strtok_ptr++;
+
+ local_str = pg_strtok_ptr;
+
+ if (*local_str != ':')
+ return false; /* not a field name */
+
+ expect_len = strlen(fldname);
+
+ Assert(expect_len > 0);
+
+ /* check if the next few bytes match the token */
+ if (strncmp(local_str, fldname, expect_len) != 0)
+ return false;
+
+ next_char = local_str[expect_len];
+
+ /*
+ * Check that the token was actually terminated at the end of the
+ * expected token with a character that is a separate token.
+ * Otherwise, we'd get positive matches for mathing the token of "is"
+ * against a local_str of "isn't", which is clearly wrong.
+ */
+ if (next_char == '\0' ||
+ next_char == ' ' ||
+ next_char == '\n' ||
+ next_char == '\t' ||
+ next_char == '(' ||
+ next_char == ')' ||
+ next_char == '{' ||
+ next_char == '}')
+ {
+ pg_strtok_ptr = local_str + expect_len;
+ *length = expect_len;
+
+ return local_str;
+ }
+ return NULL;
+}
+
/*
* debackslash -
* create a palloc'd string holding the given token.
diff --git a/src/backend/nodes/readfuncs.c b/src/backend/nodes/readfuncs.c
index b1e2f2b440..e61bdbaef4 100644
--- a/src/backend/nodes/readfuncs.c
+++ b/src/backend/nodes/readfuncs.c
@@ -57,111 +57,218 @@
READ_TEMP_LOCALS()
/* Read an integer field (anything written as ":fldname %d") */
+#define READ_INT_FIELD_DEFAULT(fldname, default_value) \
+ do { \
+ if ((token = pg_strtok_fieldname(":" CppAsString(fldname), &length))) \
+ { \
+ token = pg_strtok(&length); /* get field value */ \
+ local_node->fldname = atoi(token); \
+ } \
+ else \
+ local_node->fldname = default_value; \
+ } while (0)
#define READ_INT_FIELD(fldname) \
- token = pg_strtok(&length); /* skip :fldname */ \
- token = pg_strtok(&length); /* get field value */ \
- local_node->fldname = atoi(token)
+ READ_INT_FIELD_DEFAULT(fldname, 0)
/* Read an unsigned integer field (anything written as ":fldname %u") */
+#define READ_UINT_FIELD_DEFAULT(fldname, default_value) \
+ do { \
+ if ((token = pg_strtok_fieldname(":" CppAsString(fldname), &length))) \
+ { \
+ token = pg_strtok(&length); /* get field value */ \
+ local_node->fldname = atoui(token); \
+ } \
+ else \
+ local_node->fldname = default_value; \
+ } while (0)
#define READ_UINT_FIELD(fldname) \
- token = pg_strtok(&length); /* skip :fldname */ \
- token = pg_strtok(&length); /* get field value */ \
- local_node->fldname = atoui(token)
+ READ_UINT_FIELD_DEFAULT(fldname, 0)
/* Read an unsigned integer field (anything written using UINT64_FORMAT) */
+#define READ_UINT64_FIELD_DEFAULT(fldname, default_value) \
+ do { \
+ if ((token = pg_strtok_fieldname(":" CppAsString(fldname), &length))) \
+ { \
+ token = pg_strtok(&length); /* get field value */ \
+ local_node->fldname = strtou64(token, NULL, 10); \
+ } \
+ else \
+ local_node->fldname = default_value; \
+ } while (0)
#define READ_UINT64_FIELD(fldname) \
- token = pg_strtok(&length); /* skip :fldname */ \
- token = pg_strtok(&length); /* get field value */ \
- local_node->fldname = strtou64(token, NULL, 10)
+ READ_UINT64_FIELD_DEFAULT(fldname, 0)
/* Read a long integer field (anything written as ":fldname %ld") */
+#define READ_LONG_FIELD_DEFAULT(fldname, default_value) \
+ do { \
+ if ((token = pg_strtok_fieldname(":" CppAsString(fldname), &length))) \
+ { \
+ token = pg_strtok(&length); /* get field value */ \
+ local_node->fldname = atol(token); \
+ } \
+ else \
+ local_node->fldname = default_value; \
+ } while (0)
#define READ_LONG_FIELD(fldname) \
- token = pg_strtok(&length); /* skip :fldname */ \
- token = pg_strtok(&length); /* get field value */ \
- local_node->fldname = atol(token)
+ READ_LONG_FIELD_DEFAULT(fldname, 0)
/* Read an OID field (don't hard-wire assumption that OID is same as uint) */
+#define READ_OID_FIELD_DEFAULT(fldname, default_value) \
+ do { \
+ if ((token = pg_strtok_fieldname(":" CppAsString(fldname), &length))) \
+ { \
+ token = pg_strtok(&length); /* get field value */ \
+ local_node->fldname = atooid(token); \
+ } \
+ else \
+ local_node->fldname = default_value; \
+ } while (0)
#define READ_OID_FIELD(fldname) \
- token = pg_strtok(&length); /* skip :fldname */ \
- token = pg_strtok(&length); /* get field value */ \
- local_node->fldname = atooid(token)
+ READ_OID_FIELD_DEFAULT(fldname, 0)
/* Read a char field (ie, one ascii character) */
+#define READ_CHAR_FIELD_DEFAULT(fldname, default_value) \
+ do { \
+ if ((token = pg_strtok_fieldname(":" CppAsString(fldname), &length))) \
+ { \
+ token = pg_strtok(&length); /* get field value */ \
+ /* avoid overhead of calling debackslash() for one char */ \
+ local_node->fldname = (length == 0) ? '\0' : (token[0] == '\\' ? token[1] : token[0]); \
+ } \
+ else \
+ local_node->fldname = default_value; \
+ } while (0)
#define READ_CHAR_FIELD(fldname) \
- token = pg_strtok(&length); /* skip :fldname */ \
- token = pg_strtok(&length); /* get field value */ \
- /* avoid overhead of calling debackslash() for one char */ \
- local_node->fldname = (length == 0) ? '\0' : (token[0] == '\\' ? token[1] : token[0])
+ READ_CHAR_FIELD_DEFAULT(fldname, '\0')
/* Read an enumerated-type field that was written as an integer code */
#define READ_ENUM_FIELD(fldname, enumtype) \
- token = pg_strtok(&length); /* skip :fldname */ \
- token = pg_strtok(&length); /* get field value */ \
- local_node->fldname = (enumtype) atoi(token)
+ do { \
+ token = pg_strtok_fieldname(":" CppAsString(fldname), &length); \
+ Assert(token != NULL); \
+ token = pg_strtok(&length); /* get field value */ \
+ local_node->fldname = (enumtype) atoi(token); \
+ } while (0)
/* Read a float field */
+#define READ_FLOAT_FIELD_DEFAULT(fldname, default_value) \
+ do { \
+ if ((token = pg_strtok_fieldname(":" CppAsString(fldname), &length))) \
+ { \
+ token = pg_strtok(&length); /* get field value */ \
+ local_node->fldname = atof(token); \
+ } \
+ else \
+ local_node->fldname = default_value; \
+ } while (0)
#define READ_FLOAT_FIELD(fldname) \
- token = pg_strtok(&length); /* skip :fldname */ \
- token = pg_strtok(&length); /* get field value */ \
- local_node->fldname = atof(token)
+ READ_FLOAT_FIELD_DEFAULT(fldname, 0.0)
/* Read a boolean field */
+#define READ_BOOL_FIELD_DEFAULT(fldname, default_value) \
+ do { \
+ if ((token = pg_strtok_fieldname(":" CppAsString(fldname), &length))) \
+ { \
+ token = pg_strtok(&length); /* get field value */ \
+ local_node->fldname = strtobool(token); \
+ } \
+ else \
+ local_node->fldname = default_value; \
+ } while (0)
#define READ_BOOL_FIELD(fldname) \
- token = pg_strtok(&length); /* skip :fldname */ \
- token = pg_strtok(&length); /* get field value */ \
- local_node->fldname = strtobool(token)
+ READ_BOOL_FIELD_DEFAULT(fldname, false)
/* Read a character-string field */
+/* see WRITE_STRING_FIELD in outfuncs.c */
#define READ_STRING_FIELD(fldname) \
- token = pg_strtok(&length); /* skip :fldname */ \
- token = pg_strtok(&length); /* get field value */ \
- local_node->fldname = nullable_string(token, length)
+ do { \
+ if ((token = pg_strtok_fieldname(":" CppAsString(fldname), &length))) \
+ { \
+ token = pg_strtok(&length); /* get field value */ \
+ local_node->fldname = nullable_string(token, length); \
+ } \
+ else \
+ local_node->fldname = NULL; \
+ } while (0)
/* Read a parse location field (and possibly throw away the value) */
#ifdef WRITE_READ_PARSE_PLAN_TREES
#define READ_LOCATION_FIELD(fldname) \
- token = pg_strtok(&length); /* skip :fldname */ \
- token = pg_strtok(&length); /* get field value */ \
- local_node->fldname = restore_location_fields ? atoi(token) : -1
+ do { \
+ if ((token = pg_strtok_fieldname(":" CppAsString(fldname), &length))) \
+ { \
+ token = pg_strtok(&length); /* get field value */ \
+ local_node->fldname = restore_location_fields ? atoi(token) : -1; \
+ } \
+ else \
+ local_node->fldname = -1; \
+ } while (0)
#else
#define READ_LOCATION_FIELD(fldname) \
- token = pg_strtok(&length); /* skip :fldname */ \
- token = pg_strtok(&length); /* get field value */ \
- (void) token; /* in case not used elsewhere */ \
- local_node->fldname = -1 /* set field to "unknown" */
+ do { \
+ if ((token = pg_strtok_fieldname(":" CppAsString(fldname), &length))) \
+ { \
+ token = pg_strtok(&length); /* the field value is useless */ \
+ (void) token; \
+ } \
+ local_node->fldname = -1; \
+ } while (0)
#endif
+
/* Read a Node field */
#define READ_NODE_FIELD(fldname) \
- token = pg_strtok(&length); /* skip :fldname */ \
- (void) token; /* in case not used elsewhere */ \
- local_node->fldname = nodeRead(NULL, 0)
+ do { \
+ if ((token = pg_strtok_fieldname(":" CppAsString(fldname), &length))) \
+ local_node->fldname = nodeRead(NULL, 0); \
+ else \
+ local_node->fldname = NULL; \
+ } while (0)
/* Read a bitmapset field */
#define READ_BITMAPSET_FIELD(fldname) \
- token = pg_strtok(&length); /* skip :fldname */ \
- (void) token; /* in case not used elsewhere */ \
- local_node->fldname = _readBitmapset()
+ do { \
+ if ((token = pg_strtok_fieldname(":" CppAsString(fldname), &length))) \
+ local_node->fldname = _readBitmapset(); \
+ else \
+ local_node->fldname = NULL; \
+ } while (0)
/* Read an attribute number array */
#define READ_ATTRNUMBER_ARRAY(fldname, len) \
- token = pg_strtok(&length); /* skip :fldname */ \
- local_node->fldname = readAttrNumberCols(len)
+ do { \
+ if ((token = pg_strtok_fieldname(":" CppAsString(fldname), &length))) \
+ local_node->fldname = readAttrNumberCols(len); \
+ else \
+ local_node->fldname = NULL; \
+ } while (0)
/* Read an oid array */
#define READ_OID_ARRAY(fldname, len) \
- token = pg_strtok(&length); /* skip :fldname */ \
- local_node->fldname = readOidCols(len)
+ do { \
+ if ((token = pg_strtok_fieldname(":" CppAsString(fldname), &length))) \
+ local_node->fldname = readOidCols(len); \
+ else \
+ local_node->fldname = NULL; \
+ } while (0)
/* Read an int array */
#define READ_INT_ARRAY(fldname, len) \
- token = pg_strtok(&length); /* skip :fldname */ \
- local_node->fldname = readIntCols(len)
+ do { \
+ if ((token = pg_strtok_fieldname(":" CppAsString(fldname), &length))) \
+ local_node->fldname = readIntCols(len); \
+ else \
+ local_node->fldname = NULL; \
+ } while (0)
/* Read a bool array */
#define READ_BOOL_ARRAY(fldname, len) \
- token = pg_strtok(&length); /* skip :fldname */ \
- local_node->fldname = readBoolCols(len)
+ do { \
+ if ((token = pg_strtok_fieldname(":" CppAsString(fldname), &length))) \
+ local_node->fldname = readBoolCols(len); \
+ else \
+ local_node->fldname = NULL; \
+ } while (0)
/* Routine exit */
#define READ_DONE() \
diff --git a/src/include/nodes/readfuncs.h b/src/include/nodes/readfuncs.h
index 8466038ed0..0579c62973 100644
--- a/src/include/nodes/readfuncs.h
+++ b/src/include/nodes/readfuncs.h
@@ -27,6 +27,7 @@ extern PGDLLIMPORT bool restore_location_fields;
* prototypes for functions in read.c (the lisp token parser)
*/
extern const char *pg_strtok(int *length);
+extern const char *pg_strtok_fieldname(const char *fldname, int *length);
extern char *debackslash(const char *token, int length);
extern void *nodeRead(const char *token, int tok_len);
diff --git a/src/test/regress/expected/rowsecurity.out b/src/test/regress/expected/rowsecurity.out
index 6988128aa4..a69aa40f82 100644
--- a/src/test/regress/expected/rowsecurity.out
+++ b/src/test/regress/expected/rowsecurity.out
@@ -3937,7 +3937,10 @@ CREATE TABLE coll_t (c) AS VALUES ('bar'::text);
CREATE POLICY coll_p ON coll_t USING (c < ('foo'::text COLLATE "C"));
ALTER TABLE coll_t ENABLE ROW LEVEL SECURITY;
GRANT SELECT ON coll_t TO regress_rls_alice;
-SELECT (string_to_array(polqual, ':'))[7] AS inputcollid FROM pg_policy WHERE polrelid = 'coll_t'::regclass;
+SELECT split AS inputcollid
+FROM pg_policy,
+ lateral unnest(string_to_array(polqual, ':')) as split
+WHERE polrelid = 'coll_t'::regclass and split LIKE '%inputcollid%';
inputcollid
------------------
inputcollid 950
diff --git a/src/test/regress/sql/rowsecurity.sql b/src/test/regress/sql/rowsecurity.sql
index dec7340538..cb7a4c776b 100644
--- a/src/test/regress/sql/rowsecurity.sql
+++ b/src/test/regress/sql/rowsecurity.sql
@@ -1732,7 +1732,10 @@ CREATE TABLE coll_t (c) AS VALUES ('bar'::text);
CREATE POLICY coll_p ON coll_t USING (c < ('foo'::text COLLATE "C"));
ALTER TABLE coll_t ENABLE ROW LEVEL SECURITY;
GRANT SELECT ON coll_t TO regress_rls_alice;
-SELECT (string_to_array(polqual, ':'))[7] AS inputcollid FROM pg_policy WHERE polrelid = 'coll_t'::regclass;
+SELECT split AS inputcollid
+FROM pg_policy,
+ lateral unnest(string_to_array(polqual, ':')) as split
+WHERE polrelid = 'coll_t'::regclass and split LIKE '%inputcollid%';
SET SESSION AUTHORIZATION regress_rls_alice;
SELECT * FROM coll_t;
ROLLBACK;
--
2.40.1
[application/octet-stream] v7-0005-nodeToString-omit-serializing-NULL-datums-in-Cons.patch (1.8K, ../../CAEze2WgKVdmMLAGga8Gfh7DsvuJPxiqPvYH8fS0Px8f3w5g64g@mail.gmail.com/6-v7-0005-nodeToString-omit-serializing-NULL-datums-in-Cons.patch)
download | inline diff:
From 6edc0b73cf35b152a13bd7790ef1f8a181e3ba45 Mon Sep 17 00:00:00 2001
From: Matthias van de Meent <[email protected]>
Date: Thu, 8 Feb 2024 20:24:22 +0100
Subject: [PATCH v7 5/8] nodeToString: omit serializing NULL datums in Const
nodes
This saves some bytes in certain cases, and aligns its serialization conditions
with other field's serialization conditions.
---
src/backend/nodes/outfuncs.c | 8 ++++----
src/backend/nodes/readfuncs.c | 10 ++++++----
2 files changed, 10 insertions(+), 8 deletions(-)
diff --git a/src/backend/nodes/outfuncs.c b/src/backend/nodes/outfuncs.c
index d1395b1940..bc57b7add6 100644
--- a/src/backend/nodes/outfuncs.c
+++ b/src/backend/nodes/outfuncs.c
@@ -504,11 +504,11 @@ _outConst(StringInfo str, const Const *node, bool omitLocation)
WRITE_BOOL_FIELD(constisnull);
WRITE_LOCATION_FIELD(location);
- appendStringInfoString(str, " :constvalue ");
- if (node->constisnull)
- appendStringInfoString(str, "<>");
- else
+ if (!node->constisnull)
+ {
+ appendStringInfoString(str, " :constvalue ");
outDatum(str, node->constvalue, node->constlen, node->constbyval);
+ }
}
static void
diff --git a/src/backend/nodes/readfuncs.c b/src/backend/nodes/readfuncs.c
index 5453f53128..4f0fc4cac5 100644
--- a/src/backend/nodes/readfuncs.c
+++ b/src/backend/nodes/readfuncs.c
@@ -375,11 +375,13 @@ _readConst(void)
READ_BOOL_FIELD(constisnull);
READ_LOCATION_FIELD(location);
- token = pg_strtok(&length); /* skip :constvalue */
- if (local_node->constisnull)
- token = pg_strtok(&length); /* skip "<>" */
- else
+ if ((token = pg_strtok_fieldname(":constvalue", &length)))
local_node->constvalue = readDatum(local_node->constbyval);
+ else
+ {
+ /* value was omitted */
+ Assert(local_node->constisnull);
+ }
READ_DONE();
}
--
2.40.1
[application/octet-stream] v7-0007-gen_node_support.pl-Optimize-serialization-of-fie.patch (9.2K, ../../CAEze2WgKVdmMLAGga8Gfh7DsvuJPxiqPvYH8fS0Px8f3w5g64g@mail.gmail.com/7-v7-0007-gen_node_support.pl-Optimize-serialization-of-fie.patch)
download | inline diff:
From e96b48c0779233ebe66f6bc0b44caad067d15d36 Mon Sep 17 00:00:00 2001
From: Matthias van de Meent <[email protected]>
Date: Mon, 12 Feb 2024 17:12:02 +0100
Subject: [PATCH v7 7/8] gen_node_support.pl: Optimize serialization of fields
with copied values
The Var node's [syn] fields often contain the same data as their non-syn
counterparts. We invent a new pg_node_attr()ibute that represents this
relation, which allows us to omit these fields from serialization when they
are indeed copies of the original fields.
---
src/backend/nodes/gen_node_support.pl | 94 ++++++++++++++++++---------
src/backend/nodes/outfuncs.c | 3 +
src/backend/nodes/readfuncs.c | 3 +
src/include/nodes/primnodes.h | 4 +-
4 files changed, 70 insertions(+), 34 deletions(-)
diff --git a/src/backend/nodes/gen_node_support.pl b/src/backend/nodes/gen_node_support.pl
index fda8c3a494..66661a3881 100644
--- a/src/backend/nodes/gen_node_support.pl
+++ b/src/backend/nodes/gen_node_support.pl
@@ -363,6 +363,10 @@ foreach my $infile (@ARGV)
{
$manual_nodetag_number{$in_struct} = $1;
}
+ elsif ($attr =~ /^default_ref\(([\w\d."'-]+)\)$/)
+ {
+ # Unused at the node level
+ }
else
{
die
@@ -437,7 +441,7 @@ foreach my $infile (@ARGV)
}
# normal struct field
elsif ($line =~
- /^\s*(.+)\s*\b(\w+)(\[[\w\s+]+\])?\s*(?:pg_node_attr\(([\w(), ]*)\))?;/
+ /^\s*(.+)\s*\b(\w+)(\[[\w\s+]+\])?\s*(?:pg_node_attr\(([\w\d(), ."'-]*)\))?;/
)
{
if ($is_node_struct)
@@ -469,6 +473,7 @@ foreach my $infile (@ARGV)
if ( $attr !~ /^array_size\(\w+\)$/
&& $attr !~ /^copy_as\(\w+\)$/
&& $attr !~ /^read_as\(\w+\)$/
+ && $attr !~ /^default_ref\([\w\d."'-]+\)$/
&& !elem $attr,
qw(copy_as_scalar
equal_as_scalar
@@ -495,7 +500,7 @@ foreach my $infile (@ARGV)
}
# function pointer field
elsif ($line =~
- /^\s*([\w\s*]+)\s*\(\*(\w+)\)\s*\((.*)\)\s*(?:pg_node_attr\(([\w(), ]*)\))?;/
+ /^\s*([\w\s*]+)\s*\(\*(\w+)\)\s*\((.*)\)\s*(?:pg_node_attr\(([\w\d(), ."'-]*)\))?;/
)
{
if ($is_node_struct)
@@ -970,6 +975,15 @@ _read${n}(void)
my $array_size_field;
my $read_as_field;
my $read_write_ignore = 0;
+ # Type read/write macro suffix, "" for normal use, or "_DEFAULT" when
+ # value argument.
+ my $s = "";
+ # Default parameter to read/write macro. Includes comma separator so
+ # that MACRO_NAME$s($fieldname$defaultparam) is the full complete
+ # read/write expression for essentially all types.
+ # Note that this (currently) only works for scalar values.
+ my $d = "";
+
foreach my $a (@a)
{
if ($a =~ /^array_size\(([\w.]+)\)$/)
@@ -988,6 +1002,19 @@ _read${n}(void)
{
$read_write_ignore = 1;
}
+ elsif ($a =~ /^default_ref\(([\w\d+."'-]+)\)$/)
+ {
+ $s = "_DEFAULT";
+ $d = ", NODE_FIELD($1)";
+ }
+ }
+
+ if ($s eq "_DEFAULT")
+ {
+ die "custom defaults for non-scalar fields are not supported\n\tat $n.$f"
+ unless (elem $t, @scalar_types or elem $t, @enum_types);
+ die "custom defaults for Location fields are not supported\n\tat $n.$f"
+ if ($t eq "Location");
}
if ($read_write_ignore)
@@ -1008,8 +1035,8 @@ _read${n}(void)
# select instructions by field type
if ($t eq 'bool')
{
- print $off "\tWRITE_BOOL_FIELD($f);\n";
- print $rff "\tREAD_BOOL_FIELD($f);\n" unless $no_read;
+ print $off "\tWRITE_BOOL_FIELD$s($f$d);\n";
+ print $rff "\tREAD_BOOL_FIELD$s($f$d);\n" unless $no_read;
}
elsif ($t eq 'Location')
{
@@ -1018,8 +1045,11 @@ _read${n}(void)
}
elsif ($t eq 'TypMod')
{
- print $off "\tWRITE_INT_FIELD_DEFAULT($f, -1);\n";
- print $rff "\tREAD_INT_FIELD_DEFAULT($f, -1);\n" unless $no_read;
+ # The default value of a TypMod fields is -1, rather than 0
+ # for normal int fields.
+ $d = ", -1" if ($d eq "");
+ print $off "\tWRITE_INT_FIELD_DEFAULT($f$d);\n";
+ print $rff "\tREAD_INT_FIELD_DEFAULT($f$d);\n" unless $no_read;
}
elsif ($t eq 'int'
|| $t eq 'int16'
@@ -1027,8 +1057,8 @@ _read${n}(void)
|| $t eq 'AttrNumber'
|| $t eq 'StrategyNumber')
{
- print $off "\tWRITE_INT_FIELD($f);\n";
- print $rff "\tREAD_INT_FIELD($f);\n" unless $no_read;
+ print $off "\tWRITE_INT_FIELD$s($f$d);\n";
+ print $rff "\tREAD_INT_FIELD$s($f$d);\n" unless $no_read;
}
elsif ($t eq 'uint32'
|| $t eq 'bits32'
@@ -1036,56 +1066,56 @@ _read${n}(void)
|| $t eq 'Index'
|| $t eq 'SubTransactionId')
{
- print $off "\tWRITE_UINT_FIELD($f);\n";
- print $rff "\tREAD_UINT_FIELD($f);\n" unless $no_read;
+ print $off "\tWRITE_UINT_FIELD$s($f$d);\n";
+ print $rff "\tREAD_UINT_FIELD$s($f$d);\n" unless $no_read;
}
elsif ($t eq 'uint64'
|| $t eq 'AclMode')
{
- print $off "\tWRITE_UINT64_FIELD($f);\n";
- print $rff "\tREAD_UINT64_FIELD($f);\n" unless $no_read;
+ print $off "\tWRITE_UINT64_FIELD$s($f$d);\n";
+ print $rff "\tREAD_UINT64_FIELD$s($f$d);\n" unless $no_read;
}
elsif ($t eq 'Oid' || $t eq 'RelFileNumber')
{
- print $off "\tWRITE_OID_FIELD($f);\n";
- print $rff "\tREAD_OID_FIELD($f);\n" unless $no_read;
+ print $off "\tWRITE_OID_FIELD$s($f$d);\n";
+ print $rff "\tREAD_OID_FIELD$s($f$d);\n" unless $no_read;
}
elsif ($t eq 'long')
{
- print $off "\tWRITE_LONG_FIELD($f);\n";
- print $rff "\tREAD_LONG_FIELD($f);\n" unless $no_read;
+ print $off "\tWRITE_LONG_FIELD$s($f$d);\n";
+ print $rff "\tREAD_LONG_FIELD$s($f$d);\n" unless $no_read;
}
elsif ($t eq 'char')
{
- print $off "\tWRITE_CHAR_FIELD($f);\n";
- print $rff "\tREAD_CHAR_FIELD($f);\n" unless $no_read;
+ print $off "\tWRITE_CHAR_FIELD$s($f$d);\n";
+ print $rff "\tREAD_CHAR_FIELD$s($f$d);\n" unless $no_read;
}
elsif ($t eq 'double')
{
- print $off "\tWRITE_FLOAT_FIELD($f);\n";
- print $rff "\tREAD_FLOAT_FIELD($f);\n" unless $no_read;
+ print $off "\tWRITE_FLOAT_FIELD$s($f$d);\n";
+ print $rff "\tREAD_FLOAT_FIELD$s($f$d);\n" unless $no_read;
}
elsif ($t eq 'Cardinality')
{
- print $off "\tWRITE_FLOAT_FIELD($f);\n";
- print $rff "\tREAD_FLOAT_FIELD($f);\n" unless $no_read;
+ print $off "\tWRITE_FLOAT_FIELD$s($f$d);\n";
+ print $rff "\tREAD_FLOAT_FIELD$s($f$d);\n" unless $no_read;
}
elsif ($t eq 'Cost')
{
- print $off "\tWRITE_FLOAT_FIELD($f);\n";
- print $rff "\tREAD_FLOAT_FIELD($f);\n" unless $no_read;
+ print $off "\tWRITE_FLOAT_FIELD$s($f$d);\n";
+ print $rff "\tREAD_FLOAT_FIELD$s($f$d);\n" unless $no_read;
}
elsif ($t eq 'QualCost')
{
- print $off "\tWRITE_FLOAT_FIELD($f.startup);\n";
- print $off "\tWRITE_FLOAT_FIELD($f.per_tuple);\n";
- print $rff "\tREAD_FLOAT_FIELD($f.startup);\n" unless $no_read;
- print $rff "\tREAD_FLOAT_FIELD($f.per_tuple);\n" unless $no_read;
+ print $off "\tWRITE_FLOAT_FIELD$s($f.startup$d);\n";
+ print $off "\tWRITE_FLOAT_FIELD$s($f.per_tuple$d);\n";
+ print $rff "\tREAD_FLOAT_FIELD$s($f.startup$d);\n" unless $no_read;
+ print $rff "\tREAD_FLOAT_FIELD$s($f.per_tuple$d);\n" unless $no_read;
}
elsif ($t eq 'Selectivity')
{
- print $off "\tWRITE_FLOAT_FIELD($f);\n";
- print $rff "\tREAD_FLOAT_FIELD($f);\n" unless $no_read;
+ print $off "\tWRITE_FLOAT_FIELD$s($f$d);\n";
+ print $rff "\tREAD_FLOAT_FIELD$s($f$d);\n" unless $no_read;
}
elsif ($t eq 'char*')
{
@@ -1099,8 +1129,8 @@ _read${n}(void)
}
elsif (elem $t, @enum_types)
{
- print $off "\tWRITE_ENUM_FIELD($f, $t);\n";
- print $rff "\tREAD_ENUM_FIELD($f, $t);\n" unless $no_read;
+ print $off "\tWRITE_ENUM_FIELD$s($f, $t$d);\n";
+ print $rff "\tREAD_ENUM_FIELD$s($f, $t$d);\n" unless $no_read;
}
# arrays of scalar types
elsif ($t =~ /^(\w+)(\*|\[\w+\])$/ and elem $1, @scalar_types)
diff --git a/src/backend/nodes/outfuncs.c b/src/backend/nodes/outfuncs.c
index cfafdaa033..e64f3c5975 100644
--- a/src/backend/nodes/outfuncs.c
+++ b/src/backend/nodes/outfuncs.c
@@ -245,6 +245,9 @@ static void outDouble(StringInfo str, double d);
} \
} while (0)
+/* Accessing a node's field goes like this */
+#define NODE_FIELD(fldname) (node->fldname)
+
#define booltostr(x) ((x) ? "true" : "false")
diff --git a/src/backend/nodes/readfuncs.c b/src/backend/nodes/readfuncs.c
index 4077699cab..4c33908b66 100644
--- a/src/backend/nodes/readfuncs.c
+++ b/src/backend/nodes/readfuncs.c
@@ -270,6 +270,9 @@
local_node->fldname = NULL; \
} while (0)
+/* Accessing a node's field goes like this */
+#define NODE_FIELD(fldname) (local_node->fldname)
+
/* Routine exit */
#define READ_DONE() \
return local_node
diff --git a/src/include/nodes/primnodes.h b/src/include/nodes/primnodes.h
index 94282497d7..9dd60ad5c5 100644
--- a/src/include/nodes/primnodes.h
+++ b/src/include/nodes/primnodes.h
@@ -274,9 +274,9 @@ typedef struct Var
* their varno/varattno match.
*/
/* syntactic relation index (0 if unknown) */
- Index varnosyn pg_node_attr(equal_ignore, query_jumble_ignore);
+ Index varnosyn pg_node_attr(equal_ignore, query_jumble_ignore, default_ref(varno));
/* syntactic attribute number */
- AttrNumber varattnosyn pg_node_attr(equal_ignore, query_jumble_ignore);
+ AttrNumber varattnosyn pg_node_attr(equal_ignore, query_jumble_ignore, default_ref(varattno));
/* token location, or -1 if unknown */
Location location;
--
2.40.1
[application/octet-stream] v7-0006-nodeToString-Apply-RLE-on-Bitmapset-and-numeric-L.patch (7.9K, ../../CAEze2WgKVdmMLAGga8Gfh7DsvuJPxiqPvYH8fS0Px8f3w5g64g@mail.gmail.com/8-v7-0006-nodeToString-Apply-RLE-on-Bitmapset-and-numeric-L.patch)
download | inline diff:
From e8f8a207619af256b1efbde26688d244feb0db43 Mon Sep 17 00:00:00 2001
From: Matthias van de Meent <[email protected]>
Date: Sat, 10 Feb 2024 00:57:19 +0100
Subject: [PATCH v7 6/8] nodeToString: Apply RLE on Bitmapset and numeric List
types
This reduces the size of full serialized queries in pg_rewrite by several %,
reducing overhead in the system.
---
src/backend/nodes/outfuncs.c | 96 +++++++++++++++++++++++++++++++----
src/backend/nodes/read.c | 53 +++++++++++++++++--
src/backend/nodes/readfuncs.c | 16 +++++-
3 files changed, 149 insertions(+), 16 deletions(-)
diff --git a/src/backend/nodes/outfuncs.c b/src/backend/nodes/outfuncs.c
index bc57b7add6..cfafdaa033 100644
--- a/src/backend/nodes/outfuncs.c
+++ b/src/backend/nodes/outfuncs.c
@@ -383,23 +383,64 @@ writeNodeArray(StringInfo str, const Node *const *arr, int len,
/*
* Print a List.
+ *
+ * Notes:
+ * NodeList is formatted as (<node1> <node2> ...)
+ *
+ * OidList is formatted as (o int +int int int ...), with +int indicating N
+ * successive identical values.
+ *
+ * IntList and XidList are formatted as (i/x int +int int int ...), with +int
+ * indicating N successively increasing values. (i 9 +3) is thus equivalent
+ * to (i 9 10 11 12).
*/
static void
_outList(StringInfo str, const List *node, bool omitLocation)
{
const ListCell *lc;
+ union LCSurrogate {
+ int32 i;
+ Oid o;
+ TransactionId x;
+ } previous = { .i = 0};
+ int run = 0;
+ int run_delta;
+ const char *fmt;
appendStringInfoChar(str, '(');
if (IsA(node, IntList))
+ {
appendStringInfoChar(str, 'i');
+ fmt = " %d";
+ run_delta = 1;
+ }
else if (IsA(node, OidList))
+ {
appendStringInfoChar(str, 'o');
+ fmt = " %u";
+ run_delta = 0;
+ }
else if (IsA(node, XidList))
+ {
appendStringInfoChar(str, 'x');
+ fmt = " %u";
+ run_delta = 1;
+ }
+ else if (IsA(node, List))
+ {
+ /* silence the compiler about uninitialized variables */
+ fmt = "";
+ run_delta = 0;
+ }
+ else
+ elog(ERROR, "unrecognized list node type: %d",
+ (int) node->type);
foreach(lc, node)
{
+ union LCSurrogate val = {.i = 0};
+
/*
* For the sake of backward compatibility, we emit a slightly
* different whitespace format for lists of nodes vs. other types of
@@ -410,26 +451,41 @@ _outList(StringInfo str, const List *node, bool omitLocation)
outNode(str, lfirst(lc), omitLocation);
if (lnext(node, lc))
appendStringInfoChar(str, ' ');
+ continue;
}
else if (IsA(node, IntList))
- appendStringInfo(str, " %d", lfirst_int(lc));
+ val.i = lfirst_int(lc);
else if (IsA(node, OidList))
- appendStringInfo(str, " %u", lfirst_oid(lc));
+ val.o = lfirst_oid(lc);
else if (IsA(node, XidList))
- appendStringInfo(str, " %u", lfirst_xid(lc));
+ val.x = lfirst_xid(lc);
+
+ if (val.i == previous.i + run_delta)
+ run += 1;
else
- elog(ERROR, "unrecognized list node type: %d",
- (int) node->type);
+ {
+ if (run > 0)
+ appendStringInfo(str, " +%d", run);
+ run = 0;
+ appendStringInfo(str, fmt, val.i);
+ }
+ previous = val;
}
- appendStringInfoChar(str, ')');
-}
+ if (run > 0)
+ appendStringInfo(str, " +%d", run);
+
+ appendStringInfoChar(str, ')');}
/*
* outBitmapset -
* converts a bitmap set of integers
*
- * Note: the output format is "(b int int ...)", similar to an integer List.
+ * Note: the output format is "(b int int +int ...)", similar to an
+ * integer List. Note that consecutive runs of incremental values are
+ * indicated by [... int +int ...], where the first int is the first set bit,
+ * and the int that's tagged with the '+' sign that follows is the number of
+ * subsequent set bits (resulting in a run of N+1 set bits).
*
* We export this function for use by extensions that define extensible nodes.
* That's somewhat historical, though, because calling outNode() will work.
@@ -437,13 +493,31 @@ _outList(StringInfo str, const List *node, bool omitLocation)
void
outBitmapset(StringInfo str, const Bitmapset *bms)
{
- int x;
+ int x = -1;
+ int prev = -2;
+ int run = 0;
appendStringInfoChar(str, '(');
appendStringInfoChar(str, 'b');
- x = -1;
+
while ((x = bms_next_member(bms, x)) >= 0)
- appendStringInfo(str, " %d", x);
+ {
+ if (x - prev == 1)
+ run++;
+ else
+ {
+ if (run > 0)
+ appendStringInfo(str, " +%d", run);
+
+ run = 0;
+ appendStringInfo(str, " %d", x);
+ }
+ prev = x;
+ }
+
+ if (run > 0)
+ appendStringInfo(str, " +%d", run);
+
appendStringInfoChar(str, ')');
}
diff --git a/src/backend/nodes/read.c b/src/backend/nodes/read.c
index cffd913ba6..7456626e82 100644
--- a/src/backend/nodes/read.c
+++ b/src/backend/nodes/read.c
@@ -415,6 +415,8 @@ nodeRead(const char *token, int tok_len)
elog(ERROR, "unterminated List structure");
if (tok_len == 1 && token[0] == 'i')
{
+ int prev = 0;
+
/* List of integers */
for (;;)
{
@@ -430,12 +432,23 @@ nodeRead(const char *token, int tok_len)
if (endptr != token + tok_len)
elog(ERROR, "unrecognized integer: \"%.*s\"",
tok_len, token);
- l = lappend_int(l, val);
+
+ if (token[0] == '+')
+ {
+ for (int i = 0; i < val; i++)
+ l = lappend_int(l, ++prev);
+ }
+ else
+ {
+ prev = val;
+ l = lappend_int(l, val);
+ }
}
result = (Node *) l;
}
else if (tok_len == 1 && token[0] == 'o')
{
+ Oid prev = 0;
/* List of OIDs */
for (;;)
{
@@ -451,12 +464,23 @@ nodeRead(const char *token, int tok_len)
if (endptr != token + tok_len)
elog(ERROR, "unrecognized OID: \"%.*s\"",
tok_len, token);
- l = lappend_oid(l, val);
+
+ if (token[0] == '+')
+ {
+ for (int i = 0; i < val; i++)
+ l = lappend_oid(l, prev);
+ }
+ else
+ {
+ prev = val;
+ l = lappend_oid(l, val);
+ }
}
result = (Node *) l;
}
else if (tok_len == 1 && token[0] == 'x')
{
+ TransactionId prev = 0;
/* List of TransactionIds */
for (;;)
{
@@ -472,7 +496,17 @@ nodeRead(const char *token, int tok_len)
if (endptr != token + tok_len)
elog(ERROR, "unrecognized Xid: \"%.*s\"",
tok_len, token);
- l = lappend_xid(l, val);
+
+ if (token[0] == '+')
+ {
+ for (int i = 0; i < val; i++)
+ l = lappend_xid(l, ++prev);
+ }
+ else
+ {
+ prev = val;
+ l = lappend_xid(l, val);
+ }
}
result = (Node *) l;
}
@@ -480,6 +514,7 @@ nodeRead(const char *token, int tok_len)
{
/* Bitmapset -- see also _readBitmapset() */
Bitmapset *bms = NULL;
+ int prev = -2;
for (;;)
{
@@ -495,7 +530,17 @@ nodeRead(const char *token, int tok_len)
if (endptr != token + tok_len)
elog(ERROR, "unrecognized integer: \"%.*s\"",
tok_len, token);
- bms = bms_add_member(bms, val);
+
+ if (token[0] == '+')
+ {
+ for (int i = 0; i < val; i++)
+ bms = bms_add_member(bms, ++prev);
+ }
+ else
+ {
+ bms = bms_add_member(bms, val);
+ prev = val;
+ }
}
result = (Node *) bms;
}
diff --git a/src/backend/nodes/readfuncs.c b/src/backend/nodes/readfuncs.c
index 4f0fc4cac5..4077699cab 100644
--- a/src/backend/nodes/readfuncs.c
+++ b/src/backend/nodes/readfuncs.c
@@ -310,6 +310,7 @@ static Bitmapset *
_readBitmapset(void)
{
Bitmapset *result = NULL;
+ int prev = -2;
READ_TEMP_LOCALS();
@@ -338,7 +339,20 @@ _readBitmapset(void)
val = (int) strtol(token, &endptr, 10);
if (endptr != token + length)
elog(ERROR, "unrecognized integer: \"%.*s\"", length, token);
- result = bms_add_member(result, val);
+
+ if (token[0] == '+')
+ {
+ for (int i = 0; i < val; i++)
+ {
+ ++prev;
+ result = bms_add_member(result, prev);
+ }
+ }
+ else
+ {
+ result = bms_add_member(result, val);
+ prev = val;
+ }
}
return result;
--
2.40.1
[application/octet-stream] v7-0008-nodeToString-omit-serializing-0s-in-enum-typed-fi.patch (2.1K, ../../CAEze2WgKVdmMLAGga8Gfh7DsvuJPxiqPvYH8fS0Px8f3w5g64g@mail.gmail.com/9-v7-0008-nodeToString-omit-serializing-0s-in-enum-typed-fi.patch)
download | inline diff:
From f2a10fa0c779823a7c967f816e37074785aed001 Mon Sep 17 00:00:00 2001
From: Matthias van de Meent <[email protected]>
Date: Mon, 19 Feb 2024 13:53:00 +0100
Subject: [PATCH v7 8/8] nodeToString: omit serializing 0s in enum-typed
fields.
---
src/backend/nodes/outfuncs.c | 5 +----
src/backend/nodes/readfuncs.c | 16 +++++++++++-----
2 files changed, 12 insertions(+), 9 deletions(-)
diff --git a/src/backend/nodes/outfuncs.c b/src/backend/nodes/outfuncs.c
index e64f3c5975..0269daf484 100644
--- a/src/backend/nodes/outfuncs.c
+++ b/src/backend/nodes/outfuncs.c
@@ -112,10 +112,7 @@ static void outDouble(StringInfo str, double d);
(int) node->fldname); \
} while (0)
#define WRITE_ENUM_FIELD(fldname, enumtype) \
- do { \
- appendStringInfo(str, " :" CppAsString(fldname) " %d", \
- (int) node->fldname); \
- } while (0)
+ WRITE_ENUM_FIELD_DEFAULT(fldname, enumtype, 0)
/* Write a float field (actually, they're double) */
#define WRITE_FLOAT_FIELD_DEFAULT(fldname, default) \
diff --git a/src/backend/nodes/readfuncs.c b/src/backend/nodes/readfuncs.c
index 4c33908b66..7b532535fe 100644
--- a/src/backend/nodes/readfuncs.c
+++ b/src/backend/nodes/readfuncs.c
@@ -142,13 +142,19 @@
READ_CHAR_FIELD_DEFAULT(fldname, '\0')
/* Read an enumerated-type field that was written as an integer code */
-#define READ_ENUM_FIELD(fldname, enumtype) \
+
+#define READ_ENUM_FIELD_DEFAULT(fldname, enumtype, default) \
do { \
- token = pg_strtok_fieldname(":" CppAsString(fldname), &length); \
- Assert(token != NULL); \
- token = pg_strtok(&length); /* get field value */ \
- local_node->fldname = (enumtype) atoi(token); \
+ if ((token = pg_strtok_fieldname(":" CppAsString(fldname), &length))) \
+ { \
+ token = pg_strtok(&length); /* get field value */ \
+ local_node->fldname = (enumtype) atoi(token); \
+ } \
+ else \
+ local_node->fldname = (enumtype) default; \
} while (0)
+#define READ_ENUM_FIELD(fldname, enumtype) \
+ READ_ENUM_FIELD_DEFAULT(fldname, enumtype, 0)
/* Read a float field */
#define READ_FLOAT_FIELD_DEFAULT(fldname, default_value) \
--
2.40.1
^ permalink raw reply [nested|flat] 69+ messages in thread
* Re: Reducing output size of nodeToString
@ 2024-03-11 13:19 Peter Eisentraut <[email protected]>
parent: Matthias van de Meent <[email protected]>
0 siblings, 1 reply; 69+ messages in thread
From: Peter Eisentraut @ 2024-03-11 13:19 UTC (permalink / raw)
To: Matthias van de Meent <[email protected]>; +Cc: Robert Haas <[email protected]>; PostgreSQL Hackers <[email protected]>; Andres Freund <[email protected]>; Michel Pelletier <[email protected]>
On 22.02.24 16:07, Matthias van de Meent wrote:
> On Thu, 22 Feb 2024 at 13:37, Matthias van de Meent
> <[email protected]> wrote:
>>
>> On Mon, 19 Feb 2024 at 14:19, Matthias van de Meent
>> <[email protected]> wrote:
>>> Attached the updated version of the patch on top of 5497daf3, which
>>> incorporates this last round of feedback.
>>
>> Now attached rebased on top of 93db6cbd to fix conflicts with fbc93b8b
>> and an issue in the previous patchset: I attached one too many v3-0001
>> from a previous patch I worked on.
>
> ... and now with a fix for not overwriting newly deserialized location
> attributes with -1, which breaks test output for
> READ_WRITE_PARSE_PLAN_TREES installations. Again, no other significant
> changes since the patch of last Monday.
* v7-0002-pg_node_tree-Don-t-store-query-text-locations-in-.patch
This patch looks much more complicated than I was expecting. I had
suggested to model this after stringToNodeWithLocations(). This uses a
global variable to toggle the mode. Your patch creates a function
nodeToStringNoQLocs() -- why the different naming scheme? -- and passes
the flag down as an argument to all the output functions. I mean, in a
green field, avoiding global variables can be sensible, of course, but I
think in this limited scope here it would really be better to keep the
code for the two directions read and write the same.
Attached is a small patch that shows what I had in mind. (It doesn't
contain any callers, but your patch shows where all those would go.)
* v7-0003-gen_node_support.pl-Mark-location-fields-as-type-.patch
This looks sensible, but maybe making Location a global type is a bit
much? Maybe something more specific like ParseLocation, or ParseLoc, to
keep it under 12 characters.
From dee403f637f279813f0711bbc64c365cba82c18c Mon Sep 17 00:00:00 2001
From: Peter Eisentraut <[email protected]>
Date: Mon, 11 Mar 2024 14:07:52 +0100
Subject: [PATCH] Add nodeToStringWithoutLocations()
---
src/backend/nodes/outfuncs.c | 30 +++++++++++++++++++++++++++++-
src/include/nodes/nodes.h | 1 +
2 files changed, 30 insertions(+), 1 deletion(-)
diff --git a/src/backend/nodes/outfuncs.c b/src/backend/nodes/outfuncs.c
index 29cbc83bd9..ea0a6cb94d 100644
--- a/src/backend/nodes/outfuncs.c
+++ b/src/backend/nodes/outfuncs.c
@@ -25,6 +25,8 @@
#include "nodes/pg_list.h"
#include "utils/datum.h"
+static bool write_location_fields = true;
+
static void outChar(StringInfo str, char c);
static void outDouble(StringInfo str, double d);
@@ -88,7 +90,7 @@ static void outDouble(StringInfo str, double d);
/* Write a parse location field (actually same as INT case) */
#define WRITE_LOCATION_FIELD(fldname) \
- appendStringInfo(str, " :" CppAsString(fldname) " %d", node->fldname)
+ appendStringInfo(str, " :" CppAsString(fldname) " %d", write_location_fields ? node->fldname : -1)
/* Write a Node field */
#define WRITE_NODE_FIELD(fldname) \
@@ -770,6 +772,32 @@ nodeToString(const void *obj)
return str.data;
}
+/*
+ * nodeToStringWithoutLocations -
+ * returns the ascii representation of the Node as a palloc'd string
+ *
+ * This form prints location fields with their default value (not the actual
+ * field value). This is useful for serializing node trees for storing into
+ * catalogs, where the location information is not useful since the original
+ * query string is not preserved anyway.
+ */
+char *
+nodeToStringWithoutLocations(const void *obj)
+{
+ StringInfoData str;
+ bool save_write_location_fields;
+
+ save_write_location_fields = write_location_fields;
+ write_location_fields = false;
+
+ initStringInfo(&str);
+ outNode(&str, obj);
+
+ write_location_fields = save_write_location_fields;
+
+ return str.data;
+}
+
/*
* bmsToString -
* returns the ascii representation of the Bitmapset as a palloc'd string
diff --git a/src/include/nodes/nodes.h b/src/include/nodes/nodes.h
index 2969dd831b..3abe47af94 100644
--- a/src/include/nodes/nodes.h
+++ b/src/include/nodes/nodes.h
@@ -195,6 +195,7 @@ extern void outBitmapset(struct StringInfoData *str,
extern void outDatum(struct StringInfoData *str, uintptr_t value,
int typlen, bool typbyval);
extern char *nodeToString(const void *obj);
+extern char *nodeToStringWithoutLocations(const void *obj);
extern char *bmsToString(const struct Bitmapset *bms);
/*
--
2.44.0
Attachments:
[text/plain] 0001-Add-nodeToStringWithoutLocations.patch.nocfbot (2.5K, ../../[email protected]/2-0001-Add-nodeToStringWithoutLocations.patch.nocfbot)
download | inline diff:
From dee403f637f279813f0711bbc64c365cba82c18c Mon Sep 17 00:00:00 2001
From: Peter Eisentraut <[email protected]>
Date: Mon, 11 Mar 2024 14:07:52 +0100
Subject: [PATCH] Add nodeToStringWithoutLocations()
---
src/backend/nodes/outfuncs.c | 30 +++++++++++++++++++++++++++++-
src/include/nodes/nodes.h | 1 +
2 files changed, 30 insertions(+), 1 deletion(-)
diff --git a/src/backend/nodes/outfuncs.c b/src/backend/nodes/outfuncs.c
index 29cbc83bd9..ea0a6cb94d 100644
--- a/src/backend/nodes/outfuncs.c
+++ b/src/backend/nodes/outfuncs.c
@@ -25,6 +25,8 @@
#include "nodes/pg_list.h"
#include "utils/datum.h"
+static bool write_location_fields = true;
+
static void outChar(StringInfo str, char c);
static void outDouble(StringInfo str, double d);
@@ -88,7 +90,7 @@ static void outDouble(StringInfo str, double d);
/* Write a parse location field (actually same as INT case) */
#define WRITE_LOCATION_FIELD(fldname) \
- appendStringInfo(str, " :" CppAsString(fldname) " %d", node->fldname)
+ appendStringInfo(str, " :" CppAsString(fldname) " %d", write_location_fields ? node->fldname : -1)
/* Write a Node field */
#define WRITE_NODE_FIELD(fldname) \
@@ -770,6 +772,32 @@ nodeToString(const void *obj)
return str.data;
}
+/*
+ * nodeToStringWithoutLocations -
+ * returns the ascii representation of the Node as a palloc'd string
+ *
+ * This form prints location fields with their default value (not the actual
+ * field value). This is useful for serializing node trees for storing into
+ * catalogs, where the location information is not useful since the original
+ * query string is not preserved anyway.
+ */
+char *
+nodeToStringWithoutLocations(const void *obj)
+{
+ StringInfoData str;
+ bool save_write_location_fields;
+
+ save_write_location_fields = write_location_fields;
+ write_location_fields = false;
+
+ initStringInfo(&str);
+ outNode(&str, obj);
+
+ write_location_fields = save_write_location_fields;
+
+ return str.data;
+}
+
/*
* bmsToString -
* returns the ascii representation of the Bitmapset as a palloc'd string
diff --git a/src/include/nodes/nodes.h b/src/include/nodes/nodes.h
index 2969dd831b..3abe47af94 100644
--- a/src/include/nodes/nodes.h
+++ b/src/include/nodes/nodes.h
@@ -195,6 +195,7 @@ extern void outBitmapset(struct StringInfoData *str,
extern void outDatum(struct StringInfoData *str, uintptr_t value,
int typlen, bool typbyval);
extern char *nodeToString(const void *obj);
+extern char *nodeToStringWithoutLocations(const void *obj);
extern char *bmsToString(const struct Bitmapset *bms);
/*
--
2.44.0
^ permalink raw reply [nested|flat] 69+ messages in thread
* Re: Reducing output size of nodeToString
@ 2024-03-11 20:52 Matthias van de Meent <[email protected]>
parent: Peter Eisentraut <[email protected]>
0 siblings, 1 reply; 69+ messages in thread
From: Matthias van de Meent @ 2024-03-11 20:52 UTC (permalink / raw)
To: Peter Eisentraut <[email protected]>; +Cc: Robert Haas <[email protected]>; PostgreSQL Hackers <[email protected]>; Andres Freund <[email protected]>; Michel Pelletier <[email protected]>
On Mon, 11 Mar 2024 at 14:19, Peter Eisentraut <[email protected]> wrote:
>
> On 22.02.24 16:07, Matthias van de Meent wrote:
> > On Thu, 22 Feb 2024 at 13:37, Matthias van de Meent
> > <[email protected]> wrote:
> >>
> >> On Mon, 19 Feb 2024 at 14:19, Matthias van de Meent
> >> <[email protected]> wrote:
> >>> Attached the updated version of the patch on top of 5497daf3, which
> >>> incorporates this last round of feedback.
> >>
> >> Now attached rebased on top of 93db6cbd to fix conflicts with fbc93b8b
> >> and an issue in the previous patchset: I attached one too many v3-0001
> >> from a previous patch I worked on.
> >
> > ... and now with a fix for not overwriting newly deserialized location
> > attributes with -1, which breaks test output for
> > READ_WRITE_PARSE_PLAN_TREES installations. Again, no other significant
> > changes since the patch of last Monday.
>
> * v7-0002-pg_node_tree-Don-t-store-query-text-locations-in-.patch
>
> This patch looks much more complicated than I was expecting. I had
> suggested to model this after stringToNodeWithLocations(). This uses a
> global variable to toggle the mode. Your patch creates a function
> nodeToStringNoQLocs() -- why the different naming scheme?
It doesn't just exclude .location fields, but also Query.len, a
similar field which contains the length of the query's string. The
name has been further refined to nodeToStringNoParseLocs() in the
attached version, but feel free to replace the names in the patch to
anything else you might want.
> -- and passes
> the flag down as an argument to all the output functions. I mean, in a
> green field, avoiding global variables can be sensible, of course, but I
> think in this limited scope here it would really be better to keep the
> code for the two directions read and write the same.
I'm a big fan of _not_ using magic global variables as passed context
without resetting on subnormal exits...
For GUCs their usage is understandable (and there is infrastructure to
reset them, and you're not supposed to manually update them), but IMO
here its usage should be a function-scoped variable or in a
passed-by-reference context struct, not a file-local static.
Regardless, attached is an adapted version with the file-local
variable implementation.
> Attached is a small patch that shows what I had in mind. (It doesn't
> contain any callers, but your patch shows where all those would go.)
Attached a revised version that does it like stringToNodeInternal's
handling of restore_location_fields.
> * v7-0003-gen_node_support.pl-Mark-location-fields-as-type-.patch
>
> This looks sensible, but maybe making Location a global type is a bit
> much? Maybe something more specific like ParseLocation, or ParseLoc, to
> keep it under 12 characters.
I've gone with ParseLoc in the attached v8 patchset.
Kind regards,
Matthias van de Meent
Attachments:
[application/octet-stream] v8-0003-pg_node_tree-Don-t-store-query-text-locations-in-.patch (13.1K, ../../CAEze2Wh0oC97R0fG7s17b=YHOTittL8mHwC6TyT-Q2_NtY7RVw@mail.gmail.com/2-v8-0003-pg_node_tree-Don-t-store-query-text-locations-in-.patch)
download | inline diff:
From 36be38efe8e710a745b1f2a211555e56b57cb04c Mon Sep 17 00:00:00 2001
From: Matthias van de Meent <[email protected]>
Date: Wed, 3 Jan 2024 01:39:42 +0100
Subject: [PATCH v8 3/8] pg_node_tree: Don't store query text locations in
pg_node_tree fields.
We don't store original query texts, so any lingering query location
value can only be useful in forensic debugging. In normal operation,
however, a non-default value will show up as measurable overhead in
serialization, so by omitting serialization we save several 10s of
kBs in (amongst others) pg_rewrite.
---
src/backend/catalog/heap.c | 10 ++++++----
src/backend/catalog/index.c | 5 +++--
src/backend/catalog/pg_attrdef.c | 6 ++++--
src/backend/catalog/pg_proc.c | 11 +++++++++--
src/backend/catalog/pg_publication.c | 6 +++++-
src/backend/commands/policy.c | 11 +++++++++--
src/backend/commands/statscmds.c | 3 ++-
src/backend/commands/trigger.c | 3 ++-
src/backend/commands/typecmds.c | 8 +++++---
src/backend/nodes/outfuncs.c | 20 +++++++++++++++++++-
src/backend/rewrite/rewriteDefine.c | 5 +++--
src/include/nodes/nodes.h | 1 +
12 files changed, 68 insertions(+), 21 deletions(-)
diff --git a/src/backend/catalog/heap.c b/src/backend/catalog/heap.c
index 5e773740f4..c729196538 100644
--- a/src/backend/catalog/heap.c
+++ b/src/backend/catalog/heap.c
@@ -58,6 +58,7 @@
#include "commands/typecmds.h"
#include "common/int.h"
#include "miscadmin.h"
+#include "nodes/nodes.h"
#include "nodes/nodeFuncs.h"
#include "optimizer/optimizer.h"
#include "parser/parse_coerce.h"
@@ -2064,9 +2065,9 @@ StoreRelCheck(Relation rel, const char *ccname, Node *expr,
Oid constrOid;
/*
- * Flatten expression to string form for storage.
+ * Flatten expression to string form for storage, without query refs.
*/
- ccbin = nodeToString(expr);
+ ccbin = nodeToStringNoParseLocs(expr);
/*
* Find columns of rel that are used in expr
@@ -3676,7 +3677,7 @@ StorePartitionKey(Relation rel,
{
char *exprString;
- exprString = nodeToString(partexprs);
+ exprString = nodeToStringNoParseLocs(partexprs);
partexprDatum = CStringGetTextDatum(exprString);
pfree(exprString);
}
@@ -3834,7 +3835,8 @@ StorePartitionBound(Relation rel, Relation parent, PartitionBoundSpec *bound)
memset(new_val, 0, sizeof(new_val));
memset(new_null, false, sizeof(new_null));
memset(new_repl, false, sizeof(new_repl));
- new_val[Anum_pg_class_relpartbound - 1] = CStringGetTextDatum(nodeToString(bound));
+ new_val[Anum_pg_class_relpartbound - 1] =
+ CStringGetTextDatum(nodeToStringNoParseLocs(bound));
new_null[Anum_pg_class_relpartbound - 1] = false;
new_repl[Anum_pg_class_relpartbound - 1] = true;
newtuple = heap_modify_tuple(tuple, RelationGetDescr(classRel),
diff --git a/src/backend/catalog/index.c b/src/backend/catalog/index.c
index e6140853c0..a796d4e8b1 100644
--- a/src/backend/catalog/index.c
+++ b/src/backend/catalog/index.c
@@ -59,6 +59,7 @@
#include "executor/executor.h"
#include "miscadmin.h"
#include "nodes/makefuncs.h"
+#include "nodes/nodes.h"
#include "nodes/nodeFuncs.h"
#include "optimizer/optimizer.h"
#include "parser/parser.h"
@@ -582,7 +583,7 @@ UpdateIndexRelation(Oid indexoid,
{
char *exprsString;
- exprsString = nodeToString(indexInfo->ii_Expressions);
+ exprsString = nodeToStringNoParseLocs(indexInfo->ii_Expressions);
exprsDatum = CStringGetTextDatum(exprsString);
pfree(exprsString);
}
@@ -597,7 +598,7 @@ UpdateIndexRelation(Oid indexoid,
{
char *predString;
- predString = nodeToString(make_ands_explicit(indexInfo->ii_Predicate));
+ predString = nodeToStringNoParseLocs(make_ands_explicit(indexInfo->ii_Predicate));
predDatum = CStringGetTextDatum(predString);
pfree(predString);
}
diff --git a/src/backend/catalog/pg_attrdef.c b/src/backend/catalog/pg_attrdef.c
index 003ae70b4d..685399b2be 100644
--- a/src/backend/catalog/pg_attrdef.c
+++ b/src/backend/catalog/pg_attrdef.c
@@ -23,6 +23,7 @@
#include "catalog/objectaccess.h"
#include "catalog/pg_attrdef.h"
#include "executor/executor.h"
+#include "nodes/nodes.h"
#include "optimizer/optimizer.h"
#include "utils/array.h"
#include "utils/builtins.h"
@@ -62,9 +63,10 @@ StoreAttrDefault(Relation rel, AttrNumber attnum,
adrel = table_open(AttrDefaultRelationId, RowExclusiveLock);
/*
- * Flatten expression to string form for storage.
+ * Flatten expression to string form for storage, without references to
+ * the original query string.
*/
- adbin = nodeToString(expr);
+ adbin = nodeToStringNoParseLocs(expr);
/*
* Make the pg_attrdef entry.
diff --git a/src/backend/catalog/pg_proc.c b/src/backend/catalog/pg_proc.c
index ab2b6ca148..9317aa3c0c 100644
--- a/src/backend/catalog/pg_proc.c
+++ b/src/backend/catalog/pg_proc.c
@@ -31,6 +31,7 @@
#include "funcapi.h"
#include "mb/pg_wchar.h"
#include "miscadmin.h"
+#include "nodes/nodes.h"
#include "nodes/nodeFuncs.h"
#include "parser/parse_coerce.h"
#include "pgstat.h"
@@ -329,7 +330,10 @@ ProcedureCreate(const char *procedureName,
else
nulls[Anum_pg_proc_proargnames - 1] = true;
if (parameterDefaults != NIL)
- values[Anum_pg_proc_proargdefaults - 1] = CStringGetTextDatum(nodeToString(parameterDefaults));
+ {
+ values[Anum_pg_proc_proargdefaults - 1] =
+ CStringGetTextDatum(nodeToStringNoParseLocs(parameterDefaults));
+ }
else
nulls[Anum_pg_proc_proargdefaults - 1] = true;
if (trftypes != PointerGetDatum(NULL))
@@ -342,7 +346,10 @@ ProcedureCreate(const char *procedureName,
else
nulls[Anum_pg_proc_probin - 1] = true;
if (prosqlbody)
- values[Anum_pg_proc_prosqlbody - 1] = CStringGetTextDatum(nodeToString(prosqlbody));
+ {
+ values[Anum_pg_proc_prosqlbody - 1] =
+ CStringGetTextDatum(nodeToStringNoParseLocs(prosqlbody));
+ }
else
nulls[Anum_pg_proc_prosqlbody - 1] = true;
if (proconfig != PointerGetDatum(NULL))
diff --git a/src/backend/catalog/pg_publication.c b/src/backend/catalog/pg_publication.c
index ac05dc057f..a5647d5cd2 100644
--- a/src/backend/catalog/pg_publication.c
+++ b/src/backend/catalog/pg_publication.c
@@ -32,6 +32,7 @@
#include "catalog/pg_type.h"
#include "commands/publicationcmds.h"
#include "funcapi.h"
+#include "nodes/nodes.h"
#include "utils/array.h"
#include "utils/builtins.h"
#include "utils/catcache.h"
@@ -417,7 +418,10 @@ publication_add_relation(Oid pubid, PublicationRelInfo *pri,
/* Add qualifications, if available */
if (pri->whereClause != NULL)
- values[Anum_pg_publication_rel_prqual - 1] = CStringGetTextDatum(nodeToString(pri->whereClause));
+ {
+ values[Anum_pg_publication_rel_prqual - 1] =
+ CStringGetTextDatum(nodeToStringNoParseLocs(pri->whereClause));
+ }
else
nulls[Anum_pg_publication_rel_prqual - 1] = true;
diff --git a/src/backend/commands/policy.c b/src/backend/commands/policy.c
index 6ff3eba824..43bf0bd829 100644
--- a/src/backend/commands/policy.c
+++ b/src/backend/commands/policy.c
@@ -28,6 +28,7 @@
#include "catalog/pg_type.h"
#include "commands/policy.h"
#include "miscadmin.h"
+#include "nodes/nodes.h"
#include "nodes/pg_list.h"
#include "parser/parse_clause.h"
#include "parser/parse_collate.h"
@@ -698,13 +699,19 @@ CreatePolicy(CreatePolicyStmt *stmt)
/* Add qual if present. */
if (qual)
- values[Anum_pg_policy_polqual - 1] = CStringGetTextDatum(nodeToString(qual));
+ {
+ values[Anum_pg_policy_polqual - 1] =
+ CStringGetTextDatum(nodeToStringNoParseLocs(qual));
+ }
else
isnull[Anum_pg_policy_polqual - 1] = true;
/* Add WITH CHECK qual if present */
if (with_check_qual)
- values[Anum_pg_policy_polwithcheck - 1] = CStringGetTextDatum(nodeToString(with_check_qual));
+ {
+ values[Anum_pg_policy_polwithcheck - 1] =
+ CStringGetTextDatum(nodeToStringNoParseLocs(with_check_qual));
+ }
else
isnull[Anum_pg_policy_polwithcheck - 1] = true;
diff --git a/src/backend/commands/statscmds.c b/src/backend/commands/statscmds.c
index 6fa840fada..bdf0fda82b 100644
--- a/src/backend/commands/statscmds.c
+++ b/src/backend/commands/statscmds.c
@@ -27,6 +27,7 @@
#include "commands/comment.h"
#include "commands/defrem.h"
#include "miscadmin.h"
+#include "nodes/nodes.h"
#include "nodes/nodeFuncs.h"
#include "optimizer/optimizer.h"
#include "statistics/statistics.h"
@@ -474,7 +475,7 @@ CreateStatistics(CreateStatsStmt *stmt)
{
char *exprsString;
- exprsString = nodeToString(stxexprs);
+ exprsString = nodeToStringNoParseLocs(stxexprs);
exprsDatum = CStringGetTextDatum(exprsString);
pfree(exprsString);
}
diff --git a/src/backend/commands/trigger.c b/src/backend/commands/trigger.c
index 35eb7180f7..07be4f1994 100644
--- a/src/backend/commands/trigger.c
+++ b/src/backend/commands/trigger.c
@@ -36,6 +36,7 @@
#include "miscadmin.h"
#include "nodes/bitmapset.h"
#include "nodes/makefuncs.h"
+#include "nodes/nodes.h"
#include "optimizer/optimizer.h"
#include "parser/parse_clause.h"
#include "parser/parse_collate.h"
@@ -667,7 +668,7 @@ CreateTriggerFiringOn(CreateTrigStmt *stmt, const char *queryString,
/* we'll need the rtable for recordDependencyOnExpr */
whenRtable = pstate->p_rtable;
- qual = nodeToString(whenClause);
+ qual = nodeToStringNoParseLocs(whenClause);
free_parsestate(pstate);
}
diff --git a/src/backend/commands/typecmds.c b/src/backend/commands/typecmds.c
index f4cdec3bf2..04fa5587c6 100644
--- a/src/backend/commands/typecmds.c
+++ b/src/backend/commands/typecmds.c
@@ -59,6 +59,7 @@
#include "executor/executor.h"
#include "miscadmin.h"
#include "nodes/makefuncs.h"
+#include "nodes/nodes.h"
#include "optimizer/optimizer.h"
#include "parser/parse_coerce.h"
#include "parser/parse_collate.h"
@@ -924,7 +925,7 @@ DefineDomain(CreateDomainStmt *stmt)
defaultValue =
deparse_expression(defaultExpr,
NIL, false, false);
- defaultValueBin = nodeToString(defaultExpr);
+ defaultValueBin = nodeToStringNoParseLocs(defaultExpr);
}
}
else
@@ -3506,9 +3507,10 @@ domainAddConstraint(Oid domainOid, Oid domainNamespace, Oid baseTypeOid,
errmsg("cannot use table references in domain check constraint")));
/*
- * Convert to string form for storage.
+ * Convert to string form for storage, without references to the original
+ * query text.
*/
- ccbin = nodeToString(expr);
+ ccbin = nodeToStringNoParseLocs(expr);
/*
* Store the constraint in pg_constraint
diff --git a/src/backend/nodes/outfuncs.c b/src/backend/nodes/outfuncs.c
index 70123ea39c..b03773b7fe 100644
--- a/src/backend/nodes/outfuncs.c
+++ b/src/backend/nodes/outfuncs.c
@@ -29,6 +29,7 @@
static void outChar(StringInfo str, char c);
static void outDouble(StringInfo str, double d);
+static bool output_parse_locations = true;
/*
* Macros to simplify output of different kinds of fields. Use these
@@ -159,7 +160,7 @@ static void outDouble(StringInfo str, double d);
/* Write a parse location field (actually same as INT case) */
#define WRITE_LOCATION_FIELD(fldname) \
do { \
- if (node->fldname != -1) \
+ if (node->fldname != -1 && output_parse_locations) \
appendStringInfo(str, " :" CppAsString(fldname) " %d", \
node->fldname); \
} while (0)
@@ -877,10 +878,27 @@ char *
nodeToString(const void *obj)
{
StringInfoData str;
+ bool save_output_parse_locations = output_parse_locations;
+ output_parse_locations = true;
/* see stringinfo.h for an explanation of this maneuver */
initStringInfo(&str);
outNode(&str, obj);
+ output_parse_locations = save_output_parse_locations;
+ return str.data;
+}
+
+char *
+nodeToStringNoParseLocs(const void *obj)
+{
+ StringInfoData str;
+ bool save_output_parse_locations = output_parse_locations;
+ output_parse_locations = false;
+
+ /* see stringinfo.h for an explanation of this maneuver */
+ initStringInfo(&str);
+ outNode(&str, obj);
+ output_parse_locations = save_output_parse_locations;
return str.data;
}
diff --git a/src/backend/rewrite/rewriteDefine.c b/src/backend/rewrite/rewriteDefine.c
index 6cc9a8d8bf..f1ef7130b5 100644
--- a/src/backend/rewrite/rewriteDefine.c
+++ b/src/backend/rewrite/rewriteDefine.c
@@ -25,6 +25,7 @@
#include "catalog/pg_rewrite.h"
#include "miscadmin.h"
#include "nodes/nodeFuncs.h"
+#include "nodes/nodes.h"
#include "parser/parse_utilcmd.h"
#include "rewrite/rewriteDefine.h"
#include "rewrite/rewriteManip.h"
@@ -57,8 +58,8 @@ InsertRule(const char *rulname,
List *action,
bool replace)
{
- char *evqual = nodeToString(event_qual);
- char *actiontree = nodeToString((Node *) action);
+ char *evqual = nodeToStringNoParseLocs(event_qual);
+ char *actiontree = nodeToStringNoParseLocs((Node *) action);
Datum values[Natts_pg_rewrite];
bool nulls[Natts_pg_rewrite] = {0};
NameData rname;
diff --git a/src/include/nodes/nodes.h b/src/include/nodes/nodes.h
index 2969dd831b..3dd52d5cad 100644
--- a/src/include/nodes/nodes.h
+++ b/src/include/nodes/nodes.h
@@ -195,6 +195,7 @@ extern void outBitmapset(struct StringInfoData *str,
extern void outDatum(struct StringInfoData *str, uintptr_t value,
int typlen, bool typbyval);
extern char *nodeToString(const void *obj);
+extern char *nodeToStringNoParseLocs(const void *obj);
extern char *bmsToString(const struct Bitmapset *bms);
/*
--
2.40.1
[application/octet-stream] v8-0001-pg_node_tree-Omit-serialization-of-fields-with-de.patch (22.8K, ../../CAEze2Wh0oC97R0fG7s17b=YHOTittL8mHwC6TyT-Q2_NtY7RVw@mail.gmail.com/3-v8-0001-pg_node_tree-Omit-serialization-of-fields-with-de.patch)
download | inline diff:
From dc74da49087dc2af0d759e653ced4ef526d572e6 Mon Sep 17 00:00:00 2001
From: Matthias van de Meent <[email protected]>
Date: Tue, 2 Jan 2024 23:55:02 +0100
Subject: [PATCH v8 1/8] pg_node_tree: Omit serialization of fields with
default values.
Often, the values in nodes are their default values. By not serializing
those fields and inserting the defaults during deserialization, we
reduce the size of pg_node_tree attributes seen in e.g. pg_rewrite by a
significant factor.
Note: Enum fields are excluded from this for now, as it would hinder
debugging.
In passing, we fix a test that had a strict dependency on the
serialization of pg_node_tree; we now do the checks in a more generic
manner, making it more stable and ensuring its stability in future work.
---
src/backend/nodes/outfuncs.c | 180 ++++++++++++++----
src/backend/nodes/read.c | 58 ++++++
src/backend/nodes/readfuncs.c | 211 ++++++++++++++++------
src/include/nodes/readfuncs.h | 1 +
src/test/regress/expected/rowsecurity.out | 5 +-
src/test/regress/sql/rowsecurity.sql | 5 +-
6 files changed, 373 insertions(+), 87 deletions(-)
diff --git a/src/backend/nodes/outfuncs.c b/src/backend/nodes/outfuncs.c
index 29cbc83bd9..70123ea39c 100644
--- a/src/backend/nodes/outfuncs.c
+++ b/src/backend/nodes/outfuncs.c
@@ -15,6 +15,7 @@
#include "postgres.h"
#include <ctype.h>
+#include <math.h>
#include "access/attnum.h"
#include "common/shortest_dec.h"
@@ -41,94 +42,207 @@ static void outDouble(StringInfo str, double d);
appendStringInfoString(str, nodelabel)
/* Write an integer field (anything written as ":fldname %d") */
+#define WRITE_INT_FIELD_DEFAULT(fldname, default) \
+ do { \
+ if (node->fldname != default) \
+ appendStringInfo(str, " :" CppAsString(fldname) " %d", \
+ node->fldname); \
+ } while (0)
#define WRITE_INT_FIELD(fldname) \
- appendStringInfo(str, " :" CppAsString(fldname) " %d", node->fldname)
+ WRITE_INT_FIELD_DEFAULT(fldname, 0)
/* Write an unsigned integer field (anything written as ":fldname %u") */
+#define WRITE_UINT_FIELD_DEFAULT(fldname, default) \
+ do { \
+ if (node->fldname != default) \
+ appendStringInfo(str, " :" CppAsString(fldname) " %u", \
+ node->fldname); \
+ } while (0)
#define WRITE_UINT_FIELD(fldname) \
- appendStringInfo(str, " :" CppAsString(fldname) " %u", node->fldname)
+ WRITE_UINT_FIELD_DEFAULT(fldname, 0)
/* Write an unsigned integer field (anything written with UINT64_FORMAT) */
+#define WRITE_UINT64_FIELD_DEFAULT(fldname, default) \
+ do { \
+ if (node->fldname != default) \
+ appendStringInfo(str, " :" CppAsString(fldname) " " UINT64_FORMAT, \
+ node->fldname); \
+ } while (0)
#define WRITE_UINT64_FIELD(fldname) \
- appendStringInfo(str, " :" CppAsString(fldname) " " UINT64_FORMAT, \
- node->fldname)
+ WRITE_UINT64_FIELD_DEFAULT(fldname, 0)
/* Write an OID field (don't hard-wire assumption that OID is same as uint) */
+#define WRITE_OID_FIELD_DEFAULT(fldname, default) \
+ do { \
+ if (node->fldname != default) \
+ appendStringInfo(str, " :" CppAsString(fldname) " %u", \
+ node->fldname); \
+ } while (0)
#define WRITE_OID_FIELD(fldname) \
- appendStringInfo(str, " :" CppAsString(fldname) " %u", node->fldname)
+ WRITE_OID_FIELD_DEFAULT(fldname, 0)
/* Write a long-integer field */
+#define WRITE_LONG_FIELD_DEFAULT(fldname, default) \
+ do { \
+ if (node->fldname != default) \
+ appendStringInfo(str, " :" CppAsString(fldname) " %ld", \
+ node->fldname); \
+ } while (0)
#define WRITE_LONG_FIELD(fldname) \
- appendStringInfo(str, " :" CppAsString(fldname) " %ld", node->fldname)
+ WRITE_LONG_FIELD_DEFAULT(fldname, 0)
+
/* Write a char field (ie, one ascii character) */
+#define WRITE_CHAR_FIELD_DEFAULT(fldname, default) \
+ do { \
+ if (node->fldname != default) \
+ { \
+ appendStringInfoString(str, " :" CppAsString(fldname) " "); \
+ outChar(str, node->fldname); \
+ } \
+ } while (0)
#define WRITE_CHAR_FIELD(fldname) \
- (appendStringInfo(str, " :" CppAsString(fldname) " "), \
- outChar(str, node->fldname))
+ WRITE_CHAR_FIELD_DEFAULT(fldname, '\0')
/* Write an enumerated-type field as an integer code */
+#define WRITE_ENUM_FIELD_DEFAULT(fldname, enumtype, default) \
+ do { \
+ if (node->fldname != default) \
+ appendStringInfo(str, " :" CppAsString(fldname) " %d", \
+ (int) node->fldname); \
+ } while (0)
#define WRITE_ENUM_FIELD(fldname, enumtype) \
- appendStringInfo(str, " :" CppAsString(fldname) " %d", \
- (int) node->fldname)
+ do { \
+ appendStringInfo(str, " :" CppAsString(fldname) " %d", \
+ (int) node->fldname); \
+ } while (0)
/* Write a float field (actually, they're double) */
+#define WRITE_FLOAT_FIELD_DEFAULT(fldname, default) \
+ do { \
+ if (node->fldname != default || \
+ signbit(node->fldname) != signbit(default)) \
+ { \
+ appendStringInfoString(str, " :" CppAsString(fldname) " "); \
+ outDouble(str, node->fldname); \
+ } \
+ } while (0)
#define WRITE_FLOAT_FIELD(fldname) \
- (appendStringInfo(str, " :" CppAsString(fldname) " "), \
- outDouble(str, node->fldname))
+ WRITE_FLOAT_FIELD_DEFAULT(fldname, 0.0)
/* Write a boolean field */
+#define WRITE_BOOL_FIELD_DEFAULT(fldname, default) \
+ do { \
+ if (node->fldname != default) \
+ appendStringInfo(str, " :" CppAsString(fldname) " %s", \
+ booltostr(node->fldname)); \
+ } while (0)
#define WRITE_BOOL_FIELD(fldname) \
- appendStringInfo(str, " :" CppAsString(fldname) " %s", \
- booltostr(node->fldname))
+ WRITE_BOOL_FIELD_DEFAULT(fldname, false)
+
+/*
+ * Non-null defaults of by-ref types are exceedingly rare (if not generally
+ * nonexistent), so we don't (yet) have a specialized macro for non-NULL
+ * defaults omission.
+ */
/* Write a character-string (possibly NULL) field */
#define WRITE_STRING_FIELD(fldname) \
- (appendStringInfoString(str, " :" CppAsString(fldname) " "), \
- outToken(str, node->fldname))
+ do { \
+ if (node->fldname != NULL) \
+ { \
+ appendStringInfoString(str, " :" CppAsString(fldname) " "); \
+ outToken(str, node->fldname); \
+ } \
+ } while (0)
/* Write a parse location field (actually same as INT case) */
#define WRITE_LOCATION_FIELD(fldname) \
- appendStringInfo(str, " :" CppAsString(fldname) " %d", node->fldname)
+ do { \
+ if (node->fldname != -1) \
+ appendStringInfo(str, " :" CppAsString(fldname) " %d", \
+ node->fldname); \
+ } while (0)
/* Write a Node field */
#define WRITE_NODE_FIELD(fldname) \
- (appendStringInfoString(str, " :" CppAsString(fldname) " "), \
- outNode(str, node->fldname))
+ do { \
+ if (node->fldname != NULL) \
+ { \
+ appendStringInfoString(str, " :" CppAsString(fldname) " "); \
+ outNode(str, node->fldname); \
+ } \
+ } while (0)
/* Write a bitmapset field */
#define WRITE_BITMAPSET_FIELD(fldname) \
- (appendStringInfoString(str, " :" CppAsString(fldname) " "), \
- outBitmapset(str, node->fldname))
+ do { \
+ if (node->fldname != NULL) \
+ { \
+ appendStringInfoString(str, " :" CppAsString(fldname) " "); \
+ outBitmapset(str, node->fldname); \
+ } \
+ } while (0)
/* Write a variable-length array (not a List) of Node pointers */
#define WRITE_NODE_ARRAY(fldname, len) \
- (appendStringInfoString(str, " :" CppAsString(fldname) " "), \
- writeNodeArray(str, (const Node * const *) node->fldname, len))
+ do { \
+ if (node->fldname != NULL) \
+ { \
+ appendStringInfoString(str, " :" CppAsString(fldname) " "); \
+ writeNodeArray(str, (const Node * const *) node->fldname, len); \
+ } \
+ } while (0)
/* Write a variable-length array of AttrNumber */
#define WRITE_ATTRNUMBER_ARRAY(fldname, len) \
- (appendStringInfoString(str, " :" CppAsString(fldname) " "), \
- writeAttrNumberCols(str, node->fldname, len))
+ do { \
+ if (node->fldname != NULL) \
+ { \
+ appendStringInfoString(str, " :" CppAsString(fldname) " "); \
+ writeAttrNumberCols(str, node->fldname, len); \
+ } \
+ } while (0)
/* Write a variable-length array of Oid */
#define WRITE_OID_ARRAY(fldname, len) \
- (appendStringInfoString(str, " :" CppAsString(fldname) " "), \
- writeOidCols(str, node->fldname, len))
+ do { \
+ if (node->fldname != NULL) \
+ { \
+ appendStringInfoString(str, " :" CppAsString(fldname) " "); \
+ writeOidCols(str, node->fldname, len); \
+ } \
+ } while (0)
/* Write a variable-length array of Index */
#define WRITE_INDEX_ARRAY(fldname, len) \
- (appendStringInfoString(str, " :" CppAsString(fldname) " "), \
- writeIndexCols(str, node->fldname, len))
+ do { \
+ if (node->fldname != NULL) \
+ { \
+ appendStringInfoString(str, " :" CppAsString(fldname) " "); \
+ writeIndexCols(str, node->fldname, len); \
+ } \
+ } while (0)
/* Write a variable-length array of int */
#define WRITE_INT_ARRAY(fldname, len) \
- (appendStringInfoString(str, " :" CppAsString(fldname) " "), \
- writeIntCols(str, node->fldname, len))
+ do { \
+ if (node->fldname != NULL) \
+ { \
+ appendStringInfoString(str, " :" CppAsString(fldname) " "); \
+ writeIntCols(str, node->fldname, len); \
+ } \
+ } while (0)
/* Write a variable-length array of bool */
#define WRITE_BOOL_ARRAY(fldname, len) \
- (appendStringInfoString(str, " :" CppAsString(fldname) " "), \
- writeBoolCols(str, node->fldname, len))
+ do { \
+ if (node->fldname != NULL) \
+ { \
+ appendStringInfoString(str, " :" CppAsString(fldname) " "); \
+ writeBoolCols(str, node->fldname, len); \
+ } \
+ } while (0)
#define booltostr(x) ((x) ? "true" : "false")
diff --git a/src/backend/nodes/read.c b/src/backend/nodes/read.c
index 4eb42445c5..cffd913ba6 100644
--- a/src/backend/nodes/read.c
+++ b/src/backend/nodes/read.c
@@ -205,6 +205,64 @@ pg_strtok(int *length)
return ret_str;
}
+/*
+ * Check if the next token is the expected field name, and if so,
+ * consume and return it.
+ *
+ * It handles similar to pg_strtok, except that it is special-cased for user-
+ * provided field names, rather than arbitrary tokens.
+ */
+const char *
+pg_strtok_fieldname(const char *fldname, int *length)
+{
+ const char *local_str; /* working pointer to string */
+ int expect_len;
+ char next_char;
+
+ /* skip global state to the next token */
+ while (*pg_strtok_ptr == ' ' ||
+ *pg_strtok_ptr == '\n' ||
+ *pg_strtok_ptr == '\t')
+ pg_strtok_ptr++;
+
+ local_str = pg_strtok_ptr;
+
+ if (*local_str != ':')
+ return false; /* not a field name */
+
+ expect_len = strlen(fldname);
+
+ Assert(expect_len > 0);
+
+ /* check if the next few bytes match the token */
+ if (strncmp(local_str, fldname, expect_len) != 0)
+ return false;
+
+ next_char = local_str[expect_len];
+
+ /*
+ * Check that the token was actually terminated at the end of the
+ * expected token with a character that is a separate token.
+ * Otherwise, we'd get positive matches for mathing the token of "is"
+ * against a local_str of "isn't", which is clearly wrong.
+ */
+ if (next_char == '\0' ||
+ next_char == ' ' ||
+ next_char == '\n' ||
+ next_char == '\t' ||
+ next_char == '(' ||
+ next_char == ')' ||
+ next_char == '{' ||
+ next_char == '}')
+ {
+ pg_strtok_ptr = local_str + expect_len;
+ *length = expect_len;
+
+ return local_str;
+ }
+ return NULL;
+}
+
/*
* debackslash -
* create a palloc'd string holding the given token.
diff --git a/src/backend/nodes/readfuncs.c b/src/backend/nodes/readfuncs.c
index a122407c88..7b727c168f 100644
--- a/src/backend/nodes/readfuncs.c
+++ b/src/backend/nodes/readfuncs.c
@@ -57,111 +57,218 @@
READ_TEMP_LOCALS()
/* Read an integer field (anything written as ":fldname %d") */
+#define READ_INT_FIELD_DEFAULT(fldname, default_value) \
+ do { \
+ if ((token = pg_strtok_fieldname(":" CppAsString(fldname), &length))) \
+ { \
+ token = pg_strtok(&length); /* get field value */ \
+ local_node->fldname = atoi(token); \
+ } \
+ else \
+ local_node->fldname = default_value; \
+ } while (0)
#define READ_INT_FIELD(fldname) \
- token = pg_strtok(&length); /* skip :fldname */ \
- token = pg_strtok(&length); /* get field value */ \
- local_node->fldname = atoi(token)
+ READ_INT_FIELD_DEFAULT(fldname, 0)
/* Read an unsigned integer field (anything written as ":fldname %u") */
+#define READ_UINT_FIELD_DEFAULT(fldname, default_value) \
+ do { \
+ if ((token = pg_strtok_fieldname(":" CppAsString(fldname), &length))) \
+ { \
+ token = pg_strtok(&length); /* get field value */ \
+ local_node->fldname = atoui(token); \
+ } \
+ else \
+ local_node->fldname = default_value; \
+ } while (0)
#define READ_UINT_FIELD(fldname) \
- token = pg_strtok(&length); /* skip :fldname */ \
- token = pg_strtok(&length); /* get field value */ \
- local_node->fldname = atoui(token)
+ READ_UINT_FIELD_DEFAULT(fldname, 0)
/* Read an unsigned integer field (anything written using UINT64_FORMAT) */
+#define READ_UINT64_FIELD_DEFAULT(fldname, default_value) \
+ do { \
+ if ((token = pg_strtok_fieldname(":" CppAsString(fldname), &length))) \
+ { \
+ token = pg_strtok(&length); /* get field value */ \
+ local_node->fldname = strtou64(token, NULL, 10); \
+ } \
+ else \
+ local_node->fldname = default_value; \
+ } while (0)
#define READ_UINT64_FIELD(fldname) \
- token = pg_strtok(&length); /* skip :fldname */ \
- token = pg_strtok(&length); /* get field value */ \
- local_node->fldname = strtou64(token, NULL, 10)
+ READ_UINT64_FIELD_DEFAULT(fldname, 0)
/* Read a long integer field (anything written as ":fldname %ld") */
+#define READ_LONG_FIELD_DEFAULT(fldname, default_value) \
+ do { \
+ if ((token = pg_strtok_fieldname(":" CppAsString(fldname), &length))) \
+ { \
+ token = pg_strtok(&length); /* get field value */ \
+ local_node->fldname = atol(token); \
+ } \
+ else \
+ local_node->fldname = default_value; \
+ } while (0)
#define READ_LONG_FIELD(fldname) \
- token = pg_strtok(&length); /* skip :fldname */ \
- token = pg_strtok(&length); /* get field value */ \
- local_node->fldname = atol(token)
+ READ_LONG_FIELD_DEFAULT(fldname, 0)
/* Read an OID field (don't hard-wire assumption that OID is same as uint) */
+#define READ_OID_FIELD_DEFAULT(fldname, default_value) \
+ do { \
+ if ((token = pg_strtok_fieldname(":" CppAsString(fldname), &length))) \
+ { \
+ token = pg_strtok(&length); /* get field value */ \
+ local_node->fldname = atooid(token); \
+ } \
+ else \
+ local_node->fldname = default_value; \
+ } while (0)
#define READ_OID_FIELD(fldname) \
- token = pg_strtok(&length); /* skip :fldname */ \
- token = pg_strtok(&length); /* get field value */ \
- local_node->fldname = atooid(token)
+ READ_OID_FIELD_DEFAULT(fldname, 0)
/* Read a char field (ie, one ascii character) */
+#define READ_CHAR_FIELD_DEFAULT(fldname, default_value) \
+ do { \
+ if ((token = pg_strtok_fieldname(":" CppAsString(fldname), &length))) \
+ { \
+ token = pg_strtok(&length); /* get field value */ \
+ /* avoid overhead of calling debackslash() for one char */ \
+ local_node->fldname = (length == 0) ? '\0' : (token[0] == '\\' ? token[1] : token[0]); \
+ } \
+ else \
+ local_node->fldname = default_value; \
+ } while (0)
#define READ_CHAR_FIELD(fldname) \
- token = pg_strtok(&length); /* skip :fldname */ \
- token = pg_strtok(&length); /* get field value */ \
- /* avoid overhead of calling debackslash() for one char */ \
- local_node->fldname = (length == 0) ? '\0' : (token[0] == '\\' ? token[1] : token[0])
+ READ_CHAR_FIELD_DEFAULT(fldname, '\0')
/* Read an enumerated-type field that was written as an integer code */
#define READ_ENUM_FIELD(fldname, enumtype) \
- token = pg_strtok(&length); /* skip :fldname */ \
- token = pg_strtok(&length); /* get field value */ \
- local_node->fldname = (enumtype) atoi(token)
+ do { \
+ token = pg_strtok_fieldname(":" CppAsString(fldname), &length); \
+ Assert(token != NULL); \
+ token = pg_strtok(&length); /* get field value */ \
+ local_node->fldname = (enumtype) atoi(token); \
+ } while (0)
/* Read a float field */
+#define READ_FLOAT_FIELD_DEFAULT(fldname, default_value) \
+ do { \
+ if ((token = pg_strtok_fieldname(":" CppAsString(fldname), &length))) \
+ { \
+ token = pg_strtok(&length); /* get field value */ \
+ local_node->fldname = atof(token); \
+ } \
+ else \
+ local_node->fldname = default_value; \
+ } while (0)
#define READ_FLOAT_FIELD(fldname) \
- token = pg_strtok(&length); /* skip :fldname */ \
- token = pg_strtok(&length); /* get field value */ \
- local_node->fldname = atof(token)
+ READ_FLOAT_FIELD_DEFAULT(fldname, 0.0)
/* Read a boolean field */
+#define READ_BOOL_FIELD_DEFAULT(fldname, default_value) \
+ do { \
+ if ((token = pg_strtok_fieldname(":" CppAsString(fldname), &length))) \
+ { \
+ token = pg_strtok(&length); /* get field value */ \
+ local_node->fldname = strtobool(token); \
+ } \
+ else \
+ local_node->fldname = default_value; \
+ } while (0)
#define READ_BOOL_FIELD(fldname) \
- token = pg_strtok(&length); /* skip :fldname */ \
- token = pg_strtok(&length); /* get field value */ \
- local_node->fldname = strtobool(token)
+ READ_BOOL_FIELD_DEFAULT(fldname, false)
/* Read a character-string field */
+/* see WRITE_STRING_FIELD in outfuncs.c */
#define READ_STRING_FIELD(fldname) \
- token = pg_strtok(&length); /* skip :fldname */ \
- token = pg_strtok(&length); /* get field value */ \
- local_node->fldname = nullable_string(token, length)
+ do { \
+ if ((token = pg_strtok_fieldname(":" CppAsString(fldname), &length))) \
+ { \
+ token = pg_strtok(&length); /* get field value */ \
+ local_node->fldname = nullable_string(token, length); \
+ } \
+ else \
+ local_node->fldname = NULL; \
+ } while (0)
/* Read a parse location field (and possibly throw away the value) */
#ifdef WRITE_READ_PARSE_PLAN_TREES
#define READ_LOCATION_FIELD(fldname) \
- token = pg_strtok(&length); /* skip :fldname */ \
- token = pg_strtok(&length); /* get field value */ \
- local_node->fldname = restore_location_fields ? atoi(token) : -1
+ do { \
+ if ((token = pg_strtok_fieldname(":" CppAsString(fldname), &length))) \
+ { \
+ token = pg_strtok(&length); /* get field value */ \
+ local_node->fldname = restore_location_fields ? atoi(token) : -1; \
+ } \
+ else \
+ local_node->fldname = -1; \
+ } while (0)
#else
#define READ_LOCATION_FIELD(fldname) \
- token = pg_strtok(&length); /* skip :fldname */ \
- token = pg_strtok(&length); /* get field value */ \
- (void) token; /* in case not used elsewhere */ \
- local_node->fldname = -1 /* set field to "unknown" */
+ do { \
+ if ((token = pg_strtok_fieldname(":" CppAsString(fldname), &length))) \
+ { \
+ token = pg_strtok(&length); /* the field value is useless */ \
+ (void) token; \
+ } \
+ local_node->fldname = -1; \
+ } while (0)
#endif
+
/* Read a Node field */
#define READ_NODE_FIELD(fldname) \
- token = pg_strtok(&length); /* skip :fldname */ \
- (void) token; /* in case not used elsewhere */ \
- local_node->fldname = nodeRead(NULL, 0)
+ do { \
+ if ((token = pg_strtok_fieldname(":" CppAsString(fldname), &length))) \
+ local_node->fldname = nodeRead(NULL, 0); \
+ else \
+ local_node->fldname = NULL; \
+ } while (0)
/* Read a bitmapset field */
#define READ_BITMAPSET_FIELD(fldname) \
- token = pg_strtok(&length); /* skip :fldname */ \
- (void) token; /* in case not used elsewhere */ \
- local_node->fldname = _readBitmapset()
+ do { \
+ if ((token = pg_strtok_fieldname(":" CppAsString(fldname), &length))) \
+ local_node->fldname = _readBitmapset(); \
+ else \
+ local_node->fldname = NULL; \
+ } while (0)
/* Read an attribute number array */
#define READ_ATTRNUMBER_ARRAY(fldname, len) \
- token = pg_strtok(&length); /* skip :fldname */ \
- local_node->fldname = readAttrNumberCols(len)
+ do { \
+ if ((token = pg_strtok_fieldname(":" CppAsString(fldname), &length))) \
+ local_node->fldname = readAttrNumberCols(len); \
+ else \
+ local_node->fldname = NULL; \
+ } while (0)
/* Read an oid array */
#define READ_OID_ARRAY(fldname, len) \
- token = pg_strtok(&length); /* skip :fldname */ \
- local_node->fldname = readOidCols(len)
+ do { \
+ if ((token = pg_strtok_fieldname(":" CppAsString(fldname), &length))) \
+ local_node->fldname = readOidCols(len); \
+ else \
+ local_node->fldname = NULL; \
+ } while (0)
/* Read an int array */
#define READ_INT_ARRAY(fldname, len) \
- token = pg_strtok(&length); /* skip :fldname */ \
- local_node->fldname = readIntCols(len)
+ do { \
+ if ((token = pg_strtok_fieldname(":" CppAsString(fldname), &length))) \
+ local_node->fldname = readIntCols(len); \
+ else \
+ local_node->fldname = NULL; \
+ } while (0)
/* Read a bool array */
#define READ_BOOL_ARRAY(fldname, len) \
- token = pg_strtok(&length); /* skip :fldname */ \
- local_node->fldname = readBoolCols(len)
+ do { \
+ if ((token = pg_strtok_fieldname(":" CppAsString(fldname), &length))) \
+ local_node->fldname = readBoolCols(len); \
+ else \
+ local_node->fldname = NULL; \
+ } while (0)
/* Routine exit */
#define READ_DONE() \
diff --git a/src/include/nodes/readfuncs.h b/src/include/nodes/readfuncs.h
index 8466038ed0..0579c62973 100644
--- a/src/include/nodes/readfuncs.h
+++ b/src/include/nodes/readfuncs.h
@@ -27,6 +27,7 @@ extern PGDLLIMPORT bool restore_location_fields;
* prototypes for functions in read.c (the lisp token parser)
*/
extern const char *pg_strtok(int *length);
+extern const char *pg_strtok_fieldname(const char *fldname, int *length);
extern char *debackslash(const char *token, int length);
extern void *nodeRead(const char *token, int tok_len);
diff --git a/src/test/regress/expected/rowsecurity.out b/src/test/regress/expected/rowsecurity.out
index 6988128aa4..a69aa40f82 100644
--- a/src/test/regress/expected/rowsecurity.out
+++ b/src/test/regress/expected/rowsecurity.out
@@ -3937,7 +3937,10 @@ CREATE TABLE coll_t (c) AS VALUES ('bar'::text);
CREATE POLICY coll_p ON coll_t USING (c < ('foo'::text COLLATE "C"));
ALTER TABLE coll_t ENABLE ROW LEVEL SECURITY;
GRANT SELECT ON coll_t TO regress_rls_alice;
-SELECT (string_to_array(polqual, ':'))[7] AS inputcollid FROM pg_policy WHERE polrelid = 'coll_t'::regclass;
+SELECT split AS inputcollid
+FROM pg_policy,
+ lateral unnest(string_to_array(polqual, ':')) as split
+WHERE polrelid = 'coll_t'::regclass and split LIKE '%inputcollid%';
inputcollid
------------------
inputcollid 950
diff --git a/src/test/regress/sql/rowsecurity.sql b/src/test/regress/sql/rowsecurity.sql
index dec7340538..cb7a4c776b 100644
--- a/src/test/regress/sql/rowsecurity.sql
+++ b/src/test/regress/sql/rowsecurity.sql
@@ -1732,7 +1732,10 @@ CREATE TABLE coll_t (c) AS VALUES ('bar'::text);
CREATE POLICY coll_p ON coll_t USING (c < ('foo'::text COLLATE "C"));
ALTER TABLE coll_t ENABLE ROW LEVEL SECURITY;
GRANT SELECT ON coll_t TO regress_rls_alice;
-SELECT (string_to_array(polqual, ':'))[7] AS inputcollid FROM pg_policy WHERE polrelid = 'coll_t'::regclass;
+SELECT split AS inputcollid
+FROM pg_policy,
+ lateral unnest(string_to_array(polqual, ':')) as split
+WHERE polrelid = 'coll_t'::regclass and split LIKE '%inputcollid%';
SET SESSION AUTHORIZATION regress_rls_alice;
SELECT * FROM coll_t;
ROLLBACK;
--
2.40.1
[application/octet-stream] v8-0005-nodeToString-omit-serializing-NULL-datums-in-Cons.patch (1.7K, ../../CAEze2Wh0oC97R0fG7s17b=YHOTittL8mHwC6TyT-Q2_NtY7RVw@mail.gmail.com/4-v8-0005-nodeToString-omit-serializing-NULL-datums-in-Cons.patch)
download | inline diff:
From 1dee8b709983593f93e54e37df84ad604f6e0c78 Mon Sep 17 00:00:00 2001
From: Matthias van de Meent <[email protected]>
Date: Thu, 8 Feb 2024 20:24:22 +0100
Subject: [PATCH v8 5/8] nodeToString: omit serializing NULL datums in Const
nodes
This saves some bytes in certain cases, and aligns its serialization conditions
with other field's serialization conditions.
---
src/backend/nodes/outfuncs.c | 8 ++++----
src/backend/nodes/readfuncs.c | 10 ++++++----
2 files changed, 10 insertions(+), 8 deletions(-)
diff --git a/src/backend/nodes/outfuncs.c b/src/backend/nodes/outfuncs.c
index 3789e45c53..a6c2b5bfc5 100644
--- a/src/backend/nodes/outfuncs.c
+++ b/src/backend/nodes/outfuncs.c
@@ -503,11 +503,11 @@ _outConst(StringInfo str, const Const *node)
WRITE_BOOL_FIELD(constisnull);
WRITE_LOCATION_FIELD(location);
- appendStringInfoString(str, " :constvalue ");
- if (node->constisnull)
- appendStringInfoString(str, "<>");
- else
+ if (!node->constisnull)
+ {
+ appendStringInfoString(str, " :constvalue ");
outDatum(str, node->constvalue, node->constlen, node->constbyval);
+ }
}
static void
diff --git a/src/backend/nodes/readfuncs.c b/src/backend/nodes/readfuncs.c
index e56ab5d060..ad13545de4 100644
--- a/src/backend/nodes/readfuncs.c
+++ b/src/backend/nodes/readfuncs.c
@@ -375,11 +375,13 @@ _readConst(void)
READ_BOOL_FIELD(constisnull);
READ_LOCATION_FIELD(location);
- token = pg_strtok(&length); /* skip :constvalue */
- if (local_node->constisnull)
- token = pg_strtok(&length); /* skip "<>" */
- else
+ if ((token = pg_strtok_fieldname(":constvalue", &length)))
local_node->constvalue = readDatum(local_node->constbyval);
+ else
+ {
+ /* value was omitted */
+ Assert(local_node->constisnull);
+ }
READ_DONE();
}
--
2.40.1
[application/octet-stream] v8-0004-gen_node_support.pl-Add-a-TypMod-type-for-signall.patch (10.4K, ../../CAEze2Wh0oC97R0fG7s17b=YHOTittL8mHwC6TyT-Q2_NtY7RVw@mail.gmail.com/5-v8-0004-gen_node_support.pl-Add-a-TypMod-type-for-signall.patch)
download | inline diff:
From 080ad9bdafea2056ea23830b9baa3cddae2c473d Mon Sep 17 00:00:00 2001
From: Matthias van de Meent <[email protected]>
Date: Thu, 8 Feb 2024 17:37:39 +0100
Subject: [PATCH v8 4/8] gen_node_support.pl: Add a TypMod type for signalling
TypMod behaviour
Like Location, TypMod has its own default (-1). By using a custom typedef,
we can omit adding pg_node_attribute(default(-1)) markers to every
typmod-valued field, whilst still getting the benefits of a smaller size
in serialization.
---
src/backend/nodes/gen_node_support.pl | 8 ++++++-
src/backend/nodes/outfuncs.c | 2 +-
src/backend/nodes/readfuncs.c | 2 +-
src/include/nodes/parsenodes.h | 4 ++--
src/include/nodes/pathnodes.h | 2 +-
src/include/nodes/primnodes.h | 33 ++++++++++++++-------------
6 files changed, 29 insertions(+), 22 deletions(-)
diff --git a/src/backend/nodes/gen_node_support.pl b/src/backend/nodes/gen_node_support.pl
index d4244facbb..13a3979469 100644
--- a/src/backend/nodes/gen_node_support.pl
+++ b/src/backend/nodes/gen_node_support.pl
@@ -135,7 +135,8 @@ my @nodetag_only;
# types that are copied by straight assignment
my @scalar_types = qw(
bits32 bool char double int int8 int16 int32 int64 long uint8 uint16 uint32 uint64
- AclMode AttrNumber Cardinality Cost Index Oid RelFileNumber Selectivity Size StrategyNumber SubTransactionId TimeLineID XLogRecPtr
+ AclMode AttrNumber Cardinality Cost Index Oid RelFileNumber Selectivity Size StrategyNumber
+ SubTransactionId TimeLineID XLogRecPtr TypMod
);
# collect enum types
@@ -1015,6 +1016,11 @@ _read${n}(void)
print $off "\tWRITE_LOCATION_FIELD($f);\n";
print $rff "\tREAD_LOCATION_FIELD($f);\n" unless $no_read;
}
+ elsif ($t eq 'TypMod')
+ {
+ print $off "\tWRITE_INT_FIELD_DEFAULT($f, -1);\n";
+ print $rff "\tREAD_INT_FIELD_DEFAULT($f, -1);\n" unless $no_read;
+ }
elsif ($t eq 'int'
|| $t eq 'int16'
|| $t eq 'int32'
diff --git a/src/backend/nodes/outfuncs.c b/src/backend/nodes/outfuncs.c
index b03773b7fe..3789e45c53 100644
--- a/src/backend/nodes/outfuncs.c
+++ b/src/backend/nodes/outfuncs.c
@@ -496,7 +496,7 @@ _outConst(StringInfo str, const Const *node)
WRITE_NODE_TYPE("CONST");
WRITE_OID_FIELD(consttype);
- WRITE_INT_FIELD(consttypmod);
+ WRITE_INT_FIELD_DEFAULT(consttypmod, -1);
WRITE_OID_FIELD(constcollid);
WRITE_INT_FIELD(constlen);
WRITE_BOOL_FIELD(constbyval);
diff --git a/src/backend/nodes/readfuncs.c b/src/backend/nodes/readfuncs.c
index 7b727c168f..e56ab5d060 100644
--- a/src/backend/nodes/readfuncs.c
+++ b/src/backend/nodes/readfuncs.c
@@ -368,7 +368,7 @@ _readConst(void)
READ_LOCALS(Const);
READ_OID_FIELD(consttype);
- READ_INT_FIELD(consttypmod);
+ READ_INT_FIELD_DEFAULT(consttypmod, -1);
READ_OID_FIELD(constcollid);
READ_INT_FIELD(constlen);
READ_BOOL_FIELD(constbyval);
diff --git a/src/include/nodes/parsenodes.h b/src/include/nodes/parsenodes.h
index fe2b740fd9..80ba3cf31e 100644
--- a/src/include/nodes/parsenodes.h
+++ b/src/include/nodes/parsenodes.h
@@ -268,7 +268,7 @@ typedef struct TypeName
bool setof; /* is a set? */
bool pct_type; /* %TYPE specified? */
List *typmods; /* type modifier expression(s) */
- int32 typemod; /* prespecified type modifier */
+ TypMod typemod; /* prespecified type modifier */
List *arrayBounds; /* array bounds */
ParseLoc location; /* token location, or -1 if unknown */
} TypeName;
@@ -1619,7 +1619,7 @@ typedef struct CTECycleClause
ParseLoc location;
/* These fields are set during parse analysis: */
Oid cycle_mark_type; /* common type of _value and _default */
- int cycle_mark_typmod;
+ TypMod cycle_mark_typmod;
Oid cycle_mark_collation;
Oid cycle_mark_neop; /* <> operator for type */
} CTECycleClause;
diff --git a/src/include/nodes/pathnodes.h b/src/include/nodes/pathnodes.h
index 534692bee1..f7c2496a7f 100644
--- a/src/include/nodes/pathnodes.h
+++ b/src/include/nodes/pathnodes.h
@@ -3394,7 +3394,7 @@ typedef struct AggTransInfo
Oid aggtranstype;
/* Additional data about transtype */
- int32 aggtranstypmod;
+ TypMod aggtranstypmod;
int transtypeLen;
bool transtypeByVal;
diff --git a/src/include/nodes/primnodes.h b/src/include/nodes/primnodes.h
index ade7e09175..5fd4d54475 100644
--- a/src/include/nodes/primnodes.h
+++ b/src/include/nodes/primnodes.h
@@ -29,8 +29,9 @@ typedef enum OverridingKind
OVERRIDING_SYSTEM_VALUE,
} OverridingKind;
-
+/* Type aliases for gen_node_support */
typedef int ParseLoc;
+typedef int32 TypMod;
/* ----------------------------------------------------------------
* node definitions
@@ -250,7 +251,7 @@ typedef struct Var
/* pg_type OID for the type of this var */
Oid vartype pg_node_attr(query_jumble_ignore);
/* pg_attribute typmod value */
- int32 vartypmod pg_node_attr(query_jumble_ignore);
+ TypMod vartypmod pg_node_attr(query_jumble_ignore);
/* OID of collation, or InvalidOid if none */
Oid varcollid pg_node_attr(query_jumble_ignore);
@@ -299,7 +300,7 @@ typedef struct Const
/* pg_type OID of the constant's datatype */
Oid consttype;
/* typmod value, if any */
- int32 consttypmod pg_node_attr(query_jumble_ignore);
+ TypMod consttypmod pg_node_attr(query_jumble_ignore);
/* OID of collation, or InvalidOid if none */
Oid constcollid pg_node_attr(query_jumble_ignore);
/* typlen of the constant's datatype */
@@ -365,7 +366,7 @@ typedef struct Param
int paramid; /* numeric ID for parameter */
Oid paramtype; /* pg_type OID of parameter's datatype */
/* typmod value, if known */
- int32 paramtypmod pg_node_attr(query_jumble_ignore);
+ TypMod paramtypmod pg_node_attr(query_jumble_ignore);
/* OID of collation, or InvalidOid if none */
Oid paramcollid pg_node_attr(query_jumble_ignore);
/* token location, or -1 if unknown */
@@ -623,7 +624,7 @@ typedef struct SubscriptingRef
/* type of the SubscriptingRef's result */
Oid refrestype pg_node_attr(query_jumble_ignore);
/* typmod of the result */
- int32 reftypmod pg_node_attr(query_jumble_ignore);
+ TypMod reftypmod pg_node_attr(query_jumble_ignore);
/* collation of result, or InvalidOid if none */
Oid refcollid pg_node_attr(query_jumble_ignore);
/* expressions that evaluate to upper container indexes */
@@ -1009,7 +1010,7 @@ typedef struct SubPlan
char *plan_name; /* A name assigned during planning */
/* Extra data useful for determining subplan's output type: */
Oid firstColType; /* Type of first column of subplan result */
- int32 firstColTypmod; /* Typmod of first column of subplan result */
+ TypMod firstColTypmod; /* Typmod of first column of subplan result */
Oid firstColCollation; /* Collation of first column of subplan
* result */
/* Information about execution strategy: */
@@ -1067,7 +1068,7 @@ typedef struct FieldSelect
/* type of the field (result type of this node) */
Oid resulttype pg_node_attr(query_jumble_ignore);
/* output typmod (usually -1) */
- int32 resulttypmod pg_node_attr(query_jumble_ignore);
+ TypMod resulttypmod pg_node_attr(query_jumble_ignore);
/* OID of collation of the field */
Oid resultcollid pg_node_attr(query_jumble_ignore);
} FieldSelect;
@@ -1121,7 +1122,7 @@ typedef struct RelabelType
Expr *arg; /* input expression */
Oid resulttype; /* output type of coercion expression */
/* output typmod (usually -1) */
- int32 resulttypmod pg_node_attr(query_jumble_ignore);
+ TypMod resulttypmod pg_node_attr(query_jumble_ignore);
/* OID of collation, or InvalidOid if none */
Oid resultcollid pg_node_attr(query_jumble_ignore);
/* how to display this node */
@@ -1171,7 +1172,7 @@ typedef struct ArrayCoerceExpr
Expr *elemexpr; /* expression representing per-element work */
Oid resulttype; /* output type of coercion (an array type) */
/* output typmod (also element typmod) */
- int32 resulttypmod pg_node_attr(query_jumble_ignore);
+ TypMod resulttypmod pg_node_attr(query_jumble_ignore);
/* OID of collation, or InvalidOid if none */
Oid resultcollid pg_node_attr(query_jumble_ignore);
/* how to display this node */
@@ -1291,7 +1292,7 @@ typedef struct CaseTestExpr
Expr xpr;
Oid typeId; /* type for substituted value */
/* typemod for substituted value */
- int32 typeMod pg_node_attr(query_jumble_ignore);
+ TypMod typeMod pg_node_attr(query_jumble_ignore);
/* collation for the substituted value */
Oid collation pg_node_attr(query_jumble_ignore);
} CaseTestExpr;
@@ -1497,7 +1498,7 @@ typedef struct SQLValueFunction
* include this Oid in the query jumbling.
*/
Oid type pg_node_attr(query_jumble_ignore);
- int32 typmod;
+ TypMod typmod;
ParseLoc location; /* token location, or -1 if unknown */
} SQLValueFunction;
@@ -1549,7 +1550,7 @@ typedef struct XmlExpr
bool indent;
/* target type/typmod for XMLSERIALIZE */
Oid type pg_node_attr(query_jumble_ignore);
- int32 typmod pg_node_attr(query_jumble_ignore);
+ TypMod typmod pg_node_attr(query_jumble_ignore);
/* token location, or -1 if unknown */
ParseLoc location;
} XmlExpr;
@@ -1599,7 +1600,7 @@ typedef struct JsonReturning
NodeTag type;
JsonFormat *format; /* output JSON format */
Oid typid; /* target type Oid */
- int32 typmod; /* target type modifier */
+ TypMod typmod; /* target type modifier */
} JsonReturning;
/*
@@ -1762,7 +1763,7 @@ typedef struct CoerceToDomain
Expr *arg; /* input expression */
Oid resulttype; /* domain type ID (result type) */
/* output typmod (currently always -1) */
- int32 resulttypmod pg_node_attr(query_jumble_ignore);
+ TypMod resulttypmod pg_node_attr(query_jumble_ignore);
/* OID of collation, or InvalidOid if none */
Oid resultcollid pg_node_attr(query_jumble_ignore);
/* how to display this node */
@@ -1785,7 +1786,7 @@ typedef struct CoerceToDomainValue
/* type for substituted value */
Oid typeId;
/* typemod for substituted value */
- int32 typeMod pg_node_attr(query_jumble_ignore);
+ TypMod typeMod pg_node_attr(query_jumble_ignore);
/* collation for the substituted value */
Oid collation pg_node_attr(query_jumble_ignore);
/* token location, or -1 if unknown */
@@ -1805,7 +1806,7 @@ typedef struct SetToDefault
/* type for substituted value */
Oid typeId;
/* typemod for substituted value */
- int32 typeMod pg_node_attr(query_jumble_ignore);
+ TypMod typeMod pg_node_attr(query_jumble_ignore);
/* collation for the substituted value */
Oid collation pg_node_attr(query_jumble_ignore);
/* token location, or -1 if unknown */
--
2.40.1
[application/octet-stream] v8-0002-gen_node_support.pl-Mark-location-fields-as-type-.patch (26.6K, ../../CAEze2Wh0oC97R0fG7s17b=YHOTittL8mHwC6TyT-Q2_NtY7RVw@mail.gmail.com/6-v8-0002-gen_node_support.pl-Mark-location-fields-as-type-.patch)
download | inline diff:
From 5d089d443ce5e6c92e164833a320895b6f473e9e Mon Sep 17 00:00:00 2001
From: Matthias van de Meent <[email protected]>
Date: Thu, 8 Feb 2024 17:15:22 +0100
Subject: [PATCH v8 2/8] gen_node_support.pl: Mark location fields as type
alias ParseLoc
Instead of the rather ugly type=int + name ~= location$, we now have a
marker type for offset pointers or sizes that are only relevant when a
query text is included, which decreases the complexity required in
gen_node_support.pl for handling these values.
---
src/backend/nodes/gen_node_support.pl | 6 +-
src/include/nodes/parsenodes.h | 92 +++++++++++++--------------
src/include/nodes/primnodes.h | 72 +++++++++++----------
src/tools/pgindent/typedefs.list | 1 +
4 files changed, 87 insertions(+), 84 deletions(-)
diff --git a/src/backend/nodes/gen_node_support.pl b/src/backend/nodes/gen_node_support.pl
index 2f0a59bc87..d4244facbb 100644
--- a/src/backend/nodes/gen_node_support.pl
+++ b/src/backend/nodes/gen_node_support.pl
@@ -777,7 +777,7 @@ _equal${n}(const $n *a, const $n *b)
print $eff "\tCOMPARE_BITMAPSET_FIELD($f);\n"
unless $equal_ignore;
}
- elsif ($t eq 'int' && $f =~ 'location$')
+ elsif ($t eq 'ParseLoc')
{
print $cff "\tCOPY_LOCATION_FIELD($f);\n" unless $copy_ignore;
print $eff "\tCOMPARE_LOCATION_FIELD($f);\n" unless $equal_ignore;
@@ -1010,7 +1010,7 @@ _read${n}(void)
print $off "\tWRITE_BOOL_FIELD($f);\n";
print $rff "\tREAD_BOOL_FIELD($f);\n" unless $no_read;
}
- elsif ($t eq 'int' && $f =~ 'location$')
+ elsif ($t eq 'ParseLoc')
{
print $off "\tWRITE_LOCATION_FIELD($f);\n";
print $rff "\tREAD_LOCATION_FIELD($f);\n" unless $no_read;
@@ -1303,7 +1303,7 @@ _jumble${n}(JumbleState *jstate, Node *node)
print $jff "\tJUMBLE_NODE($f);\n"
unless $query_jumble_ignore;
}
- elsif ($t eq 'int' && $f =~ 'location$')
+ elsif ($t eq 'ParseLoc')
{
# Track the node's location only if directly requested.
if ($query_jumble_location)
diff --git a/src/include/nodes/parsenodes.h b/src/include/nodes/parsenodes.h
index 2380821600..fe2b740fd9 100644
--- a/src/include/nodes/parsenodes.h
+++ b/src/include/nodes/parsenodes.h
@@ -233,9 +233,9 @@ typedef struct Query
* both be -1 meaning "unknown".
*/
/* start location, or -1 if unknown */
- int stmt_location;
+ ParseLoc stmt_location;
/* length in bytes; 0 means "rest of string" */
- int stmt_len pg_node_attr(query_jumble_ignore);
+ ParseLoc stmt_len pg_node_attr(query_jumble_ignore);
} Query;
@@ -270,7 +270,7 @@ typedef struct TypeName
List *typmods; /* type modifier expression(s) */
int32 typemod; /* prespecified type modifier */
List *arrayBounds; /* array bounds */
- int location; /* token location, or -1 if unknown */
+ ParseLoc location; /* token location, or -1 if unknown */
} TypeName;
/*
@@ -290,7 +290,7 @@ typedef struct ColumnRef
{
NodeTag type;
List *fields; /* field names (String nodes) or A_Star */
- int location; /* token location, or -1 if unknown */
+ ParseLoc location; /* token location, or -1 if unknown */
} ColumnRef;
/*
@@ -300,7 +300,7 @@ typedef struct ParamRef
{
NodeTag type;
int number; /* the number of the parameter */
- int location; /* token location, or -1 if unknown */
+ ParseLoc location; /* token location, or -1 if unknown */
} ParamRef;
/*
@@ -333,7 +333,7 @@ typedef struct A_Expr
List *name; /* possibly-qualified name of operator */
Node *lexpr; /* left argument, or NULL if none */
Node *rexpr; /* right argument, or NULL if none */
- int location; /* token location, or -1 if unknown */
+ ParseLoc location; /* token location, or -1 if unknown */
} A_Expr;
/*
@@ -359,7 +359,7 @@ typedef struct A_Const
NodeTag type;
union ValUnion val;
bool isnull; /* SQL NULL constant */
- int location; /* token location, or -1 if unknown */
+ ParseLoc location; /* token location, or -1 if unknown */
} A_Const;
/*
@@ -370,7 +370,7 @@ typedef struct TypeCast
NodeTag type;
Node *arg; /* the expression being casted */
TypeName *typeName; /* the target type */
- int location; /* token location, or -1 if unknown */
+ ParseLoc location; /* token location, or -1 if unknown */
} TypeCast;
/*
@@ -381,7 +381,7 @@ typedef struct CollateClause
NodeTag type;
Node *arg; /* input expression */
List *collname; /* possibly-qualified collation name */
- int location; /* token location, or -1 if unknown */
+ ParseLoc location; /* token location, or -1 if unknown */
} CollateClause;
/*
@@ -401,7 +401,7 @@ typedef struct RoleSpec
NodeTag type;
RoleSpecType roletype; /* Type of this rolespec */
char *rolename; /* filled only for ROLESPEC_CSTRING */
- int location; /* token location, or -1 if unknown */
+ ParseLoc location; /* token location, or -1 if unknown */
} RoleSpec;
/*
@@ -431,7 +431,7 @@ typedef struct FuncCall
bool agg_distinct; /* arguments were labeled DISTINCT */
bool func_variadic; /* last argument was labeled VARIADIC */
CoercionForm funcformat; /* how to display this node */
- int location; /* token location, or -1 if unknown */
+ ParseLoc location; /* token location, or -1 if unknown */
} FuncCall;
/*
@@ -488,7 +488,7 @@ typedef struct A_ArrayExpr
{
NodeTag type;
List *elements; /* array element expressions */
- int location; /* token location, or -1 if unknown */
+ ParseLoc location; /* token location, or -1 if unknown */
} A_ArrayExpr;
/*
@@ -515,7 +515,7 @@ typedef struct ResTarget
char *name; /* column name or NULL */
List *indirection; /* subscripts, field names, and '*', or NIL */
Node *val; /* the value expression to compute or assign */
- int location; /* token location, or -1 if unknown */
+ ParseLoc location; /* token location, or -1 if unknown */
} ResTarget;
/*
@@ -545,7 +545,7 @@ typedef struct SortBy
SortByDir sortby_dir; /* ASC/DESC/USING/default */
SortByNulls sortby_nulls; /* NULLS FIRST/LAST */
List *useOp; /* name of op to use, if SORTBY_USING */
- int location; /* operator location, or -1 if none/unknown */
+ ParseLoc location; /* operator location, or -1 if none/unknown */
} SortBy;
/*
@@ -566,7 +566,7 @@ typedef struct WindowDef
int frameOptions; /* frame_clause options, see below */
Node *startOffset; /* expression for starting bound, if any */
Node *endOffset; /* expression for ending bound, if any */
- int location; /* parse location, or -1 if none/unknown */
+ ParseLoc location; /* parse location, or -1 if none/unknown */
} WindowDef;
/*
@@ -656,7 +656,7 @@ typedef struct RangeTableFunc
List *namespaces; /* list of namespaces as ResTarget */
List *columns; /* list of RangeTableFuncCol */
Alias *alias; /* table alias & optional column aliases */
- int location; /* token location, or -1 if unknown */
+ ParseLoc location; /* token location, or -1 if unknown */
} RangeTableFunc;
/*
@@ -674,7 +674,7 @@ typedef struct RangeTableFuncCol
bool is_not_null; /* does it have NOT NULL? */
Node *colexpr; /* column filter expression */
Node *coldefexpr; /* column default value expression */
- int location; /* token location, or -1 if unknown */
+ ParseLoc location; /* token location, or -1 if unknown */
} RangeTableFuncCol;
/*
@@ -694,7 +694,7 @@ typedef struct RangeTableSample
List *method; /* sampling method name (possibly qualified) */
List *args; /* argument(s) for sampling method */
Node *repeatable; /* REPEATABLE expression, or NULL if none */
- int location; /* method name location, or -1 if unknown */
+ ParseLoc location; /* method name location, or -1 if unknown */
} RangeTableSample;
/*
@@ -737,7 +737,7 @@ typedef struct ColumnDef
Oid collOid; /* collation OID (InvalidOid if not set) */
List *constraints; /* other constraints on column */
List *fdwoptions; /* per-column FDW options */
- int location; /* parse location, or -1 if none/unknown */
+ ParseLoc location; /* parse location, or -1 if none/unknown */
} ColumnDef;
/*
@@ -811,7 +811,7 @@ typedef struct DefElem
Node *arg; /* typically Integer, Float, String, or
* TypeName */
DefElemAction defaction; /* unspecified action, or SET/ADD/DROP */
- int location; /* token location, or -1 if unknown */
+ ParseLoc location; /* token location, or -1 if unknown */
} DefElem;
/*
@@ -841,7 +841,7 @@ typedef struct XmlSerialize
Node *expr;
TypeName *typeName;
bool indent; /* [NO] INDENT */
- int location; /* token location, or -1 if unknown */
+ ParseLoc location; /* token location, or -1 if unknown */
} XmlSerialize;
/* Partitioning related definitions */
@@ -859,7 +859,7 @@ typedef struct PartitionElem
Node *expr; /* expression to partition on, or NULL */
List *collation; /* name of collation; NIL = default */
List *opclass; /* name of desired opclass; NIL = default */
- int location; /* token location, or -1 if unknown */
+ ParseLoc location; /* token location, or -1 if unknown */
} PartitionElem;
typedef enum PartitionStrategy
@@ -879,7 +879,7 @@ typedef struct PartitionSpec
NodeTag type;
PartitionStrategy strategy;
List *partParams; /* List of PartitionElems */
- int location; /* token location, or -1 if unknown */
+ ParseLoc location; /* token location, or -1 if unknown */
} PartitionSpec;
/*
@@ -906,7 +906,7 @@ struct PartitionBoundSpec
List *lowerdatums; /* List of PartitionRangeDatums */
List *upperdatums; /* List of PartitionRangeDatums */
- int location; /* token location, or -1 if unknown */
+ ParseLoc location; /* token location, or -1 if unknown */
};
/*
@@ -929,7 +929,7 @@ typedef struct PartitionRangeDatum
Node *value; /* Const (or A_Const in raw tree), if kind is
* PARTITION_RANGE_DATUM_VALUE, else NULL */
- int location; /* token location, or -1 if unknown */
+ ParseLoc location; /* token location, or -1 if unknown */
} PartitionRangeDatum;
/*
@@ -1465,7 +1465,7 @@ typedef struct GroupingSet
NodeTag type;
GroupingSetKind kind pg_node_attr(query_jumble_ignore);
List *content;
- int location;
+ ParseLoc location;
} GroupingSet;
/*
@@ -1553,7 +1553,7 @@ typedef struct WithClause
NodeTag type;
List *ctes; /* list of CommonTableExprs */
bool recursive; /* true = WITH RECURSIVE */
- int location; /* token location, or -1 if unknown */
+ ParseLoc location; /* token location, or -1 if unknown */
} WithClause;
/*
@@ -1568,7 +1568,7 @@ typedef struct InferClause
List *indexElems; /* IndexElems to infer unique index */
Node *whereClause; /* qualification (partial-index predicate) */
char *conname; /* Constraint name, or NULL if unnamed */
- int location; /* token location, or -1 if unknown */
+ ParseLoc location; /* token location, or -1 if unknown */
} InferClause;
/*
@@ -1584,7 +1584,7 @@ typedef struct OnConflictClause
InferClause *infer; /* Optional index inference clause */
List *targetList; /* the target list (of ResTarget) */
Node *whereClause; /* qualifications */
- int location; /* token location, or -1 if unknown */
+ ParseLoc location; /* token location, or -1 if unknown */
} OnConflictClause;
/*
@@ -1605,7 +1605,7 @@ typedef struct CTESearchClause
List *search_col_list;
bool search_breadth_first;
char *search_seq_column;
- int location;
+ ParseLoc location;
} CTESearchClause;
typedef struct CTECycleClause
@@ -1616,7 +1616,7 @@ typedef struct CTECycleClause
Node *cycle_mark_value;
Node *cycle_mark_default;
char *cycle_path_column;
- int location;
+ ParseLoc location;
/* These fields are set during parse analysis: */
Oid cycle_mark_type; /* common type of _value and _default */
int cycle_mark_typmod;
@@ -1640,7 +1640,7 @@ typedef struct CommonTableExpr
Node *ctequery; /* the CTE's subquery */
CTESearchClause *search_clause pg_node_attr(query_jumble_ignore);
CTECycleClause *cycle_clause pg_node_attr(query_jumble_ignore);
- int location; /* token location, or -1 if unknown */
+ ParseLoc location; /* token location, or -1 if unknown */
/* These fields are set during parse analysis: */
/* is this CTE actually recursive? */
bool cterecursive pg_node_attr(query_jumble_ignore);
@@ -1736,7 +1736,7 @@ typedef struct JsonParseExpr
JsonValueExpr *expr; /* string expression */
JsonOutput *output; /* RETURNING clause, if specified */
bool unique_keys; /* WITH UNIQUE KEYS? */
- int location; /* token location, or -1 if unknown */
+ ParseLoc location; /* token location, or -1 if unknown */
} JsonParseExpr;
/*
@@ -1748,7 +1748,7 @@ typedef struct JsonScalarExpr
NodeTag type;
Expr *expr; /* scalar expression */
JsonOutput *output; /* RETURNING clause, if specified */
- int location; /* token location, or -1 if unknown */
+ ParseLoc location; /* token location, or -1 if unknown */
} JsonScalarExpr;
/*
@@ -1760,7 +1760,7 @@ typedef struct JsonSerializeExpr
NodeTag type;
JsonValueExpr *expr; /* json value expression */
JsonOutput *output; /* RETURNING clause, if specified */
- int location; /* token location, or -1 if unknown */
+ ParseLoc location; /* token location, or -1 if unknown */
} JsonSerializeExpr;
/*
@@ -1774,7 +1774,7 @@ typedef struct JsonObjectConstructor
JsonOutput *output; /* RETURNING clause, if specified */
bool absent_on_null; /* skip NULL values? */
bool unique; /* check key uniqueness? */
- int location; /* token location, or -1 if unknown */
+ ParseLoc location; /* token location, or -1 if unknown */
} JsonObjectConstructor;
/*
@@ -1787,7 +1787,7 @@ typedef struct JsonArrayConstructor
List *exprs; /* list of JsonValueExpr elements */
JsonOutput *output; /* RETURNING clause, if specified */
bool absent_on_null; /* skip NULL elements? */
- int location; /* token location, or -1 if unknown */
+ ParseLoc location; /* token location, or -1 if unknown */
} JsonArrayConstructor;
/*
@@ -1801,7 +1801,7 @@ typedef struct JsonArrayQueryConstructor
JsonOutput *output; /* RETURNING clause, if specified */
JsonFormat *format; /* FORMAT clause for subquery, if specified */
bool absent_on_null; /* skip NULL elements? */
- int location; /* token location, or -1 if unknown */
+ ParseLoc location; /* token location, or -1 if unknown */
} JsonArrayQueryConstructor;
/*
@@ -1816,7 +1816,7 @@ typedef struct JsonAggConstructor
Node *agg_filter; /* FILTER clause, if any */
List *agg_order; /* ORDER BY clause, if any */
struct WindowDef *over; /* OVER clause, if any */
- int location; /* token location, or -1 if unknown */
+ ParseLoc location; /* token location, or -1 if unknown */
} JsonAggConstructor;
/*
@@ -1870,8 +1870,8 @@ typedef struct RawStmt
NodeTag type;
Node *stmt; /* raw parse tree */
- int stmt_location; /* start location, or -1 if unknown */
- int stmt_len; /* length in bytes; 0 means "rest of string" */
+ ParseLoc stmt_location; /* start location, or -1 if unknown */
+ ParseLoc stmt_len; /* length in bytes; 0 means "rest of string" */
} RawStmt;
/*****************************************************************************
@@ -2078,7 +2078,7 @@ typedef struct PLAssignStmt
List *indirection; /* subscripts and field names, if any */
int nnames; /* number of names to use in ColumnRef */
SelectStmt *val; /* the PL/pgSQL expression to assign */
- int location; /* name's token location, or -1 if unknown */
+ ParseLoc location; /* name's token location, or -1 if unknown */
} PLAssignStmt;
@@ -2620,7 +2620,7 @@ typedef struct Constraint
Oid old_pktable_oid; /* pg_constraint.confrelid of my former
* self */
- int location; /* token location, or -1 if unknown */
+ ParseLoc location; /* token location, or -1 if unknown */
} Constraint;
/* ----------------------
@@ -3527,7 +3527,7 @@ typedef struct TransactionStmt
char *gid pg_node_attr(query_jumble_ignore);
bool chain; /* AND CHAIN option */
/* token location, or -1 if unknown */
- int location pg_node_attr(query_jumble_location);
+ ParseLoc location pg_node_attr(query_jumble_location);
} TransactionStmt;
/* ----------------------
@@ -3913,7 +3913,7 @@ typedef struct DeallocateStmt
/* true if DEALLOCATE ALL */
bool isall;
/* token location, or -1 if unknown */
- int location pg_node_attr(query_jumble_location);
+ ParseLoc location pg_node_attr(query_jumble_location);
} DeallocateStmt;
/*
@@ -4001,7 +4001,7 @@ typedef struct PublicationObjSpec
PublicationObjSpecType pubobjtype; /* type of this publication object */
char *name;
PublicationTable *pubtable;
- int location; /* token location, or -1 if unknown */
+ ParseLoc location; /* token location, or -1 if unknown */
} PublicationObjSpec;
typedef struct CreatePublicationStmt
diff --git a/src/include/nodes/primnodes.h b/src/include/nodes/primnodes.h
index 4a154606d2..ade7e09175 100644
--- a/src/include/nodes/primnodes.h
+++ b/src/include/nodes/primnodes.h
@@ -30,6 +30,8 @@ typedef enum OverridingKind
} OverridingKind;
+typedef int ParseLoc;
+
/* ----------------------------------------------------------------
* node definitions
* ----------------------------------------------------------------
@@ -91,7 +93,7 @@ typedef struct RangeVar
Alias *alias;
/* token location, or -1 if unknown */
- int location;
+ ParseLoc location;
} RangeVar;
/*
@@ -128,7 +130,7 @@ typedef struct TableFunc
/* counts from 0; -1 if none specified */
int ordinalitycol pg_node_attr(query_jumble_ignore);
/* token location, or -1 if unknown */
- int location;
+ ParseLoc location;
} TableFunc;
/*
@@ -276,7 +278,7 @@ typedef struct Var
AttrNumber varattnosyn pg_node_attr(equal_ignore, query_jumble_ignore);
/* token location, or -1 if unknown */
- int location;
+ ParseLoc location;
} Var;
/*
@@ -318,7 +320,7 @@ typedef struct Const
* token location, or -1 if unknown. All constants are tracked as
* locations in query jumbling, to be marked as parameters.
*/
- int location pg_node_attr(query_jumble_location);
+ ParseLoc location pg_node_attr(query_jumble_location);
} Const;
/*
@@ -367,7 +369,7 @@ typedef struct Param
/* OID of collation, or InvalidOid if none */
Oid paramcollid pg_node_attr(query_jumble_ignore);
/* token location, or -1 if unknown */
- int location;
+ ParseLoc location;
} Param;
/*
@@ -490,7 +492,7 @@ typedef struct Aggref
int aggtransno pg_node_attr(query_jumble_ignore);
/* token location, or -1 if unknown */
- int location;
+ ParseLoc location;
} Aggref;
/*
@@ -537,7 +539,7 @@ typedef struct GroupingFunc
Index agglevelsup;
/* token location */
- int location;
+ ParseLoc location;
} GroupingFunc;
/*
@@ -568,7 +570,7 @@ typedef struct WindowFunc
/* is function a simple aggregate? */
bool winagg pg_node_attr(query_jumble_ignore);
/* token location, or -1 if unknown */
- int location;
+ ParseLoc location;
} WindowFunc;
/*
@@ -702,7 +704,7 @@ typedef struct FuncExpr
/* arguments to the function */
List *args;
/* token location, or -1 if unknown */
- int location;
+ ParseLoc location;
} FuncExpr;
/*
@@ -729,7 +731,7 @@ typedef struct NamedArgExpr
/* argument's number in positional notation */
int argnumber;
/* argument name location, or -1 if unknown */
- int location;
+ ParseLoc location;
} NamedArgExpr;
/*
@@ -771,7 +773,7 @@ typedef struct OpExpr
List *args;
/* token location, or -1 if unknown */
- int location;
+ ParseLoc location;
} OpExpr;
/*
@@ -851,7 +853,7 @@ typedef struct ScalarArrayOpExpr
List *args;
/* token location, or -1 if unknown */
- int location;
+ ParseLoc location;
} ScalarArrayOpExpr;
/*
@@ -873,7 +875,7 @@ typedef struct BoolExpr
Expr xpr;
BoolExprType boolop;
List *args; /* arguments to this expression */
- int location; /* token location, or -1 if unknown */
+ ParseLoc location; /* token location, or -1 if unknown */
} BoolExpr;
/*
@@ -950,7 +952,7 @@ typedef struct SubLink
List *operName pg_node_attr(query_jumble_ignore);
/* subselect as Query* or raw parsetree */
Node *subselect;
- int location; /* token location, or -1 if unknown */
+ ParseLoc location; /* token location, or -1 if unknown */
} SubLink;
/*
@@ -1124,7 +1126,7 @@ typedef struct RelabelType
Oid resultcollid pg_node_attr(query_jumble_ignore);
/* how to display this node */
CoercionForm relabelformat pg_node_attr(query_jumble_ignore);
- int location; /* token location, or -1 if unknown */
+ ParseLoc location; /* token location, or -1 if unknown */
} RelabelType;
/* ----------------
@@ -1146,7 +1148,7 @@ typedef struct CoerceViaIO
Oid resultcollid pg_node_attr(query_jumble_ignore);
/* how to display this node */
CoercionForm coerceformat pg_node_attr(query_jumble_ignore);
- int location; /* token location, or -1 if unknown */
+ ParseLoc location; /* token location, or -1 if unknown */
} CoerceViaIO;
/* ----------------
@@ -1174,7 +1176,7 @@ typedef struct ArrayCoerceExpr
Oid resultcollid pg_node_attr(query_jumble_ignore);
/* how to display this node */
CoercionForm coerceformat pg_node_attr(query_jumble_ignore);
- int location; /* token location, or -1 if unknown */
+ ParseLoc location; /* token location, or -1 if unknown */
} ArrayCoerceExpr;
/* ----------------
@@ -1198,7 +1200,7 @@ typedef struct ConvertRowtypeExpr
/* Like RowExpr, we deliberately omit a typmod and collation here */
/* how to display this node */
CoercionForm convertformat pg_node_attr(query_jumble_ignore);
- int location; /* token location, or -1 if unknown */
+ ParseLoc location; /* token location, or -1 if unknown */
} ConvertRowtypeExpr;
/*----------
@@ -1213,7 +1215,7 @@ typedef struct CollateExpr
Expr xpr;
Expr *arg; /* input expression */
Oid collOid; /* collation's OID */
- int location; /* token location, or -1 if unknown */
+ ParseLoc location; /* token location, or -1 if unknown */
} CollateExpr;
/*----------
@@ -1248,7 +1250,7 @@ typedef struct CaseExpr
Expr *arg; /* implicit equality comparison argument */
List *args; /* the arguments (list of WHEN clauses) */
Expr *defresult; /* the default result (ELSE clause) */
- int location; /* token location, or -1 if unknown */
+ ParseLoc location; /* token location, or -1 if unknown */
} CaseExpr;
/*
@@ -1259,7 +1261,7 @@ typedef struct CaseWhen
Expr xpr;
Expr *expr; /* condition expression */
Expr *result; /* substitution result */
- int location; /* token location, or -1 if unknown */
+ ParseLoc location; /* token location, or -1 if unknown */
} CaseWhen;
/*
@@ -1316,7 +1318,7 @@ typedef struct ArrayExpr
/* true if elements are sub-arrays */
bool multidims pg_node_attr(query_jumble_ignore);
/* token location, or -1 if unknown */
- int location;
+ ParseLoc location;
} ArrayExpr;
/*
@@ -1367,7 +1369,7 @@ typedef struct RowExpr
/* list of String, or NIL */
List *colnames pg_node_attr(query_jumble_ignore);
- int location; /* token location, or -1 if unknown */
+ ParseLoc location; /* token location, or -1 if unknown */
} RowExpr;
/*
@@ -1426,7 +1428,7 @@ typedef struct CoalesceExpr
/* the arguments */
List *args;
/* token location, or -1 if unknown */
- int location;
+ ParseLoc location;
} CoalesceExpr;
/*
@@ -1452,7 +1454,7 @@ typedef struct MinMaxExpr
/* the arguments */
List *args;
/* token location, or -1 if unknown */
- int location;
+ ParseLoc location;
} MinMaxExpr;
/*
@@ -1496,7 +1498,7 @@ typedef struct SQLValueFunction
*/
Oid type pg_node_attr(query_jumble_ignore);
int32 typmod;
- int location; /* token location, or -1 if unknown */
+ ParseLoc location; /* token location, or -1 if unknown */
} SQLValueFunction;
/*
@@ -1549,7 +1551,7 @@ typedef struct XmlExpr
Oid type pg_node_attr(query_jumble_ignore);
int32 typmod pg_node_attr(query_jumble_ignore);
/* token location, or -1 if unknown */
- int location;
+ ParseLoc location;
} XmlExpr;
/*
@@ -1585,7 +1587,7 @@ typedef struct JsonFormat
NodeTag type;
JsonFormatType format_type; /* format type */
JsonEncoding encoding; /* JSON encoding */
- int location; /* token location, or -1 if unknown */
+ ParseLoc location; /* token location, or -1 if unknown */
} JsonFormat;
/*
@@ -1641,7 +1643,7 @@ typedef struct JsonConstructorExpr
JsonReturning *returning; /* RETURNING clause */
bool absent_on_null; /* ABSENT ON NULL? */
bool unique; /* WITH UNIQUE KEYS? (JSON_OBJECT[AGG] only) */
- int location;
+ ParseLoc location;
} JsonConstructorExpr;
/*
@@ -1667,7 +1669,7 @@ typedef struct JsonIsPredicate
JsonFormat *format; /* FORMAT clause, if specified */
JsonValueType item_type; /* JSON item type */
bool unique_keys; /* check key uniqueness? */
- int location; /* token location, or -1 if unknown */
+ ParseLoc location; /* token location, or -1 if unknown */
} JsonIsPredicate;
/* ----------------
@@ -1701,7 +1703,7 @@ typedef struct NullTest
NullTestType nulltesttype; /* IS NULL, IS NOT NULL */
/* T to perform field-by-field null checks */
bool argisrow pg_node_attr(query_jumble_ignore);
- int location; /* token location, or -1 if unknown */
+ ParseLoc location; /* token location, or -1 if unknown */
} NullTest;
/*
@@ -1723,7 +1725,7 @@ typedef struct BooleanTest
Expr xpr;
Expr *arg; /* input expression */
BoolTestType booltesttype; /* test type */
- int location; /* token location, or -1 if unknown */
+ ParseLoc location; /* token location, or -1 if unknown */
} BooleanTest;
@@ -1765,7 +1767,7 @@ typedef struct CoerceToDomain
Oid resultcollid pg_node_attr(query_jumble_ignore);
/* how to display this node */
CoercionForm coercionformat pg_node_attr(query_jumble_ignore);
- int location; /* token location, or -1 if unknown */
+ ParseLoc location; /* token location, or -1 if unknown */
} CoerceToDomain;
/*
@@ -1787,7 +1789,7 @@ typedef struct CoerceToDomainValue
/* collation for the substituted value */
Oid collation pg_node_attr(query_jumble_ignore);
/* token location, or -1 if unknown */
- int location;
+ ParseLoc location;
} CoerceToDomainValue;
/*
@@ -1807,7 +1809,7 @@ typedef struct SetToDefault
/* collation for the substituted value */
Oid collation pg_node_attr(query_jumble_ignore);
/* token location, or -1 if unknown */
- int location;
+ ParseLoc location;
} SetToDefault;
/*
diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list
index d3a7f75b08..4aeba2a4ff 100644
--- a/src/tools/pgindent/typedefs.list
+++ b/src/tools/pgindent/typedefs.list
@@ -2218,6 +2218,7 @@ QueryEnvironment
QueryInfo
QueryItem
QueryItemType
+QueryLoc
QueryMode
QueryOperand
QueryOperator
--
2.40.1
[application/octet-stream] v8-0006-nodeToString-Apply-RLE-on-Bitmapset-and-numeric-L.patch (7.8K, ../../CAEze2Wh0oC97R0fG7s17b=YHOTittL8mHwC6TyT-Q2_NtY7RVw@mail.gmail.com/7-v8-0006-nodeToString-Apply-RLE-on-Bitmapset-and-numeric-L.patch)
download | inline diff:
From 56ebc620f8d3bd7fae310b19a094f03241ce7d94 Mon Sep 17 00:00:00 2001
From: Matthias van de Meent <[email protected]>
Date: Sat, 10 Feb 2024 00:57:19 +0100
Subject: [PATCH v8 6/8] nodeToString: Apply RLE on Bitmapset and numeric List
types
This reduces the size of full serialized queries in pg_rewrite by several %,
reducing overhead in the system.
---
src/backend/nodes/outfuncs.c | 96 +++++++++++++++++++++++++++++++----
src/backend/nodes/read.c | 53 +++++++++++++++++--
src/backend/nodes/readfuncs.c | 16 +++++-
3 files changed, 149 insertions(+), 16 deletions(-)
diff --git a/src/backend/nodes/outfuncs.c b/src/backend/nodes/outfuncs.c
index a6c2b5bfc5..90a7d73d3b 100644
--- a/src/backend/nodes/outfuncs.c
+++ b/src/backend/nodes/outfuncs.c
@@ -382,23 +382,64 @@ writeNodeArray(StringInfo str, const Node *const *arr, int len)
/*
* Print a List.
+ *
+ * Notes:
+ * NodeList is formatted as (<node1> <node2> ...)
+ *
+ * OidList is formatted as (o int +int int int ...), with +int indicating N
+ * successive identical values.
+ *
+ * IntList and XidList are formatted as (i/x int +int int int ...), with +int
+ * indicating N successively increasing values. (i 9 +3) is thus equivalent
+ * to (i 9 10 11 12).
*/
static void
_outList(StringInfo str, const List *node)
{
const ListCell *lc;
+ union LCSurrogate {
+ int32 i;
+ Oid o;
+ TransactionId x;
+ } previous = { .i = 0};
+ int run = 0;
+ int run_delta;
+ const char *fmt;
appendStringInfoChar(str, '(');
if (IsA(node, IntList))
+ {
appendStringInfoChar(str, 'i');
+ fmt = " %d";
+ run_delta = 1;
+ }
else if (IsA(node, OidList))
+ {
appendStringInfoChar(str, 'o');
+ fmt = " %u";
+ run_delta = 0;
+ }
else if (IsA(node, XidList))
+ {
appendStringInfoChar(str, 'x');
+ fmt = " %u";
+ run_delta = 1;
+ }
+ else if (IsA(node, List))
+ {
+ /* silence the compiler about uninitialized variables */
+ fmt = "";
+ run_delta = 0;
+ }
+ else
+ elog(ERROR, "unrecognized list node type: %d",
+ (int) node->type);
foreach(lc, node)
{
+ union LCSurrogate val = {.i = 0};
+
/*
* For the sake of backward compatibility, we emit a slightly
* different whitespace format for lists of nodes vs. other types of
@@ -409,26 +450,41 @@ _outList(StringInfo str, const List *node)
outNode(str, lfirst(lc));
if (lnext(node, lc))
appendStringInfoChar(str, ' ');
+ continue;
}
else if (IsA(node, IntList))
- appendStringInfo(str, " %d", lfirst_int(lc));
+ val.i = lfirst_int(lc);
else if (IsA(node, OidList))
- appendStringInfo(str, " %u", lfirst_oid(lc));
+ val.o = lfirst_oid(lc);
else if (IsA(node, XidList))
- appendStringInfo(str, " %u", lfirst_xid(lc));
+ val.x = lfirst_xid(lc);
+
+ if (val.i == previous.i + run_delta)
+ run += 1;
else
- elog(ERROR, "unrecognized list node type: %d",
- (int) node->type);
+ {
+ if (run > 0)
+ appendStringInfo(str, " +%d", run);
+ run = 0;
+ appendStringInfo(str, fmt, val.i);
+ }
+ previous = val;
}
- appendStringInfoChar(str, ')');
-}
+ if (run > 0)
+ appendStringInfo(str, " +%d", run);
+
+ appendStringInfoChar(str, ')');}
/*
* outBitmapset -
* converts a bitmap set of integers
*
- * Note: the output format is "(b int int ...)", similar to an integer List.
+ * Note: the output format is "(b int int +int ...)", similar to an
+ * integer List. Note that consecutive runs of incremental values are
+ * indicated by [... int +int ...], where the first int is the first set bit,
+ * and the int that's tagged with the '+' sign that follows is the number of
+ * subsequent set bits (resulting in a run of N+1 set bits).
*
* We export this function for use by extensions that define extensible nodes.
* That's somewhat historical, though, because calling outNode() will work.
@@ -436,13 +492,31 @@ _outList(StringInfo str, const List *node)
void
outBitmapset(StringInfo str, const Bitmapset *bms)
{
- int x;
+ int x = -1;
+ int prev = -2;
+ int run = 0;
appendStringInfoChar(str, '(');
appendStringInfoChar(str, 'b');
- x = -1;
+
while ((x = bms_next_member(bms, x)) >= 0)
- appendStringInfo(str, " %d", x);
+ {
+ if (x - prev == 1)
+ run++;
+ else
+ {
+ if (run > 0)
+ appendStringInfo(str, " +%d", run);
+
+ run = 0;
+ appendStringInfo(str, " %d", x);
+ }
+ prev = x;
+ }
+
+ if (run > 0)
+ appendStringInfo(str, " +%d", run);
+
appendStringInfoChar(str, ')');
}
diff --git a/src/backend/nodes/read.c b/src/backend/nodes/read.c
index cffd913ba6..7456626e82 100644
--- a/src/backend/nodes/read.c
+++ b/src/backend/nodes/read.c
@@ -415,6 +415,8 @@ nodeRead(const char *token, int tok_len)
elog(ERROR, "unterminated List structure");
if (tok_len == 1 && token[0] == 'i')
{
+ int prev = 0;
+
/* List of integers */
for (;;)
{
@@ -430,12 +432,23 @@ nodeRead(const char *token, int tok_len)
if (endptr != token + tok_len)
elog(ERROR, "unrecognized integer: \"%.*s\"",
tok_len, token);
- l = lappend_int(l, val);
+
+ if (token[0] == '+')
+ {
+ for (int i = 0; i < val; i++)
+ l = lappend_int(l, ++prev);
+ }
+ else
+ {
+ prev = val;
+ l = lappend_int(l, val);
+ }
}
result = (Node *) l;
}
else if (tok_len == 1 && token[0] == 'o')
{
+ Oid prev = 0;
/* List of OIDs */
for (;;)
{
@@ -451,12 +464,23 @@ nodeRead(const char *token, int tok_len)
if (endptr != token + tok_len)
elog(ERROR, "unrecognized OID: \"%.*s\"",
tok_len, token);
- l = lappend_oid(l, val);
+
+ if (token[0] == '+')
+ {
+ for (int i = 0; i < val; i++)
+ l = lappend_oid(l, prev);
+ }
+ else
+ {
+ prev = val;
+ l = lappend_oid(l, val);
+ }
}
result = (Node *) l;
}
else if (tok_len == 1 && token[0] == 'x')
{
+ TransactionId prev = 0;
/* List of TransactionIds */
for (;;)
{
@@ -472,7 +496,17 @@ nodeRead(const char *token, int tok_len)
if (endptr != token + tok_len)
elog(ERROR, "unrecognized Xid: \"%.*s\"",
tok_len, token);
- l = lappend_xid(l, val);
+
+ if (token[0] == '+')
+ {
+ for (int i = 0; i < val; i++)
+ l = lappend_xid(l, ++prev);
+ }
+ else
+ {
+ prev = val;
+ l = lappend_xid(l, val);
+ }
}
result = (Node *) l;
}
@@ -480,6 +514,7 @@ nodeRead(const char *token, int tok_len)
{
/* Bitmapset -- see also _readBitmapset() */
Bitmapset *bms = NULL;
+ int prev = -2;
for (;;)
{
@@ -495,7 +530,17 @@ nodeRead(const char *token, int tok_len)
if (endptr != token + tok_len)
elog(ERROR, "unrecognized integer: \"%.*s\"",
tok_len, token);
- bms = bms_add_member(bms, val);
+
+ if (token[0] == '+')
+ {
+ for (int i = 0; i < val; i++)
+ bms = bms_add_member(bms, ++prev);
+ }
+ else
+ {
+ bms = bms_add_member(bms, val);
+ prev = val;
+ }
}
result = (Node *) bms;
}
diff --git a/src/backend/nodes/readfuncs.c b/src/backend/nodes/readfuncs.c
index ad13545de4..970b4987bb 100644
--- a/src/backend/nodes/readfuncs.c
+++ b/src/backend/nodes/readfuncs.c
@@ -310,6 +310,7 @@ static Bitmapset *
_readBitmapset(void)
{
Bitmapset *result = NULL;
+ int prev = -2;
READ_TEMP_LOCALS();
@@ -338,7 +339,20 @@ _readBitmapset(void)
val = (int) strtol(token, &endptr, 10);
if (endptr != token + length)
elog(ERROR, "unrecognized integer: \"%.*s\"", length, token);
- result = bms_add_member(result, val);
+
+ if (token[0] == '+')
+ {
+ for (int i = 0; i < val; i++)
+ {
+ ++prev;
+ result = bms_add_member(result, prev);
+ }
+ }
+ else
+ {
+ result = bms_add_member(result, val);
+ prev = val;
+ }
}
return result;
--
2.40.1
[application/octet-stream] v8-0008-nodeToString-omit-serializing-0s-in-enum-typed-fi.patch (2.1K, ../../CAEze2Wh0oC97R0fG7s17b=YHOTittL8mHwC6TyT-Q2_NtY7RVw@mail.gmail.com/8-v8-0008-nodeToString-omit-serializing-0s-in-enum-typed-fi.patch)
download | inline diff:
From 2bf517aac3b9579ada20a95c5e698462d72ef07f Mon Sep 17 00:00:00 2001
From: Matthias van de Meent <[email protected]>
Date: Mon, 19 Feb 2024 13:53:00 +0100
Subject: [PATCH v8 8/8] nodeToString: omit serializing 0s in enum-typed
fields.
---
src/backend/nodes/outfuncs.c | 5 +----
src/backend/nodes/readfuncs.c | 16 +++++++++++-----
2 files changed, 12 insertions(+), 9 deletions(-)
diff --git a/src/backend/nodes/outfuncs.c b/src/backend/nodes/outfuncs.c
index 986620f7f3..322d0a5f10 100644
--- a/src/backend/nodes/outfuncs.c
+++ b/src/backend/nodes/outfuncs.c
@@ -113,10 +113,7 @@ static bool output_parse_locations = true;
(int) node->fldname); \
} while (0)
#define WRITE_ENUM_FIELD(fldname, enumtype) \
- do { \
- appendStringInfo(str, " :" CppAsString(fldname) " %d", \
- (int) node->fldname); \
- } while (0)
+ WRITE_ENUM_FIELD_DEFAULT(fldname, enumtype, 0)
/* Write a float field (actually, they're double) */
#define WRITE_FLOAT_FIELD_DEFAULT(fldname, default) \
diff --git a/src/backend/nodes/readfuncs.c b/src/backend/nodes/readfuncs.c
index 7c000371f0..d5ddafe968 100644
--- a/src/backend/nodes/readfuncs.c
+++ b/src/backend/nodes/readfuncs.c
@@ -142,13 +142,19 @@
READ_CHAR_FIELD_DEFAULT(fldname, '\0')
/* Read an enumerated-type field that was written as an integer code */
-#define READ_ENUM_FIELD(fldname, enumtype) \
+
+#define READ_ENUM_FIELD_DEFAULT(fldname, enumtype, default) \
do { \
- token = pg_strtok_fieldname(":" CppAsString(fldname), &length); \
- Assert(token != NULL); \
- token = pg_strtok(&length); /* get field value */ \
- local_node->fldname = (enumtype) atoi(token); \
+ if ((token = pg_strtok_fieldname(":" CppAsString(fldname), &length))) \
+ { \
+ token = pg_strtok(&length); /* get field value */ \
+ local_node->fldname = (enumtype) atoi(token); \
+ } \
+ else \
+ local_node->fldname = (enumtype) default; \
} while (0)
+#define READ_ENUM_FIELD(fldname, enumtype) \
+ READ_ENUM_FIELD_DEFAULT(fldname, enumtype, 0)
/* Read a float field */
#define READ_FLOAT_FIELD_DEFAULT(fldname, default_value) \
--
2.40.1
[application/octet-stream] v8-0007-gen_node_support.pl-Optimize-serialization-of-fie.patch (9.2K, ../../CAEze2Wh0oC97R0fG7s17b=YHOTittL8mHwC6TyT-Q2_NtY7RVw@mail.gmail.com/9-v8-0007-gen_node_support.pl-Optimize-serialization-of-fie.patch)
download | inline diff:
From c2ab602d6f2dc245f1504a6dcf592f3e42751f5b Mon Sep 17 00:00:00 2001
From: Matthias van de Meent <[email protected]>
Date: Mon, 12 Feb 2024 17:12:02 +0100
Subject: [PATCH v8 7/8] gen_node_support.pl: Optimize serialization of fields
with copied values
The Var node's [syn] fields often contain the same data as their non-syn
counterparts. We invent a new pg_node_attr()ibute that represents this
relation, which allows us to omit these fields from serialization when they
are indeed copies of the original fields.
---
src/backend/nodes/gen_node_support.pl | 94 ++++++++++++++++++---------
src/backend/nodes/outfuncs.c | 3 +
src/backend/nodes/readfuncs.c | 3 +
src/include/nodes/primnodes.h | 4 +-
4 files changed, 70 insertions(+), 34 deletions(-)
diff --git a/src/backend/nodes/gen_node_support.pl b/src/backend/nodes/gen_node_support.pl
index 13a3979469..62cb5d671b 100644
--- a/src/backend/nodes/gen_node_support.pl
+++ b/src/backend/nodes/gen_node_support.pl
@@ -363,6 +363,10 @@ foreach my $infile (@ARGV)
{
$manual_nodetag_number{$in_struct} = $1;
}
+ elsif ($attr =~ /^default_ref\(([\w\d."'-]+)\)$/)
+ {
+ # Unused at the node level
+ }
else
{
die
@@ -437,7 +441,7 @@ foreach my $infile (@ARGV)
}
# normal struct field
elsif ($line =~
- /^\s*(.+)\s*\b(\w+)(\[[\w\s+]+\])?\s*(?:pg_node_attr\(([\w(), ]*)\))?;/
+ /^\s*(.+)\s*\b(\w+)(\[[\w\s+]+\])?\s*(?:pg_node_attr\(([\w\d(), ."'-]*)\))?;/
)
{
if ($is_node_struct)
@@ -469,6 +473,7 @@ foreach my $infile (@ARGV)
if ( $attr !~ /^array_size\(\w+\)$/
&& $attr !~ /^copy_as\(\w+\)$/
&& $attr !~ /^read_as\(\w+\)$/
+ && $attr !~ /^default_ref\([\w\d."'-]+\)$/
&& !elem $attr,
qw(copy_as_scalar
equal_as_scalar
@@ -495,7 +500,7 @@ foreach my $infile (@ARGV)
}
# function pointer field
elsif ($line =~
- /^\s*([\w\s*]+)\s*\(\*(\w+)\)\s*\((.*)\)\s*(?:pg_node_attr\(([\w(), ]*)\))?;/
+ /^\s*([\w\s*]+)\s*\(\*(\w+)\)\s*\((.*)\)\s*(?:pg_node_attr\(([\w\d(), ."'-]*)\))?;/
)
{
if ($is_node_struct)
@@ -970,6 +975,15 @@ _read${n}(void)
my $array_size_field;
my $read_as_field;
my $read_write_ignore = 0;
+ # Type read/write macro suffix, "" for normal use, or "_DEFAULT" when
+ # value argument.
+ my $s = "";
+ # Default parameter to read/write macro. Includes comma separator so
+ # that MACRO_NAME$s($fieldname$defaultparam) is the full complete
+ # read/write expression for essentially all types.
+ # Note that this (currently) only works for scalar values.
+ my $d = "";
+
foreach my $a (@a)
{
if ($a =~ /^array_size\(([\w.]+)\)$/)
@@ -988,6 +1002,19 @@ _read${n}(void)
{
$read_write_ignore = 1;
}
+ elsif ($a =~ /^default_ref\(([\w\d+."'-]+)\)$/)
+ {
+ $s = "_DEFAULT";
+ $d = ", NODE_FIELD($1)";
+ }
+ }
+
+ if ($s eq "_DEFAULT")
+ {
+ die "custom defaults for non-scalar fields are not supported\n\tat $n.$f"
+ unless (elem $t, @scalar_types or elem $t, @enum_types);
+ die "custom defaults for Location fields are not supported\n\tat $n.$f"
+ if ($t eq "Location");
}
if ($read_write_ignore)
@@ -1008,8 +1035,8 @@ _read${n}(void)
# select instructions by field type
if ($t eq 'bool')
{
- print $off "\tWRITE_BOOL_FIELD($f);\n";
- print $rff "\tREAD_BOOL_FIELD($f);\n" unless $no_read;
+ print $off "\tWRITE_BOOL_FIELD$s($f$d);\n";
+ print $rff "\tREAD_BOOL_FIELD$s($f$d);\n" unless $no_read;
}
elsif ($t eq 'ParseLoc')
{
@@ -1018,8 +1045,11 @@ _read${n}(void)
}
elsif ($t eq 'TypMod')
{
- print $off "\tWRITE_INT_FIELD_DEFAULT($f, -1);\n";
- print $rff "\tREAD_INT_FIELD_DEFAULT($f, -1);\n" unless $no_read;
+ # The default value of a TypMod fields is -1, rather than 0
+ # for normal int fields.
+ $d = ", -1" if ($d eq "");
+ print $off "\tWRITE_INT_FIELD_DEFAULT($f$d);\n";
+ print $rff "\tREAD_INT_FIELD_DEFAULT($f$d);\n" unless $no_read;
}
elsif ($t eq 'int'
|| $t eq 'int16'
@@ -1027,8 +1057,8 @@ _read${n}(void)
|| $t eq 'AttrNumber'
|| $t eq 'StrategyNumber')
{
- print $off "\tWRITE_INT_FIELD($f);\n";
- print $rff "\tREAD_INT_FIELD($f);\n" unless $no_read;
+ print $off "\tWRITE_INT_FIELD$s($f$d);\n";
+ print $rff "\tREAD_INT_FIELD$s($f$d);\n" unless $no_read;
}
elsif ($t eq 'uint32'
|| $t eq 'bits32'
@@ -1036,56 +1066,56 @@ _read${n}(void)
|| $t eq 'Index'
|| $t eq 'SubTransactionId')
{
- print $off "\tWRITE_UINT_FIELD($f);\n";
- print $rff "\tREAD_UINT_FIELD($f);\n" unless $no_read;
+ print $off "\tWRITE_UINT_FIELD$s($f$d);\n";
+ print $rff "\tREAD_UINT_FIELD$s($f$d);\n" unless $no_read;
}
elsif ($t eq 'uint64'
|| $t eq 'AclMode')
{
- print $off "\tWRITE_UINT64_FIELD($f);\n";
- print $rff "\tREAD_UINT64_FIELD($f);\n" unless $no_read;
+ print $off "\tWRITE_UINT64_FIELD$s($f$d);\n";
+ print $rff "\tREAD_UINT64_FIELD$s($f$d);\n" unless $no_read;
}
elsif ($t eq 'Oid' || $t eq 'RelFileNumber')
{
- print $off "\tWRITE_OID_FIELD($f);\n";
- print $rff "\tREAD_OID_FIELD($f);\n" unless $no_read;
+ print $off "\tWRITE_OID_FIELD$s($f$d);\n";
+ print $rff "\tREAD_OID_FIELD$s($f$d);\n" unless $no_read;
}
elsif ($t eq 'long')
{
- print $off "\tWRITE_LONG_FIELD($f);\n";
- print $rff "\tREAD_LONG_FIELD($f);\n" unless $no_read;
+ print $off "\tWRITE_LONG_FIELD$s($f$d);\n";
+ print $rff "\tREAD_LONG_FIELD$s($f$d);\n" unless $no_read;
}
elsif ($t eq 'char')
{
- print $off "\tWRITE_CHAR_FIELD($f);\n";
- print $rff "\tREAD_CHAR_FIELD($f);\n" unless $no_read;
+ print $off "\tWRITE_CHAR_FIELD$s($f$d);\n";
+ print $rff "\tREAD_CHAR_FIELD$s($f$d);\n" unless $no_read;
}
elsif ($t eq 'double')
{
- print $off "\tWRITE_FLOAT_FIELD($f);\n";
- print $rff "\tREAD_FLOAT_FIELD($f);\n" unless $no_read;
+ print $off "\tWRITE_FLOAT_FIELD$s($f$d);\n";
+ print $rff "\tREAD_FLOAT_FIELD$s($f$d);\n" unless $no_read;
}
elsif ($t eq 'Cardinality')
{
- print $off "\tWRITE_FLOAT_FIELD($f);\n";
- print $rff "\tREAD_FLOAT_FIELD($f);\n" unless $no_read;
+ print $off "\tWRITE_FLOAT_FIELD$s($f$d);\n";
+ print $rff "\tREAD_FLOAT_FIELD$s($f$d);\n" unless $no_read;
}
elsif ($t eq 'Cost')
{
- print $off "\tWRITE_FLOAT_FIELD($f);\n";
- print $rff "\tREAD_FLOAT_FIELD($f);\n" unless $no_read;
+ print $off "\tWRITE_FLOAT_FIELD$s($f$d);\n";
+ print $rff "\tREAD_FLOAT_FIELD$s($f$d);\n" unless $no_read;
}
elsif ($t eq 'QualCost')
{
- print $off "\tWRITE_FLOAT_FIELD($f.startup);\n";
- print $off "\tWRITE_FLOAT_FIELD($f.per_tuple);\n";
- print $rff "\tREAD_FLOAT_FIELD($f.startup);\n" unless $no_read;
- print $rff "\tREAD_FLOAT_FIELD($f.per_tuple);\n" unless $no_read;
+ print $off "\tWRITE_FLOAT_FIELD$s($f.startup$d);\n";
+ print $off "\tWRITE_FLOAT_FIELD$s($f.per_tuple$d);\n";
+ print $rff "\tREAD_FLOAT_FIELD$s($f.startup$d);\n" unless $no_read;
+ print $rff "\tREAD_FLOAT_FIELD$s($f.per_tuple$d);\n" unless $no_read;
}
elsif ($t eq 'Selectivity')
{
- print $off "\tWRITE_FLOAT_FIELD($f);\n";
- print $rff "\tREAD_FLOAT_FIELD($f);\n" unless $no_read;
+ print $off "\tWRITE_FLOAT_FIELD$s($f$d);\n";
+ print $rff "\tREAD_FLOAT_FIELD$s($f$d);\n" unless $no_read;
}
elsif ($t eq 'char*')
{
@@ -1099,8 +1129,8 @@ _read${n}(void)
}
elsif (elem $t, @enum_types)
{
- print $off "\tWRITE_ENUM_FIELD($f, $t);\n";
- print $rff "\tREAD_ENUM_FIELD($f, $t);\n" unless $no_read;
+ print $off "\tWRITE_ENUM_FIELD$s($f, $t$d);\n";
+ print $rff "\tREAD_ENUM_FIELD$s($f, $t$d);\n" unless $no_read;
}
# arrays of scalar types
elsif ($t =~ /^(\w+)(\*|\[\w+\])$/ and elem $1, @scalar_types)
diff --git a/src/backend/nodes/outfuncs.c b/src/backend/nodes/outfuncs.c
index 90a7d73d3b..986620f7f3 100644
--- a/src/backend/nodes/outfuncs.c
+++ b/src/backend/nodes/outfuncs.c
@@ -245,6 +245,9 @@ static bool output_parse_locations = true;
} \
} while (0)
+/* Accessing a node's field goes like this */
+#define NODE_FIELD(fldname) (node->fldname)
+
#define booltostr(x) ((x) ? "true" : "false")
diff --git a/src/backend/nodes/readfuncs.c b/src/backend/nodes/readfuncs.c
index 970b4987bb..7c000371f0 100644
--- a/src/backend/nodes/readfuncs.c
+++ b/src/backend/nodes/readfuncs.c
@@ -270,6 +270,9 @@
local_node->fldname = NULL; \
} while (0)
+/* Accessing a node's field goes like this */
+#define NODE_FIELD(fldname) (local_node->fldname)
+
/* Routine exit */
#define READ_DONE() \
return local_node
diff --git a/src/include/nodes/primnodes.h b/src/include/nodes/primnodes.h
index 5fd4d54475..970faa002e 100644
--- a/src/include/nodes/primnodes.h
+++ b/src/include/nodes/primnodes.h
@@ -274,9 +274,9 @@ typedef struct Var
* their varno/varattno match.
*/
/* syntactic relation index (0 if unknown) */
- Index varnosyn pg_node_attr(equal_ignore, query_jumble_ignore);
+ Index varnosyn pg_node_attr(equal_ignore, query_jumble_ignore, default_ref(varno));
/* syntactic attribute number */
- AttrNumber varattnosyn pg_node_attr(equal_ignore, query_jumble_ignore);
+ AttrNumber varattnosyn pg_node_attr(equal_ignore, query_jumble_ignore, default_ref(varattno));
/* token location, or -1 if unknown */
ParseLoc location;
--
2.40.1
^ permalink raw reply [nested|flat] 69+ messages in thread
* Re: Reducing output size of nodeToString
@ 2024-03-19 16:13 Peter Eisentraut <[email protected]>
parent: Matthias van de Meent <[email protected]>
0 siblings, 2 replies; 69+ messages in thread
From: Peter Eisentraut @ 2024-03-19 16:13 UTC (permalink / raw)
To: Matthias van de Meent <[email protected]>; +Cc: Robert Haas <[email protected]>; PostgreSQL Hackers <[email protected]>; Andres Freund <[email protected]>; Michel Pelletier <[email protected]>
On 11.03.24 21:52, Matthias van de Meent wrote:
>> * v7-0003-gen_node_support.pl-Mark-location-fields-as-type-.patch
>>
>> This looks sensible, but maybe making Location a global type is a bit
>> much? Maybe something more specific like ParseLocation, or ParseLoc, to
>> keep it under 12 characters.
> I've gone with ParseLoc in the attached v8 patchset.
I have committed this one.
I moved the typedef to nodes/nodes.h, where we already had similar
typdefs (Cardinality, etc.). The fields stmt_location and stmt_len in
PlannedStmt were not converted, so I fixed that. Also, between you
writing your patch and now, at least one new node type was added, so I
fixed that one up, too. (I diffed the generated node support functions
to check.) Hopefully, future hackers will apply the new type when
appropriate.
^ permalink raw reply [nested|flat] 69+ messages in thread
* Re: Reducing output size of nodeToString
@ 2024-03-19 16:46 Matthias van de Meent <[email protected]>
parent: Peter Eisentraut <[email protected]>
1 sibling, 0 replies; 69+ messages in thread
From: Matthias van de Meent @ 2024-03-19 16:46 UTC (permalink / raw)
To: Peter Eisentraut <[email protected]>; +Cc: Robert Haas <[email protected]>; PostgreSQL Hackers <[email protected]>; Andres Freund <[email protected]>; Michel Pelletier <[email protected]>
On Tue, 19 Mar 2024 at 17:13, Peter Eisentraut <[email protected]> wrote:
>
> On 11.03.24 21:52, Matthias van de Meent wrote:
> >> * v7-0003-gen_node_support.pl-Mark-location-fields-as-type-.patch
> >>
> >> This looks sensible, but maybe making Location a global type is a bit
> >> much? Maybe something more specific like ParseLocation, or ParseLoc, to
> >> keep it under 12 characters.
> > I've gone with ParseLoc in the attached v8 patchset.
>
> I have committed this one.
Thanks!
> I moved the typedef to nodes/nodes.h, where we already had similar
> typdefs (Cardinality, etc.). The fields stmt_location and stmt_len in
> PlannedStmt were not converted, so I fixed that. Also, between you
> writing your patch and now, at least one new node type was added, so I
> fixed that one up, too.
Good points, thank you for fixing that.
> (I diffed the generated node support functions
> to check.) Hopefully, future hackers will apply the new type when
> appropriate.
Are you also planning on committing some of the other patches later,
or should I rebase the set to keep CFBot happy?
-Matthias
^ permalink raw reply [nested|flat] 69+ messages in thread
* Re: Reducing output size of nodeToString
@ 2024-03-20 11:49 Peter Eisentraut <[email protected]>
parent: Peter Eisentraut <[email protected]>
1 sibling, 1 reply; 69+ messages in thread
From: Peter Eisentraut @ 2024-03-20 11:49 UTC (permalink / raw)
To: Matthias van de Meent <[email protected]>; +Cc: Robert Haas <[email protected]>; PostgreSQL Hackers <[email protected]>; Andres Freund <[email protected]>; Michel Pelletier <[email protected]>
On 19.03.24 17:13, Peter Eisentraut wrote:
> On 11.03.24 21:52, Matthias van de Meent wrote:
>>> * v7-0003-gen_node_support.pl-Mark-location-fields-as-type-.patch
>>>
>>> This looks sensible, but maybe making Location a global type is a bit
>>> much? Maybe something more specific like ParseLocation, or ParseLoc, to
>>> keep it under 12 characters.
>> I've gone with ParseLoc in the attached v8 patchset.
>
> I have committed this one.
Next, I was looking at
v8-0003-pg_node_tree-Don-t-store-query-text-locations-in-.patch. After
applying that, I was looking how many uses of nodeToString() (with
locations) were left. I think your patch forgot to convert a number of
them, and there also might have been a few new ones that came in with
other recent patches. Might be hard to make sure all new developments
do this right. Plus, there are various mentions in the documentation
that should be updated. After considering all that, there weren't
really many callers of nodeToString() left. It's really only debugging
support in postgres.c and print.c, and a few places were it doesn't
matter, like the few places where it initializes "cooked expressions",
which were in turn already stripped of location fields at some earlier time.
So anyway, my idea was that we should turn this around and make
nodeToString() always drop location information, and instead add
nodeToStringWithLocations() for the few debugging uses. And this would
also be nice because then it matches exactly with the existing
stringToNodeWithLocations().
Attached patch shows this.
From 009c7359a5f0bf8c1ea9d685ec92f73a8dc4a345 Mon Sep 17 00:00:00 2001
From: Peter Eisentraut <[email protected]>
Date: Wed, 20 Mar 2024 12:33:35 +0100
Subject: [PATCH] Do not output actual value of location fields in node
serialization by default
This changes nodeToString() to not output the actual value of location
fields in nodes, but instead it writes -1. This mirrors the fact that
stringToNode() also does not read location field values but always
stores -1.
For most uses of nodeToString(), which is to store nodes in catalog
fields, this is more useful. We don't store original query texts in
catalogs, so any lingering query location values are not meaningful.
For debugging purposes, there is a new nodeToStringWithLocations(),
which mirrors the existing stringToNodeWithLocations(). This is used
for WRITE_READ_PARSE_PLAN_TREES and nodes/print.c functions, which
covers all the debugging uses.
---
src/backend/nodes/outfuncs.c | 37 +++++++++++++++++++++++++++++++++---
src/backend/nodes/print.c | 6 +++---
src/backend/tcop/postgres.c | 6 +++---
src/include/nodes/nodes.h | 1 +
4 files changed, 41 insertions(+), 9 deletions(-)
diff --git a/src/backend/nodes/outfuncs.c b/src/backend/nodes/outfuncs.c
index 29cbc83bd9f..b25ca952b45 100644
--- a/src/backend/nodes/outfuncs.c
+++ b/src/backend/nodes/outfuncs.c
@@ -25,6 +25,9 @@
#include "nodes/pg_list.h"
#include "utils/datum.h"
+/* State flag that determines how nodeToStringInternal() should treat location fields */
+static bool write_location_fields = false;
+
static void outChar(StringInfo str, char c);
static void outDouble(StringInfo str, double d);
@@ -88,7 +91,7 @@ static void outDouble(StringInfo str, double d);
/* Write a parse location field (actually same as INT case) */
#define WRITE_LOCATION_FIELD(fldname) \
- appendStringInfo(str, " :" CppAsString(fldname) " %d", node->fldname)
+ appendStringInfo(str, " :" CppAsString(fldname) " %d", write_location_fields ? node->fldname : -1)
/* Write a Node field */
#define WRITE_NODE_FIELD(fldname) \
@@ -758,18 +761,46 @@ outNode(StringInfo str, const void *obj)
/*
* nodeToString -
* returns the ascii representation of the Node as a palloc'd string
+ *
+ * write_loc_fields determines whether location fields are output with their
+ * actual value rather than -1. The actual value can be useful for debugging,
+ * but for most uses, the actual value is not useful, since the original query
+ * string is no longer available.
*/
-char *
-nodeToString(const void *obj)
+static char *
+nodeToStringInternal(const void *obj, bool write_loc_fields)
{
StringInfoData str;
+ bool save_write_location_fields;
+
+ save_write_location_fields = write_location_fields;
+ write_location_fields = write_loc_fields;
/* see stringinfo.h for an explanation of this maneuver */
initStringInfo(&str);
outNode(&str, obj);
+
+ write_location_fields = save_write_location_fields;
+
return str.data;
}
+/*
+ * Externally visible entry points
+ */
+char *
+nodeToString(const void *obj)
+{
+ return nodeToStringInternal(obj, false);
+}
+
+char *
+nodeToStringWithLocations(const void *obj)
+{
+ return nodeToStringInternal(obj, true);
+}
+
+
/*
* bmsToString -
* returns the ascii representation of the Bitmapset as a palloc'd string
diff --git a/src/backend/nodes/print.c b/src/backend/nodes/print.c
index d2a58a5956a..02798f4482d 100644
--- a/src/backend/nodes/print.c
+++ b/src/backend/nodes/print.c
@@ -38,7 +38,7 @@ print(const void *obj)
char *s;
char *f;
- s = nodeToString(obj);
+ s = nodeToStringWithLocations(obj);
f = format_node_dump(s);
pfree(s);
printf("%s\n", f);
@@ -56,7 +56,7 @@ pprint(const void *obj)
char *s;
char *f;
- s = nodeToString(obj);
+ s = nodeToStringWithLocations(obj);
f = pretty_format_node_dump(s);
pfree(s);
printf("%s\n", f);
@@ -74,7 +74,7 @@ elog_node_display(int lev, const char *title, const void *obj, bool pretty)
char *s;
char *f;
- s = nodeToString(obj);
+ s = nodeToStringWithLocations(obj);
if (pretty)
f = pretty_format_node_dump(s);
else
diff --git a/src/backend/tcop/postgres.c b/src/backend/tcop/postgres.c
index aec1b194424..22577390c44 100644
--- a/src/backend/tcop/postgres.c
+++ b/src/backend/tcop/postgres.c
@@ -640,7 +640,7 @@ pg_parse_query(const char *query_string)
*/
#ifdef WRITE_READ_PARSE_PLAN_TREES
{
- char *str = nodeToString(raw_parsetree_list);
+ char *str = nodeToStringWithLocations(raw_parsetree_list);
List *new_list = stringToNodeWithLocations(str);
pfree(str);
@@ -848,7 +848,7 @@ pg_rewrite_query(Query *query)
foreach(lc, querytree_list)
{
Query *curr_query = lfirst_node(Query, lc);
- char *str = nodeToString(curr_query);
+ char *str = nodeToStringWithLocations(curr_query);
Query *new_query = stringToNodeWithLocations(str);
/*
@@ -930,7 +930,7 @@ pg_plan_query(Query *querytree, const char *query_string, int cursorOptions,
char *str;
PlannedStmt *new_plan;
- str = nodeToString(plan);
+ str = nodeToStringWithLocations(plan);
new_plan = stringToNodeWithLocations(str);
pfree(str);
diff --git a/src/include/nodes/nodes.h b/src/include/nodes/nodes.h
index 2969dd831b2..c52788abe3e 100644
--- a/src/include/nodes/nodes.h
+++ b/src/include/nodes/nodes.h
@@ -195,6 +195,7 @@ extern void outBitmapset(struct StringInfoData *str,
extern void outDatum(struct StringInfoData *str, uintptr_t value,
int typlen, bool typbyval);
extern char *nodeToString(const void *obj);
+extern char *nodeToStringWithLocations(const void *obj);
extern char *bmsToString(const struct Bitmapset *bms);
/*
--
2.44.0
Attachments:
[text/plain] 0001-Do-not-output-actual-value-of-location-fields-in-nod.patch (5.6K, ../../[email protected]/2-0001-Do-not-output-actual-value-of-location-fields-in-nod.patch)
download | inline diff:
From 009c7359a5f0bf8c1ea9d685ec92f73a8dc4a345 Mon Sep 17 00:00:00 2001
From: Peter Eisentraut <[email protected]>
Date: Wed, 20 Mar 2024 12:33:35 +0100
Subject: [PATCH] Do not output actual value of location fields in node
serialization by default
This changes nodeToString() to not output the actual value of location
fields in nodes, but instead it writes -1. This mirrors the fact that
stringToNode() also does not read location field values but always
stores -1.
For most uses of nodeToString(), which is to store nodes in catalog
fields, this is more useful. We don't store original query texts in
catalogs, so any lingering query location values are not meaningful.
For debugging purposes, there is a new nodeToStringWithLocations(),
which mirrors the existing stringToNodeWithLocations(). This is used
for WRITE_READ_PARSE_PLAN_TREES and nodes/print.c functions, which
covers all the debugging uses.
---
src/backend/nodes/outfuncs.c | 37 +++++++++++++++++++++++++++++++++---
src/backend/nodes/print.c | 6 +++---
src/backend/tcop/postgres.c | 6 +++---
src/include/nodes/nodes.h | 1 +
4 files changed, 41 insertions(+), 9 deletions(-)
diff --git a/src/backend/nodes/outfuncs.c b/src/backend/nodes/outfuncs.c
index 29cbc83bd9f..b25ca952b45 100644
--- a/src/backend/nodes/outfuncs.c
+++ b/src/backend/nodes/outfuncs.c
@@ -25,6 +25,9 @@
#include "nodes/pg_list.h"
#include "utils/datum.h"
+/* State flag that determines how nodeToStringInternal() should treat location fields */
+static bool write_location_fields = false;
+
static void outChar(StringInfo str, char c);
static void outDouble(StringInfo str, double d);
@@ -88,7 +91,7 @@ static void outDouble(StringInfo str, double d);
/* Write a parse location field (actually same as INT case) */
#define WRITE_LOCATION_FIELD(fldname) \
- appendStringInfo(str, " :" CppAsString(fldname) " %d", node->fldname)
+ appendStringInfo(str, " :" CppAsString(fldname) " %d", write_location_fields ? node->fldname : -1)
/* Write a Node field */
#define WRITE_NODE_FIELD(fldname) \
@@ -758,18 +761,46 @@ outNode(StringInfo str, const void *obj)
/*
* nodeToString -
* returns the ascii representation of the Node as a palloc'd string
+ *
+ * write_loc_fields determines whether location fields are output with their
+ * actual value rather than -1. The actual value can be useful for debugging,
+ * but for most uses, the actual value is not useful, since the original query
+ * string is no longer available.
*/
-char *
-nodeToString(const void *obj)
+static char *
+nodeToStringInternal(const void *obj, bool write_loc_fields)
{
StringInfoData str;
+ bool save_write_location_fields;
+
+ save_write_location_fields = write_location_fields;
+ write_location_fields = write_loc_fields;
/* see stringinfo.h for an explanation of this maneuver */
initStringInfo(&str);
outNode(&str, obj);
+
+ write_location_fields = save_write_location_fields;
+
return str.data;
}
+/*
+ * Externally visible entry points
+ */
+char *
+nodeToString(const void *obj)
+{
+ return nodeToStringInternal(obj, false);
+}
+
+char *
+nodeToStringWithLocations(const void *obj)
+{
+ return nodeToStringInternal(obj, true);
+}
+
+
/*
* bmsToString -
* returns the ascii representation of the Bitmapset as a palloc'd string
diff --git a/src/backend/nodes/print.c b/src/backend/nodes/print.c
index d2a58a5956a..02798f4482d 100644
--- a/src/backend/nodes/print.c
+++ b/src/backend/nodes/print.c
@@ -38,7 +38,7 @@ print(const void *obj)
char *s;
char *f;
- s = nodeToString(obj);
+ s = nodeToStringWithLocations(obj);
f = format_node_dump(s);
pfree(s);
printf("%s\n", f);
@@ -56,7 +56,7 @@ pprint(const void *obj)
char *s;
char *f;
- s = nodeToString(obj);
+ s = nodeToStringWithLocations(obj);
f = pretty_format_node_dump(s);
pfree(s);
printf("%s\n", f);
@@ -74,7 +74,7 @@ elog_node_display(int lev, const char *title, const void *obj, bool pretty)
char *s;
char *f;
- s = nodeToString(obj);
+ s = nodeToStringWithLocations(obj);
if (pretty)
f = pretty_format_node_dump(s);
else
diff --git a/src/backend/tcop/postgres.c b/src/backend/tcop/postgres.c
index aec1b194424..22577390c44 100644
--- a/src/backend/tcop/postgres.c
+++ b/src/backend/tcop/postgres.c
@@ -640,7 +640,7 @@ pg_parse_query(const char *query_string)
*/
#ifdef WRITE_READ_PARSE_PLAN_TREES
{
- char *str = nodeToString(raw_parsetree_list);
+ char *str = nodeToStringWithLocations(raw_parsetree_list);
List *new_list = stringToNodeWithLocations(str);
pfree(str);
@@ -848,7 +848,7 @@ pg_rewrite_query(Query *query)
foreach(lc, querytree_list)
{
Query *curr_query = lfirst_node(Query, lc);
- char *str = nodeToString(curr_query);
+ char *str = nodeToStringWithLocations(curr_query);
Query *new_query = stringToNodeWithLocations(str);
/*
@@ -930,7 +930,7 @@ pg_plan_query(Query *querytree, const char *query_string, int cursorOptions,
char *str;
PlannedStmt *new_plan;
- str = nodeToString(plan);
+ str = nodeToStringWithLocations(plan);
new_plan = stringToNodeWithLocations(str);
pfree(str);
diff --git a/src/include/nodes/nodes.h b/src/include/nodes/nodes.h
index 2969dd831b2..c52788abe3e 100644
--- a/src/include/nodes/nodes.h
+++ b/src/include/nodes/nodes.h
@@ -195,6 +195,7 @@ extern void outBitmapset(struct StringInfoData *str,
extern void outDatum(struct StringInfoData *str, uintptr_t value,
int typlen, bool typbyval);
extern char *nodeToString(const void *obj);
+extern char *nodeToStringWithLocations(const void *obj);
extern char *bmsToString(const struct Bitmapset *bms);
/*
--
2.44.0
^ permalink raw reply [nested|flat] 69+ messages in thread
* Re: Reducing output size of nodeToString
@ 2024-03-20 12:03 Matthias van de Meent <[email protected]>
parent: Peter Eisentraut <[email protected]>
0 siblings, 1 reply; 69+ messages in thread
From: Matthias van de Meent @ 2024-03-20 12:03 UTC (permalink / raw)
To: Peter Eisentraut <[email protected]>; +Cc: Robert Haas <[email protected]>; PostgreSQL Hackers <[email protected]>; Andres Freund <[email protected]>; Michel Pelletier <[email protected]>
On Wed, 20 Mar 2024 at 12:49, Peter Eisentraut <[email protected]> wrote:
>
> On 19.03.24 17:13, Peter Eisentraut wrote:
> > On 11.03.24 21:52, Matthias van de Meent wrote:
> >>> * v7-0003-gen_node_support.pl-Mark-location-fields-as-type-.patch
> >>>
> >>> This looks sensible, but maybe making Location a global type is a bit
> >>> much? Maybe something more specific like ParseLocation, or ParseLoc, to
> >>> keep it under 12 characters.
> >> I've gone with ParseLoc in the attached v8 patchset.
> >
> > I have committed this one.
>
> Next, I was looking at
> v8-0003-pg_node_tree-Don-t-store-query-text-locations-in-.patch.
[...]
> So anyway, my idea was that we should turn this around and make
> nodeToString() always drop location information, and instead add
> nodeToStringWithLocations() for the few debugging uses. And this would
> also be nice because then it matches exactly with the existing
> stringToNodeWithLocations().
That seems reasonable, yes.
-Matthias
^ permalink raw reply [nested|flat] 69+ messages in thread
* Re: Reducing output size of nodeToString
@ 2024-03-22 09:18 Peter Eisentraut <[email protected]>
parent: Matthias van de Meent <[email protected]>
0 siblings, 0 replies; 69+ messages in thread
From: Peter Eisentraut @ 2024-03-22 09:18 UTC (permalink / raw)
To: Matthias van de Meent <[email protected]>; +Cc: Robert Haas <[email protected]>; PostgreSQL Hackers <[email protected]>; Andres Freund <[email protected]>; Michel Pelletier <[email protected]>
On 20.03.24 13:03, Matthias van de Meent wrote:
> On Wed, 20 Mar 2024 at 12:49, Peter Eisentraut <[email protected]> wrote:
>>
>> On 19.03.24 17:13, Peter Eisentraut wrote:
>>> On 11.03.24 21:52, Matthias van de Meent wrote:
>>>>> * v7-0003-gen_node_support.pl-Mark-location-fields-as-type-.patch
>>>>>
>>>>> This looks sensible, but maybe making Location a global type is a bit
>>>>> much? Maybe something more specific like ParseLocation, or ParseLoc, to
>>>>> keep it under 12 characters.
>>>> I've gone with ParseLoc in the attached v8 patchset.
>>>
>>> I have committed this one.
>>
>> Next, I was looking at
>> v8-0003-pg_node_tree-Don-t-store-query-text-locations-in-.patch.
>
> [...]
>
>> So anyway, my idea was that we should turn this around and make
>> nodeToString() always drop location information, and instead add
>> nodeToStringWithLocations() for the few debugging uses. And this would
>> also be nice because then it matches exactly with the existing
>> stringToNodeWithLocations().
>
> That seems reasonable, yes.
I have committed that one.
This takes care of your patches v8-0002 and v8-0003.
About the rest of your patch set:
As long as we have only one output format, we need to balance several
uses, including debugging, storage size, (de)serialization performance.
Your patches v8-0005 and up are clearly positive for storage size but
negative for debugging. So I think we can't consider them now.
Your patches v8-0001 ("pg_node_tree: Omit serialization of fields with
default values.") and v8-0004 ("gen_node_support.pl: Add a TypMod type
for signalling TypMod behaviour") are also good for storage size. I
don't know how they affect serialization performance. I also don't know
how good they are for debugging. I have argued here and there that
omitting unset fields can make node dumps more readable. But that's
just me. I have looked at a lot of Query and RangeTblEntry nodes
lately, which contain many rarely used fields. But other people might
have completely different experiences, with other node and tree types.
We didn't get much feedback from anyone else in this thread, so I'm very
hesitant to impose this on everyone without any consensus.
I could see "Omit serialization of fields with default values" as a
separate toggle for debug node dumps.
Also, there is clearly some lingering interesting in a separate
binary-ish serialization format for internal use. This should probably
also take a look at (de)serialization performance, which we haven't
really done in this thread. In a way, with the omit default values
patch, the serialization and deserialization does more work, so it could
have an adverse impact. But we don't know.
I think to proceed we need more buy-in on the "what do I want from my
node dumps" side, and more performance numbers on the other side.
Saving a few hundred kilobytes on view storage is fine but isn't by
itself that useful, I think, if it potentially negatively affects other
uses.
^ permalink raw reply [nested|flat] 69+ messages in thread
* Small fixes for CHECKPOINT FLUSH_UNLOGGED
@ 2026-05-25 07:07 Chao Li <[email protected]>
0 siblings, 1 reply; 69+ messages in thread
From: Chao Li @ 2026-05-25 07:07 UTC (permalink / raw)
To: PostgreSQL Hackers <[email protected]>; +Cc: Nathan Bossart <[email protected]>; Christoph Berg <[email protected]>
Hi,
I just tested “Add FLUSH_UNLOGGED option to CHECKPOINT command”, and found no functional issue. I only noticed a couple of small issues.
1. Tab-completion for FLUSH_UNLOGGED misses its boolean value
FLUSH_UNLOGGED can take a boolean value, but tab completion does not suggest ON/OFF for it. For comparison, REPACK does:
```
evantest=# checkpoint (FLUSH_UNLOGGED o
evantest=# repack (concurrently O
OFF ON
```The fix just follows the existing code for REPACK. After the fix:
```
evantest=# checkpoint (flush_unlogged O
OFF ON
```
2. The CHECKPOINT doc still says “All data files will be flushed to disk”
After introducing FLUSH_UNLOGGED, CHECKPOINT by default no longer flushes dirty buffers of unlogged relations, but checkpoint.sgml still says “All data files will be flushed to disk.” That seems stale, so I updated the doc.
These are two small changes, but to make the patch easier to process, I split them into two commits.
Best reagards,
--
Chao Li (Evan)
HighGo Software Co., Ltd.
https://www.highgo.com/
Attachments:
[application/octet-stream] v1-0002-doc-Clarify-CHECKPOINT-handling-of-unlogged-buffe.patch (1.6K, ../../[email protected]/2-v1-0002-doc-Clarify-CHECKPOINT-handling-of-unlogged-buffe.patch)
download | inline diff:
From 8e18462c160a45ebf0cec535b6ab48c0fed942c4 Mon Sep 17 00:00:00 2001
From: "Chao Li (Evan)" <[email protected]>
Date: Mon, 25 May 2026 14:30:41 +0800
Subject: [PATCH v1 2/2] doc: Clarify CHECKPOINT handling of unlogged buffers
The CHECKPOINT reference page still described checkpoints as flushing all
data files, which became misleading after adding the FLUSH_UNLOGGED
option. By default, manual checkpoints flush dirty buffers for permanent
relations but do not flush dirty buffers of unlogged relations.
Update the description to mention that exception and point readers to
FLUSH_UNLOGGED when they want unlogged buffers flushed too.
Author: Chao Li <[email protected]>
---
doc/src/sgml/ref/checkpoint.sgml | 6 ++++--
1 file changed, 4 insertions(+), 2 deletions(-)
diff --git a/doc/src/sgml/ref/checkpoint.sgml b/doc/src/sgml/ref/checkpoint.sgml
index cd981cf2cab..a43a1913b41 100644
--- a/doc/src/sgml/ref/checkpoint.sgml
+++ b/doc/src/sgml/ref/checkpoint.sgml
@@ -35,8 +35,10 @@ CHECKPOINT [ ( option [, ...] ) ]
<para>
A checkpoint is a point in the write-ahead log sequence at which
- all data files have been updated to reflect the information in the
- log. All data files will be flushed to disk. Refer to
+ data files for permanent relations have been updated to reflect the
+ information in the log. Those data files will be flushed to disk.
+ Dirty buffers of unlogged relations are normally not flushed unless
+ <literal>FLUSH_UNLOGGED</literal> is specified. Refer to
<xref linkend="wal-configuration"/> for more details about what happens
during a checkpoint.
</para>
--
2.50.1 (Apple Git-155)
[application/octet-stream] v1-0001-psql-Complete-boolean-values-for-CHECKPOINT-FLUSH.patch (1.1K, ../../[email protected]/3-v1-0001-psql-Complete-boolean-values-for-CHECKPOINT-FLUSH.patch)
download | inline diff:
From afbc48bcc06d5668b8a3db3a0b3caba4a0f8a4a0 Mon Sep 17 00:00:00 2001
From: "Chao Li (Evan)" <[email protected]>
Date: Mon, 25 May 2026 14:14:26 +0800
Subject: [PATCH v1 1/2] psql: Complete boolean values for CHECKPOINT
FLUSH_UNLOGGED
Tab completion for CHECKPOINT option lists suggested FLUSH_UNLOGGED, but
once that option had been entered it did not offer the accepted boolean
values. Add ON/OFF completion, matching other parenthesized boolean
option-list completions.
Author: Chao Li <[email protected]>
---
src/bin/psql/tab-complete.in.c | 2 ++
1 file changed, 2 insertions(+)
diff --git a/src/bin/psql/tab-complete.in.c b/src/bin/psql/tab-complete.in.c
index 364546c13e8..de547a8cb37 100644
--- a/src/bin/psql/tab-complete.in.c
+++ b/src/bin/psql/tab-complete.in.c
@@ -3274,6 +3274,8 @@ match_previous_words(int pattern_id,
COMPLETE_WITH("MODE", "FLUSH_UNLOGGED");
else if (TailMatches("MODE"))
COMPLETE_WITH("FAST", "SPREAD");
+ else if (TailMatches("FLUSH_UNLOGGED"))
+ COMPLETE_WITH("ON", "OFF");
}
/* CLOSE */
else if (Matches("CLOSE"))
--
2.50.1 (Apple Git-155)
^ permalink raw reply [nested|flat] 69+ messages in thread
* Re: Small fixes for CHECKPOINT FLUSH_UNLOGGED
@ 2026-05-25 07:57 Daniel Gustafsson <[email protected]>
parent: Chao Li <[email protected]>
0 siblings, 0 replies; 69+ messages in thread
From: Daniel Gustafsson @ 2026-05-25 07:57 UTC (permalink / raw)
To: Chao Li <[email protected]>; +Cc: PostgreSQL Hackers <[email protected]>; Nathan Bossart <[email protected]>; Christoph Berg <[email protected]>
> On 25 May 2026, at 09:07, Chao Li <[email protected]> wrote:
> These are two small changes, but to make the patch easier to process, I split them into two commits.
Thanks for testing and fixing. Both LGTM, will test a little and then apply
(perhaps with some editorializing on the docs patch).
--
Daniel Gustafsson
^ permalink raw reply [nested|flat] 69+ messages in thread
end of thread, other threads:[~2026-05-25 07:57 UTC | newest]
Thread overview: 69+ messages (download: mbox mbox.gz follow: Atom feed)
-- links below jump to the message on this page --
2019-02-01 00:45 [PATCH 1/2] Add start timestamp to PGPROC Kyotaro Horiguchi <[email protected]>
2019-09-05 11:21 [PATCH v15 1/4] Move callback-call from ReadPageInternal to XLogReadRecord. Kyotaro Horiguchi <[email protected]>
2019-09-05 11:21 [PATCH v17 1/4] Move callback-call from ReadPageInternal to XLogReadRecord. Kyotaro Horiguchi <[email protected]>
2019-09-05 11:21 [PATCH v13 1/4] Move callback-call from ReadPageInternal to XLogReadRecord. Kyotaro Horiguchi <[email protected]>
2019-09-05 11:21 [PATCH v17 1/4] Move callback-call from ReadPageInternal to XLogReadRecord. Kyotaro Horiguchi <[email protected]>
2019-09-05 11:21 [PATCH v17 1/4] Move callback-call from ReadPageInternal to XLogReadRecord. Kyotaro Horiguchi <[email protected]>
2019-09-05 11:21 [PATCH v16 1/4] Move callback-call from ReadPageInternal to XLogReadRecord. Kyotaro Horiguchi <[email protected]>
2019-09-05 11:21 [PATCH v15 1/4] Move callback-call from ReadPageInternal to XLogReadRecord. Kyotaro Horiguchi <[email protected]>
2019-09-05 11:21 [PATCH v15 1/4] Move callback-call from ReadPageInternal to XLogReadRecord. Kyotaro Horiguchi <[email protected]>
2019-09-05 11:21 [PATCH v15 1/4] Move callback-call from ReadPageInternal to XLogReadRecord. Kyotaro Horiguchi <[email protected]>
2019-09-05 11:21 [PATCH v17 1/4] Move callback-call from ReadPageInternal to XLogReadRecord. Kyotaro Horiguchi <[email protected]>
2019-09-05 11:21 [PATCH v15 1/4] Move callback-call from ReadPageInternal to XLogReadRecord. Kyotaro Horiguchi <[email protected]>
2019-09-05 11:21 [PATCH v15 1/4] Move callback-call from ReadPageInternal to XLogReadRecord. Kyotaro Horiguchi <[email protected]>
2019-09-05 11:21 [PATCH v15 1/4] Move callback-call from ReadPageInternal to XLogReadRecord. Kyotaro Horiguchi <[email protected]>
2019-09-05 11:21 [PATCH v15 1/4] Move callback-call from ReadPageInternal to XLogReadRecord. Kyotaro Horiguchi <[email protected]>
2019-09-05 11:21 [PATCH v17 1/4] Move callback-call from ReadPageInternal to XLogReadRecord. Kyotaro Horiguchi <[email protected]>
2019-09-05 11:21 [PATCH v15 1/4] Move callback-call from ReadPageInternal to XLogReadRecord. Kyotaro Horiguchi <[email protected]>
2019-09-05 11:21 [PATCH v17 1/4] Move callback-call from ReadPageInternal to XLogReadRecord. Kyotaro Horiguchi <[email protected]>
2019-09-05 11:21 [PATCH v19 1/5] Move callback-call from ReadPageInternal to XLogReadRecord. Kyotaro Horiguchi <[email protected]>
2019-09-05 11:21 [PATCH v15 1/4] Move callback-call from ReadPageInternal to XLogReadRecord. Kyotaro Horiguchi <[email protected]>
2019-09-05 11:21 [PATCH v9 1/4] Move callback-call from ReadPageInternal to XLogReadRecord. Kyotaro Horiguchi <[email protected]>
2019-09-05 11:21 [PATCH v17 1/4] Move callback-call from ReadPageInternal to XLogReadRecord. Kyotaro Horiguchi <[email protected]>
2019-09-05 11:21 [PATCH v17 1/4] Move callback-call from ReadPageInternal to XLogReadRecord. Kyotaro Horiguchi <[email protected]>
2019-09-05 11:21 [PATCH v15 1/4] Move callback-call from ReadPageInternal to XLogReadRecord. Kyotaro Horiguchi <[email protected]>
2019-09-05 11:21 [PATCH v17 1/4] Move callback-call from ReadPageInternal to XLogReadRecord. Kyotaro Horiguchi <[email protected]>
2019-09-05 11:21 [PATCH v17 1/4] Move callback-call from ReadPageInternal to XLogReadRecord. Kyotaro Horiguchi <[email protected]>
2019-09-05 11:21 [PATCH v17 1/4] Move callback-call from ReadPageInternal to XLogReadRecord. Kyotaro Horiguchi <[email protected]>
2019-09-05 11:21 [PATCH v10 1/4] Move callback-call from ReadPageInternal to XLogReadRecord. Kyotaro Horiguchi <[email protected]>
2019-09-05 11:21 [PATCH v17 1/4] Move callback-call from ReadPageInternal to XLogReadRecord. Kyotaro Horiguchi <[email protected]>
2019-09-05 11:21 [PATCH v17 1/4] Move callback-call from ReadPageInternal to XLogReadRecord. Kyotaro Horiguchi <[email protected]>
2019-09-05 11:21 [PATCH v17 1/4] Move callback-call from ReadPageInternal to XLogReadRecord. Kyotaro Horiguchi <[email protected]>
2019-09-05 11:21 [PATCH v17 1/4] Move callback-call from ReadPageInternal to XLogReadRecord. Kyotaro Horiguchi <[email protected]>
2019-09-05 11:21 [PATCH v17 1/4] Move callback-call from ReadPageInternal to XLogReadRecord. Kyotaro Horiguchi <[email protected]>
2019-09-05 11:21 [PATCH v17 1/4] Move callback-call from ReadPageInternal to XLogReadRecord. Kyotaro Horiguchi <[email protected]>
2019-09-05 11:21 [PATCH v17 1/4] Move callback-call from ReadPageInternal to XLogReadRecord. Kyotaro Horiguchi <[email protected]>
2019-09-05 11:21 [PATCH v15 1/4] Move callback-call from ReadPageInternal to XLogReadRecord. Kyotaro Horiguchi <[email protected]>
2019-09-05 11:21 [PATCH v15 1/4] Move callback-call from ReadPageInternal to XLogReadRecord. Kyotaro Horiguchi <[email protected]>
2019-09-05 11:21 [PATCH v17 1/4] Move callback-call from ReadPageInternal to XLogReadRecord. Kyotaro Horiguchi <[email protected]>
2019-09-05 11:21 [PATCH v15 1/4] Move callback-call from ReadPageInternal to XLogReadRecord. Kyotaro Horiguchi <[email protected]>
2019-09-05 11:21 [PATCH v15 1/4] Move callback-call from ReadPageInternal to XLogReadRecord. Kyotaro Horiguchi <[email protected]>
2019-09-05 11:21 [PATCH v8 1/4] Move callback-call from ReadPageInternal to XLogReadRecord. Kyotaro Horiguchi <[email protected]>
2019-09-05 11:21 [PATCH v15 1/4] Move callback-call from ReadPageInternal to XLogReadRecord. Kyotaro Horiguchi <[email protected]>
2019-09-05 11:21 [PATCH v15 1/4] Move callback-call from ReadPageInternal to XLogReadRecord. Kyotaro Horiguchi <[email protected]>
2019-09-05 11:21 [PATCH v17 1/4] Move callback-call from ReadPageInternal to XLogReadRecord. Kyotaro Horiguchi <[email protected]>
2019-09-05 11:21 [PATCH v17 1/4] Move callback-call from ReadPageInternal to XLogReadRecord. Kyotaro Horiguchi <[email protected]>
2019-09-05 11:21 [PATCH v17 1/4] Move callback-call from ReadPageInternal to XLogReadRecord. Kyotaro Horiguchi <[email protected]>
2019-09-05 11:21 [PATCH v15 1/4] Move callback-call from ReadPageInternal to XLogReadRecord. Kyotaro Horiguchi <[email protected]>
2019-09-05 11:21 [PATCH v17 1/4] Move callback-call from ReadPageInternal to XLogReadRecord. Kyotaro Horiguchi <[email protected]>
2019-09-05 11:21 [PATCH v17 1/4] Move callback-call from ReadPageInternal to XLogReadRecord. Kyotaro Horiguchi <[email protected]>
2019-09-05 11:21 [PATCH v15 1/4] Move callback-call from ReadPageInternal to XLogReadRecord. Kyotaro Horiguchi <[email protected]>
2019-09-05 11:21 [PATCH v15 1/4] Move callback-call from ReadPageInternal to XLogReadRecord. Kyotaro Horiguchi <[email protected]>
2021-09-30 02:48 [PATCH v18 1/5] Move callback-call from ReadPageInternal to XLogReadRecord. Kyotaro Horiguchi <[email protected]>
2024-02-12 18:03 Re: Reducing output size of nodeToString Matthias van de Meent <[email protected]>
2024-02-12 19:32 ` Re: Reducing output size of nodeToString Matthias van de Meent <[email protected]>
2024-02-12 23:10 ` Re: Reducing output size of nodeToString Matthias van de Meent <[email protected]>
2024-02-15 12:59 ` Re: Reducing output size of nodeToString Peter Eisentraut <[email protected]>
2024-02-15 14:37 ` Re: Reducing output size of nodeToString Matthias van de Meent <[email protected]>
2024-02-19 13:19 ` Re: Reducing output size of nodeToString Matthias van de Meent <[email protected]>
2024-02-22 12:37 ` Re: Reducing output size of nodeToString Matthias van de Meent <[email protected]>
2024-02-22 15:07 ` Re: Reducing output size of nodeToString Matthias van de Meent <[email protected]>
2024-03-11 13:19 ` Re: Reducing output size of nodeToString Peter Eisentraut <[email protected]>
2024-03-11 20:52 ` Re: Reducing output size of nodeToString Matthias van de Meent <[email protected]>
2024-03-19 16:13 ` Re: Reducing output size of nodeToString Peter Eisentraut <[email protected]>
2024-03-19 16:46 ` Re: Reducing output size of nodeToString Matthias van de Meent <[email protected]>
2024-03-20 11:49 ` Re: Reducing output size of nodeToString Peter Eisentraut <[email protected]>
2024-03-20 12:03 ` Re: Reducing output size of nodeToString Matthias van de Meent <[email protected]>
2024-03-22 09:18 ` Re: Reducing output size of nodeToString Peter Eisentraut <[email protected]>
2026-05-25 07:07 Small fixes for CHECKPOINT FLUSH_UNLOGGED Chao Li <[email protected]>
2026-05-25 07:57 ` Re: Small fixes for CHECKPOINT FLUSH_UNLOGGED Daniel Gustafsson <[email protected]>
This inbox is served by agora; see mirroring instructions
for how to clone and mirror all data and code used for this inbox