public inbox for [email protected]
help / color / mirror / Atom feed[PATCH 2/2] Remove the old implemenations of XLogRead().
22+ messages / 8 participants
[nested] [flat]
* [PATCH 2/2] Remove the old implemenations of XLogRead().
@ 2019-09-26 11:51 Antonin Houska <[email protected]>
0 siblings, 0 replies; 22+ messages in thread
From: Antonin Houska @ 2019-09-26 11:51 UTC (permalink / raw)
Done in a separate patch because the diff looks harder to read if one function
(XLogRead) is removed and another one (the WALSegmentOpen callback) is added
nearby at the same time (the addition and removal of code can get mixed in the
diff).
---
src/backend/access/transam/xlogutils.c | 122 ----------------
src/backend/replication/walsender.c | 188 -------------------------
src/bin/pg_waldump/pg_waldump.c | 122 ----------------
3 files changed, 432 deletions(-)
diff --git a/src/backend/access/transam/xlogutils.c b/src/backend/access/transam/xlogutils.c
index 09d42d3112..002b8f72a9 100644
--- a/src/backend/access/transam/xlogutils.c
+++ b/src/backend/access/transam/xlogutils.c
@@ -639,128 +639,6 @@ XLogTruncateRelation(RelFileNode rnode, ForkNumber forkNum,
forget_invalid_pages(rnode, forkNum, nblocks);
}
-/*
- * Read 'count' bytes from WAL into 'buf', starting at location 'startptr'
- * in timeline 'tli'.
- *
- * Will open, and keep open, one WAL segment stored in the static file
- * descriptor 'sendFile'. This means if XLogRead is used once, there will
- * always be one descriptor left open until the process ends, but never
- * more than one.
- *
- * XXX This is very similar to pg_waldump's XLogDumpXLogRead and to XLogRead
- * in walsender.c but for small differences (such as lack of elog() in
- * frontend). Probably these should be merged at some point.
- */
-static void
-XLogReadOld(char *buf, int segsize, TimeLineID tli, XLogRecPtr startptr,
- Size count)
-{
- char *p;
- XLogRecPtr recptr;
- Size nbytes;
-
- /* state maintained across calls */
- static int sendFile = -1;
- static XLogSegNo sendSegNo = 0;
- static TimeLineID sendTLI = 0;
- static uint32 sendOff = 0;
-
- Assert(segsize == wal_segment_size);
-
- p = buf;
- recptr = startptr;
- nbytes = count;
-
- while (nbytes > 0)
- {
- uint32 startoff;
- int segbytes;
- int readbytes;
-
- startoff = XLogSegmentOffset(recptr, segsize);
-
- /* Do we need to switch to a different xlog segment? */
- if (sendFile < 0 || !XLByteInSeg(recptr, sendSegNo, segsize) ||
- sendTLI != tli)
- {
- char path[MAXPGPATH];
-
- if (sendFile >= 0)
- close(sendFile);
-
- XLByteToSeg(recptr, sendSegNo, segsize);
-
- XLogFilePath(path, tli, sendSegNo, segsize);
-
- sendFile = BasicOpenFile(path, O_RDONLY | PG_BINARY);
-
- if (sendFile < 0)
- {
- if (errno == ENOENT)
- ereport(ERROR,
- (errcode_for_file_access(),
- errmsg("requested WAL segment %s has already been removed",
- path)));
- else
- ereport(ERROR,
- (errcode_for_file_access(),
- errmsg("could not open file \"%s\": %m",
- path)));
- }
- sendOff = 0;
- sendTLI = tli;
- }
-
- /* Need to seek in the file? */
- if (sendOff != startoff)
- {
- if (lseek(sendFile, (off_t) startoff, SEEK_SET) < 0)
- {
- char path[MAXPGPATH];
- int save_errno = errno;
-
- XLogFilePath(path, tli, sendSegNo, segsize);
- errno = save_errno;
- ereport(ERROR,
- (errcode_for_file_access(),
- errmsg("could not seek in log segment %s to offset %u: %m",
- path, startoff)));
- }
- sendOff = startoff;
- }
-
- /* How many bytes are within this segment? */
- if (nbytes > (segsize - startoff))
- segbytes = segsize - startoff;
- else
- segbytes = nbytes;
-
- pgstat_report_wait_start(WAIT_EVENT_WAL_READ);
- readbytes = read(sendFile, p, segbytes);
- pgstat_report_wait_end();
- if (readbytes <= 0)
- {
- char path[MAXPGPATH];
- int save_errno = errno;
-
- XLogFilePath(path, tli, sendSegNo, segsize);
- errno = save_errno;
- ereport(ERROR,
- (errcode_for_file_access(),
- errmsg("could not read from log segment %s, offset %u, length %lu: %m",
- path, sendOff, (unsigned long) segbytes)));
- }
-
- /* Update state for read */
- recptr += readbytes;
-
- sendOff += readbytes;
- nbytes -= readbytes;
- p += readbytes;
- }
-}
-
/*
* Determine which timeline to read an xlog page from and set the
* XLogReaderState's currTLI to that timeline ID.
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index dcb84693fc..d7df72fb87 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -253,9 +253,6 @@ static void WalSndSegmentOpen(XLogSegNo nextSegNo, TimeLineID *tli_p,
WALSegmentContext *segcxt);
-static void XLogReadOld(WALSegmentContext *segcxt, char *buf,
- XLogRecPtr startptr, Size count);
-
/* Initialize walsender process before entering the main command loop */
void
InitWalSender(void)
@@ -2375,191 +2372,6 @@ WalSndKill(int code, Datum arg)
SpinLockRelease(&walsnd->mutex);
}
-/*
- * Read 'count' bytes from WAL into 'buf', starting at location 'startptr'
- *
- * XXX probably this should be improved to suck data directly from the
- * WAL buffers when possible.
- *
- * Will open, and keep open, one WAL segment stored in the global file
- * descriptor sendFile. This means if XLogRead is used once, there will
- * always be one descriptor left open until the process ends, but never
- * more than one.
- */
-static void
-XLogReadOld(WALSegmentContext *segcxt, char *buf, XLogRecPtr startptr, Size count)
-{
- char *p;
- XLogRecPtr recptr;
- Size nbytes;
- XLogSegNo segno;
-
-retry:
- p = buf;
- recptr = startptr;
- nbytes = count;
-
- while (nbytes > 0)
- {
- uint32 startoff;
- int segbytes;
- int readbytes;
-
- startoff = XLogSegmentOffset(recptr, segcxt->ws_segsize);
-
- if (sendSeg->ws_file < 0 ||
- !XLByteInSeg(recptr, sendSeg->ws_segno, segcxt->ws_segsize))
- {
- char path[MAXPGPATH];
-
- /* Switch to another logfile segment */
- if (sendSeg->ws_file >= 0)
- close(sendSeg->ws_file);
-
- XLByteToSeg(recptr, sendSeg->ws_segno, segcxt->ws_segsize);
-
- /*-------
- * When reading from a historic timeline, and there is a timeline
- * switch within this segment, read from the WAL segment belonging
- * to the new timeline.
- *
- * For example, imagine that this server is currently on timeline
- * 5, and we're streaming timeline 4. The switch from timeline 4
- * to 5 happened at 0/13002088. In pg_wal, we have these files:
- *
- * ...
- * 000000040000000000000012
- * 000000040000000000000013
- * 000000050000000000000013
- * 000000050000000000000014
- * ...
- *
- * In this situation, when requested to send the WAL from
- * segment 0x13, on timeline 4, we read the WAL from file
- * 000000050000000000000013. Archive recovery prefers files from
- * newer timelines, so if the segment was restored from the
- * archive on this server, the file belonging to the old timeline,
- * 000000040000000000000013, might not exist. Their contents are
- * equal up to the switchpoint, because at a timeline switch, the
- * used portion of the old segment is copied to the new file.
- *-------
- */
- sendSeg->ws_tli = sendTimeLine;
- if (sendTimeLineIsHistoric)
- {
- XLogSegNo endSegNo;
-
- XLByteToSeg(sendTimeLineValidUpto, endSegNo, segcxt->ws_segsize);
- if (sendSeg->ws_segno == endSegNo)
- sendSeg->ws_tli = sendTimeLineNextTLI;
- }
-
- XLogFilePath(path, sendSeg->ws_tli, sendSeg->ws_segno, segcxt->ws_segsize);
-
- sendSeg->ws_file = BasicOpenFile(path, O_RDONLY | PG_BINARY);
- if (sendSeg->ws_file < 0)
- {
- /*
- * If the file is not found, assume it's because the standby
- * asked for a too old WAL segment that has already been
- * removed or recycled.
- */
- if (errno == ENOENT)
- ereport(ERROR,
- (errcode_for_file_access(),
- errmsg("requested WAL segment %s has already been removed",
- XLogFileNameP(sendSeg->ws_tli, sendSeg->ws_segno))));
- else
- ereport(ERROR,
- (errcode_for_file_access(),
- errmsg("could not open file \"%s\": %m",
- path)));
- }
- sendSeg->ws_off = 0;
- }
-
- /* Need to seek in the file? */
- if (sendSeg->ws_off != startoff)
- {
- if (lseek(sendSeg->ws_file, (off_t) startoff, SEEK_SET) < 0)
- ereport(ERROR,
- (errcode_for_file_access(),
- errmsg("could not seek in log segment %s to offset %u: %m",
- XLogFileNameP(sendSeg->ws_tli, sendSeg->ws_segno),
- startoff)));
- sendSeg->ws_off = startoff;
- }
-
- /* How many bytes are within this segment? */
- if (nbytes > (segcxt->ws_segsize - startoff))
- segbytes = segcxt->ws_segsize - startoff;
- else
- segbytes = nbytes;
-
- pgstat_report_wait_start(WAIT_EVENT_WAL_READ);
- readbytes = read(sendSeg->ws_file, p, segbytes);
- pgstat_report_wait_end();
- if (readbytes < 0)
- {
- ereport(ERROR,
- (errcode_for_file_access(),
- errmsg("could not read from log segment %s, offset %u, length %zu: %m",
- XLogFileNameP(sendSeg->ws_tli, sendSeg->ws_segno),
- sendSeg->ws_off, (Size) segbytes)));
- }
- else if (readbytes == 0)
- {
- ereport(ERROR,
- (errcode(ERRCODE_DATA_CORRUPTED),
- errmsg("could not read from log segment %s, offset %u: read %d of %zu",
- XLogFileNameP(sendSeg->ws_tli, sendSeg->ws_segno),
- sendSeg->ws_off, readbytes, (Size) segbytes)));
- }
-
- /* Update state for read */
- recptr += readbytes;
-
- sendSeg->ws_off += readbytes;
- nbytes -= readbytes;
- p += readbytes;
- }
-
- /*
- * After reading into the buffer, check that what we read was valid. We do
- * this after reading, because even though the segment was present when we
- * opened it, it might get recycled or removed while we read it. The
- * read() succeeds in that case, but the data we tried to read might
- * already have been overwritten with new WAL records.
- */
- XLByteToSeg(startptr, segno, segcxt->ws_segsize);
- CheckXLogRemoved(segno, ThisTimeLineID);
-
- /*
- * During recovery, the currently-open WAL file might be replaced with the
- * file of the same name retrieved from archive. So we always need to
- * check what we read was valid after reading into the buffer. If it's
- * invalid, we try to open and read the file again.
- */
- if (am_cascading_walsender)
- {
- WalSnd *walsnd = MyWalSnd;
- bool reload;
-
- SpinLockAcquire(&walsnd->mutex);
- reload = walsnd->needreload;
- walsnd->needreload = false;
- SpinLockRelease(&walsnd->mutex);
-
- if (reload && sendSeg->ws_file >= 0)
- {
- close(sendSeg->ws_file);
- sendSeg->ws_file = -1;
-
- goto retry;
- }
- }
-}
-
/*
* Callback for XLogRead() to open the next segment.
*/
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index c97376101c..57dd7df56f 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -321,128 +321,6 @@ WALDumpOpenSegment(XLogSegNo nextSegNo, TimeLineID *tli_p, int *file_p,
fname, strerror(errno));
}
-/*
- * Read count bytes from a segment file in the specified directory, for the
- * given timeline, containing the specified record pointer; store the data in
- * the passed buffer.
- */
-static void
-XLogDumpXLogRead(const char *directory, TimeLineID timeline_id,
- XLogRecPtr startptr, char *buf, Size count)
-{
- char *p;
- XLogRecPtr recptr;
- Size nbytes;
-
- static int sendFile = -1;
- static XLogSegNo sendSegNo = 0;
- static uint32 sendOff = 0;
-
- p = buf;
- recptr = startptr;
- nbytes = count;
-
- while (nbytes > 0)
- {
- uint32 startoff;
- int segbytes;
- int readbytes;
-
- startoff = XLogSegmentOffset(recptr, WalSegSz);
-
- if (sendFile < 0 || !XLByteInSeg(recptr, sendSegNo, WalSegSz))
- {
- char fname[MAXFNAMELEN];
- int tries;
-
- /* Switch to another logfile segment */
- if (sendFile >= 0)
- close(sendFile);
-
- XLByteToSeg(recptr, sendSegNo, WalSegSz);
-
- XLogFileName(fname, timeline_id, sendSegNo, WalSegSz);
-
- /*
- * In follow mode there is a short period of time after the server
- * has written the end of the previous file before the new file is
- * available. So we loop for 5 seconds looking for the file to
- * appear before giving up.
- */
- for (tries = 0; tries < 10; tries++)
- {
- sendFile = open_file_in_directory(directory, fname);
- if (sendFile >= 0)
- break;
- if (errno == ENOENT)
- {
- int save_errno = errno;
-
- /* File not there yet, try again */
- pg_usleep(500 * 1000);
-
- errno = save_errno;
- continue;
- }
- /* Any other error, fall through and fail */
- break;
- }
-
- if (sendFile < 0)
- fatal_error("could not find file \"%s\": %s",
- fname, strerror(errno));
- sendOff = 0;
- }
-
- /* Need to seek in the file? */
- if (sendOff != startoff)
- {
- if (lseek(sendFile, (off_t) startoff, SEEK_SET) < 0)
- {
- int err = errno;
- char fname[MAXPGPATH];
-
- XLogFileName(fname, timeline_id, sendSegNo, WalSegSz);
-
- fatal_error("could not seek in log file %s to offset %u: %s",
- fname, startoff, strerror(err));
- }
- sendOff = startoff;
- }
-
- /* How many bytes are within this segment? */
- if (nbytes > (WalSegSz - startoff))
- segbytes = WalSegSz - startoff;
- else
- segbytes = nbytes;
-
- readbytes = read(sendFile, p, segbytes);
- if (readbytes <= 0)
- {
- int err = errno;
- char fname[MAXPGPATH];
- int save_errno = errno;
-
- XLogFileName(fname, timeline_id, sendSegNo, WalSegSz);
- errno = save_errno;
-
- if (readbytes < 0)
- fatal_error("could not read from log file %s, offset %u, length %d: %s",
- fname, sendOff, segbytes, strerror(err));
- else if (readbytes == 0)
- fatal_error("could not read from log file %s, offset %u: read %d of %zu",
- fname, sendOff, readbytes, (Size) segbytes);
- }
-
- /* Update state for read */
- recptr += readbytes;
-
- sendOff += readbytes;
- nbytes -= readbytes;
- p += readbytes;
- }
-}
-
/*
* XLogReader read_page callback
*/
--
2.20.1
--=-=-=--
^ permalink raw reply [nested|flat] 22+ messages in thread
* Improve WALRead() to suck data directly from WAL buffers when possible
@ 2022-12-09 09:03 Bharath Rupireddy <[email protected]>
0 siblings, 1 reply; 22+ messages in thread
From: Bharath Rupireddy @ 2022-12-09 09:03 UTC (permalink / raw)
To: PostgreSQL Hackers <[email protected]>
Hi,
WALRead() currently reads WAL from the WAL file on the disk, which
means, the walsenders serving streaming and logical replication
(callers of WALRead()) will have to hit the disk/OS's page cache for
reading the WAL. This may increase the amount of read IO required for
all the walsenders put together as one typically maintains many
standbys/subscribers on production servers for high availability,
disaster recovery, read-replicas and so on. Also, it may increase
replication lag if all the WAL reads are always hitting the disk.
It may happen that WAL buffers contain the requested WAL, if so, the
WALRead() can attempt to read from the WAL buffers first before
reading from the file. If the read hits the WAL buffers, then reading
from the file on disk is avoided. This mainly reduces the read IO/read
system calls. It also enables us to do other features specified
elsewhere [1].
I'm attaching a patch that implements the idea which is also noted
elsewhere [2]. I've run some tests [3]. The WAL buffers hit ratio with
the patch stood at 95%, in other words, the walsenders avoided 95% of
the time reading from the file. The benefit, if measured in terms of
the amount of data - 79% (13.5GB out of total 17GB) of the requested
WAL is read from the WAL buffers as opposed to 21% from the file. Note
that the WAL buffers hit ratio can be very low for write-heavy
workloads, in which case, file reads are inevitable.
The patch introduces concurrent readers for the WAL buffers, so far
only there are concurrent writers. In the patch, WALRead() takes just
one lock (WALBufMappingLock) in shared mode to enable concurrent
readers and does minimal things - checks if the requested WAL page is
present in WAL buffers, if so, copies the page and releases the lock.
I think taking just WALBufMappingLock is enough here as the concurrent
writers depend on it to initialize and replace a page in WAL buffers.
I'll add this to the next commitfest.
Thoughts?
[1] https://www.postgresql.org/message-id/CALj2ACXCSM%2BsTR%3D5NNRtmSQr3g1Vnr-yR91azzkZCaCJ7u4d4w%40mail...
[2]
* XXX probably this should be improved to suck data directly from the
* WAL buffers when possible.
*/
bool
WALRead(XLogReaderState *state,
[3]
1 primary, 1 sync standby, 1 async standby
./pgbench --initialize --scale=300 postgres
./pgbench --jobs=16 --progress=300 --client=32 --time=900
--username=ubuntu postgres
PATCHED:
-[ RECORD 1 ]----------+----------------
application_name | assb1
wal_read | 31005
wal_read_bytes | 3800607104
wal_read_time | 779.402
wal_read_buffers | 610611
wal_read_bytes_buffers | 14493226440
wal_read_time_buffers | 3033.309
sync_state | async
-[ RECORD 2 ]----------+----------------
application_name | ssb1
wal_read | 31027
wal_read_bytes | 3800932712
wal_read_time | 696.365
wal_read_buffers | 610580
wal_read_bytes_buffers | 14492900832
wal_read_time_buffers | 2989.507
sync_state | sync
HEAD:
-[ RECORD 1 ]----+----------------
application_name | assb1
wal_read | 705627
wal_read_bytes | 18343480640
wal_read_time | 7607.783
sync_state | async
-[ RECORD 2 ]----+------------
application_name | ssb1
wal_read | 705625
wal_read_bytes | 18343480640
wal_read_time | 4539.058
sync_state | sync
--
Bharath Rupireddy
PostgreSQL Contributors Team
RDS Open Source Databases
Amazon Web Services: https://aws.amazon.com
Attachments:
[application/octet-stream] v1-0001-Improve-WALRead-to-suck-data-directly-from-WAL-bu.patch (9.1K, ../../CALj2ACXKKK=wbiG5_t6dGao5GoecMwRkhr7GjVBM_jg54+Na=Q@mail.gmail.com/2-v1-0001-Improve-WALRead-to-suck-data-directly-from-WAL-bu.patch)
download | inline diff:
From d93a6c97bad19d3718f0e4f06caeac5ce9937b37 Mon Sep 17 00:00:00 2001
From: Bharath Rupireddy <[email protected]>
Date: Thu, 8 Dec 2022 09:37:01 +0000
Subject: [PATCH v1] Improve WALRead() to suck data directly from WAL buffers
when possible
---
src/backend/access/transam/xlog.c | 184 ++++++++++++++++++++++++
src/backend/access/transam/xlogreader.c | 58 +++++++-
src/include/access/xlog.h | 9 ++
3 files changed, 249 insertions(+), 2 deletions(-)
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index a31fbbff78..196be98591 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -689,6 +689,7 @@ static bool ReserveXLogSwitch(XLogRecPtr *StartPos, XLogRecPtr *EndPos,
XLogRecPtr *PrevPtr);
static XLogRecPtr WaitXLogInsertionsToFinish(XLogRecPtr upto);
static char *GetXLogBuffer(XLogRecPtr ptr, TimeLineID tli);
+static char *GetXLogBufferForRead(XLogRecPtr ptr, TimeLineID tli, char *page);
static XLogRecPtr XLogBytePosToRecPtr(uint64 bytepos);
static XLogRecPtr XLogBytePosToEndRecPtr(uint64 bytepos);
static uint64 XLogRecPtrToBytePos(XLogRecPtr ptr);
@@ -1639,6 +1640,189 @@ GetXLogBuffer(XLogRecPtr ptr, TimeLineID tli)
return cachedPos + ptr % XLOG_BLCKSZ;
}
+/*
+ * Get the WAL buffer page containing passed in WAL record and also return the
+ * record's location within that buffer page.
+ */
+static char *
+GetXLogBufferForRead(XLogRecPtr ptr, TimeLineID tli, char *page)
+{
+ XLogRecPtr expectedEndPtr;
+ XLogRecPtr endptr;
+ int idx;
+ char *recptr = NULL;
+
+ idx = XLogRecPtrToBufIdx(ptr);
+ expectedEndPtr = ptr;
+ expectedEndPtr += XLOG_BLCKSZ - ptr % XLOG_BLCKSZ;
+
+ /*
+ * Hold WALBufMappingLock in shared mode so that the other concurrent WAL
+ * readers are also allowed. We try to do as less work as possible while
+ * holding the lock as it might impact concurrent WAL writers.
+ *
+ * XXX: Perhaps, measuring the immediate lock availability and its impact
+ * on concurrent WAL writers is a good idea here.
+ *
+ * XXX: Perhaps, returning if lock is not immediately available a good idea
+ * here. The caller can then go ahead with reading WAL from WAL file.
+ *
+ * XXX: Perhaps, quickly finding if the given WAL record is in WAL buffers
+ * a good idea here. This avoids unnecessary lock acquire-release cycles.
+ * One way to do that is by maintaining oldest WAL record that's currently
+ * present in WAL buffers.
+ */
+ LWLockAcquire(WALBufMappingLock, LW_SHARED);
+
+ /*
+ * Holding WALBufMappingLock ensures inserters don't overwrite this value
+ * while we are reading it.
+ */
+ endptr = XLogCtl->xlblocks[idx];
+
+ if (expectedEndPtr == endptr)
+ {
+ XLogPageHeader phdr;
+
+ /*
+ * We have found the WAL buffer page holding the given LSN. Read from a pointer
+ * to the right offset within the page.
+ */
+ memcpy(page, (XLogCtl->pages + idx * (Size) XLOG_BLCKSZ),
+ (Size) XLOG_BLCKSZ);
+
+ /*
+ * Release the lock as early as possible to avoid any possible
+ * contention.
+ */
+ LWLockRelease(WALBufMappingLock);
+
+ /*
+ * Despite we read the WAL buffer page by holding all necessary locks,
+ * we still want to be extra cautious here and serve the valid WAL
+ * buffer page.
+ *
+ * XXX: Perhaps, we can further go and validate the found page header,
+ * record header and record at least in assert builds, something like
+ * the xlogreader.c does and return if any of those validity checks
+ * fail. Having said that, we stick to the minimal checks for now.
+ */
+ phdr = (XLogPageHeader) page;
+
+ if (phdr->xlp_magic == XLOG_PAGE_MAGIC &&
+ phdr->xlp_pageaddr == (ptr - (ptr % XLOG_BLCKSZ)) &&
+ phdr->xlp_tli == tli)
+ {
+ /*
+ * Page looks valid, so return the page and the requested record's
+ * LSN.
+ */
+ recptr = page + ptr % XLOG_BLCKSZ;
+ }
+ }
+ else
+ {
+ /* We have not found anything. */
+ LWLockRelease(WALBufMappingLock);
+ }
+
+ return recptr;
+}
+
+/*
+ * When possible, read WAL starting at 'startptr' of size 'count' bytes from
+ * WAL buffers into buffer passed in by the caller 'buf'. Read as much WAL as
+ * possible from the WAL buffers, remaining WAL, if any, the caller will take
+ * care of reading from WAL files directly.
+ *
+ * This function sets read bytes to 'read_bytes' and sets 'hit', 'partial_hit'
+ * and 'miss' accordingly.
+ */
+void
+XLogReadFromBuffers(XLogRecPtr startptr,
+ TimeLineID tli,
+ Size count,
+ char *buf,
+ Size *read_bytes,
+ bool *hit,
+ bool *partial_hit,
+ bool *miss)
+{
+ XLogRecPtr ptr;
+ char *dst;
+ Size nbytes;
+
+ Assert(!XLogRecPtrIsInvalid(startptr));
+ Assert(count > 0);
+ Assert(startptr <= GetFlushRecPtr(NULL));
+ Assert(!RecoveryInProgress());
+
+ ptr = startptr;
+ nbytes = count;
+ dst = buf;
+ *read_bytes = 0;
+ *hit = false;
+ *partial_hit = false;
+ *miss = false;
+
+ while (nbytes > 0)
+ {
+ char page[XLOG_BLCKSZ] = {0};
+ char *recptr;
+
+ recptr = GetXLogBufferForRead(ptr, tli, page);
+
+ if (recptr == NULL)
+ break;
+
+ if ((recptr + nbytes) <= (page + XLOG_BLCKSZ))
+ {
+ /* All the bytes are in one page. */
+ memcpy(dst, recptr, nbytes);
+ dst += nbytes;
+ *read_bytes += nbytes;
+ ptr += nbytes;
+ nbytes = 0;
+ }
+ else if ((recptr + nbytes) > (page + XLOG_BLCKSZ))
+ {
+ /* All the bytes are not in one page. */
+ Size bytes_remaining;
+
+ /*
+ * Compute the remaining bytes on the current page, copy them over
+ * to output buffer and move forward to read further.
+ */
+ bytes_remaining = XLOG_BLCKSZ - (recptr - page);
+ memcpy(dst, recptr, bytes_remaining);
+ dst += bytes_remaining;
+ nbytes -= bytes_remaining;
+ *read_bytes += bytes_remaining;
+ ptr += bytes_remaining;
+ }
+ }
+
+ if (*read_bytes == count)
+ {
+ /* It's a buffer hit. */
+ *hit = true;
+ }
+ else if (*read_bytes > 0 &&
+ *read_bytes < count)
+ {
+ /* It's a buffer partial hit. */
+ *partial_hit = true;
+ }
+ else if (*read_bytes == 0)
+ {
+ /* It's a buffer miss. */
+ *miss = true;
+ }
+
+ elog(DEBUG1, "read %zu bytes out of %zu bytes from WAL buffers for given LSN %X/%X",
+ *read_bytes, count, LSN_FORMAT_ARGS(startptr));
+}
+
/*
* Converts a "usable byte position" to XLogRecPtr. A usable byte position
* is the position starting from the beginning of WAL, excluding all WAL
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index a38a80e049..7ec94a0535 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -1485,8 +1485,7 @@ err:
* Returns true if succeeded, false if an error occurs, in which case
* 'errinfo' receives error details.
*
- * XXX probably this should be improved to suck data directly from the
- * WAL buffers when possible.
+ * When possible, this function reads data directly from WAL buffers.
*/
bool
WALRead(XLogReaderState *state,
@@ -1497,6 +1496,61 @@ WALRead(XLogReaderState *state,
XLogRecPtr recptr;
Size nbytes;
+#ifndef FRONTEND
+ /* Frontend tools have no idea of WAL buffers. */
+ Size read_bytes;
+ bool hit;
+ bool partial_hit;
+ bool miss;
+
+ /*
+ * When possible, read WAL from WAL buffers. We skip this step and continue
+ * the usual way, that is to read from WAL file, either when the server is
+ * in recovery (standby mode, archive or crash recovery), in which case the
+ * WAL buffers are not used or when the server is inserting in a different
+ * timeline from that of the timeline that we're trying to read WAL from.
+ */
+ if (!RecoveryInProgress() &&
+ tli == GetWALInsertionTimeLine())
+ {
+ pgstat_report_wait_start(WAIT_EVENT_WAL_READ);
+ XLogReadFromBuffers(startptr, tli, count, buf, &read_bytes,
+ &hit, &partial_hit, &miss);
+ pgstat_report_wait_end();
+
+ if (hit)
+ {
+ /*
+ * We have fully read the requested WAL from WAL buffers, so
+ * return.
+ */
+ Assert(count == read_bytes);
+ return true;
+ }
+ else if (partial_hit)
+ {
+ /*
+ * We have partially read from WAL buffers, so reset the state and
+ * read the remaining bytes the usual way.
+ */
+ Assert(read_bytes > 0 && count > read_bytes);
+ buf += read_bytes;
+ startptr += read_bytes;
+ count -= read_bytes;
+ }
+#ifdef USE_ASSERT_CHECKING
+ else if (miss)
+ {
+ /*
+ * We have not read anything from WAL buffers, so read the usual way,
+ * that is to read from WAL file.
+ */
+ Assert(read_bytes == 0);
+ }
+#endif /* USE_ASSERT_CHECKING */
+ }
+#endif /* FRONTEND */
+
p = buf;
recptr = startptr;
nbytes = count;
diff --git a/src/include/access/xlog.h b/src/include/access/xlog.h
index 1fbd48fbda..968608353e 100644
--- a/src/include/access/xlog.h
+++ b/src/include/access/xlog.h
@@ -247,6 +247,15 @@ extern XLogRecPtr GetLastImportantRecPtr(void);
extern void SetWalWriterSleeping(bool sleeping);
+extern void XLogReadFromBuffers(XLogRecPtr startptr,
+ TimeLineID tli,
+ Size count,
+ char *buf,
+ Size *read_bytes,
+ bool *hit,
+ bool *partial_hit,
+ bool *miss);
+
/*
* Routines used by xlogrecovery.c to call back into xlog.c during recovery.
*/
--
2.34.1
^ permalink raw reply [nested|flat] 22+ messages in thread
* Re: Improve WALRead() to suck data directly from WAL buffers when possible
@ 2022-12-12 02:57 Kyotaro Horiguchi <[email protected]>
parent: Bharath Rupireddy <[email protected]>
0 siblings, 2 replies; 22+ messages in thread
From: Kyotaro Horiguchi @ 2022-12-12 02:57 UTC (permalink / raw)
To: [email protected]; +Cc: [email protected]
At Fri, 9 Dec 2022 14:33:39 +0530, Bharath Rupireddy <[email protected]> wrote in
> The patch introduces concurrent readers for the WAL buffers, so far
> only there are concurrent writers. In the patch, WALRead() takes just
> one lock (WALBufMappingLock) in shared mode to enable concurrent
> readers and does minimal things - checks if the requested WAL page is
> present in WAL buffers, if so, copies the page and releases the lock.
> I think taking just WALBufMappingLock is enough here as the concurrent
> writers depend on it to initialize and replace a page in WAL buffers.
>
> I'll add this to the next commitfest.
>
> Thoughts?
This adds copying of the whole page (at least) at every WAL *record*
read, fighting all WAL writers by taking WALBufMappingLock on a very
busy page while the copying. I'm a bit doubtful that it results in an
overall improvement. It seems to me almost all pread()s here happens
on file buffer so it is unclear to me that copying a whole WAL page
(then copying the target record again) wins over a pread() call that
copies only the record to read. Do you have an actual number of how
frequent WAL reads go to disk, or the actual number of performance
gain or real I/O reduction this patch offers?
This patch copies the bleeding edge WAL page without recording the
(next) insertion point nor checking whether all in-progress insertion
behind the target LSN have finished. Thus the copied page may have
holes. That being said, the sequential-reading nature and the fact
that WAL buffers are zero-initialized may make it work for recovery,
but I don't think this also works for replication.
I remember that the one of the advantage of reading the on-memory WAL
records is that that allows walsender to presend the unwritten
records. So perhaps we should manage how far the buffer is filled with
valid content (or how far we can presend) in this feature.
regards.
--
Kyotaro Horiguchi
NTT Open Source Software Center
^ permalink raw reply [nested|flat] 22+ messages in thread
* Re: Improve WALRead() to suck data directly from WAL buffers when possible
@ 2022-12-12 03:06 Kyotaro Horiguchi <[email protected]>
parent: Kyotaro Horiguchi <[email protected]>
1 sibling, 1 reply; 22+ messages in thread
From: Kyotaro Horiguchi @ 2022-12-12 03:06 UTC (permalink / raw)
To: [email protected]; +Cc: [email protected]
At Mon, 12 Dec 2022 11:57:17 +0900 (JST), Kyotaro Horiguchi <[email protected]> wrote in
> This patch copies the bleeding edge WAL page without recording the
> (next) insertion point nor checking whether all in-progress insertion
> behind the target LSN have finished. Thus the copied page may have
> holes. That being said, the sequential-reading nature and the fact
> that WAL buffers are zero-initialized may make it work for recovery,
> but I don't think this also works for replication.
Mmm. I'm a bit dim. Recovery doesn't read concurrently-written
records. Please forget about recovery.
> I remember that the one of the advantage of reading the on-memory WAL
> records is that that allows walsender to presend the unwritten
> records. So perhaps we should manage how far the buffer is filled with
> valid content (or how far we can presend) in this feature.
--
Kyotaro Horiguchi
NTT Open Source Software Center
^ permalink raw reply [nested|flat] 22+ messages in thread
* Re: Improve WALRead() to suck data directly from WAL buffers when possible
@ 2022-12-12 03:08 Kyotaro Horiguchi <[email protected]>
parent: Kyotaro Horiguchi <[email protected]>
0 siblings, 0 replies; 22+ messages in thread
From: Kyotaro Horiguchi @ 2022-12-12 03:08 UTC (permalink / raw)
To: [email protected]; +Cc: [email protected]
Sorry for the confusion.
At Mon, 12 Dec 2022 12:06:36 +0900 (JST), Kyotaro Horiguchi <[email protected]> wrote in
> At Mon, 12 Dec 2022 11:57:17 +0900 (JST), Kyotaro Horiguchi <[email protected]> wrote in
> > This patch copies the bleeding edge WAL page without recording the
> > (next) insertion point nor checking whether all in-progress insertion
> > behind the target LSN have finished. Thus the copied page may have
> > holes. That being said, the sequential-reading nature and the fact
> > that WAL buffers are zero-initialized may make it work for recovery,
> > but I don't think this also works for replication.
>
> Mmm. I'm a bit dim. Recovery doesn't read concurrently-written
> records. Please forget about recovery.
NO... Logical walsenders do that. So, please forget about this...
> > I remember that the one of the advantage of reading the on-memory WAL
> > records is that that allows walsender to presend the unwritten
> > records. So perhaps we should manage how far the buffer is filled with
> > valid content (or how far we can presend) in this feature.
regards.
--
Kyotaro Horiguchi
NTT Open Source Software Center
^ permalink raw reply [nested|flat] 22+ messages in thread
* Re: Improve WALRead() to suck data directly from WAL buffers when possible
@ 2022-12-23 10:15 Bharath Rupireddy <[email protected]>
parent: Kyotaro Horiguchi <[email protected]>
1 sibling, 1 reply; 22+ messages in thread
From: Bharath Rupireddy @ 2022-12-23 10:15 UTC (permalink / raw)
To: Kyotaro Horiguchi <[email protected]>; +Cc: [email protected]
On Mon, Dec 12, 2022 at 8:27 AM Kyotaro Horiguchi
<[email protected]> wrote:
>
Thanks for providing thoughts.
> At Fri, 9 Dec 2022 14:33:39 +0530, Bharath Rupireddy <[email protected]> wrote in
> > The patch introduces concurrent readers for the WAL buffers, so far
> > only there are concurrent writers. In the patch, WALRead() takes just
> > one lock (WALBufMappingLock) in shared mode to enable concurrent
> > readers and does minimal things - checks if the requested WAL page is
> > present in WAL buffers, if so, copies the page and releases the lock.
> > I think taking just WALBufMappingLock is enough here as the concurrent
> > writers depend on it to initialize and replace a page in WAL buffers.
> >
> > I'll add this to the next commitfest.
> >
> > Thoughts?
>
> This adds copying of the whole page (at least) at every WAL *record*
> read,
In the worst case yes, but that may not always be true. On a typical
production server with decent write traffic, it happens that the
callers of WALRead() read a full WAL page of size XLOG_BLCKSZ bytes or
MAX_SEND_SIZE bytes.
> fighting all WAL writers by taking WALBufMappingLock on a very
> busy page while the copying. I'm a bit doubtful that it results in an
> overall improvement.
Well, the tests don't reflect that [1], I've run an insert work load
[2]. The WAL is being read from WAL buffers 99% of the time, which is
pretty cool. If you have any use-cases in mind, please share them
and/or feel free to run at your end.
> It seems to me almost all pread()s here happens
> on file buffer so it is unclear to me that copying a whole WAL page
> (then copying the target record again) wins over a pread() call that
> copies only the record to read.
That's not always guaranteed. Imagine a typical production server with
decent write traffic and heavy analytical queries (which fills OS page
cache with the table pages accessed for the queries), the WAL pread()
calls turn to IOPS. Despite the WAL being present in WAL buffers,
customers will be paying unnecessarily for these IOPS too. With the
patch, we are basically avoiding the pread() system calls which may
turn into IOPS on production servers (99% of the time for the insert
use case [1][2], 95% of the time for pgbench use case specified
upthread). With the patch, WAL buffers can act as L1 cache, if one
calls OS page cache as L2 cache (of course this illustration is not
related to the typical processor L1 and L2 ... caches).
> Do you have an actual number of how
> frequent WAL reads go to disk, or the actual number of performance
> gain or real I/O reduction this patch offers?
It might be a bit tough to generate such heavy traffic. An idea is to
ensure the WAL page/file goes out of the OS page cache before
WALRead() - these might help here - 0002 patch from
https://www.postgresql.org/message-id/CA%2BhUKGLmeyrDcUYAty90V_YTcoo5kAFfQjRQ-_1joS_%3DX7HztA%40mail...
and tool https://github.com/klando/pgfincore.
> This patch copies the bleeding edge WAL page without recording the
> (next) insertion point nor checking whether all in-progress insertion
> behind the target LSN have finished. Thus the copied page may have
> holes. That being said, the sequential-reading nature and the fact
> that WAL buffers are zero-initialized may make it work for recovery,
> but I don't think this also works for replication.
WALRead() callers are smart enough to take the flushed bytes only.
Although they read the whole WAL page, they calculate the valid bytes.
> I remember that the one of the advantage of reading the on-memory WAL
> records is that that allows walsender to presend the unwritten
> records. So perhaps we should manage how far the buffer is filled with
> valid content (or how far we can presend) in this feature.
Yes, the non-flushed WAL can be read and sent across if one wishes to
to make replication faster and parallel flushing on primary and
standbys at the cost of a bit of extra crash handling, that's
mentioned here https://www.postgresql.org/message-id/CALj2ACXCSM%2BsTR%3D5NNRtmSQr3g1Vnr-yR91azzkZCaCJ7u4d4w%40mail....
However, this can be a separate discussion.
I also want to reiterate that the patch implemented a TODO item:
* XXX probably this should be improved to suck data directly from the
* WAL buffers when possible.
*/
bool
WALRead(XLogReaderState *state,
[1]
PATCHED:
1 1470.329907
2 1437.096329
4 2966.096948
8 5978.441040
16 11405.538255
32 22933.546058
64 43341.870038
128 73623.837068
256 104754.248661
512 115746.359530
768 106106.691455
1024 91900.079086
2048 84134.278589
4096 62580.875507
-[ RECORD 1 ]----------+-----------
application_name | assb1
sent_lsn | 0/1B8106A8
write_lsn | 0/1B8106A8
flush_lsn | 0/1B8106A8
replay_lsn | 0/1B8106A8
write_lag |
flush_lag |
replay_lag |
wal_read | 104
wal_read_bytes | 10733008
wal_read_time | 1.845
wal_read_buffers | 76662
wal_read_bytes_buffers | 383598808
wal_read_time_buffers | 205.418
sync_state | async
HEAD:
1 1312.054496
2 1449.429321
4 2717.496207
8 5913.361540
16 10762.978907
32 19653.449728
64 41086.124269
128 68548.061171
256 104468.415361
512 114328.943598
768 91751.279309
1024 96403.736757
2048 82155.140270
4096 66160.659511
-[ RECORD 1 ]----+-----------
application_name | assb1
sent_lsn | 0/1AB5BCB8
write_lsn | 0/1AB5BCB8
flush_lsn | 0/1AB5BCB8
replay_lsn | 0/1AB5BCB8
write_lag |
flush_lag |
replay_lag |
wal_read | 71967
wal_read_bytes | 381009080
wal_read_time | 243.616
sync_state | async
[2] Test details:
./configure --prefix=$PWD/inst/ CFLAGS="-O3" > install.log && make -j
8 install > install.log 2>&1 &
1 primary, 1 async standby
cd inst/bin
./pg_ctl -D data -l logfile stop
./pg_ctl -D assbdata -l logfile1 stop
rm -rf data assbdata
rm logfile logfile1
free -m
sudo su -c 'sync; echo 3 > /proc/sys/vm/drop_caches'
free -m
./initdb -D data
rm -rf /home/ubuntu/archived_wal
mkdir /home/ubuntu/archived_wal
cat << EOF >> data/postgresql.conf
shared_buffers = '8GB'
wal_buffers = '1GB'
max_wal_size = '16GB'
max_connections = '5000'
archive_mode = 'on'
archive_command='cp %p /home/ubuntu/archived_wal/%f'
track_wal_io_timing = 'on'
EOF
./pg_ctl -D data -l logfile start
./psql -c "select
pg_create_physical_replication_slot('assb1_repl_slot', true, false)"
postgres
./pg_ctl -D data -l logfile restart
./pg_basebackup -D assbdata
./pg_ctl -D data -l logfile stop
cat << EOF >> assbdata/postgresql.conf
port=5433
primary_conninfo='host=localhost port=5432 dbname=postgres user=ubuntu
application_name=assb1'
primary_slot_name='assb1_repl_slot'
restore_command='cp /home/ubuntu/archived_wal/%f %p'
EOF
touch assbdata/standby.signal
./pg_ctl -D data -l logfile start
./pg_ctl -D assbdata -l logfile1 start
./pgbench -i -s 1 -d postgres
./psql -d postgres -c "ALTER TABLE pgbench_accounts DROP CONSTRAINT
pgbench_accounts_pkey;"
cat << EOF >> insert.sql
\set aid random(1, 10 * :scale)
\set delta random(1, 100000 * :scale)
INSERT INTO pgbench_accounts (aid, bid, abalance) VALUES (:aid, :aid, :delta);
EOF
ulimit -S -n 5000
for c in 1 2 4 8 16 32 64 128 256 512 768 1024 2048 4096; do echo -n
"$c ";./pgbench -n -M prepared -U ubuntu postgres -f insert.sql -c$c
-j$c -T5 2>&1|grep '^tps'|awk '{print $3}';done
--
Bharath Rupireddy
PostgreSQL Contributors Team
RDS Open Source Databases
Amazon Web Services: https://aws.amazon.com
^ permalink raw reply [nested|flat] 22+ messages in thread
* Re: Improve WALRead() to suck data directly from WAL buffers when possible
@ 2022-12-25 11:25 Dilip Kumar <[email protected]>
parent: Bharath Rupireddy <[email protected]>
0 siblings, 1 reply; 22+ messages in thread
From: Dilip Kumar @ 2022-12-25 11:25 UTC (permalink / raw)
To: Bharath Rupireddy <[email protected]>; +Cc: Kyotaro Horiguchi <[email protected]>; [email protected]
On Fri, Dec 23, 2022 at 3:46 PM Bharath Rupireddy
<[email protected]> wrote:
>
> On Mon, Dec 12, 2022 at 8:27 AM Kyotaro Horiguchi
> <[email protected]> wrote:
> >
>
> Thanks for providing thoughts.
>
> > At Fri, 9 Dec 2022 14:33:39 +0530, Bharath Rupireddy <[email protected]> wrote in
> > > The patch introduces concurrent readers for the WAL buffers, so far
> > > only there are concurrent writers. In the patch, WALRead() takes just
> > > one lock (WALBufMappingLock) in shared mode to enable concurrent
> > > readers and does minimal things - checks if the requested WAL page is
> > > present in WAL buffers, if so, copies the page and releases the lock.
> > > I think taking just WALBufMappingLock is enough here as the concurrent
> > > writers depend on it to initialize and replace a page in WAL buffers.
> > >
> > > I'll add this to the next commitfest.
> > >
> > > Thoughts?
> >
> > This adds copying of the whole page (at least) at every WAL *record*
> > read,
>
> In the worst case yes, but that may not always be true. On a typical
> production server with decent write traffic, it happens that the
> callers of WALRead() read a full WAL page of size XLOG_BLCKSZ bytes or
> MAX_SEND_SIZE bytes.
I agree with this.
> > This patch copies the bleeding edge WAL page without recording the
> > (next) insertion point nor checking whether all in-progress insertion
> > behind the target LSN have finished. Thus the copied page may have
> > holes. That being said, the sequential-reading nature and the fact
> > that WAL buffers are zero-initialized may make it work for recovery,
> > but I don't think this also works for replication.
>
> WALRead() callers are smart enough to take the flushed bytes only.
> Although they read the whole WAL page, they calculate the valid bytes.
Right
On first read the patch looks good, although it needs some more
thoughts on 'XXX' comments in the patch.
And also I do not like that XLogReadFromBuffers() is using 3 bools
hit/partial hit/miss, instead of this we can use an enum or some
tristate variable, I think that will be cleaner.
--
Regards,
Dilip Kumar
EnterpriseDB: http://www.enterprisedb.com
^ permalink raw reply [nested|flat] 22+ messages in thread
* Re: Improve WALRead() to suck data directly from WAL buffers when possible
@ 2022-12-26 08:50 Bharath Rupireddy <[email protected]>
parent: Dilip Kumar <[email protected]>
0 siblings, 0 replies; 22+ messages in thread
From: Bharath Rupireddy @ 2022-12-26 08:50 UTC (permalink / raw)
To: Dilip Kumar <[email protected]>; +Cc: Kyotaro Horiguchi <[email protected]>; [email protected]
On Sun, Dec 25, 2022 at 4:55 PM Dilip Kumar <[email protected]> wrote:
>
> > > This adds copying of the whole page (at least) at every WAL *record*
> > > read,
> >
> > In the worst case yes, but that may not always be true. On a typical
> > production server with decent write traffic, it happens that the
> > callers of WALRead() read a full WAL page of size XLOG_BLCKSZ bytes or
> > MAX_SEND_SIZE bytes.
>
> I agree with this.
>
> > > This patch copies the bleeding edge WAL page without recording the
> > > (next) insertion point nor checking whether all in-progress insertion
> > > behind the target LSN have finished. Thus the copied page may have
> > > holes. That being said, the sequential-reading nature and the fact
> > > that WAL buffers are zero-initialized may make it work for recovery,
> > > but I don't think this also works for replication.
> >
> > WALRead() callers are smart enough to take the flushed bytes only.
> > Although they read the whole WAL page, they calculate the valid bytes.
>
> Right
>
> On first read the patch looks good, although it needs some more
> thoughts on 'XXX' comments in the patch.
Thanks a lot for reviewing.
Here are some open points that I mentioned in v1 patch:
1.
+ * XXX: Perhaps, measuring the immediate lock availability and its impact
+ * on concurrent WAL writers is a good idea here.
It was shown in my testng upthread [1] that the patch does no harm in
this regard. It will be great if other members try testing in their
respective environments and use cases.
2.
+ * XXX: Perhaps, returning if lock is not immediately available a good idea
+ * here. The caller can then go ahead with reading WAL from WAL file.
After thinking a bit more on this, ISTM that doing the above is right
to not cause any contention when the lock is busy. I've done so in the
v2 patch.
3.
+ * XXX: Perhaps, quickly finding if the given WAL record is in WAL buffers
+ * a good idea here. This avoids unnecessary lock acquire-release cycles.
+ * One way to do that is by maintaining oldest WAL record that's currently
+ * present in WAL buffers.
I think by doing the above we might end up creating a new point of
contention. Because shared variables to track min and max available
LSNs in the WAL buffers will need to be protected against all the
concurrent writers. Also, with the change that's done in (2) above,
that is, quickly exiting if the lock was busy, this comment seems
unnecessary to worry about. Hence, I decided to leave it there.
4.
+ * XXX: Perhaps, we can further go and validate the found page header,
+ * record header and record at least in assert builds, something like
+ * the xlogreader.c does and return if any of those validity checks
+ * fail. Having said that, we stick to the minimal checks for now.
I was being over-cautious initially. The fact that we acquire
WALBufMappingLock while reading the needed WAL buffer page itself
guarantees that no one else initializes it/makes it ready for next use
in AdvanceXLInsertBuffer(). The checks that we have for page header
(xlp_magic, xlp_pageaddr and xlp_tli) in the patch are enough for us
to ensure that we're not reading a page that got just initialized. The
callers will anyway perform extensive checks on page and record in
XLogReaderValidatePageHeader() and ValidXLogRecordHeader()
respectively. If any such failures occur after reading WAL from WAL
buffers, then that must be treated as a bug IMO. Hence, I don't think
we need to do the above.
> And also I do not like that XLogReadFromBuffers() is using 3 bools
> hit/partial hit/miss, instead of this we can use an enum or some
> tristate variable, I think that will be cleaner.
Yeah, that seems more verbose, all that information can be deduced
from requested bytes and read bytes, I've done so in the v2 patch.
Please review the attached v2 patch further.
I'm also attaching two helper patches (as .txt files) herewith for
testing that basically adds WAL read stats -
USE-ON-HEAD-Collect-WAL-read-from-file-stats.txt - apply on HEAD and
monitor pg_stat_replication for per-walsender WAL read from WAL file
stats. USE-ON-PATCH-Collect-WAL-read-from-buffers-and-file-stats.txt -
apply on v2 patch and monitor pg_stat_replication for per-walsender
WAL read from WAL buffers and WAL file stats.
[1] https://www.postgresql.org/message-id/CALj2ACXUbvON86vgwTkum8ab3bf1%3DHkMxQ5hZJZS3ZcJn8NEXQ%40mail.g...
--
Bharath Rupireddy
PostgreSQL Contributors Team
RDS Open Source Databases
Amazon Web Services: https://aws.amazon.com
From 6517e50f482f88ea5185609ff4dcf0e0256475d5 Mon Sep 17 00:00:00 2001
From: Bharath Rupireddy <[email protected]>
Date: Mon, 26 Dec 2022 08:14:11 +0000
Subject: [PATCH v2] Collect WAL read from file stats for WAL senders
---
doc/src/sgml/monitoring.sgml | 31 +++++++++++
src/backend/access/transam/xlogreader.c | 33 ++++++++++--
src/backend/access/transam/xlogutils.c | 2 +-
src/backend/catalog/system_views.sql | 5 +-
src/backend/replication/walsender.c | 58 +++++++++++++++++++--
src/bin/pg_waldump/pg_waldump.c | 2 +-
src/include/access/xlogreader.h | 21 ++++++--
src/include/catalog/pg_proc.dat | 6 +--
src/include/replication/walsender_private.h | 4 ++
src/test/regress/expected/rules.out | 7 ++-
10 files changed, 151 insertions(+), 18 deletions(-)
diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml
index 363b183e5f..fdf4c7d774 100644
--- a/doc/src/sgml/monitoring.sgml
+++ b/doc/src/sgml/monitoring.sgml
@@ -2615,6 +2615,37 @@ SELECT pid, wait_event_type, wait_event FROM pg_stat_activity WHERE wait_event i
Send time of last reply message received from standby server
</para></entry>
</row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>wal_read</structfield> <type>bigint</type>
+ </para>
+ <para>
+ Number of times WAL data is read from disk
+ </para></entry>
+ </row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>wal_read_bytes</structfield> <type>numeric</type>
+ </para>
+ <para>
+ Total amount of WAL read from disk in bytes
+ </para></entry>
+ </row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>wal_read_time</structfield> <type>double precision</type>
+ </para>
+ <para>
+ Total amount of time spent reading WAL from disk via
+ <function>WALRead</function> request, in milliseconds
+ (if <xref linkend="guc-track-wal-io-timing"/> is enabled,
+ otherwise zero).
+ </para></entry>
+ </row>
+
</tbody>
</tgroup>
</table>
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index a38a80e049..7453724a07 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -31,6 +31,7 @@
#include "access/xlogrecord.h"
#include "catalog/pg_control.h"
#include "common/pg_lzcompress.h"
+#include "portability/instr_time.h"
#include "replication/origin.h"
#ifndef FRONTEND
@@ -1489,9 +1490,9 @@ err:
* WAL buffers when possible.
*/
bool
-WALRead(XLogReaderState *state,
- char *buf, XLogRecPtr startptr, Size count, TimeLineID tli,
- WALReadError *errinfo)
+WALRead(XLogReaderState *state, char *buf, XLogRecPtr startptr, Size count,
+ TimeLineID tli, WALReadError *errinfo, WALReadStats *stats,
+ bool capture_wal_io_timing)
{
char *p;
XLogRecPtr recptr;
@@ -1506,6 +1507,7 @@ WALRead(XLogReaderState *state,
uint32 startoff;
int segbytes;
int readbytes;
+ instr_time start;
startoff = XLogSegmentOffset(recptr, state->segcxt.ws_segsize);
@@ -1540,6 +1542,10 @@ WALRead(XLogReaderState *state,
else
segbytes = nbytes;
+ /* Measure I/O timing to read WAL data if requested by the caller. */
+ if (stats != NULL && capture_wal_io_timing)
+ INSTR_TIME_SET_CURRENT(start);
+
#ifndef FRONTEND
pgstat_report_wait_start(WAIT_EVENT_WAL_READ);
#endif
@@ -1552,6 +1558,27 @@ WALRead(XLogReaderState *state,
pgstat_report_wait_end();
#endif
+ /* Collect I/O stats if requested by the caller. */
+ if (stats != NULL)
+ {
+ /* Increment the number of times WAL is read from disk. */
+ stats->wal_read++;
+
+ /* Collect bytes read. */
+ if (readbytes > 0)
+ stats->wal_read_bytes += readbytes;
+
+ /* Increment the I/O timing. */
+ if (capture_wal_io_timing)
+ {
+ instr_time duration;
+
+ INSTR_TIME_SET_CURRENT(duration);
+ INSTR_TIME_SUBTRACT(duration, start);
+ stats->wal_read_time += INSTR_TIME_GET_MICROSEC(duration);
+ }
+ }
+
if (readbytes <= 0)
{
errinfo->wre_errno = errno;
diff --git a/src/backend/access/transam/xlogutils.c b/src/backend/access/transam/xlogutils.c
index 563cba258d..372de2c7d8 100644
--- a/src/backend/access/transam/xlogutils.c
+++ b/src/backend/access/transam/xlogutils.c
@@ -1027,7 +1027,7 @@ read_local_xlog_page_guts(XLogReaderState *state, XLogRecPtr targetPagePtr,
* zero-padded up to the page boundary if it's incomplete.
*/
if (!WALRead(state, cur_page, targetPagePtr, XLOG_BLCKSZ, tli,
- &errinfo))
+ &errinfo, NULL, false))
WALReadRaiseError(&errinfo);
/* number of valid bytes in the buffer */
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index 2d8104b090..b47f44a852 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -892,7 +892,10 @@ CREATE VIEW pg_stat_replication AS
W.replay_lag,
W.sync_priority,
W.sync_state,
- W.reply_time
+ W.reply_time,
+ W.wal_read,
+ W.wal_read_bytes,
+ W.wal_read_time
FROM pg_stat_get_activity(NULL) AS S
JOIN pg_stat_get_wal_senders() AS W ON (S.pid = W.pid)
LEFT JOIN pg_authid AS U ON (S.usesysid = U.oid);
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index c11bb3716f..fa02e327f2 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -259,7 +259,7 @@ static bool TransactionIdInRecentPast(TransactionId xid, uint32 epoch);
static void WalSndSegmentOpen(XLogReaderState *state, XLogSegNo nextSegNo,
TimeLineID *tli_p);
-
+static void WalSndAccumulateWalReadStats(WALReadStats *stats);
/* Initialize walsender process before entering the main command loop */
void
@@ -907,6 +907,7 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
WALReadError errinfo;
XLogSegNo segno;
TimeLineID currTLI = GetWALInsertionTimeLine();
+ WALReadStats stats;
/*
* Since logical decoding is only permitted on a primary server, we know
@@ -932,6 +933,8 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
else
count = flushptr - targetPagePtr; /* part of the page available */
+ MemSet(&stats, 0, sizeof(WALReadStats));
+
/* now actually read the data, we know it's there */
if (!WALRead(state,
cur_page,
@@ -940,9 +943,13 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
state->seg.ws_tli, /* Pass the current TLI because only
* WalSndSegmentOpen controls whether new
* TLI is needed. */
- &errinfo))
+ &errinfo,
+ &stats,
+ track_wal_io_timing))
WALReadRaiseError(&errinfo);
+ WalSndAccumulateWalReadStats(&stats);
+
/*
* After reading into the buffer, check that what we read was valid. We do
* this after reading, because even though the segment was present when we
@@ -2610,6 +2617,9 @@ InitWalSenderSlot(void)
walsnd->sync_standby_priority = 0;
walsnd->latch = &MyProc->procLatch;
walsnd->replyTime = 0;
+ walsnd->wal_read_stats.wal_read = 0;
+ walsnd->wal_read_stats.wal_read_bytes = 0;
+ walsnd->wal_read_stats.wal_read_time = 0;
SpinLockRelease(&walsnd->mutex);
/* don't need the lock anymore */
MyWalSnd = (WalSnd *) walsnd;
@@ -2730,6 +2740,7 @@ XLogSendPhysical(void)
Size nbytes;
XLogSegNo segno;
WALReadError errinfo;
+ WALReadStats stats;
/* If requested switch the WAL sender to the stopping state. */
if (got_STOPPING)
@@ -2945,6 +2956,8 @@ XLogSendPhysical(void)
enlargeStringInfo(&output_message, nbytes);
retry:
+ MemSet(&stats, 0, sizeof(WALReadStats));
+
if (!WALRead(xlogreader,
&output_message.data[output_message.len],
startptr,
@@ -2952,9 +2965,13 @@ retry:
xlogreader->seg.ws_tli, /* Pass the current TLI because
* only WalSndSegmentOpen controls
* whether new TLI is needed. */
- &errinfo))
+ &errinfo,
+ &stats,
+ track_wal_io_timing))
WALReadRaiseError(&errinfo);
+ WalSndAccumulateWalReadStats(&stats);
+
/* See logical_read_xlog_page(). */
XLByteToSeg(startptr, segno, xlogreader->segcxt.ws_segsize);
CheckXLogRemoved(segno, xlogreader->seg.ws_tli);
@@ -3458,7 +3475,7 @@ offset_to_interval(TimeOffset offset)
Datum
pg_stat_get_wal_senders(PG_FUNCTION_ARGS)
{
-#define PG_STAT_GET_WAL_SENDERS_COLS 12
+#define PG_STAT_GET_WAL_SENDERS_COLS 15
ReturnSetInfo *rsinfo = (ReturnSetInfo *) fcinfo->resultinfo;
SyncRepStandbyData *sync_standbys;
int num_standbys;
@@ -3487,9 +3504,13 @@ pg_stat_get_wal_senders(PG_FUNCTION_ARGS)
WalSndState state;
TimestampTz replyTime;
bool is_sync_standby;
+ int64 wal_read;
+ uint64 wal_read_bytes;
+ int64 wal_read_time;
Datum values[PG_STAT_GET_WAL_SENDERS_COLS];
bool nulls[PG_STAT_GET_WAL_SENDERS_COLS] = {0};
int j;
+ char buf[256];
/* Collect data from shared memory */
SpinLockAcquire(&walsnd->mutex);
@@ -3509,6 +3530,9 @@ pg_stat_get_wal_senders(PG_FUNCTION_ARGS)
applyLag = walsnd->applyLag;
priority = walsnd->sync_standby_priority;
replyTime = walsnd->replyTime;
+ wal_read = walsnd->wal_read_stats.wal_read;
+ wal_read_bytes = walsnd->wal_read_stats.wal_read_bytes;
+ wal_read_time = walsnd->wal_read_stats.wal_read_time;
SpinLockRelease(&walsnd->mutex);
/*
@@ -3605,6 +3629,18 @@ pg_stat_get_wal_senders(PG_FUNCTION_ARGS)
nulls[11] = true;
else
values[11] = TimestampTzGetDatum(replyTime);
+
+ values[12] = Int64GetDatum(wal_read);
+
+ /* Convert to numeric. */
+ snprintf(buf, sizeof buf, UINT64_FORMAT, wal_read_bytes);
+ values[13] = DirectFunctionCall3(numeric_in,
+ CStringGetDatum(buf),
+ ObjectIdGetDatum(0),
+ Int32GetDatum(-1));
+
+ /* Convert counter from microsec to millisec for display. */
+ values[14] = Float8GetDatum(((double) wal_read_time) / 1000.0);
}
tuplestore_putvalues(rsinfo->setResult, rsinfo->setDesc,
@@ -3849,3 +3885,17 @@ LagTrackerRead(int head, XLogRecPtr lsn, TimestampTz now)
Assert(time != 0);
return now - time;
}
+
+/*
+ * Function to accumulate WAL Read stats for WAL sender.
+ */
+static void
+WalSndAccumulateWalReadStats(WALReadStats *stats)
+{
+ /* Collect I/O stats for walsender. */
+ SpinLockAcquire(&MyWalSnd->mutex);
+ MyWalSnd->wal_read_stats.wal_read += stats->wal_read;
+ MyWalSnd->wal_read_stats.wal_read_bytes += stats->wal_read_bytes;
+ MyWalSnd->wal_read_stats.wal_read_time += stats->wal_read_time;
+ SpinLockRelease(&MyWalSnd->mutex);
+}
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index 9993378ca5..698ce1e9f7 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -364,7 +364,7 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
}
if (!WALRead(state, readBuff, targetPagePtr, count, private->timeline,
- &errinfo))
+ &errinfo, NULL, false))
{
WALOpenSegment *seg = &errinfo.wre_seg;
char fname[MAXPGPATH];
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index e87f91316a..26a2c975de 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -389,9 +389,24 @@ typedef struct WALReadError
WALOpenSegment wre_seg; /* Segment we tried to read from. */
} WALReadError;
-extern bool WALRead(XLogReaderState *state,
- char *buf, XLogRecPtr startptr, Size count,
- TimeLineID tli, WALReadError *errinfo);
+/*
+ * WAL read stats from WALRead that the callers can use.
+ */
+typedef struct WALReadStats
+{
+ /* Number of times WAL read from disk. */
+ int64 wal_read;
+
+ /* Total amount of WAL read from disk in bytes. */
+ uint64 wal_read_bytes;
+
+ /* Total amount of time spent reading WAL from disk. */
+ int64 wal_read_time;
+} WALReadStats;
+
+extern bool WALRead(XLogReaderState *state, char *buf, XLogRecPtr startptr,
+ Size count, TimeLineID tli, WALReadError *errinfo,
+ WALReadStats *stats, bool capture_wal_io_timing);
/* Functions for decoding an XLogRecord */
diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat
index 7056c95371..18320cf846 100644
--- a/src/include/catalog/pg_proc.dat
+++ b/src/include/catalog/pg_proc.dat
@@ -5391,9 +5391,9 @@
proname => 'pg_stat_get_wal_senders', prorows => '10', proisstrict => 'f',
proretset => 't', provolatile => 's', proparallel => 'r',
prorettype => 'record', proargtypes => '',
- proallargtypes => '{int4,text,pg_lsn,pg_lsn,pg_lsn,pg_lsn,interval,interval,interval,int4,text,timestamptz}',
- proargmodes => '{o,o,o,o,o,o,o,o,o,o,o,o}',
- proargnames => '{pid,state,sent_lsn,write_lsn,flush_lsn,replay_lsn,write_lag,flush_lag,replay_lag,sync_priority,sync_state,reply_time}',
+ proallargtypes => '{int4,text,pg_lsn,pg_lsn,pg_lsn,pg_lsn,interval,interval,interval,int4,text,timestamptz,int8,numeric,float8}',
+ proargmodes => '{o,o,o,o,o,o,o,o,o,o,o,o,o,o,o}',
+ proargnames => '{pid,state,sent_lsn,write_lsn,flush_lsn,replay_lsn,write_lag,flush_lag,replay_lag,sync_priority,sync_state,reply_time,wal_read,wal_read_bytes,wal_read_time}',
prosrc => 'pg_stat_get_wal_senders' },
{ oid => '3317', descr => 'statistics: information about WAL receiver',
proname => 'pg_stat_get_wal_receiver', proisstrict => 'f', provolatile => 's',
diff --git a/src/include/replication/walsender_private.h b/src/include/replication/walsender_private.h
index 7897c74589..35413ea0d2 100644
--- a/src/include/replication/walsender_private.h
+++ b/src/include/replication/walsender_private.h
@@ -13,6 +13,7 @@
#define _WALSENDER_PRIVATE_H
#include "access/xlog.h"
+#include "access/xlogreader.h"
#include "nodes/nodes.h"
#include "replication/syncrep.h"
#include "storage/latch.h"
@@ -78,6 +79,9 @@ typedef struct WalSnd
* Timestamp of the last message received from standby.
*/
TimestampTz replyTime;
+
+ /* WAL read stats for walsender. */
+ WALReadStats wal_read_stats;
} WalSnd;
extern PGDLLIMPORT WalSnd *MyWalSnd;
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index fb9f936d43..fd9d298e79 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -2054,9 +2054,12 @@ pg_stat_replication| SELECT s.pid,
w.replay_lag,
w.sync_priority,
w.sync_state,
- w.reply_time
+ w.reply_time,
+ w.wal_read,
+ w.wal_read_bytes,
+ w.wal_read_time
FROM ((pg_stat_get_activity(NULL::integer) s(datid, pid, usesysid, application_name, state, query, wait_event_type, wait_event, xact_start, query_start, backend_start, state_change, client_addr, client_hostname, client_port, backend_xid, backend_xmin, backend_type, ssl, sslversion, sslcipher, sslbits, ssl_client_dn, ssl_client_serial, ssl_issuer_dn, gss_auth, gss_princ, gss_enc, leader_pid, query_id)
- JOIN pg_stat_get_wal_senders() w(pid, state, sent_lsn, write_lsn, flush_lsn, replay_lsn, write_lag, flush_lag, replay_lag, sync_priority, sync_state, reply_time) ON ((s.pid = w.pid)))
+ JOIN pg_stat_get_wal_senders() w(pid, state, sent_lsn, write_lsn, flush_lsn, replay_lsn, write_lag, flush_lag, replay_lag, sync_priority, sync_state, reply_time, wal_read, wal_read_bytes, wal_read_time) ON ((s.pid = w.pid)))
LEFT JOIN pg_authid u ON ((s.usesysid = u.oid)));
pg_stat_replication_slots| SELECT s.slot_name,
s.spill_txns,
--
2.34.1
From f90dfcbd1968280feec6d116568697225854ac40 Mon Sep 17 00:00:00 2001
From: Bharath Rupireddy <[email protected]>
Date: Mon, 26 Dec 2022 08:13:07 +0000
Subject: [PATCH v2] Collect WAL read from buffers and file stats for WAL
senders
---
doc/src/sgml/monitoring.sgml | 61 +++++++++++++++
src/backend/access/transam/xlogreader.c | 56 +++++++++++++-
src/backend/access/transam/xlogutils.c | 2 +-
src/backend/catalog/system_views.sql | 8 +-
src/backend/replication/walsender.c | 85 ++++++++++++++++++++-
src/bin/pg_waldump/pg_waldump.c | 2 +-
src/include/access/xlogreader.h | 30 +++++++-
src/include/catalog/pg_proc.dat | 6 +-
src/include/replication/walsender_private.h | 4 +
src/test/regress/expected/rules.out | 10 ++-
10 files changed, 246 insertions(+), 18 deletions(-)
diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml
index 363b183e5f..239e0b0db9 100644
--- a/doc/src/sgml/monitoring.sgml
+++ b/doc/src/sgml/monitoring.sgml
@@ -2615,6 +2615,67 @@ SELECT pid, wait_event_type, wait_event FROM pg_stat_activity WHERE wait_event i
Send time of last reply message received from standby server
</para></entry>
</row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>wal_read</structfield> <type>bigint</type>
+ </para>
+ <para>
+ Number of times WAL data is read from disk
+ </para></entry>
+ </row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>wal_read_bytes</structfield> <type>numeric</type>
+ </para>
+ <para>
+ Total amount of WAL read from disk in bytes
+ </para></entry>
+ </row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>wal_read_time</structfield> <type>double precision</type>
+ </para>
+ <para>
+ Total amount of time spent reading WAL from disk via
+ <function>WALRead</function> request, in milliseconds
+ (if <xref linkend="guc-track-wal-io-timing"/> is enabled,
+ otherwise zero).
+ </para></entry>
+ </row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>wal_read_buffers</structfield> <type>bigint</type>
+ </para>
+ <para>
+ Number of times WAL data is read from WAL buffers
+ </para></entry>
+ </row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>wal_read_bytes_buffers</structfield> <type>numeric</type>
+ </para>
+ <para>
+ Total amount of WAL read from WAL buffers in bytes
+ </para></entry>
+ </row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>wal_read_time_buffers</structfield> <type>double precision</type>
+ </para>
+ <para>
+ Total amount of time spent reading WAL from WAL buffers via
+ <function>WALRead</function> request, in milliseconds
+ (if <xref linkend="guc-track-wal-io-timing"/> is enabled,
+ otherwise zero).
+ </para></entry>
+ </row>
+
</tbody>
</tgroup>
</table>
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index 4a2e7af169..b9dfd4fde7 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -31,6 +31,7 @@
#include "access/xlogrecord.h"
#include "catalog/pg_control.h"
#include "common/pg_lzcompress.h"
+#include "portability/instr_time.h"
#include "replication/origin.h"
#ifndef FRONTEND
@@ -1488,9 +1489,9 @@ err:
* When possible, this function reads data directly from WAL buffers.
*/
bool
-WALRead(XLogReaderState *state,
- char *buf, XLogRecPtr startptr, Size count, TimeLineID tli,
- WALReadError *errinfo)
+WALRead(XLogReaderState *state, char *buf, XLogRecPtr startptr, Size count,
+ TimeLineID tli, WALReadError *errinfo, WALReadStats *stats,
+ bool capture_wal_io_timing)
{
char *p;
XLogRecPtr recptr;
@@ -1510,10 +1511,33 @@ WALRead(XLogReaderState *state,
if (!RecoveryInProgress() &&
tli == GetWALInsertionTimeLine())
{
+ instr_time start;
+
+ /* Measure I/O timing to read WAL data if requested by the caller. */
+ if (stats != NULL && capture_wal_io_timing)
+ INSTR_TIME_SET_CURRENT(start);
+
pgstat_report_wait_start(WAIT_EVENT_WAL_READ);
XLogReadFromBuffers(startptr, tli, count, buf, &read_bytes);
pgstat_report_wait_end();
+ /* Collect I/O stats if requested by the caller. */
+ if (stats != NULL && read_bytes > 0)
+ {
+ stats->wal_read_buffers++;
+ stats->wal_read_bytes_buffers += read_bytes;
+
+ /* Increment the I/O timing. */
+ if (capture_wal_io_timing)
+ {
+ instr_time duration;
+
+ INSTR_TIME_SET_CURRENT(duration);
+ INSTR_TIME_SUBTRACT(duration, start);
+ stats->wal_read_time_buffers += INSTR_TIME_GET_MICROSEC(duration);
+ }
+ }
+
/*
* Check if we have read fully (hit), partially (partial hit) or
* nothing (miss) from WAL buffers. If we have read either partially or
@@ -1549,6 +1573,7 @@ WALRead(XLogReaderState *state,
uint32 startoff;
int segbytes;
int readbytes;
+ instr_time start;
startoff = XLogSegmentOffset(recptr, state->segcxt.ws_segsize);
@@ -1583,6 +1608,10 @@ WALRead(XLogReaderState *state,
else
segbytes = nbytes;
+ /* Measure I/O timing to read WAL data if requested by the caller. */
+ if (stats != NULL && capture_wal_io_timing)
+ INSTR_TIME_SET_CURRENT(start);
+
#ifndef FRONTEND
pgstat_report_wait_start(WAIT_EVENT_WAL_READ);
#endif
@@ -1595,6 +1624,27 @@ WALRead(XLogReaderState *state,
pgstat_report_wait_end();
#endif
+ /* Collect I/O stats if requested by the caller. */
+ if (stats != NULL)
+ {
+ /* Increment the number of times WAL is read from disk. */
+ stats->wal_read++;
+
+ /* Collect bytes read. */
+ if (readbytes > 0)
+ stats->wal_read_bytes += readbytes;
+
+ /* Increment the I/O timing. */
+ if (capture_wal_io_timing)
+ {
+ instr_time duration;
+
+ INSTR_TIME_SET_CURRENT(duration);
+ INSTR_TIME_SUBTRACT(duration, start);
+ stats->wal_read_time += INSTR_TIME_GET_MICROSEC(duration);
+ }
+ }
+
if (readbytes <= 0)
{
errinfo->wre_errno = errno;
diff --git a/src/backend/access/transam/xlogutils.c b/src/backend/access/transam/xlogutils.c
index 563cba258d..372de2c7d8 100644
--- a/src/backend/access/transam/xlogutils.c
+++ b/src/backend/access/transam/xlogutils.c
@@ -1027,7 +1027,7 @@ read_local_xlog_page_guts(XLogReaderState *state, XLogRecPtr targetPagePtr,
* zero-padded up to the page boundary if it's incomplete.
*/
if (!WALRead(state, cur_page, targetPagePtr, XLOG_BLCKSZ, tli,
- &errinfo))
+ &errinfo, NULL, false))
WALReadRaiseError(&errinfo);
/* number of valid bytes in the buffer */
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index 2d8104b090..bf6315df27 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -892,7 +892,13 @@ CREATE VIEW pg_stat_replication AS
W.replay_lag,
W.sync_priority,
W.sync_state,
- W.reply_time
+ W.reply_time,
+ W.wal_read,
+ W.wal_read_bytes,
+ W.wal_read_time,
+ W.wal_read_buffers,
+ W.wal_read_bytes_buffers,
+ W.wal_read_time_buffers
FROM pg_stat_get_activity(NULL) AS S
JOIN pg_stat_get_wal_senders() AS W ON (S.pid = W.pid)
LEFT JOIN pg_authid AS U ON (S.usesysid = U.oid);
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index c11bb3716f..d3393b2b63 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -259,7 +259,7 @@ static bool TransactionIdInRecentPast(TransactionId xid, uint32 epoch);
static void WalSndSegmentOpen(XLogReaderState *state, XLogSegNo nextSegNo,
TimeLineID *tli_p);
-
+static void WalSndAccumulateWalReadStats(WALReadStats *stats);
/* Initialize walsender process before entering the main command loop */
void
@@ -907,6 +907,7 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
WALReadError errinfo;
XLogSegNo segno;
TimeLineID currTLI = GetWALInsertionTimeLine();
+ WALReadStats stats;
/*
* Since logical decoding is only permitted on a primary server, we know
@@ -932,6 +933,8 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
else
count = flushptr - targetPagePtr; /* part of the page available */
+ MemSet(&stats, 0, sizeof(WALReadStats));
+
/* now actually read the data, we know it's there */
if (!WALRead(state,
cur_page,
@@ -940,9 +943,13 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
state->seg.ws_tli, /* Pass the current TLI because only
* WalSndSegmentOpen controls whether new
* TLI is needed. */
- &errinfo))
+ &errinfo,
+ &stats,
+ track_wal_io_timing))
WALReadRaiseError(&errinfo);
+ WalSndAccumulateWalReadStats(&stats);
+
/*
* After reading into the buffer, check that what we read was valid. We do
* this after reading, because even though the segment was present when we
@@ -2610,6 +2617,12 @@ InitWalSenderSlot(void)
walsnd->sync_standby_priority = 0;
walsnd->latch = &MyProc->procLatch;
walsnd->replyTime = 0;
+ walsnd->wal_read_stats.wal_read = 0;
+ walsnd->wal_read_stats.wal_read_bytes = 0;
+ walsnd->wal_read_stats.wal_read_time = 0;
+ walsnd->wal_read_stats.wal_read_buffers = 0;
+ walsnd->wal_read_stats.wal_read_bytes_buffers = 0;
+ walsnd->wal_read_stats.wal_read_time_buffers = 0;
SpinLockRelease(&walsnd->mutex);
/* don't need the lock anymore */
MyWalSnd = (WalSnd *) walsnd;
@@ -2730,6 +2743,7 @@ XLogSendPhysical(void)
Size nbytes;
XLogSegNo segno;
WALReadError errinfo;
+ WALReadStats stats;
/* If requested switch the WAL sender to the stopping state. */
if (got_STOPPING)
@@ -2945,6 +2959,8 @@ XLogSendPhysical(void)
enlargeStringInfo(&output_message, nbytes);
retry:
+ MemSet(&stats, 0, sizeof(WALReadStats));
+
if (!WALRead(xlogreader,
&output_message.data[output_message.len],
startptr,
@@ -2952,9 +2968,13 @@ retry:
xlogreader->seg.ws_tli, /* Pass the current TLI because
* only WalSndSegmentOpen controls
* whether new TLI is needed. */
- &errinfo))
+ &errinfo,
+ &stats,
+ track_wal_io_timing))
WALReadRaiseError(&errinfo);
+ WalSndAccumulateWalReadStats(&stats);
+
/* See logical_read_xlog_page(). */
XLByteToSeg(startptr, segno, xlogreader->segcxt.ws_segsize);
CheckXLogRemoved(segno, xlogreader->seg.ws_tli);
@@ -3458,7 +3478,7 @@ offset_to_interval(TimeOffset offset)
Datum
pg_stat_get_wal_senders(PG_FUNCTION_ARGS)
{
-#define PG_STAT_GET_WAL_SENDERS_COLS 12
+#define PG_STAT_GET_WAL_SENDERS_COLS 18
ReturnSetInfo *rsinfo = (ReturnSetInfo *) fcinfo->resultinfo;
SyncRepStandbyData *sync_standbys;
int num_standbys;
@@ -3487,9 +3507,16 @@ pg_stat_get_wal_senders(PG_FUNCTION_ARGS)
WalSndState state;
TimestampTz replyTime;
bool is_sync_standby;
+ int64 wal_read;
+ uint64 wal_read_bytes;
+ int64 wal_read_time;
+ int64 wal_read_buffers;
+ uint64 wal_read_bytes_buffers;
+ int64 wal_read_time_buffers;
Datum values[PG_STAT_GET_WAL_SENDERS_COLS];
bool nulls[PG_STAT_GET_WAL_SENDERS_COLS] = {0};
int j;
+ char buf[256];
/* Collect data from shared memory */
SpinLockAcquire(&walsnd->mutex);
@@ -3509,6 +3536,12 @@ pg_stat_get_wal_senders(PG_FUNCTION_ARGS)
applyLag = walsnd->applyLag;
priority = walsnd->sync_standby_priority;
replyTime = walsnd->replyTime;
+ wal_read = walsnd->wal_read_stats.wal_read;
+ wal_read_bytes = walsnd->wal_read_stats.wal_read_bytes;
+ wal_read_time = walsnd->wal_read_stats.wal_read_time;
+ wal_read_buffers = walsnd->wal_read_stats.wal_read_buffers;
+ wal_read_bytes_buffers = walsnd->wal_read_stats.wal_read_bytes_buffers;
+ wal_read_time_buffers = walsnd->wal_read_stats.wal_read_time_buffers;
SpinLockRelease(&walsnd->mutex);
/*
@@ -3605,6 +3638,31 @@ pg_stat_get_wal_senders(PG_FUNCTION_ARGS)
nulls[11] = true;
else
values[11] = TimestampTzGetDatum(replyTime);
+
+ values[12] = Int64GetDatum(wal_read);
+
+ /* Convert to numeric. */
+ snprintf(buf, sizeof buf, UINT64_FORMAT, wal_read_bytes);
+ values[13] = DirectFunctionCall3(numeric_in,
+ CStringGetDatum(buf),
+ ObjectIdGetDatum(0),
+ Int32GetDatum(-1));
+
+ /* Convert counter from microsec to millisec for display. */
+ values[14] = Float8GetDatum(((double) wal_read_time) / 1000.0);
+
+ values[15] = Int64GetDatum(wal_read_buffers);
+
+ /* Convert to numeric. */
+ MemSet(buf, '\0', sizeof buf);
+ snprintf(buf, sizeof buf, UINT64_FORMAT, wal_read_bytes_buffers);
+ values[16] = DirectFunctionCall3(numeric_in,
+ CStringGetDatum(buf),
+ ObjectIdGetDatum(0),
+ Int32GetDatum(-1));
+
+ /* Convert counter from microsec to millisec for display. */
+ values[17] = Float8GetDatum(((double) wal_read_time_buffers) / 1000.0);
}
tuplestore_putvalues(rsinfo->setResult, rsinfo->setDesc,
@@ -3849,3 +3907,22 @@ LagTrackerRead(int head, XLogRecPtr lsn, TimestampTz now)
Assert(time != 0);
return now - time;
}
+
+/*
+ * Function to accumulate WAL Read stats for WAL sender.
+ */
+static void
+WalSndAccumulateWalReadStats(WALReadStats *stats)
+{
+ /* Collect I/O stats for walsender. */
+ SpinLockAcquire(&MyWalSnd->mutex);
+ MyWalSnd->wal_read_stats.wal_read += stats->wal_read;
+ MyWalSnd->wal_read_stats.wal_read_bytes += stats->wal_read_bytes;
+ MyWalSnd->wal_read_stats.wal_read_time += stats->wal_read_time;
+ MyWalSnd->wal_read_stats.wal_read_buffers += stats->wal_read_buffers;
+ MyWalSnd->wal_read_stats.wal_read_bytes_buffers +=
+ stats->wal_read_bytes_buffers;
+ MyWalSnd->wal_read_stats.wal_read_time_buffers +=
+ stats->wal_read_time_buffers;
+ SpinLockRelease(&MyWalSnd->mutex);
+}
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index 9993378ca5..698ce1e9f7 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -364,7 +364,7 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
}
if (!WALRead(state, readBuff, targetPagePtr, count, private->timeline,
- &errinfo))
+ &errinfo, NULL, false))
{
WALOpenSegment *seg = &errinfo.wre_seg;
char fname[MAXPGPATH];
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index e87f91316a..9287114779 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -389,9 +389,33 @@ typedef struct WALReadError
WALOpenSegment wre_seg; /* Segment we tried to read from. */
} WALReadError;
-extern bool WALRead(XLogReaderState *state,
- char *buf, XLogRecPtr startptr, Size count,
- TimeLineID tli, WALReadError *errinfo);
+/*
+ * WAL read stats from WALRead that the callers can use.
+ */
+typedef struct WALReadStats
+{
+ /* Number of times WAL read from disk. */
+ int64 wal_read;
+
+ /* Total amount of WAL read from disk in bytes. */
+ uint64 wal_read_bytes;
+
+ /* Total amount of time spent reading WAL from disk. */
+ int64 wal_read_time;
+
+ /* Number of times WAL read from WAL buffers. */
+ int64 wal_read_buffers;
+
+ /* Total amount of WAL read from WAL buffers in bytes. */
+ uint64 wal_read_bytes_buffers;
+
+ /* Total amount of time spent reading WAL from WAL buffers. */
+ int64 wal_read_time_buffers;
+} WALReadStats;
+
+extern bool WALRead(XLogReaderState *state, char *buf, XLogRecPtr startptr,
+ Size count, TimeLineID tli, WALReadError *errinfo,
+ WALReadStats *stats, bool capture_wal_io_timing);
/* Functions for decoding an XLogRecord */
diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat
index 7056c95371..706a005c2b 100644
--- a/src/include/catalog/pg_proc.dat
+++ b/src/include/catalog/pg_proc.dat
@@ -5391,9 +5391,9 @@
proname => 'pg_stat_get_wal_senders', prorows => '10', proisstrict => 'f',
proretset => 't', provolatile => 's', proparallel => 'r',
prorettype => 'record', proargtypes => '',
- proallargtypes => '{int4,text,pg_lsn,pg_lsn,pg_lsn,pg_lsn,interval,interval,interval,int4,text,timestamptz}',
- proargmodes => '{o,o,o,o,o,o,o,o,o,o,o,o}',
- proargnames => '{pid,state,sent_lsn,write_lsn,flush_lsn,replay_lsn,write_lag,flush_lag,replay_lag,sync_priority,sync_state,reply_time}',
+ proallargtypes => '{int4,text,pg_lsn,pg_lsn,pg_lsn,pg_lsn,interval,interval,interval,int4,text,timestamptz,int8,numeric,float8,int8,numeric,float8}',
+ proargmodes => '{o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o}',
+ proargnames => '{pid,state,sent_lsn,write_lsn,flush_lsn,replay_lsn,write_lag,flush_lag,replay_lag,sync_priority,sync_state,reply_time,wal_read,wal_read_bytes,wal_read_time,wal_read_buffers,wal_read_bytes_buffers,wal_read_time_buffers}',
prosrc => 'pg_stat_get_wal_senders' },
{ oid => '3317', descr => 'statistics: information about WAL receiver',
proname => 'pg_stat_get_wal_receiver', proisstrict => 'f', provolatile => 's',
diff --git a/src/include/replication/walsender_private.h b/src/include/replication/walsender_private.h
index 7897c74589..35413ea0d2 100644
--- a/src/include/replication/walsender_private.h
+++ b/src/include/replication/walsender_private.h
@@ -13,6 +13,7 @@
#define _WALSENDER_PRIVATE_H
#include "access/xlog.h"
+#include "access/xlogreader.h"
#include "nodes/nodes.h"
#include "replication/syncrep.h"
#include "storage/latch.h"
@@ -78,6 +79,9 @@ typedef struct WalSnd
* Timestamp of the last message received from standby.
*/
TimestampTz replyTime;
+
+ /* WAL read stats for walsender. */
+ WALReadStats wal_read_stats;
} WalSnd;
extern PGDLLIMPORT WalSnd *MyWalSnd;
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index fb9f936d43..6ae65981c2 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -2054,9 +2054,15 @@ pg_stat_replication| SELECT s.pid,
w.replay_lag,
w.sync_priority,
w.sync_state,
- w.reply_time
+ w.reply_time,
+ w.wal_read,
+ w.wal_read_bytes,
+ w.wal_read_time,
+ w.wal_read_buffers,
+ w.wal_read_bytes_buffers,
+ w.wal_read_time_buffers
FROM ((pg_stat_get_activity(NULL::integer) s(datid, pid, usesysid, application_name, state, query, wait_event_type, wait_event, xact_start, query_start, backend_start, state_change, client_addr, client_hostname, client_port, backend_xid, backend_xmin, backend_type, ssl, sslversion, sslcipher, sslbits, ssl_client_dn, ssl_client_serial, ssl_issuer_dn, gss_auth, gss_princ, gss_enc, leader_pid, query_id)
- JOIN pg_stat_get_wal_senders() w(pid, state, sent_lsn, write_lsn, flush_lsn, replay_lsn, write_lag, flush_lag, replay_lag, sync_priority, sync_state, reply_time) ON ((s.pid = w.pid)))
+ JOIN pg_stat_get_wal_senders() w(pid, state, sent_lsn, write_lsn, flush_lsn, replay_lsn, write_lag, flush_lag, replay_lag, sync_priority, sync_state, reply_time, wal_read, wal_read_bytes, wal_read_time, wal_read_buffers, wal_read_bytes_buffers, wal_read_time_buffers) ON ((s.pid = w.pid)))
LEFT JOIN pg_authid u ON ((s.usesysid = u.oid)));
pg_stat_replication_slots| SELECT s.slot_name,
s.spill_txns,
--
2.34.1
Attachments:
[text/plain] USE-ON-HEAD-Collect-WAL-read-from-file-stats.txt (15.9K, ../../CALj2ACU9cfAcfVsGwUqXMace_7rfSBJ7+hXVJfVV1jnspTDGHQ@mail.gmail.com/2-USE-ON-HEAD-Collect-WAL-read-from-file-stats.txt)
download | inline diff:
From 6517e50f482f88ea5185609ff4dcf0e0256475d5 Mon Sep 17 00:00:00 2001
From: Bharath Rupireddy <[email protected]>
Date: Mon, 26 Dec 2022 08:14:11 +0000
Subject: [PATCH v2] Collect WAL read from file stats for WAL senders
---
doc/src/sgml/monitoring.sgml | 31 +++++++++++
src/backend/access/transam/xlogreader.c | 33 ++++++++++--
src/backend/access/transam/xlogutils.c | 2 +-
src/backend/catalog/system_views.sql | 5 +-
src/backend/replication/walsender.c | 58 +++++++++++++++++++--
src/bin/pg_waldump/pg_waldump.c | 2 +-
src/include/access/xlogreader.h | 21 ++++++--
src/include/catalog/pg_proc.dat | 6 +--
src/include/replication/walsender_private.h | 4 ++
src/test/regress/expected/rules.out | 7 ++-
10 files changed, 151 insertions(+), 18 deletions(-)
diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml
index 363b183e5f..fdf4c7d774 100644
--- a/doc/src/sgml/monitoring.sgml
+++ b/doc/src/sgml/monitoring.sgml
@@ -2615,6 +2615,37 @@ SELECT pid, wait_event_type, wait_event FROM pg_stat_activity WHERE wait_event i
Send time of last reply message received from standby server
</para></entry>
</row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>wal_read</structfield> <type>bigint</type>
+ </para>
+ <para>
+ Number of times WAL data is read from disk
+ </para></entry>
+ </row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>wal_read_bytes</structfield> <type>numeric</type>
+ </para>
+ <para>
+ Total amount of WAL read from disk in bytes
+ </para></entry>
+ </row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>wal_read_time</structfield> <type>double precision</type>
+ </para>
+ <para>
+ Total amount of time spent reading WAL from disk via
+ <function>WALRead</function> request, in milliseconds
+ (if <xref linkend="guc-track-wal-io-timing"/> is enabled,
+ otherwise zero).
+ </para></entry>
+ </row>
+
</tbody>
</tgroup>
</table>
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index a38a80e049..7453724a07 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -31,6 +31,7 @@
#include "access/xlogrecord.h"
#include "catalog/pg_control.h"
#include "common/pg_lzcompress.h"
+#include "portability/instr_time.h"
#include "replication/origin.h"
#ifndef FRONTEND
@@ -1489,9 +1490,9 @@ err:
* WAL buffers when possible.
*/
bool
-WALRead(XLogReaderState *state,
- char *buf, XLogRecPtr startptr, Size count, TimeLineID tli,
- WALReadError *errinfo)
+WALRead(XLogReaderState *state, char *buf, XLogRecPtr startptr, Size count,
+ TimeLineID tli, WALReadError *errinfo, WALReadStats *stats,
+ bool capture_wal_io_timing)
{
char *p;
XLogRecPtr recptr;
@@ -1506,6 +1507,7 @@ WALRead(XLogReaderState *state,
uint32 startoff;
int segbytes;
int readbytes;
+ instr_time start;
startoff = XLogSegmentOffset(recptr, state->segcxt.ws_segsize);
@@ -1540,6 +1542,10 @@ WALRead(XLogReaderState *state,
else
segbytes = nbytes;
+ /* Measure I/O timing to read WAL data if requested by the caller. */
+ if (stats != NULL && capture_wal_io_timing)
+ INSTR_TIME_SET_CURRENT(start);
+
#ifndef FRONTEND
pgstat_report_wait_start(WAIT_EVENT_WAL_READ);
#endif
@@ -1552,6 +1558,27 @@ WALRead(XLogReaderState *state,
pgstat_report_wait_end();
#endif
+ /* Collect I/O stats if requested by the caller. */
+ if (stats != NULL)
+ {
+ /* Increment the number of times WAL is read from disk. */
+ stats->wal_read++;
+
+ /* Collect bytes read. */
+ if (readbytes > 0)
+ stats->wal_read_bytes += readbytes;
+
+ /* Increment the I/O timing. */
+ if (capture_wal_io_timing)
+ {
+ instr_time duration;
+
+ INSTR_TIME_SET_CURRENT(duration);
+ INSTR_TIME_SUBTRACT(duration, start);
+ stats->wal_read_time += INSTR_TIME_GET_MICROSEC(duration);
+ }
+ }
+
if (readbytes <= 0)
{
errinfo->wre_errno = errno;
diff --git a/src/backend/access/transam/xlogutils.c b/src/backend/access/transam/xlogutils.c
index 563cba258d..372de2c7d8 100644
--- a/src/backend/access/transam/xlogutils.c
+++ b/src/backend/access/transam/xlogutils.c
@@ -1027,7 +1027,7 @@ read_local_xlog_page_guts(XLogReaderState *state, XLogRecPtr targetPagePtr,
* zero-padded up to the page boundary if it's incomplete.
*/
if (!WALRead(state, cur_page, targetPagePtr, XLOG_BLCKSZ, tli,
- &errinfo))
+ &errinfo, NULL, false))
WALReadRaiseError(&errinfo);
/* number of valid bytes in the buffer */
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index 2d8104b090..b47f44a852 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -892,7 +892,10 @@ CREATE VIEW pg_stat_replication AS
W.replay_lag,
W.sync_priority,
W.sync_state,
- W.reply_time
+ W.reply_time,
+ W.wal_read,
+ W.wal_read_bytes,
+ W.wal_read_time
FROM pg_stat_get_activity(NULL) AS S
JOIN pg_stat_get_wal_senders() AS W ON (S.pid = W.pid)
LEFT JOIN pg_authid AS U ON (S.usesysid = U.oid);
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index c11bb3716f..fa02e327f2 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -259,7 +259,7 @@ static bool TransactionIdInRecentPast(TransactionId xid, uint32 epoch);
static void WalSndSegmentOpen(XLogReaderState *state, XLogSegNo nextSegNo,
TimeLineID *tli_p);
-
+static void WalSndAccumulateWalReadStats(WALReadStats *stats);
/* Initialize walsender process before entering the main command loop */
void
@@ -907,6 +907,7 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
WALReadError errinfo;
XLogSegNo segno;
TimeLineID currTLI = GetWALInsertionTimeLine();
+ WALReadStats stats;
/*
* Since logical decoding is only permitted on a primary server, we know
@@ -932,6 +933,8 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
else
count = flushptr - targetPagePtr; /* part of the page available */
+ MemSet(&stats, 0, sizeof(WALReadStats));
+
/* now actually read the data, we know it's there */
if (!WALRead(state,
cur_page,
@@ -940,9 +943,13 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
state->seg.ws_tli, /* Pass the current TLI because only
* WalSndSegmentOpen controls whether new
* TLI is needed. */
- &errinfo))
+ &errinfo,
+ &stats,
+ track_wal_io_timing))
WALReadRaiseError(&errinfo);
+ WalSndAccumulateWalReadStats(&stats);
+
/*
* After reading into the buffer, check that what we read was valid. We do
* this after reading, because even though the segment was present when we
@@ -2610,6 +2617,9 @@ InitWalSenderSlot(void)
walsnd->sync_standby_priority = 0;
walsnd->latch = &MyProc->procLatch;
walsnd->replyTime = 0;
+ walsnd->wal_read_stats.wal_read = 0;
+ walsnd->wal_read_stats.wal_read_bytes = 0;
+ walsnd->wal_read_stats.wal_read_time = 0;
SpinLockRelease(&walsnd->mutex);
/* don't need the lock anymore */
MyWalSnd = (WalSnd *) walsnd;
@@ -2730,6 +2740,7 @@ XLogSendPhysical(void)
Size nbytes;
XLogSegNo segno;
WALReadError errinfo;
+ WALReadStats stats;
/* If requested switch the WAL sender to the stopping state. */
if (got_STOPPING)
@@ -2945,6 +2956,8 @@ XLogSendPhysical(void)
enlargeStringInfo(&output_message, nbytes);
retry:
+ MemSet(&stats, 0, sizeof(WALReadStats));
+
if (!WALRead(xlogreader,
&output_message.data[output_message.len],
startptr,
@@ -2952,9 +2965,13 @@ retry:
xlogreader->seg.ws_tli, /* Pass the current TLI because
* only WalSndSegmentOpen controls
* whether new TLI is needed. */
- &errinfo))
+ &errinfo,
+ &stats,
+ track_wal_io_timing))
WALReadRaiseError(&errinfo);
+ WalSndAccumulateWalReadStats(&stats);
+
/* See logical_read_xlog_page(). */
XLByteToSeg(startptr, segno, xlogreader->segcxt.ws_segsize);
CheckXLogRemoved(segno, xlogreader->seg.ws_tli);
@@ -3458,7 +3475,7 @@ offset_to_interval(TimeOffset offset)
Datum
pg_stat_get_wal_senders(PG_FUNCTION_ARGS)
{
-#define PG_STAT_GET_WAL_SENDERS_COLS 12
+#define PG_STAT_GET_WAL_SENDERS_COLS 15
ReturnSetInfo *rsinfo = (ReturnSetInfo *) fcinfo->resultinfo;
SyncRepStandbyData *sync_standbys;
int num_standbys;
@@ -3487,9 +3504,13 @@ pg_stat_get_wal_senders(PG_FUNCTION_ARGS)
WalSndState state;
TimestampTz replyTime;
bool is_sync_standby;
+ int64 wal_read;
+ uint64 wal_read_bytes;
+ int64 wal_read_time;
Datum values[PG_STAT_GET_WAL_SENDERS_COLS];
bool nulls[PG_STAT_GET_WAL_SENDERS_COLS] = {0};
int j;
+ char buf[256];
/* Collect data from shared memory */
SpinLockAcquire(&walsnd->mutex);
@@ -3509,6 +3530,9 @@ pg_stat_get_wal_senders(PG_FUNCTION_ARGS)
applyLag = walsnd->applyLag;
priority = walsnd->sync_standby_priority;
replyTime = walsnd->replyTime;
+ wal_read = walsnd->wal_read_stats.wal_read;
+ wal_read_bytes = walsnd->wal_read_stats.wal_read_bytes;
+ wal_read_time = walsnd->wal_read_stats.wal_read_time;
SpinLockRelease(&walsnd->mutex);
/*
@@ -3605,6 +3629,18 @@ pg_stat_get_wal_senders(PG_FUNCTION_ARGS)
nulls[11] = true;
else
values[11] = TimestampTzGetDatum(replyTime);
+
+ values[12] = Int64GetDatum(wal_read);
+
+ /* Convert to numeric. */
+ snprintf(buf, sizeof buf, UINT64_FORMAT, wal_read_bytes);
+ values[13] = DirectFunctionCall3(numeric_in,
+ CStringGetDatum(buf),
+ ObjectIdGetDatum(0),
+ Int32GetDatum(-1));
+
+ /* Convert counter from microsec to millisec for display. */
+ values[14] = Float8GetDatum(((double) wal_read_time) / 1000.0);
}
tuplestore_putvalues(rsinfo->setResult, rsinfo->setDesc,
@@ -3849,3 +3885,17 @@ LagTrackerRead(int head, XLogRecPtr lsn, TimestampTz now)
Assert(time != 0);
return now - time;
}
+
+/*
+ * Function to accumulate WAL Read stats for WAL sender.
+ */
+static void
+WalSndAccumulateWalReadStats(WALReadStats *stats)
+{
+ /* Collect I/O stats for walsender. */
+ SpinLockAcquire(&MyWalSnd->mutex);
+ MyWalSnd->wal_read_stats.wal_read += stats->wal_read;
+ MyWalSnd->wal_read_stats.wal_read_bytes += stats->wal_read_bytes;
+ MyWalSnd->wal_read_stats.wal_read_time += stats->wal_read_time;
+ SpinLockRelease(&MyWalSnd->mutex);
+}
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index 9993378ca5..698ce1e9f7 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -364,7 +364,7 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
}
if (!WALRead(state, readBuff, targetPagePtr, count, private->timeline,
- &errinfo))
+ &errinfo, NULL, false))
{
WALOpenSegment *seg = &errinfo.wre_seg;
char fname[MAXPGPATH];
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index e87f91316a..26a2c975de 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -389,9 +389,24 @@ typedef struct WALReadError
WALOpenSegment wre_seg; /* Segment we tried to read from. */
} WALReadError;
-extern bool WALRead(XLogReaderState *state,
- char *buf, XLogRecPtr startptr, Size count,
- TimeLineID tli, WALReadError *errinfo);
+/*
+ * WAL read stats from WALRead that the callers can use.
+ */
+typedef struct WALReadStats
+{
+ /* Number of times WAL read from disk. */
+ int64 wal_read;
+
+ /* Total amount of WAL read from disk in bytes. */
+ uint64 wal_read_bytes;
+
+ /* Total amount of time spent reading WAL from disk. */
+ int64 wal_read_time;
+} WALReadStats;
+
+extern bool WALRead(XLogReaderState *state, char *buf, XLogRecPtr startptr,
+ Size count, TimeLineID tli, WALReadError *errinfo,
+ WALReadStats *stats, bool capture_wal_io_timing);
/* Functions for decoding an XLogRecord */
diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat
index 7056c95371..18320cf846 100644
--- a/src/include/catalog/pg_proc.dat
+++ b/src/include/catalog/pg_proc.dat
@@ -5391,9 +5391,9 @@
proname => 'pg_stat_get_wal_senders', prorows => '10', proisstrict => 'f',
proretset => 't', provolatile => 's', proparallel => 'r',
prorettype => 'record', proargtypes => '',
- proallargtypes => '{int4,text,pg_lsn,pg_lsn,pg_lsn,pg_lsn,interval,interval,interval,int4,text,timestamptz}',
- proargmodes => '{o,o,o,o,o,o,o,o,o,o,o,o}',
- proargnames => '{pid,state,sent_lsn,write_lsn,flush_lsn,replay_lsn,write_lag,flush_lag,replay_lag,sync_priority,sync_state,reply_time}',
+ proallargtypes => '{int4,text,pg_lsn,pg_lsn,pg_lsn,pg_lsn,interval,interval,interval,int4,text,timestamptz,int8,numeric,float8}',
+ proargmodes => '{o,o,o,o,o,o,o,o,o,o,o,o,o,o,o}',
+ proargnames => '{pid,state,sent_lsn,write_lsn,flush_lsn,replay_lsn,write_lag,flush_lag,replay_lag,sync_priority,sync_state,reply_time,wal_read,wal_read_bytes,wal_read_time}',
prosrc => 'pg_stat_get_wal_senders' },
{ oid => '3317', descr => 'statistics: information about WAL receiver',
proname => 'pg_stat_get_wal_receiver', proisstrict => 'f', provolatile => 's',
diff --git a/src/include/replication/walsender_private.h b/src/include/replication/walsender_private.h
index 7897c74589..35413ea0d2 100644
--- a/src/include/replication/walsender_private.h
+++ b/src/include/replication/walsender_private.h
@@ -13,6 +13,7 @@
#define _WALSENDER_PRIVATE_H
#include "access/xlog.h"
+#include "access/xlogreader.h"
#include "nodes/nodes.h"
#include "replication/syncrep.h"
#include "storage/latch.h"
@@ -78,6 +79,9 @@ typedef struct WalSnd
* Timestamp of the last message received from standby.
*/
TimestampTz replyTime;
+
+ /* WAL read stats for walsender. */
+ WALReadStats wal_read_stats;
} WalSnd;
extern PGDLLIMPORT WalSnd *MyWalSnd;
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index fb9f936d43..fd9d298e79 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -2054,9 +2054,12 @@ pg_stat_replication| SELECT s.pid,
w.replay_lag,
w.sync_priority,
w.sync_state,
- w.reply_time
+ w.reply_time,
+ w.wal_read,
+ w.wal_read_bytes,
+ w.wal_read_time
FROM ((pg_stat_get_activity(NULL::integer) s(datid, pid, usesysid, application_name, state, query, wait_event_type, wait_event, xact_start, query_start, backend_start, state_change, client_addr, client_hostname, client_port, backend_xid, backend_xmin, backend_type, ssl, sslversion, sslcipher, sslbits, ssl_client_dn, ssl_client_serial, ssl_issuer_dn, gss_auth, gss_princ, gss_enc, leader_pid, query_id)
- JOIN pg_stat_get_wal_senders() w(pid, state, sent_lsn, write_lsn, flush_lsn, replay_lsn, write_lag, flush_lag, replay_lag, sync_priority, sync_state, reply_time) ON ((s.pid = w.pid)))
+ JOIN pg_stat_get_wal_senders() w(pid, state, sent_lsn, write_lsn, flush_lsn, replay_lsn, write_lag, flush_lag, replay_lag, sync_priority, sync_state, reply_time, wal_read, wal_read_bytes, wal_read_time) ON ((s.pid = w.pid)))
LEFT JOIN pg_authid u ON ((s.usesysid = u.oid)));
pg_stat_replication_slots| SELECT s.slot_name,
s.spill_txns,
--
2.34.1
[application/octet-stream] v2-0001-Improve-WALRead-to-suck-data-directly-from-WAL-bu.patch (8.1K, ../../CALj2ACU9cfAcfVsGwUqXMace_7rfSBJ7+hXVJfVV1jnspTDGHQ@mail.gmail.com/3-v2-0001-Improve-WALRead-to-suck-data-directly-from-WAL-bu.patch)
download | inline diff:
From 047c58df6aaae586efe58b6a4068b17f25976b0a Mon Sep 17 00:00:00 2001
From: Bharath Rupireddy <[email protected]>
Date: Mon, 26 Dec 2022 08:36:28 +0000
Subject: [PATCH v2] Improve WALRead() to suck data directly from WAL buffers
when possible
---
src/backend/access/transam/xlog.c | 154 ++++++++++++++++++++++++
src/backend/access/transam/xlogreader.c | 47 +++++++-
src/include/access/xlog.h | 6 +
3 files changed, 205 insertions(+), 2 deletions(-)
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 91473b00d9..c3138493be 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -689,6 +689,7 @@ static bool ReserveXLogSwitch(XLogRecPtr *StartPos, XLogRecPtr *EndPos,
XLogRecPtr *PrevPtr);
static XLogRecPtr WaitXLogInsertionsToFinish(XLogRecPtr upto);
static char *GetXLogBuffer(XLogRecPtr ptr, TimeLineID tli);
+static char *GetXLogBufferForRead(XLogRecPtr ptr, TimeLineID tli, char *page);
static XLogRecPtr XLogBytePosToRecPtr(uint64 bytepos);
static XLogRecPtr XLogBytePosToEndRecPtr(uint64 bytepos);
static uint64 XLogRecPtrToBytePos(XLogRecPtr ptr);
@@ -1639,6 +1640,159 @@ GetXLogBuffer(XLogRecPtr ptr, TimeLineID tli)
return cachedPos + ptr % XLOG_BLCKSZ;
}
+/*
+ * Get the WAL buffer page containing passed in WAL record and also return the
+ * record's location within that buffer page.
+ */
+static char *
+GetXLogBufferForRead(XLogRecPtr ptr, TimeLineID tli, char *page)
+{
+ XLogRecPtr expectedEndPtr;
+ XLogRecPtr endptr;
+ int idx;
+ char *recptr = NULL;
+
+ idx = XLogRecPtrToBufIdx(ptr);
+ expectedEndPtr = ptr;
+ expectedEndPtr += XLOG_BLCKSZ - ptr % XLOG_BLCKSZ;
+
+ /*
+ * Try to acquire WALBufMappingLock in shared mode so that the other
+ * concurrent WAL readers are also allowed. We try to do as less work as
+ * possible while holding the lock as it might impact concurrent WAL
+ * writers.
+ *
+ * If we cannot immediately acquire the lock, meaning the lock was busy,
+ * then exit quickly to not cause any contention. The caller can then
+ * fallback to reading WAL from WAL file.
+ */
+ if (!LWLockConditionalAcquire(WALBufMappingLock, LW_SHARED))
+ return recptr;
+
+ /*
+ * Holding WALBufMappingLock ensures inserters don't overwrite this value
+ * while we are reading it.
+ */
+ endptr = XLogCtl->xlblocks[idx];
+
+ if (expectedEndPtr == endptr)
+ {
+ XLogPageHeader phdr;
+
+ /*
+ * We have found the WAL buffer page holding the given LSN. Read from a
+ * pointer to the right offset within the page.
+ */
+ memcpy(page, (XLogCtl->pages + idx * (Size) XLOG_BLCKSZ),
+ (Size) XLOG_BLCKSZ);
+
+ /*
+ * Release the lock as early as possible to avoid creating any possible
+ * contention.
+ */
+ LWLockRelease(WALBufMappingLock);
+
+ /*
+ * The fact that we acquire WALBufMappingLock while reading the WAL
+ * buffer page itself guarantees that no one else initializes it or
+ * makes it ready for next use in AdvanceXLInsertBuffer().
+ *
+ * However, we perform basic page header checks for ensuring that we
+ * are not reading a page that got just initialized. The callers will
+ * anyway perform extensive page-level and record-level checks.
+ */
+ phdr = (XLogPageHeader) page;
+
+ if (phdr->xlp_magic == XLOG_PAGE_MAGIC &&
+ phdr->xlp_pageaddr == (ptr - (ptr % XLOG_BLCKSZ)) &&
+ phdr->xlp_tli == tli)
+ {
+ /*
+ * Page looks valid, so return the page and the requested record's
+ * LSN.
+ */
+ recptr = page + ptr % XLOG_BLCKSZ;
+ }
+ }
+ else
+ {
+ /* We have found nothing. */
+ LWLockRelease(WALBufMappingLock);
+ }
+
+ return recptr;
+}
+
+/*
+ * When possible, read WAL starting at 'startptr' of size 'count' bytes from
+ * WAL buffers into buffer passed in by the caller 'buf'. Read as much WAL as
+ * possible from the WAL buffers, remaining WAL, if any, the caller will take
+ * care of reading from WAL files directly.
+ *
+ * This function sets read bytes to 'read_bytes'.
+ */
+void
+XLogReadFromBuffers(XLogRecPtr startptr,
+ TimeLineID tli,
+ Size count,
+ char *buf,
+ Size *read_bytes)
+{
+ XLogRecPtr ptr;
+ char *dst;
+ Size nbytes;
+
+ Assert(!XLogRecPtrIsInvalid(startptr));
+ Assert(count > 0);
+ Assert(startptr <= GetFlushRecPtr(NULL));
+ Assert(!RecoveryInProgress());
+
+ ptr = startptr;
+ nbytes = count;
+ dst = buf;
+ *read_bytes = 0;
+
+ while (nbytes > 0)
+ {
+ char page[XLOG_BLCKSZ] = {0};
+ char *recptr;
+
+ recptr = GetXLogBufferForRead(ptr, tli, page);
+
+ if (recptr == NULL)
+ break;
+
+ if ((recptr + nbytes) <= (page + XLOG_BLCKSZ))
+ {
+ /* All the bytes are in one page. */
+ memcpy(dst, recptr, nbytes);
+ dst += nbytes;
+ *read_bytes += nbytes;
+ ptr += nbytes;
+ nbytes = 0;
+ }
+ else if ((recptr + nbytes) > (page + XLOG_BLCKSZ))
+ {
+ /* All the bytes are not in one page. */
+ Size bytes_remaining;
+
+ /*
+ * Compute the remaining bytes on the current page, copy them over
+ * to output buffer and move forward to read further.
+ */
+ bytes_remaining = XLOG_BLCKSZ - (recptr - page);
+ memcpy(dst, recptr, bytes_remaining);
+ dst += bytes_remaining;
+ nbytes -= bytes_remaining;
+ *read_bytes += bytes_remaining;
+ ptr += bytes_remaining;
+ }
+ }
+
+ elog(DEBUG1, "read %zu bytes out of %zu bytes from WAL buffers for given LSN %X/%X, Timeline ID %u",
+ *read_bytes, count, LSN_FORMAT_ARGS(startptr), tli);
+}
+
/*
* Converts a "usable byte position" to XLogRecPtr. A usable byte position
* is the position starting from the beginning of WAL, excluding all WAL
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index a38a80e049..4a2e7af169 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -1485,8 +1485,7 @@ err:
* Returns true if succeeded, false if an error occurs, in which case
* 'errinfo' receives error details.
*
- * XXX probably this should be improved to suck data directly from the
- * WAL buffers when possible.
+ * When possible, this function reads data directly from WAL buffers.
*/
bool
WALRead(XLogReaderState *state,
@@ -1497,6 +1496,50 @@ WALRead(XLogReaderState *state,
XLogRecPtr recptr;
Size nbytes;
+#ifndef FRONTEND
+ /* Frontend tools have no idea of WAL buffers. */
+ Size read_bytes;
+
+ /*
+ * When possible, read WAL from WAL buffers. We skip this step and continue
+ * the usual way, that is to read from WAL file, either when the server is
+ * in recovery (standby mode, archive or crash recovery), in which case the
+ * WAL buffers are not used or when the server is inserting in a different
+ * timeline from that of the timeline that we're trying to read WAL from.
+ */
+ if (!RecoveryInProgress() &&
+ tli == GetWALInsertionTimeLine())
+ {
+ pgstat_report_wait_start(WAIT_EVENT_WAL_READ);
+ XLogReadFromBuffers(startptr, tli, count, buf, &read_bytes);
+ pgstat_report_wait_end();
+
+ /*
+ * Check if we have read fully (hit), partially (partial hit) or
+ * nothing (miss) from WAL buffers. If we have read either partially or
+ * nothing, then continue to read the remaining bytes the usual way,
+ * that is, read from WAL file.
+ */
+ if (count == read_bytes)
+ {
+ /* Buffer hit, so return. */
+ return true;
+ }
+ else if (read_bytes > 0 && count > read_bytes)
+ {
+ /*
+ * Buffer partial hit, so reset the state to count the read bytes
+ * and continue.
+ */
+ buf += read_bytes;
+ startptr += read_bytes;
+ count -= read_bytes;
+ }
+
+ /* Buffer miss i.e., read_bytes = 0, so continue */
+ }
+#endif /* FRONTEND */
+
p = buf;
recptr = startptr;
nbytes = count;
diff --git a/src/include/access/xlog.h b/src/include/access/xlog.h
index 1fbd48fbda..f4e1c46b23 100644
--- a/src/include/access/xlog.h
+++ b/src/include/access/xlog.h
@@ -247,6 +247,12 @@ extern XLogRecPtr GetLastImportantRecPtr(void);
extern void SetWalWriterSleeping(bool sleeping);
+extern void XLogReadFromBuffers(XLogRecPtr startptr,
+ TimeLineID tli,
+ Size count,
+ char *buf,
+ Size *read_bytes);
+
/*
* Routines used by xlogrecovery.c to call back into xlog.c during recovery.
*/
--
2.34.1
[text/plain] USE-ON-PATCH-Collect-WAL-read-from-buffers-and-file-stats.txt (19.9K, ../../CALj2ACU9cfAcfVsGwUqXMace_7rfSBJ7+hXVJfVV1jnspTDGHQ@mail.gmail.com/4-USE-ON-PATCH-Collect-WAL-read-from-buffers-and-file-stats.txt)
download | inline diff:
From f90dfcbd1968280feec6d116568697225854ac40 Mon Sep 17 00:00:00 2001
From: Bharath Rupireddy <[email protected]>
Date: Mon, 26 Dec 2022 08:13:07 +0000
Subject: [PATCH v2] Collect WAL read from buffers and file stats for WAL
senders
---
doc/src/sgml/monitoring.sgml | 61 +++++++++++++++
src/backend/access/transam/xlogreader.c | 56 +++++++++++++-
src/backend/access/transam/xlogutils.c | 2 +-
src/backend/catalog/system_views.sql | 8 +-
src/backend/replication/walsender.c | 85 ++++++++++++++++++++-
src/bin/pg_waldump/pg_waldump.c | 2 +-
src/include/access/xlogreader.h | 30 +++++++-
src/include/catalog/pg_proc.dat | 6 +-
src/include/replication/walsender_private.h | 4 +
src/test/regress/expected/rules.out | 10 ++-
10 files changed, 246 insertions(+), 18 deletions(-)
diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml
index 363b183e5f..239e0b0db9 100644
--- a/doc/src/sgml/monitoring.sgml
+++ b/doc/src/sgml/monitoring.sgml
@@ -2615,6 +2615,67 @@ SELECT pid, wait_event_type, wait_event FROM pg_stat_activity WHERE wait_event i
Send time of last reply message received from standby server
</para></entry>
</row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>wal_read</structfield> <type>bigint</type>
+ </para>
+ <para>
+ Number of times WAL data is read from disk
+ </para></entry>
+ </row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>wal_read_bytes</structfield> <type>numeric</type>
+ </para>
+ <para>
+ Total amount of WAL read from disk in bytes
+ </para></entry>
+ </row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>wal_read_time</structfield> <type>double precision</type>
+ </para>
+ <para>
+ Total amount of time spent reading WAL from disk via
+ <function>WALRead</function> request, in milliseconds
+ (if <xref linkend="guc-track-wal-io-timing"/> is enabled,
+ otherwise zero).
+ </para></entry>
+ </row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>wal_read_buffers</structfield> <type>bigint</type>
+ </para>
+ <para>
+ Number of times WAL data is read from WAL buffers
+ </para></entry>
+ </row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>wal_read_bytes_buffers</structfield> <type>numeric</type>
+ </para>
+ <para>
+ Total amount of WAL read from WAL buffers in bytes
+ </para></entry>
+ </row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>wal_read_time_buffers</structfield> <type>double precision</type>
+ </para>
+ <para>
+ Total amount of time spent reading WAL from WAL buffers via
+ <function>WALRead</function> request, in milliseconds
+ (if <xref linkend="guc-track-wal-io-timing"/> is enabled,
+ otherwise zero).
+ </para></entry>
+ </row>
+
</tbody>
</tgroup>
</table>
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index 4a2e7af169..b9dfd4fde7 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -31,6 +31,7 @@
#include "access/xlogrecord.h"
#include "catalog/pg_control.h"
#include "common/pg_lzcompress.h"
+#include "portability/instr_time.h"
#include "replication/origin.h"
#ifndef FRONTEND
@@ -1488,9 +1489,9 @@ err:
* When possible, this function reads data directly from WAL buffers.
*/
bool
-WALRead(XLogReaderState *state,
- char *buf, XLogRecPtr startptr, Size count, TimeLineID tli,
- WALReadError *errinfo)
+WALRead(XLogReaderState *state, char *buf, XLogRecPtr startptr, Size count,
+ TimeLineID tli, WALReadError *errinfo, WALReadStats *stats,
+ bool capture_wal_io_timing)
{
char *p;
XLogRecPtr recptr;
@@ -1510,10 +1511,33 @@ WALRead(XLogReaderState *state,
if (!RecoveryInProgress() &&
tli == GetWALInsertionTimeLine())
{
+ instr_time start;
+
+ /* Measure I/O timing to read WAL data if requested by the caller. */
+ if (stats != NULL && capture_wal_io_timing)
+ INSTR_TIME_SET_CURRENT(start);
+
pgstat_report_wait_start(WAIT_EVENT_WAL_READ);
XLogReadFromBuffers(startptr, tli, count, buf, &read_bytes);
pgstat_report_wait_end();
+ /* Collect I/O stats if requested by the caller. */
+ if (stats != NULL && read_bytes > 0)
+ {
+ stats->wal_read_buffers++;
+ stats->wal_read_bytes_buffers += read_bytes;
+
+ /* Increment the I/O timing. */
+ if (capture_wal_io_timing)
+ {
+ instr_time duration;
+
+ INSTR_TIME_SET_CURRENT(duration);
+ INSTR_TIME_SUBTRACT(duration, start);
+ stats->wal_read_time_buffers += INSTR_TIME_GET_MICROSEC(duration);
+ }
+ }
+
/*
* Check if we have read fully (hit), partially (partial hit) or
* nothing (miss) from WAL buffers. If we have read either partially or
@@ -1549,6 +1573,7 @@ WALRead(XLogReaderState *state,
uint32 startoff;
int segbytes;
int readbytes;
+ instr_time start;
startoff = XLogSegmentOffset(recptr, state->segcxt.ws_segsize);
@@ -1583,6 +1608,10 @@ WALRead(XLogReaderState *state,
else
segbytes = nbytes;
+ /* Measure I/O timing to read WAL data if requested by the caller. */
+ if (stats != NULL && capture_wal_io_timing)
+ INSTR_TIME_SET_CURRENT(start);
+
#ifndef FRONTEND
pgstat_report_wait_start(WAIT_EVENT_WAL_READ);
#endif
@@ -1595,6 +1624,27 @@ WALRead(XLogReaderState *state,
pgstat_report_wait_end();
#endif
+ /* Collect I/O stats if requested by the caller. */
+ if (stats != NULL)
+ {
+ /* Increment the number of times WAL is read from disk. */
+ stats->wal_read++;
+
+ /* Collect bytes read. */
+ if (readbytes > 0)
+ stats->wal_read_bytes += readbytes;
+
+ /* Increment the I/O timing. */
+ if (capture_wal_io_timing)
+ {
+ instr_time duration;
+
+ INSTR_TIME_SET_CURRENT(duration);
+ INSTR_TIME_SUBTRACT(duration, start);
+ stats->wal_read_time += INSTR_TIME_GET_MICROSEC(duration);
+ }
+ }
+
if (readbytes <= 0)
{
errinfo->wre_errno = errno;
diff --git a/src/backend/access/transam/xlogutils.c b/src/backend/access/transam/xlogutils.c
index 563cba258d..372de2c7d8 100644
--- a/src/backend/access/transam/xlogutils.c
+++ b/src/backend/access/transam/xlogutils.c
@@ -1027,7 +1027,7 @@ read_local_xlog_page_guts(XLogReaderState *state, XLogRecPtr targetPagePtr,
* zero-padded up to the page boundary if it's incomplete.
*/
if (!WALRead(state, cur_page, targetPagePtr, XLOG_BLCKSZ, tli,
- &errinfo))
+ &errinfo, NULL, false))
WALReadRaiseError(&errinfo);
/* number of valid bytes in the buffer */
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index 2d8104b090..bf6315df27 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -892,7 +892,13 @@ CREATE VIEW pg_stat_replication AS
W.replay_lag,
W.sync_priority,
W.sync_state,
- W.reply_time
+ W.reply_time,
+ W.wal_read,
+ W.wal_read_bytes,
+ W.wal_read_time,
+ W.wal_read_buffers,
+ W.wal_read_bytes_buffers,
+ W.wal_read_time_buffers
FROM pg_stat_get_activity(NULL) AS S
JOIN pg_stat_get_wal_senders() AS W ON (S.pid = W.pid)
LEFT JOIN pg_authid AS U ON (S.usesysid = U.oid);
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index c11bb3716f..d3393b2b63 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -259,7 +259,7 @@ static bool TransactionIdInRecentPast(TransactionId xid, uint32 epoch);
static void WalSndSegmentOpen(XLogReaderState *state, XLogSegNo nextSegNo,
TimeLineID *tli_p);
-
+static void WalSndAccumulateWalReadStats(WALReadStats *stats);
/* Initialize walsender process before entering the main command loop */
void
@@ -907,6 +907,7 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
WALReadError errinfo;
XLogSegNo segno;
TimeLineID currTLI = GetWALInsertionTimeLine();
+ WALReadStats stats;
/*
* Since logical decoding is only permitted on a primary server, we know
@@ -932,6 +933,8 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
else
count = flushptr - targetPagePtr; /* part of the page available */
+ MemSet(&stats, 0, sizeof(WALReadStats));
+
/* now actually read the data, we know it's there */
if (!WALRead(state,
cur_page,
@@ -940,9 +943,13 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
state->seg.ws_tli, /* Pass the current TLI because only
* WalSndSegmentOpen controls whether new
* TLI is needed. */
- &errinfo))
+ &errinfo,
+ &stats,
+ track_wal_io_timing))
WALReadRaiseError(&errinfo);
+ WalSndAccumulateWalReadStats(&stats);
+
/*
* After reading into the buffer, check that what we read was valid. We do
* this after reading, because even though the segment was present when we
@@ -2610,6 +2617,12 @@ InitWalSenderSlot(void)
walsnd->sync_standby_priority = 0;
walsnd->latch = &MyProc->procLatch;
walsnd->replyTime = 0;
+ walsnd->wal_read_stats.wal_read = 0;
+ walsnd->wal_read_stats.wal_read_bytes = 0;
+ walsnd->wal_read_stats.wal_read_time = 0;
+ walsnd->wal_read_stats.wal_read_buffers = 0;
+ walsnd->wal_read_stats.wal_read_bytes_buffers = 0;
+ walsnd->wal_read_stats.wal_read_time_buffers = 0;
SpinLockRelease(&walsnd->mutex);
/* don't need the lock anymore */
MyWalSnd = (WalSnd *) walsnd;
@@ -2730,6 +2743,7 @@ XLogSendPhysical(void)
Size nbytes;
XLogSegNo segno;
WALReadError errinfo;
+ WALReadStats stats;
/* If requested switch the WAL sender to the stopping state. */
if (got_STOPPING)
@@ -2945,6 +2959,8 @@ XLogSendPhysical(void)
enlargeStringInfo(&output_message, nbytes);
retry:
+ MemSet(&stats, 0, sizeof(WALReadStats));
+
if (!WALRead(xlogreader,
&output_message.data[output_message.len],
startptr,
@@ -2952,9 +2968,13 @@ retry:
xlogreader->seg.ws_tli, /* Pass the current TLI because
* only WalSndSegmentOpen controls
* whether new TLI is needed. */
- &errinfo))
+ &errinfo,
+ &stats,
+ track_wal_io_timing))
WALReadRaiseError(&errinfo);
+ WalSndAccumulateWalReadStats(&stats);
+
/* See logical_read_xlog_page(). */
XLByteToSeg(startptr, segno, xlogreader->segcxt.ws_segsize);
CheckXLogRemoved(segno, xlogreader->seg.ws_tli);
@@ -3458,7 +3478,7 @@ offset_to_interval(TimeOffset offset)
Datum
pg_stat_get_wal_senders(PG_FUNCTION_ARGS)
{
-#define PG_STAT_GET_WAL_SENDERS_COLS 12
+#define PG_STAT_GET_WAL_SENDERS_COLS 18
ReturnSetInfo *rsinfo = (ReturnSetInfo *) fcinfo->resultinfo;
SyncRepStandbyData *sync_standbys;
int num_standbys;
@@ -3487,9 +3507,16 @@ pg_stat_get_wal_senders(PG_FUNCTION_ARGS)
WalSndState state;
TimestampTz replyTime;
bool is_sync_standby;
+ int64 wal_read;
+ uint64 wal_read_bytes;
+ int64 wal_read_time;
+ int64 wal_read_buffers;
+ uint64 wal_read_bytes_buffers;
+ int64 wal_read_time_buffers;
Datum values[PG_STAT_GET_WAL_SENDERS_COLS];
bool nulls[PG_STAT_GET_WAL_SENDERS_COLS] = {0};
int j;
+ char buf[256];
/* Collect data from shared memory */
SpinLockAcquire(&walsnd->mutex);
@@ -3509,6 +3536,12 @@ pg_stat_get_wal_senders(PG_FUNCTION_ARGS)
applyLag = walsnd->applyLag;
priority = walsnd->sync_standby_priority;
replyTime = walsnd->replyTime;
+ wal_read = walsnd->wal_read_stats.wal_read;
+ wal_read_bytes = walsnd->wal_read_stats.wal_read_bytes;
+ wal_read_time = walsnd->wal_read_stats.wal_read_time;
+ wal_read_buffers = walsnd->wal_read_stats.wal_read_buffers;
+ wal_read_bytes_buffers = walsnd->wal_read_stats.wal_read_bytes_buffers;
+ wal_read_time_buffers = walsnd->wal_read_stats.wal_read_time_buffers;
SpinLockRelease(&walsnd->mutex);
/*
@@ -3605,6 +3638,31 @@ pg_stat_get_wal_senders(PG_FUNCTION_ARGS)
nulls[11] = true;
else
values[11] = TimestampTzGetDatum(replyTime);
+
+ values[12] = Int64GetDatum(wal_read);
+
+ /* Convert to numeric. */
+ snprintf(buf, sizeof buf, UINT64_FORMAT, wal_read_bytes);
+ values[13] = DirectFunctionCall3(numeric_in,
+ CStringGetDatum(buf),
+ ObjectIdGetDatum(0),
+ Int32GetDatum(-1));
+
+ /* Convert counter from microsec to millisec for display. */
+ values[14] = Float8GetDatum(((double) wal_read_time) / 1000.0);
+
+ values[15] = Int64GetDatum(wal_read_buffers);
+
+ /* Convert to numeric. */
+ MemSet(buf, '\0', sizeof buf);
+ snprintf(buf, sizeof buf, UINT64_FORMAT, wal_read_bytes_buffers);
+ values[16] = DirectFunctionCall3(numeric_in,
+ CStringGetDatum(buf),
+ ObjectIdGetDatum(0),
+ Int32GetDatum(-1));
+
+ /* Convert counter from microsec to millisec for display. */
+ values[17] = Float8GetDatum(((double) wal_read_time_buffers) / 1000.0);
}
tuplestore_putvalues(rsinfo->setResult, rsinfo->setDesc,
@@ -3849,3 +3907,22 @@ LagTrackerRead(int head, XLogRecPtr lsn, TimestampTz now)
Assert(time != 0);
return now - time;
}
+
+/*
+ * Function to accumulate WAL Read stats for WAL sender.
+ */
+static void
+WalSndAccumulateWalReadStats(WALReadStats *stats)
+{
+ /* Collect I/O stats for walsender. */
+ SpinLockAcquire(&MyWalSnd->mutex);
+ MyWalSnd->wal_read_stats.wal_read += stats->wal_read;
+ MyWalSnd->wal_read_stats.wal_read_bytes += stats->wal_read_bytes;
+ MyWalSnd->wal_read_stats.wal_read_time += stats->wal_read_time;
+ MyWalSnd->wal_read_stats.wal_read_buffers += stats->wal_read_buffers;
+ MyWalSnd->wal_read_stats.wal_read_bytes_buffers +=
+ stats->wal_read_bytes_buffers;
+ MyWalSnd->wal_read_stats.wal_read_time_buffers +=
+ stats->wal_read_time_buffers;
+ SpinLockRelease(&MyWalSnd->mutex);
+}
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index 9993378ca5..698ce1e9f7 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -364,7 +364,7 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
}
if (!WALRead(state, readBuff, targetPagePtr, count, private->timeline,
- &errinfo))
+ &errinfo, NULL, false))
{
WALOpenSegment *seg = &errinfo.wre_seg;
char fname[MAXPGPATH];
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index e87f91316a..9287114779 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -389,9 +389,33 @@ typedef struct WALReadError
WALOpenSegment wre_seg; /* Segment we tried to read from. */
} WALReadError;
-extern bool WALRead(XLogReaderState *state,
- char *buf, XLogRecPtr startptr, Size count,
- TimeLineID tli, WALReadError *errinfo);
+/*
+ * WAL read stats from WALRead that the callers can use.
+ */
+typedef struct WALReadStats
+{
+ /* Number of times WAL read from disk. */
+ int64 wal_read;
+
+ /* Total amount of WAL read from disk in bytes. */
+ uint64 wal_read_bytes;
+
+ /* Total amount of time spent reading WAL from disk. */
+ int64 wal_read_time;
+
+ /* Number of times WAL read from WAL buffers. */
+ int64 wal_read_buffers;
+
+ /* Total amount of WAL read from WAL buffers in bytes. */
+ uint64 wal_read_bytes_buffers;
+
+ /* Total amount of time spent reading WAL from WAL buffers. */
+ int64 wal_read_time_buffers;
+} WALReadStats;
+
+extern bool WALRead(XLogReaderState *state, char *buf, XLogRecPtr startptr,
+ Size count, TimeLineID tli, WALReadError *errinfo,
+ WALReadStats *stats, bool capture_wal_io_timing);
/* Functions for decoding an XLogRecord */
diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat
index 7056c95371..706a005c2b 100644
--- a/src/include/catalog/pg_proc.dat
+++ b/src/include/catalog/pg_proc.dat
@@ -5391,9 +5391,9 @@
proname => 'pg_stat_get_wal_senders', prorows => '10', proisstrict => 'f',
proretset => 't', provolatile => 's', proparallel => 'r',
prorettype => 'record', proargtypes => '',
- proallargtypes => '{int4,text,pg_lsn,pg_lsn,pg_lsn,pg_lsn,interval,interval,interval,int4,text,timestamptz}',
- proargmodes => '{o,o,o,o,o,o,o,o,o,o,o,o}',
- proargnames => '{pid,state,sent_lsn,write_lsn,flush_lsn,replay_lsn,write_lag,flush_lag,replay_lag,sync_priority,sync_state,reply_time}',
+ proallargtypes => '{int4,text,pg_lsn,pg_lsn,pg_lsn,pg_lsn,interval,interval,interval,int4,text,timestamptz,int8,numeric,float8,int8,numeric,float8}',
+ proargmodes => '{o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o}',
+ proargnames => '{pid,state,sent_lsn,write_lsn,flush_lsn,replay_lsn,write_lag,flush_lag,replay_lag,sync_priority,sync_state,reply_time,wal_read,wal_read_bytes,wal_read_time,wal_read_buffers,wal_read_bytes_buffers,wal_read_time_buffers}',
prosrc => 'pg_stat_get_wal_senders' },
{ oid => '3317', descr => 'statistics: information about WAL receiver',
proname => 'pg_stat_get_wal_receiver', proisstrict => 'f', provolatile => 's',
diff --git a/src/include/replication/walsender_private.h b/src/include/replication/walsender_private.h
index 7897c74589..35413ea0d2 100644
--- a/src/include/replication/walsender_private.h
+++ b/src/include/replication/walsender_private.h
@@ -13,6 +13,7 @@
#define _WALSENDER_PRIVATE_H
#include "access/xlog.h"
+#include "access/xlogreader.h"
#include "nodes/nodes.h"
#include "replication/syncrep.h"
#include "storage/latch.h"
@@ -78,6 +79,9 @@ typedef struct WalSnd
* Timestamp of the last message received from standby.
*/
TimestampTz replyTime;
+
+ /* WAL read stats for walsender. */
+ WALReadStats wal_read_stats;
} WalSnd;
extern PGDLLIMPORT WalSnd *MyWalSnd;
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index fb9f936d43..6ae65981c2 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -2054,9 +2054,15 @@ pg_stat_replication| SELECT s.pid,
w.replay_lag,
w.sync_priority,
w.sync_state,
- w.reply_time
+ w.reply_time,
+ w.wal_read,
+ w.wal_read_bytes,
+ w.wal_read_time,
+ w.wal_read_buffers,
+ w.wal_read_bytes_buffers,
+ w.wal_read_time_buffers
FROM ((pg_stat_get_activity(NULL::integer) s(datid, pid, usesysid, application_name, state, query, wait_event_type, wait_event, xact_start, query_start, backend_start, state_change, client_addr, client_hostname, client_port, backend_xid, backend_xmin, backend_type, ssl, sslversion, sslcipher, sslbits, ssl_client_dn, ssl_client_serial, ssl_issuer_dn, gss_auth, gss_princ, gss_enc, leader_pid, query_id)
- JOIN pg_stat_get_wal_senders() w(pid, state, sent_lsn, write_lsn, flush_lsn, replay_lsn, write_lag, flush_lag, replay_lag, sync_priority, sync_state, reply_time) ON ((s.pid = w.pid)))
+ JOIN pg_stat_get_wal_senders() w(pid, state, sent_lsn, write_lsn, flush_lsn, replay_lsn, write_lag, flush_lag, replay_lag, sync_priority, sync_state, reply_time, wal_read, wal_read_bytes, wal_read_time, wal_read_buffers, wal_read_bytes_buffers, wal_read_time_buffers) ON ((s.pid = w.pid)))
LEFT JOIN pg_authid u ON ((s.usesysid = u.oid)));
pg_stat_replication_slots| SELECT s.slot_name,
s.spill_txns,
--
2.34.1
^ permalink raw reply [nested|flat] 22+ messages in thread
* Re: Improve WALRead() to suck data directly from WAL buffers when possible
@ 2024-01-10 14:29 Bharath Rupireddy <[email protected]>
0 siblings, 3 replies; 22+ messages in thread
From: Bharath Rupireddy @ 2024-01-10 14:29 UTC (permalink / raw)
To: Jeff Davis <[email protected]>; +Cc: Andres Freund <[email protected]>; Dilip Kumar <[email protected]>; Kyotaro Horiguchi <[email protected]>; [email protected]; Nathan Bossart <[email protected]>; Masahiko Sawada <[email protected]>
On Fri, Jan 5, 2024 at 7:20 AM Jeff Davis <[email protected]> wrote:
>
> On Wed, 2023-12-20 at 15:36 +0530, Bharath Rupireddy wrote:
> > Thanks. Attaching remaining patches as v18 patch-set after commits
> > c3a8e2a7cb16 and 766571be1659.
>
> Comments:
Thanks for reviewing.
> I still think the right thing for this patch is to call
> XLogReadFromBuffers() directly from the callers who need it, and not
> change WALRead(). I am open to changing this later, but for now that
> makes sense to me so that we can clearly identify which callers benefit
> and why. I have brought this up a few times before[1][2], so there must
> be some reason that I don't understand -- can you explain it?
IMO, WALRead() is the best place to have XLogReadFromBuffers() for 2
reasons: 1) All of the WALRead() callers (except FRONTEND tools) will
benefit if WAL is read from WAL buffers. I don't see any reason for a
caller to skip reading from WAL buffers. If there's a caller (in
future) wanting to skip reading from WAL buffers, I'm open to adding a
flag in XLogReaderState to skip. 2) The amount of code is reduced if
XLogReadFromBuffers() sits in WALRead().
> The XLogReadFromBuffersResult is never used. I can see how it might be
> useful for testing or asserts, but it's not used even in the test
> module. I don't think we should clutter the API with that kind of thing
> -- let's just return the nread.
Removed.
> I also do not like the terminology "partial hit" to be used in this
> way. Perhaps "short read" or something about hitting the end of
> readable WAL would be better?
"short read" seems good. Done that way in the new patch.
> I like how the callers of WALRead() are being more precise about the
> bytes they are requesting.
>
> You've added several spinlock acquisitions to the loop. Two explicitly,
> and one implicitly in WaitXLogInsertionsToFinish(). These may allow you
> to read slightly further, but introduce performance risk. Was this
> discussed?
I opted to read slightly further thinking that the loops aren't going
to get longer for spinlocks to appear costly. Basically, I wasn't sure
which approach was the best. Now that there's an opinion to keep them
outside, I'd agree with it. Done that way in the new patch.
> The callers are not checking for XLREADBUGS_UNINITIALIZED_WAL, so it
> seems like there's a risk of getting partially-written data? And it's
> not clear to me the check of the wal page headers is the right one
> anyway.
>
> It seems like all of this would be simpler if you checked first how far
> you can safely read data, and then just loop and read that far. I'm not
> sure that it's worth it to try to mix the validity checks with the
> reading of the data.
XLogReadFromBuffers needs the page header check in after reading the
page from WAL buffers. Typically, we must not read a WAL buffer page
that just got initialized. Because we waited enough for the
in-progress WAL insertions to finish above. However, there can exist a
slight window after the above wait finishes in which the read buffer
page can get replaced especially under high WAL generation rates.
After all, we are reading from WAL buffers without any locks here. So,
let's not count such a page in.
I've addressed the above review comments and attached v19 patch-set.
--
Bharath Rupireddy
PostgreSQL Contributors Team
RDS Open Source Databases
Amazon Web Services: https://aws.amazon.com
Attachments:
[application/x-patch] v19-0001-Allow-WAL-reading-from-WAL-buffers.patch (12.6K, ../../CALj2ACWPKuoUdA7r4VFKhsTZeNekyUr=ZwM6ckyvJ0qv2f6Whg@mail.gmail.com/2-v19-0001-Allow-WAL-reading-from-WAL-buffers.patch)
download | inline diff:
From e03af5726957437c15361bdb1b373fe8982f5c7c Mon Sep 17 00:00:00 2001
From: Bharath Rupireddy <[email protected]>
Date: Wed, 10 Jan 2024 14:12:17 +0000
Subject: [PATCH v19] Allow WAL reading from WAL buffers
This commit adds postgres the capability to read WAL from WAL
buffers. When requested WAL isn't available in WAL buffers, the
WAL is read from the WAL file as usual.
This commit benefits the callers of WALRead(), that are
walsenders, pg_walinspect etc. They all can now avoid reading WAL
from the WAL file (possibly avoiding disk IO). Tests show that the
WAL buffers hit ratio stood at 95% for 1 primary, 1 sync standby,
1 async standby, with pgbench --scale=300 --client=32 --time=900.
In other words, the walsenders avoided 95% of the time reading from
the file/avoided pread system calls:
https://www.postgresql.org/message-id/CALj2ACXKKK%3DwbiG5_t6dGao5GoecMwRkhr7GjVBM_jg54%2BNa%3DQ%40mail.gmail.com
This commit also benefits when direct IO is enabled for WAL.
Reading WAL from WAL buffers puts back the performance close to
that of without direct IO for WAL:
https://www.postgresql.org/message-id/CALj2ACV6rS%2B7iZx5%2BoAvyXJaN4AG-djAQeM1mrM%3DYSDkVrUs7g%40mail.gmail.com
This commit paves the way for the following features in future:
- Improves synchronous replication performance by replicating
directly from WAL buffers.
- A opt-in way for the walreceivers to receive unflushed WAL.
More details here:
https://www.postgresql.org/message-id/20231011224353.cl7c2s222dw3de4j%40awork3.anarazel.de
Author: Bharath Rupireddy
Reviewed-by: Dilip Kumar, Andres Freund
Reviewed-by: Nathan Bossart, Kuntal Ghosh
Discussion: https://www.postgresql.org/message-id/CALj2ACXKKK%3DwbiG5_t6dGao5GoecMwRkhr7GjVBM_jg54%2BNa%3DQ%40mail.gmail.com
---
src/backend/access/transam/xlog.c | 173 ++++++++++++++++++++++++
src/backend/access/transam/xlogreader.c | 40 +++++-
src/backend/access/transam/xlogutils.c | 11 +-
src/backend/postmaster/walsummarizer.c | 10 +-
src/backend/replication/walsender.c | 10 +-
src/include/access/xlog.h | 3 +
6 files changed, 231 insertions(+), 16 deletions(-)
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 478377c4a2..886eaf12e3 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -1705,6 +1705,179 @@ GetXLogBuffer(XLogRecPtr ptr, TimeLineID tli)
return cachedPos + ptr % XLOG_BLCKSZ;
}
+/*
+ * Read WAL from WAL buffers.
+ *
+ * This function reads 'count' bytes of WAL from WAL buffers into 'buf'
+ * starting at location 'startptr' on timeline 'tli' and returns total bytes
+ * read.
+ *
+ * Points to note:
+ *
+ * - This function reads as much as it can from WAL buffers, meaning, it may
+ * not read all the requested 'count' bytes. Caller must be aware of this and
+ * deal with it.
+ *
+ * - This function reads WAL from WAL buffers without holding any lock. First
+ * it reads xlblocks atomically for checking page existence, then it reads the
+ * page contents and validates. Finally, it rechecks the page existence by
+ * re-reading xlblocks; if the read page is replaced, it discards it and
+ * returns.
+ *
+ * - This function is not available for frontend code as WAL buffers are
+ * internal to the server.
+ *
+ * - This function waits for any in-progress WAL insertions to WAL buffers to
+ * finish.
+ */
+Size
+XLogReadFromBuffers(XLogRecPtr startptr, TimeLineID tli, Size count,
+ char *buf)
+{
+ XLogRecPtr ptr;
+ Size nbytes;
+ Size ntotal = 0;
+ char *dst;
+ uint64 bytepos;
+ XLogRecPtr reservedUpto;
+ XLogwrtResult LogwrtResult;
+
+ /*
+ * Fast paths for the following reasons: 1) WAL buffers aren't in use when
+ * server is in recovery. 2) WAL is inserted into WAL buffers on current
+ * server's insertion TLI. 3) Invalid starting WAL location.
+ */
+ if (RecoveryInProgress() ||
+ tli != GetWALInsertionTimeLine() ||
+ XLogRecPtrIsInvalid(startptr))
+ return ntotal;
+
+ /* Read the current insert position */
+ SpinLockAcquire(&XLogCtl->Insert.insertpos_lck);
+ bytepos = XLogCtl->Insert.CurrBytePos;
+ SpinLockRelease(&XLogCtl->Insert.insertpos_lck);
+
+ reservedUpto = XLogBytePosToEndRecPtr(bytepos);
+
+ /*
+ * WAL being read doesn't yet exist i.e. past the current insert position.
+ */
+ if ((startptr + count) > reservedUpto)
+ return ntotal;
+
+ SpinLockAcquire(&XLogCtl->info_lck);
+ LogwrtResult = XLogCtl->LogwrtResult;
+ SpinLockRelease(&XLogCtl->info_lck);
+
+ /* Wait for any in-progress WAL insertions to WAL buffers to finish. */
+ if ((startptr + count) > LogwrtResult.Write &&
+ (startptr + count) <= reservedUpto)
+ WaitXLogInsertionsToFinish(startptr + count);
+
+ ptr = startptr;
+ nbytes = count;
+ dst = buf;
+
+ while (nbytes > 0)
+ {
+ XLogRecPtr expectedEndPtr;
+ XLogRecPtr endptr;
+ int idx;
+ char *page;
+ char *data;
+ Size nread;
+ XLogPageHeader phdr;
+
+ idx = XLogRecPtrToBufIdx(ptr);
+ expectedEndPtr = ptr;
+ expectedEndPtr += XLOG_BLCKSZ - ptr % XLOG_BLCKSZ;
+ endptr = pg_atomic_read_u64(&XLogCtl->xlblocks[idx]);
+
+ /* Requested WAL isn't available in WAL buffers. */
+ if (expectedEndPtr != endptr)
+ break;
+
+ /*
+ * We found WAL buffer page containing given XLogRecPtr. Get starting
+ * address of the page and a pointer to the right location of given
+ * XLogRecPtr in that page.
+ */
+ page = XLogCtl->pages + idx * (Size) XLOG_BLCKSZ;
+ data = page + ptr % XLOG_BLCKSZ;
+
+ /*
+ * Make sure we don't read xlblocks up above before the page contents
+ * down below.
+ */
+ pg_read_barrier();
+
+ nread = 0;
+
+ /* Read what is wanted, not the whole page. */
+ if ((data + nbytes) <= (page + XLOG_BLCKSZ))
+ {
+ /* All the bytes are in one page. */
+ nread = nbytes;
+ }
+ else
+ {
+ /*
+ * All the bytes are not in one page. Read available bytes on the
+ * current page, copy them over to output buffer and continue to
+ * read remaining bytes.
+ */
+ nread = XLOG_BLCKSZ - (data - page);
+ Assert(nread > 0 && nread <= nbytes);
+ }
+
+ Assert(nread > 0);
+ memcpy(dst, data, nread);
+
+ /*
+ * Make sure we don't read xlblocks down below before the page
+ * contents up above.
+ */
+ pg_read_barrier();
+
+ /* Recheck if the read page still exists in WAL buffers. */
+ endptr = pg_atomic_read_u64(&XLogCtl->xlblocks[idx]);
+
+ /* Return if the page got initalized while we were reading it. */
+ if (expectedEndPtr != endptr)
+ break;
+
+ /*
+ * Typically, we must not read a WAL buffer page that just got
+ * initialized. Because we waited enough for the in-progress WAL
+ * insertions to finish above. However, there can exist a slight
+ * window after the above wait finishes in which the read buffer page
+ * can get replaced especially under high WAL generation rates. After
+ * all, we are reading from WAL buffers without any locks here. So,
+ * let's not count such a page in.
+ */
+ phdr = (XLogPageHeader) page;
+ if (!(phdr->xlp_magic == XLOG_PAGE_MAGIC &&
+ phdr->xlp_pageaddr == (ptr - (ptr % XLOG_BLCKSZ)) &&
+ phdr->xlp_tli == tli))
+ break;
+
+ dst += nread;
+ ptr += nread;
+ ntotal += nread;
+ nbytes -= nread;
+ }
+
+ /* We never read more than what the caller has asked for. */
+ Assert(ntotal <= count);
+
+ ereport(DEBUG1,
+ errmsg_internal("read %zu bytes out of %zu bytes from WAL buffers for given start LSN %X/%X, timeline ID %u",
+ ntotal, count,
+ LSN_FORMAT_ARGS(startptr), tli));
+
+ return ntotal;
+}
+
/*
* Converts a "usable byte position" to XLogRecPtr. A usable byte position
* is the position starting from the beginning of WAL, excluding all WAL
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index 7190156f2f..639bba2ad9 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -1501,17 +1501,47 @@ err:
* Returns true if succeeded, false if an error occurs, in which case
* 'errinfo' receives error details.
*
- * XXX probably this should be improved to suck data directly from the
- * WAL buffers when possible.
+ * When possible, this function reads WAL from WAL buffers. When requested WAL
+ * isn't available in WAL buffers, it is read from the WAL file as usual.
*/
bool
-WALRead(XLogReaderState *state,
- char *buf, XLogRecPtr startptr, Size count, TimeLineID tli,
- WALReadError *errinfo)
+WALRead(XLogReaderState *state, char *buf, XLogRecPtr startptr,
+ Size count, TimeLineID tli, WALReadError *errinfo)
{
char *p;
XLogRecPtr recptr;
Size nbytes;
+#ifndef FRONTEND
+ Size nread;
+#endif
+
+#ifndef FRONTEND
+
+ /*
+ * Try reading WAL from WAL buffers. Frontend code has no idea of WAL
+ * buffers.
+ */
+ nread = XLogReadFromBuffers(startptr, tli, count, buf);
+
+ if (nread > 0)
+ {
+ /*
+ * Check if its a full read, short read or no read from WAL buffers.
+ * For short read or no read, continue to read the remaining bytes
+ * from WAL file.
+ *
+ * XXX: It might be worth to expose WAL buffer read stats.
+ */
+ if (nread == count) /* full read */
+ return true;
+ else if (nread < count) /* short read */
+ {
+ buf += nread;
+ startptr += nread;
+ count -= nread;
+ }
+ }
+#endif
p = buf;
recptr = startptr;
diff --git a/src/backend/access/transam/xlogutils.c b/src/backend/access/transam/xlogutils.c
index aa8667abd1..fafab9aa32 100644
--- a/src/backend/access/transam/xlogutils.c
+++ b/src/backend/access/transam/xlogutils.c
@@ -1007,12 +1007,13 @@ read_local_xlog_page_guts(XLogReaderState *state, XLogRecPtr targetPagePtr,
}
/*
- * Even though we just determined how much of the page can be validly read
- * as 'count', read the whole page anyway. It's guaranteed to be
- * zero-padded up to the page boundary if it's incomplete.
+ * We determined how much of the page can be validly read as 'count', read
+ * that much only, not the entire page. Since WALRead() can read the page
+ * from WAL buffers, in which case, the page is not guaranteed to be
+ * zero-padded up to the page boundary because of the concurrent
+ * insertions.
*/
- if (!WALRead(state, cur_page, targetPagePtr, XLOG_BLCKSZ, tli,
- &errinfo))
+ if (!WALRead(state, cur_page, targetPagePtr, count, tli, &errinfo))
WALReadRaiseError(&errinfo);
/* number of valid bytes in the buffer */
diff --git a/src/backend/postmaster/walsummarizer.c b/src/backend/postmaster/walsummarizer.c
index f828cc436a..d465848bc9 100644
--- a/src/backend/postmaster/walsummarizer.c
+++ b/src/backend/postmaster/walsummarizer.c
@@ -1254,11 +1254,13 @@ summarizer_read_local_xlog_page(XLogReaderState *state,
}
/*
- * Even though we just determined how much of the page can be validly read
- * as 'count', read the whole page anyway. It's guaranteed to be
- * zero-padded up to the page boundary if it's incomplete.
+ * We determined how much of the page can be validly read as 'count', read
+ * that much only, not the entire page. Since WALRead() can read the page
+ * from WAL buffers, in which case, the page is not guaranteed to be
+ * zero-padded up to the page boundary because of the concurrent
+ * insertions.
*/
- if (!WALRead(state, cur_page, targetPagePtr, XLOG_BLCKSZ,
+ if (!WALRead(state, cur_page, targetPagePtr, count,
private_data->tli, &errinfo))
WALReadRaiseError(&errinfo);
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index 087031e9dc..b35406bcdf 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -1095,11 +1095,17 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
else
count = flushptr - targetPagePtr; /* part of the page available */
- /* now actually read the data, we know it's there */
+ /*
+ * We determined how much of the page can be validly read as 'count', read
+ * that much only, not the entire page. Since WALRead() can read the page
+ * from WAL buffers, in which case, the page is not guaranteed to be
+ * zero-padded up to the page boundary because of the concurrent
+ * insertions.
+ */
if (!WALRead(state,
cur_page,
targetPagePtr,
- XLOG_BLCKSZ,
+ count,
currTLI, /* Pass the current TLI because only
* WalSndSegmentOpen controls whether new TLI
* is needed. */
diff --git a/src/include/access/xlog.h b/src/include/access/xlog.h
index 301c5fa11f..fa760a92d5 100644
--- a/src/include/access/xlog.h
+++ b/src/include/access/xlog.h
@@ -252,6 +252,9 @@ extern XLogRecPtr GetLastImportantRecPtr(void);
extern void SetWalWriterSleeping(bool sleeping);
+extern Size XLogReadFromBuffers(XLogRecPtr startptr, TimeLineID tli,
+ Size count, char *buf);
+
/*
* Routines used by xlogrecovery.c to call back into xlog.c during recovery.
*/
--
2.34.1
[application/x-patch] v19-0002-Add-test-module-for-verifying-WAL-read-from-WAL-.patch (9.5K, ../../CALj2ACWPKuoUdA7r4VFKhsTZeNekyUr=ZwM6ckyvJ0qv2f6Whg@mail.gmail.com/3-v19-0002-Add-test-module-for-verifying-WAL-read-from-WAL-.patch)
download | inline diff:
From 35e9c0afe130d79e4f74dfbe3a445cf3d594ec14 Mon Sep 17 00:00:00 2001
From: Bharath Rupireddy <[email protected]>
Date: Wed, 10 Jan 2024 13:36:15 +0000
Subject: [PATCH v19] Add test module for verifying WAL read from WAL buffers
This commit adds a test module to verify WAL read from WAL
buffers.
Author: Bharath Rupireddy
Reviewed-by: Dilip Kumar
Discussion: https://www.postgresql.org/message-id/CALj2ACXKKK%3DwbiG5_t6dGao5GoecMwRkhr7GjVBM_jg54%2BNa%3DQ%40mail.gmail.com
---
src/test/modules/Makefile | 1 +
src/test/modules/meson.build | 1 +
.../test_wal_read_from_buffers/.gitignore | 4 ++
.../test_wal_read_from_buffers/Makefile | 23 ++++++++
.../test_wal_read_from_buffers/meson.build | 33 ++++++++++++
.../test_wal_read_from_buffers/t/001_basic.pl | 54 +++++++++++++++++++
.../test_wal_read_from_buffers--1.0.sql | 16 ++++++
.../test_wal_read_from_buffers.c | 44 +++++++++++++++
.../test_wal_read_from_buffers.control | 4 ++
9 files changed, 180 insertions(+)
create mode 100644 src/test/modules/test_wal_read_from_buffers/.gitignore
create mode 100644 src/test/modules/test_wal_read_from_buffers/Makefile
create mode 100644 src/test/modules/test_wal_read_from_buffers/meson.build
create mode 100644 src/test/modules/test_wal_read_from_buffers/t/001_basic.pl
create mode 100644 src/test/modules/test_wal_read_from_buffers/test_wal_read_from_buffers--1.0.sql
create mode 100644 src/test/modules/test_wal_read_from_buffers/test_wal_read_from_buffers.c
create mode 100644 src/test/modules/test_wal_read_from_buffers/test_wal_read_from_buffers.control
diff --git a/src/test/modules/Makefile b/src/test/modules/Makefile
index 5d33fa6a9a..64a051ce1c 100644
--- a/src/test/modules/Makefile
+++ b/src/test/modules/Makefile
@@ -33,6 +33,7 @@ SUBDIRS = \
test_rls_hooks \
test_shm_mq \
test_slru \
+ test_wal_read_from_buffers \
unsafe_tests \
worker_spi \
xid_wraparound
diff --git a/src/test/modules/meson.build b/src/test/modules/meson.build
index 00ff1d77d1..d5ec3bd3a9 100644
--- a/src/test/modules/meson.build
+++ b/src/test/modules/meson.build
@@ -30,6 +30,7 @@ subdir('test_resowner')
subdir('test_rls_hooks')
subdir('test_shm_mq')
subdir('test_slru')
+subdir('test_wal_read_from_buffers')
subdir('unsafe_tests')
subdir('worker_spi')
subdir('xid_wraparound')
diff --git a/src/test/modules/test_wal_read_from_buffers/.gitignore b/src/test/modules/test_wal_read_from_buffers/.gitignore
new file mode 100644
index 0000000000..5dcb3ff972
--- /dev/null
+++ b/src/test/modules/test_wal_read_from_buffers/.gitignore
@@ -0,0 +1,4 @@
+# Generated subdirectories
+/log/
+/results/
+/tmp_check/
diff --git a/src/test/modules/test_wal_read_from_buffers/Makefile b/src/test/modules/test_wal_read_from_buffers/Makefile
new file mode 100644
index 0000000000..7472494501
--- /dev/null
+++ b/src/test/modules/test_wal_read_from_buffers/Makefile
@@ -0,0 +1,23 @@
+# src/test/modules/test_wal_read_from_buffers/Makefile
+
+MODULE_big = test_wal_read_from_buffers
+OBJS = \
+ $(WIN32RES) \
+ test_wal_read_from_buffers.o
+PGFILEDESC = "test_wal_read_from_buffers - test module to read WAL from WAL buffers"
+
+EXTENSION = test_wal_read_from_buffers
+DATA = test_wal_read_from_buffers--1.0.sql
+
+TAP_TESTS = 1
+
+ifdef USE_PGXS
+PG_CONFIG = pg_config
+PGXS := $(shell $(PG_CONFIG) --pgxs)
+include $(PGXS)
+else
+subdir = src/test/modules/test_wal_read_from_buffers
+top_builddir = ../../../..
+include $(top_builddir)/src/Makefile.global
+include $(top_srcdir)/contrib/contrib-global.mk
+endif
diff --git a/src/test/modules/test_wal_read_from_buffers/meson.build b/src/test/modules/test_wal_read_from_buffers/meson.build
new file mode 100644
index 0000000000..40bd5dcd33
--- /dev/null
+++ b/src/test/modules/test_wal_read_from_buffers/meson.build
@@ -0,0 +1,33 @@
+# Copyright (c) 2023, PostgreSQL Global Development Group
+
+test_wal_read_from_buffers_sources = files(
+ 'test_wal_read_from_buffers.c',
+)
+
+if host_system == 'windows'
+ test_wal_read_from_buffers_sources += rc_lib_gen.process(win32ver_rc, extra_args: [
+ '--NAME', 'test_wal_read_from_buffers',
+ '--FILEDESC', 'test_wal_read_from_buffers - test module to read WAL from WAL buffers',])
+endif
+
+test_wal_read_from_buffers = shared_module('test_wal_read_from_buffers',
+ test_wal_read_from_buffers_sources,
+ kwargs: pg_test_mod_args,
+)
+test_install_libs += test_wal_read_from_buffers
+
+test_install_data += files(
+ 'test_wal_read_from_buffers.control',
+ 'test_wal_read_from_buffers--1.0.sql',
+)
+
+tests += {
+ 'name': 'test_wal_read_from_buffers',
+ 'sd': meson.current_source_dir(),
+ 'bd': meson.current_build_dir(),
+ 'tap': {
+ 'tests': [
+ 't/001_basic.pl',
+ ],
+ },
+}
diff --git a/src/test/modules/test_wal_read_from_buffers/t/001_basic.pl b/src/test/modules/test_wal_read_from_buffers/t/001_basic.pl
new file mode 100644
index 0000000000..1d842bb02e
--- /dev/null
+++ b/src/test/modules/test_wal_read_from_buffers/t/001_basic.pl
@@ -0,0 +1,54 @@
+# Copyright (c) 2021-2023, PostgreSQL Global Development Group
+
+use strict;
+use warnings;
+
+use PostgreSQL::Test::Cluster;
+use PostgreSQL::Test::Utils;
+use Test::More;
+
+my $node = PostgreSQL::Test::Cluster->new('test');
+
+$node->init;
+
+# Ensure nobody interferes with us so that the WAL in WAL buffers don't get
+# overwritten while running tests.
+$node->append_conf(
+ 'postgresql.conf', qq(
+autovacuum = off
+checkpoint_timeout = 1h
+wal_writer_delay = 10000ms
+wal_writer_flush_after = 1GB
+));
+$node->start;
+
+# Setup.
+$node->safe_psql('postgres', 'CREATE EXTENSION test_wal_read_from_buffers;');
+
+# Get current insert LSN. After this, we generate some WAL which is guranteed
+# to be in WAL buffers as there is no other WAL generating activity is
+# happening on the server. We then verify if we can read the WAL from WAL
+# buffers using this LSN.
+my $lsn = $node->safe_psql('postgres', 'SELECT pg_current_wal_insert_lsn();');
+
+# Generate minimal WAL so that WAL buffers don't get overwritten.
+$node->safe_psql('postgres',
+ "CREATE TABLE t (c int); INSERT INTO t VALUES (1);");
+
+# Check if WAL is successfully read from WAL buffers.
+my $result = $node->safe_psql('postgres',
+ qq{SELECT test_wal_read_from_buffers('$lsn');});
+is($result, 't', "WAL is successfully read from WAL buffers");
+
+# Check with a WAL that doesn't yet exist.
+$lsn = $node->safe_psql('postgres', 'SELECT pg_current_wal_flush_lsn()+8192;');
+$result = $node->safe_psql('postgres',
+ qq{SELECT test_wal_read_from_buffers('$lsn');});
+is($result, 'f', "WAL that doesn't yet exist is not read from WAL buffers");
+
+# Check with invalid input.
+$result = $node->safe_psql('postgres',
+ qq{SELECT test_wal_read_from_buffers('0/0');});
+is($result, 'f', "WAL is not read from WAL buffers with invalid input");
+
+done_testing();
diff --git a/src/test/modules/test_wal_read_from_buffers/test_wal_read_from_buffers--1.0.sql b/src/test/modules/test_wal_read_from_buffers/test_wal_read_from_buffers--1.0.sql
new file mode 100644
index 0000000000..c6ffb3fa65
--- /dev/null
+++ b/src/test/modules/test_wal_read_from_buffers/test_wal_read_from_buffers--1.0.sql
@@ -0,0 +1,16 @@
+/* src/test/modules/test_wal_read_from_buffers/test_wal_read_from_buffers--1.0.sql */
+
+-- complain if script is sourced in psql, rather than via CREATE EXTENSION
+\echo Use "CREATE EXTENSION test_wal_read_from_buffers" to load this file. \quit
+
+--
+-- test_wal_read_from_buffers()
+--
+-- Returns true if WAL data at a given LSN can be read from WAL buffers.
+-- Otherwise returns false.
+--
+CREATE FUNCTION test_wal_read_from_buffers(IN lsn pg_lsn,
+ read_successful OUT boolean
+)
+AS 'MODULE_PATHNAME', 'test_wal_read_from_buffers'
+LANGUAGE C STRICT;
diff --git a/src/test/modules/test_wal_read_from_buffers/test_wal_read_from_buffers.c b/src/test/modules/test_wal_read_from_buffers/test_wal_read_from_buffers.c
new file mode 100644
index 0000000000..e54c64236d
--- /dev/null
+++ b/src/test/modules/test_wal_read_from_buffers/test_wal_read_from_buffers.c
@@ -0,0 +1,44 @@
+/*--------------------------------------------------------------------------
+ *
+ * test_wal_read_from_buffers.c
+ * Test module to read WAL from WAL buffers.
+ *
+ * Portions Copyright (c) 2023, PostgreSQL Global Development Group
+ *
+ * IDENTIFICATION
+ * src/test/modules/test_wal_read_from_buffers/test_wal_read_from_buffers.c
+ * -------------------------------------------------------------------------
+ */
+
+#include "postgres.h"
+
+#include "access/xlog.h"
+#include "fmgr.h"
+#include "utils/pg_lsn.h"
+
+PG_MODULE_MAGIC;
+
+/*
+ * SQL function for verifying that WAL data at a given LSN can be read from WAL
+ * buffers. Returns true if read from WAL buffers, otherwise false.
+ */
+PG_FUNCTION_INFO_V1(test_wal_read_from_buffers);
+Datum
+test_wal_read_from_buffers(PG_FUNCTION_ARGS)
+{
+ char data[XLOG_BLCKSZ] = {0};
+ Size nread;
+ bool is_read;
+
+ nread = XLogReadFromBuffers(PG_GETARG_LSN(0),
+ GetWALInsertionTimeLine(),
+ XLOG_BLCKSZ,
+ data);
+
+ if (nread > 0)
+ is_read = true;
+ else
+ is_read = false;
+
+ PG_RETURN_BOOL(is_read);
+}
diff --git a/src/test/modules/test_wal_read_from_buffers/test_wal_read_from_buffers.control b/src/test/modules/test_wal_read_from_buffers/test_wal_read_from_buffers.control
new file mode 100644
index 0000000000..eda8d47954
--- /dev/null
+++ b/src/test/modules/test_wal_read_from_buffers/test_wal_read_from_buffers.control
@@ -0,0 +1,4 @@
+comment = 'Test module to read WAL from WAL buffers'
+default_version = '1.0'
+module_pathname = '$libdir/test_wal_read_from_buffers'
+relocatable = true
--
2.34.1
^ permalink raw reply [nested|flat] 22+ messages in thread
* Re: Improve WALRead() to suck data directly from WAL buffers when possible
@ 2024-01-20 01:47 Jeff Davis <[email protected]>
parent: Bharath Rupireddy <[email protected]>
2 siblings, 0 replies; 22+ messages in thread
From: Jeff Davis @ 2024-01-20 01:47 UTC (permalink / raw)
To: Bharath Rupireddy <[email protected]>; +Cc: Andres Freund <[email protected]>; Dilip Kumar <[email protected]>; Kyotaro Horiguchi <[email protected]>; [email protected]; Nathan Bossart <[email protected]>; Masahiko Sawada <[email protected]>
On Wed, 2024-01-10 at 19:59 +0530, Bharath Rupireddy wrote:
> I've addressed the above review comments and attached v19 patch-set.
Regarding:
- if (!WALRead(state, cur_page, targetPagePtr, XLOG_BLCKSZ, tli,
- &errinfo))
+ if (!WALRead(state, cur_page, targetPagePtr, count, tli,
&errinfo))
I'd like to understand the reason it was using XLOG_BLCKSZ before. Was
it a performance optimization? Or was it to zero the remainder of the
caller's buffer (readBuf)? Or something else?
If it was to zero the remainder of the caller's buffer, then we should
explicitly make that the caller's responsibility.
Regards,
Jeff Davis
^ permalink raw reply [nested|flat] 22+ messages in thread
* Re: Improve WALRead() to suck data directly from WAL buffers when possible
@ 2024-01-22 14:03 Melih Mutlu <[email protected]>
parent: Bharath Rupireddy <[email protected]>
2 siblings, 0 replies; 22+ messages in thread
From: Melih Mutlu @ 2024-01-22 14:03 UTC (permalink / raw)
To: Bharath Rupireddy <[email protected]>; +Cc: Jeff Davis <[email protected]>; Andres Freund <[email protected]>; Dilip Kumar <[email protected]>; Kyotaro Horiguchi <[email protected]>; [email protected]; Nathan Bossart <[email protected]>; Masahiko Sawada <[email protected]>
Hi Bharath,
Thanks for working on this. It seems like a nice improvement to have.
Here are some comments on 0001 patch.
1- xlog.c
+ /*
+ * Fast paths for the following reasons: 1) WAL buffers aren't in use when
+ * server is in recovery. 2) WAL is inserted into WAL buffers on current
+ * server's insertion TLI. 3) Invalid starting WAL location.
+ */
Shouldn't the comment be something like "2) WAL is *not* inserted into WAL
buffers on current server's insertion TLI" since the condition to break is tli
!= GetWALInsertionTimeLine()
2-
+ /*
+ * WAL being read doesn't yet exist i.e. past the current insert position.
+ */
+ if ((startptr + count) > reservedUpto)
+ return ntotal;
This question may not even make sense but I wonder whether we can read from
startptr only to reservedUpto in case of startptr+count exceeds
reservedUpto?
3-
+ /* Wait for any in-progress WAL insertions to WAL buffers to finish. */
+ if ((startptr + count) > LogwrtResult.Write &&
+ (startptr + count) <= reservedUpto)
+ WaitXLogInsertionsToFinish(startptr + count);
Do we need to check if (startptr + count) <= reservedUpto as we already
verified this condition a few lines above?
4-
+ Assert(nread > 0);
+ memcpy(dst, data, nread);
+
+ /*
+ * Make sure we don't read xlblocks down below before the page
+ * contents up above.
+ */
+ pg_read_barrier();
+
+ /* Recheck if the read page still exists in WAL buffers. */
+ endptr = pg_atomic_read_u64(&XLogCtl->xlblocks[idx]);
+
+ /* Return if the page got initalized while we were reading it. */
+ if (expectedEndPtr != endptr)
+ break;
+
+ /*
+ * Typically, we must not read a WAL buffer page that just got
+ * initialized. Because we waited enough for the in-progress WAL
+ * insertions to finish above. However, there can exist a slight
+ * window after the above wait finishes in which the read buffer page
+ * can get replaced especially under high WAL generation rates. After
+ * all, we are reading from WAL buffers without any locks here. So,
+ * let's not count such a page in.
+ */
+ phdr = (XLogPageHeader) page;
+ if (!(phdr->xlp_magic == XLOG_PAGE_MAGIC &&
+ phdr->xlp_pageaddr == (ptr - (ptr % XLOG_BLCKSZ)) &&
+ phdr->xlp_tli == tli))
+ break;
I see that you recheck if the page still exists and so at the end. What
would you think about memcpy'ing only after being sure that we will need
and use the recently read data? If we break the loop during the recheck, we
simply discard the data read in the latest attempt. I guess that this may
not be a big deal but the data would be unnecessarily copied into the
destination in such a case.
5- xlogreader.c
+ nread = XLogReadFromBuffers(startptr, tli, count, buf);
+
+ if (nread > 0)
+ {
+ /*
+ * Check if its a full read, short read or no read from WAL buffers.
+ * For short read or no read, continue to read the remaining bytes
+ * from WAL file.
+ *
+ * XXX: It might be worth to expose WAL buffer read stats.
+ */
+ if (nread == count) /* full read */
+ return true;
+ else if (nread < count) /* short read */
+ {
+ buf += nread;
+ startptr += nread;
+ count -= nread;
+ }
Typo in the comment. Should be like "Check if *it's* a full read, short
read or no read from WAL buffers."
Also I don't think XLogReadFromBuffers() returns anything less than 0 and
more than count. Is verifying nread > 0 necessary? I think if nread does
not equal to count, we can simply assume that it's a short read. (or no
read at all in case nread is 0 which we don't need to handle specifically)
6-
+ /*
+ * We determined how much of the page can be validly read as 'count', read
+ * that much only, not the entire page. Since WALRead() can read the page
+ * from WAL buffers, in which case, the page is not guaranteed to be
+ * zero-padded up to the page boundary because of the concurrent
+ * insertions.
+ */
I'm not sure about pasting this into the most places we call WalRead().
Wouldn't it be better if we mention this somewhere around WALRead() only
once?
Best,
--
Melih Mutlu
Microsoft
^ permalink raw reply [nested|flat] 22+ messages in thread
* Re: Improve WALRead() to suck data directly from WAL buffers when possible
@ 2024-01-22 20:12 Andres Freund <[email protected]>
parent: Bharath Rupireddy <[email protected]>
2 siblings, 1 reply; 22+ messages in thread
From: Andres Freund @ 2024-01-22 20:12 UTC (permalink / raw)
To: Bharath Rupireddy <[email protected]>; +Cc: Jeff Davis <[email protected]>; Dilip Kumar <[email protected]>; Kyotaro Horiguchi <[email protected]>; [email protected]; Nathan Bossart <[email protected]>; Masahiko Sawada <[email protected]>
Hi,
On 2024-01-10 19:59:29 +0530, Bharath Rupireddy wrote:
> + /*
> + * Typically, we must not read a WAL buffer page that just got
> + * initialized. Because we waited enough for the in-progress WAL
> + * insertions to finish above. However, there can exist a slight
> + * window after the above wait finishes in which the read buffer page
> + * can get replaced especially under high WAL generation rates. After
> + * all, we are reading from WAL buffers without any locks here. So,
> + * let's not count such a page in.
> + */
> + phdr = (XLogPageHeader) page;
> + if (!(phdr->xlp_magic == XLOG_PAGE_MAGIC &&
> + phdr->xlp_pageaddr == (ptr - (ptr % XLOG_BLCKSZ)) &&
> + phdr->xlp_tli == tli))
> + break;
I still think that anything that requires such checks shouldn't be
merged. It's completely bogus to check page contents for validity when we
should have metadata telling us which range of the buffers is valid and which
not.
Greetings,
Andres Freund
^ permalink raw reply [nested|flat] 22+ messages in thread
* Re: Improve WALRead() to suck data directly from WAL buffers when possible
@ 2024-01-23 04:07 Jeff Davis <[email protected]>
parent: Andres Freund <[email protected]>
0 siblings, 1 reply; 22+ messages in thread
From: Jeff Davis @ 2024-01-23 04:07 UTC (permalink / raw)
To: Andres Freund <[email protected]>; Bharath Rupireddy <[email protected]>; +Cc: Dilip Kumar <[email protected]>; Kyotaro Horiguchi <[email protected]>; [email protected]; Nathan Bossart <[email protected]>; Masahiko Sawada <[email protected]>
On Mon, 2024-01-22 at 12:12 -0800, Andres Freund wrote:
> I still think that anything that requires such checks shouldn't be
> merged. It's completely bogus to check page contents for validity
> when we
> should have metadata telling us which range of the buffers is valid
> and which
> not.
The check seems entirely unnecessary, to me. A leftover from v18?
I have attached a new patch (version "19j") to illustrate some of my
previous suggestions. I didn't spend a lot of time on it so it's not
ready for commit, but I believe my suggestions are easier to understand
in code form.
Note that, right now, it only works for XLogSendPhysical(). I believe
it's best to just make it work for 1-3 callers that we understand well,
and we can generalize later if it makes sense.
I'm still not clear on why some callers are reading XLOG_BLCKSZ
(expecting zeros at the end), and if it's OK to just change them to use
the exact byte count.
Also, if we've detected that the first requested buffer has been
evicted, is there any value in continuing the loop to see if more
recent buffers are available? For example, if the requested LSNs range
over buffers 4, 5, and 6, and 4 has already been evicted, should we try
to return LSN data from 5 and 6 at the proper offset in the dest
buffer? If so, we'd need to adjust the API so the caller knows what
parts of the dest buffer were filled in.
Regards,
Jeff Davis
Attachments:
[text/x-patch] v19j-0001-Add-XLogReadFromBuffers.patch (8.8K, ../../[email protected]/2-v19j-0001-Add-XLogReadFromBuffers.patch)
download | inline diff:
From 34d89d6b869a454fd15097d8a0f0b6d3997a74da Mon Sep 17 00:00:00 2001
From: Jeff Davis <[email protected]>
Date: Mon, 22 Jan 2024 16:21:53 -0800
Subject: [PATCH v19j 1/2] Add XLogReadFromBuffers().
Allows reading directly from WAL buffers without a lock, avoiding the
need to wait for WAL flushing and read from the filesystem.
For now, the only caller is physical replication, but we can consider
expanding it to other callers as needed.
Author: Bharath Rupireddy
---
src/backend/access/transam/xlog.c | 145 ++++++++++++++++++++++++++--
src/backend/replication/walsender.c | 14 +++
src/include/access/xlog.h | 2 +
3 files changed, 152 insertions(+), 9 deletions(-)
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 478377c4a2..c9619139af 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -698,7 +698,7 @@ static void ReserveXLogInsertLocation(int size, XLogRecPtr *StartPos,
XLogRecPtr *EndPos, XLogRecPtr *PrevPtr);
static bool ReserveXLogSwitch(XLogRecPtr *StartPos, XLogRecPtr *EndPos,
XLogRecPtr *PrevPtr);
-static XLogRecPtr WaitXLogInsertionsToFinish(XLogRecPtr upto);
+static XLogRecPtr WaitXLogInsertionsToFinish(XLogRecPtr upto, bool emitLog);
static char *GetXLogBuffer(XLogRecPtr ptr, TimeLineID tli);
static XLogRecPtr XLogBytePosToRecPtr(uint64 bytepos);
static XLogRecPtr XLogBytePosToEndRecPtr(uint64 bytepos);
@@ -1494,7 +1494,7 @@ WALInsertLockUpdateInsertingAt(XLogRecPtr insertingAt)
* to make room for a new one, which in turn requires WALWriteLock.
*/
static XLogRecPtr
-WaitXLogInsertionsToFinish(XLogRecPtr upto)
+WaitXLogInsertionsToFinish(XLogRecPtr upto, bool emitLog)
{
uint64 bytepos;
XLogRecPtr reservedUpto;
@@ -1521,9 +1521,10 @@ WaitXLogInsertionsToFinish(XLogRecPtr upto)
*/
if (upto > reservedUpto)
{
- ereport(LOG,
- (errmsg("request to flush past end of generated WAL; request %X/%X, current position %X/%X",
- LSN_FORMAT_ARGS(upto), LSN_FORMAT_ARGS(reservedUpto))));
+ if (emitLog)
+ ereport(LOG,
+ (errmsg("request to flush past end of generated WAL; request %X/%X, current position %X/%X",
+ LSN_FORMAT_ARGS(upto), LSN_FORMAT_ARGS(reservedUpto))));
upto = reservedUpto;
}
@@ -1705,6 +1706,132 @@ GetXLogBuffer(XLogRecPtr ptr, TimeLineID tli)
return cachedPos + ptr % XLOG_BLCKSZ;
}
+/*
+ * Read WAL directly from WAL buffers, if available.
+ *
+ * This function reads 'count' bytes of WAL from WAL buffers into 'buf'
+ * starting at location 'startptr' and returns total bytes read.
+ *
+ * The bytes read may be fewer than requested if any of the WAL buffers in the
+ * requested range have been evicted, or if the last requested byte is beyond
+ * the Insert pointer.
+ *
+ * If reading beyond the Write pointer, this function will wait for concurent
+ * inserters to finish. Otherwise, it does not wait at all.
+ *
+ * The caller must ensure that it's reasonable to read from the WAL buffers,
+ * i.e. that the requested data is from the current timeline, that we're not
+ * in recovery, etc.
+ */
+Size
+XLogReadFromBuffers(char *buf, XLogRecPtr startptr, Size count)
+{
+ XLogRecPtr ptr = startptr;
+ XLogRecPtr upto = startptr + count;
+ Size nbytes = count;
+ Size ntotal = 0;
+ char *dst = buf;
+
+ Assert(!RecoveryInProgress());
+ Assert(!XLogRecPtrIsInvalid(startptr));
+
+ /*
+ * Caller requested very recent WAL data. Wait for any in-progress WAL
+ * insertions to WAL buffers to finish.
+ *
+ * Most callers will have already updated LogwrtResult when determining
+ * how far to read, but it's OK if it's out of date. (XXX: is it worth
+ * taking a spinlock to update LogwrtResult and check again before calling
+ * WaitXLogInsertionsToFinish()?)
+ */
+ if (upto > LogwrtResult.Write)
+ {
+ XLogRecPtr writtenUpto = WaitXLogInsertionsToFinish(upto, false);
+
+ upto = Min(upto, writtenUpto);
+ nbytes = upto - startptr;
+ }
+
+ /*
+ * Loop through the buffers without a lock. For each buffer, atomically
+ * read and verify the end pointer, then copy the data out, and finally
+ * re-read and re-verify the end pointer.
+ *
+ * Once a page is evicted, it never returns to the WAL buffers, so if the
+ * end pointer matches the expected end pointer before and after we copy
+ * the data, then the right page must have been present during the data
+ * copy. Read barriers are necessary to ensure that the data copy actually
+ * happens between the two verification steps.
+ *
+ * If the verification fails, we simply terminate the loop and return the
+ * data had been already copied out successfully.
+ */
+ while (nbytes > 0)
+ {
+ XLogRecPtr expectedEndPtr;
+ XLogRecPtr endptr;
+ int idx;
+ const char *page;
+ const char *data;
+ Size nread;
+
+ idx = XLogRecPtrToBufIdx(ptr);
+ expectedEndPtr = ptr;
+ expectedEndPtr += XLOG_BLCKSZ - ptr % XLOG_BLCKSZ;
+
+ /*
+ * First verification step: check that the correct page is present in
+ * the WAL buffers
+ */
+ endptr = pg_atomic_read_u64(&XLogCtl->xlblocks[idx]);
+ if (expectedEndPtr != endptr)
+ break;
+
+ /*
+ * We found WAL buffer page containing given XLogRecPtr. Get starting
+ * address of the page and a pointer to the right location of given
+ * XLogRecPtr in that page.
+ */
+ page = XLogCtl->pages + idx * (Size) XLOG_BLCKSZ;
+ data = page + ptr % XLOG_BLCKSZ;
+
+ /*
+ * Ensure that the data copy and the first verification step are not
+ * reordered
+ */
+ pg_read_barrier();
+
+ /* how much is available on this page to read? */
+ nread = Min(nbytes, XLOG_BLCKSZ - (data - page));
+
+ /* data copy */
+ memcpy(dst, data, nread);
+
+ /*
+ * Ensure that the data copy and the second verification step are not
+ * reordered.
+ */
+ pg_read_barrier();
+
+ /*
+ * Second verification step: check that the page we read from wasn't
+ * evicted while we were copying the data.
+ */
+ endptr = pg_atomic_read_u64(&XLogCtl->xlblocks[idx]);
+ if (expectedEndPtr != endptr)
+ break;
+
+ dst += nread;
+ ptr += nread;
+ ntotal += nread;
+ nbytes -= nread;
+ }
+
+ Assert(ntotal <= count);
+
+ return ntotal;
+}
+
/*
* Converts a "usable byte position" to XLogRecPtr. A usable byte position
* is the position starting from the beginning of WAL, excluding all WAL
@@ -1895,7 +2022,7 @@ AdvanceXLInsertBuffer(XLogRecPtr upto, TimeLineID tli, bool opportunistic)
*/
LWLockRelease(WALBufMappingLock);
- WaitXLogInsertionsToFinish(OldPageRqstPtr);
+ WaitXLogInsertionsToFinish(OldPageRqstPtr, true);
LWLockAcquire(WALWriteLock, LW_EXCLUSIVE);
@@ -2689,7 +2816,7 @@ XLogFlush(XLogRecPtr record)
* Before actually performing the write, wait for all in-flight
* insertions to the pages we're about to write to finish.
*/
- insertpos = WaitXLogInsertionsToFinish(WriteRqstPtr);
+ insertpos = WaitXLogInsertionsToFinish(WriteRqstPtr, true);
/*
* Try to get the write lock. If we can't get it immediately, wait
@@ -2740,7 +2867,7 @@ XLogFlush(XLogRecPtr record)
* We're only calling it again to allow insertpos to be moved
* further forward, not to actually wait for anyone.
*/
- insertpos = WaitXLogInsertionsToFinish(insertpos);
+ insertpos = WaitXLogInsertionsToFinish(insertpos, true);
}
/* try to write/flush later additions to XLOG as well */
@@ -2919,7 +3046,7 @@ XLogBackgroundFlush(void)
START_CRIT_SECTION();
/* now wait for any in-progress insertions to finish and get write lock */
- WaitXLogInsertionsToFinish(WriteRqst.Write);
+ WaitXLogInsertionsToFinish(WriteRqst.Write, true);
LWLockAcquire(WALWriteLock, LW_EXCLUSIVE);
LogwrtResult = XLogCtl->LogwrtResult;
if (WriteRqst.Write > LogwrtResult.Write ||
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index 087031e9dc..b06f5e75d6 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -3125,6 +3125,20 @@ XLogSendPhysical(void)
enlargeStringInfo(&output_message, nbytes);
retry:
+ /*
+ * Read from WAL buffers, if available.
+ */
+ if (!RecoveryInProgress() &&
+ xlogreader->seg.ws_tli == GetWALInsertionTimeLine())
+ {
+ Size rbytes = XLogReadFromBuffers(
+ &output_message.data[output_message.len],
+ startptr, nbytes);
+ output_message.len += rbytes;
+ startptr += rbytes;
+ nbytes -= rbytes;
+ }
+
if (!WALRead(xlogreader,
&output_message.data[output_message.len],
startptr,
diff --git a/src/include/access/xlog.h b/src/include/access/xlog.h
index 301c5fa11f..5f2621b0b9 100644
--- a/src/include/access/xlog.h
+++ b/src/include/access/xlog.h
@@ -252,6 +252,8 @@ extern XLogRecPtr GetLastImportantRecPtr(void);
extern void SetWalWriterSleeping(bool sleeping);
+extern Size XLogReadFromBuffers(char *buf, XLogRecPtr startptr, Size count);
+
/*
* Routines used by xlogrecovery.c to call back into xlog.c during recovery.
*/
--
2.34.1
[text/x-patch] v19j-0002-Add-test-module-for-verifying-WAL-read-from-WAL.patch (9.5K, ../../[email protected]/3-v19j-0002-Add-test-module-for-verifying-WAL-read-from-WAL.patch)
download | inline diff:
From fc9bbbc28c59f5c0a88d658f76bfc421f9b9e34d Mon Sep 17 00:00:00 2001
From: Bharath Rupireddy <[email protected]>
Date: Wed, 10 Jan 2024 13:36:15 +0000
Subject: [PATCH v19j 2/2] Add test module for verifying WAL read from WAL
buffers
This commit adds a test module to verify WAL read from WAL
buffers.
Author: Bharath Rupireddy
Reviewed-by: Dilip Kumar
Discussion: https://www.postgresql.org/message-id/CALj2ACXKKK%3DwbiG5_t6dGao5GoecMwRkhr7GjVBM_jg54%2BNa%3DQ%40mail.gmail.com
---
src/test/modules/Makefile | 1 +
src/test/modules/meson.build | 1 +
.../test_wal_read_from_buffers/.gitignore | 4 ++
.../test_wal_read_from_buffers/Makefile | 23 ++++++++
.../test_wal_read_from_buffers/meson.build | 33 ++++++++++++
.../test_wal_read_from_buffers/t/001_basic.pl | 54 +++++++++++++++++++
.../test_wal_read_from_buffers--1.0.sql | 16 ++++++
.../test_wal_read_from_buffers.c | 37 +++++++++++++
.../test_wal_read_from_buffers.control | 4 ++
9 files changed, 173 insertions(+)
create mode 100644 src/test/modules/test_wal_read_from_buffers/.gitignore
create mode 100644 src/test/modules/test_wal_read_from_buffers/Makefile
create mode 100644 src/test/modules/test_wal_read_from_buffers/meson.build
create mode 100644 src/test/modules/test_wal_read_from_buffers/t/001_basic.pl
create mode 100644 src/test/modules/test_wal_read_from_buffers/test_wal_read_from_buffers--1.0.sql
create mode 100644 src/test/modules/test_wal_read_from_buffers/test_wal_read_from_buffers.c
create mode 100644 src/test/modules/test_wal_read_from_buffers/test_wal_read_from_buffers.control
diff --git a/src/test/modules/Makefile b/src/test/modules/Makefile
index e32c8925f6..c6e1f01dca 100644
--- a/src/test/modules/Makefile
+++ b/src/test/modules/Makefile
@@ -34,6 +34,7 @@ SUBDIRS = \
test_rls_hooks \
test_shm_mq \
test_slru \
+ test_wal_read_from_buffers \
unsafe_tests \
worker_spi \
xid_wraparound
diff --git a/src/test/modules/meson.build b/src/test/modules/meson.build
index 397e0906e6..9595bbc342 100644
--- a/src/test/modules/meson.build
+++ b/src/test/modules/meson.build
@@ -32,6 +32,7 @@ subdir('test_resowner')
subdir('test_rls_hooks')
subdir('test_shm_mq')
subdir('test_slru')
+subdir('test_wal_read_from_buffers')
subdir('unsafe_tests')
subdir('worker_spi')
subdir('xid_wraparound')
diff --git a/src/test/modules/test_wal_read_from_buffers/.gitignore b/src/test/modules/test_wal_read_from_buffers/.gitignore
new file mode 100644
index 0000000000..5dcb3ff972
--- /dev/null
+++ b/src/test/modules/test_wal_read_from_buffers/.gitignore
@@ -0,0 +1,4 @@
+# Generated subdirectories
+/log/
+/results/
+/tmp_check/
diff --git a/src/test/modules/test_wal_read_from_buffers/Makefile b/src/test/modules/test_wal_read_from_buffers/Makefile
new file mode 100644
index 0000000000..7472494501
--- /dev/null
+++ b/src/test/modules/test_wal_read_from_buffers/Makefile
@@ -0,0 +1,23 @@
+# src/test/modules/test_wal_read_from_buffers/Makefile
+
+MODULE_big = test_wal_read_from_buffers
+OBJS = \
+ $(WIN32RES) \
+ test_wal_read_from_buffers.o
+PGFILEDESC = "test_wal_read_from_buffers - test module to read WAL from WAL buffers"
+
+EXTENSION = test_wal_read_from_buffers
+DATA = test_wal_read_from_buffers--1.0.sql
+
+TAP_TESTS = 1
+
+ifdef USE_PGXS
+PG_CONFIG = pg_config
+PGXS := $(shell $(PG_CONFIG) --pgxs)
+include $(PGXS)
+else
+subdir = src/test/modules/test_wal_read_from_buffers
+top_builddir = ../../../..
+include $(top_builddir)/src/Makefile.global
+include $(top_srcdir)/contrib/contrib-global.mk
+endif
diff --git a/src/test/modules/test_wal_read_from_buffers/meson.build b/src/test/modules/test_wal_read_from_buffers/meson.build
new file mode 100644
index 0000000000..40bd5dcd33
--- /dev/null
+++ b/src/test/modules/test_wal_read_from_buffers/meson.build
@@ -0,0 +1,33 @@
+# Copyright (c) 2023, PostgreSQL Global Development Group
+
+test_wal_read_from_buffers_sources = files(
+ 'test_wal_read_from_buffers.c',
+)
+
+if host_system == 'windows'
+ test_wal_read_from_buffers_sources += rc_lib_gen.process(win32ver_rc, extra_args: [
+ '--NAME', 'test_wal_read_from_buffers',
+ '--FILEDESC', 'test_wal_read_from_buffers - test module to read WAL from WAL buffers',])
+endif
+
+test_wal_read_from_buffers = shared_module('test_wal_read_from_buffers',
+ test_wal_read_from_buffers_sources,
+ kwargs: pg_test_mod_args,
+)
+test_install_libs += test_wal_read_from_buffers
+
+test_install_data += files(
+ 'test_wal_read_from_buffers.control',
+ 'test_wal_read_from_buffers--1.0.sql',
+)
+
+tests += {
+ 'name': 'test_wal_read_from_buffers',
+ 'sd': meson.current_source_dir(),
+ 'bd': meson.current_build_dir(),
+ 'tap': {
+ 'tests': [
+ 't/001_basic.pl',
+ ],
+ },
+}
diff --git a/src/test/modules/test_wal_read_from_buffers/t/001_basic.pl b/src/test/modules/test_wal_read_from_buffers/t/001_basic.pl
new file mode 100644
index 0000000000..1d842bb02e
--- /dev/null
+++ b/src/test/modules/test_wal_read_from_buffers/t/001_basic.pl
@@ -0,0 +1,54 @@
+# Copyright (c) 2021-2023, PostgreSQL Global Development Group
+
+use strict;
+use warnings;
+
+use PostgreSQL::Test::Cluster;
+use PostgreSQL::Test::Utils;
+use Test::More;
+
+my $node = PostgreSQL::Test::Cluster->new('test');
+
+$node->init;
+
+# Ensure nobody interferes with us so that the WAL in WAL buffers don't get
+# overwritten while running tests.
+$node->append_conf(
+ 'postgresql.conf', qq(
+autovacuum = off
+checkpoint_timeout = 1h
+wal_writer_delay = 10000ms
+wal_writer_flush_after = 1GB
+));
+$node->start;
+
+# Setup.
+$node->safe_psql('postgres', 'CREATE EXTENSION test_wal_read_from_buffers;');
+
+# Get current insert LSN. After this, we generate some WAL which is guranteed
+# to be in WAL buffers as there is no other WAL generating activity is
+# happening on the server. We then verify if we can read the WAL from WAL
+# buffers using this LSN.
+my $lsn = $node->safe_psql('postgres', 'SELECT pg_current_wal_insert_lsn();');
+
+# Generate minimal WAL so that WAL buffers don't get overwritten.
+$node->safe_psql('postgres',
+ "CREATE TABLE t (c int); INSERT INTO t VALUES (1);");
+
+# Check if WAL is successfully read from WAL buffers.
+my $result = $node->safe_psql('postgres',
+ qq{SELECT test_wal_read_from_buffers('$lsn');});
+is($result, 't', "WAL is successfully read from WAL buffers");
+
+# Check with a WAL that doesn't yet exist.
+$lsn = $node->safe_psql('postgres', 'SELECT pg_current_wal_flush_lsn()+8192;');
+$result = $node->safe_psql('postgres',
+ qq{SELECT test_wal_read_from_buffers('$lsn');});
+is($result, 'f', "WAL that doesn't yet exist is not read from WAL buffers");
+
+# Check with invalid input.
+$result = $node->safe_psql('postgres',
+ qq{SELECT test_wal_read_from_buffers('0/0');});
+is($result, 'f', "WAL is not read from WAL buffers with invalid input");
+
+done_testing();
diff --git a/src/test/modules/test_wal_read_from_buffers/test_wal_read_from_buffers--1.0.sql b/src/test/modules/test_wal_read_from_buffers/test_wal_read_from_buffers--1.0.sql
new file mode 100644
index 0000000000..c6ffb3fa65
--- /dev/null
+++ b/src/test/modules/test_wal_read_from_buffers/test_wal_read_from_buffers--1.0.sql
@@ -0,0 +1,16 @@
+/* src/test/modules/test_wal_read_from_buffers/test_wal_read_from_buffers--1.0.sql */
+
+-- complain if script is sourced in psql, rather than via CREATE EXTENSION
+\echo Use "CREATE EXTENSION test_wal_read_from_buffers" to load this file. \quit
+
+--
+-- test_wal_read_from_buffers()
+--
+-- Returns true if WAL data at a given LSN can be read from WAL buffers.
+-- Otherwise returns false.
+--
+CREATE FUNCTION test_wal_read_from_buffers(IN lsn pg_lsn,
+ read_successful OUT boolean
+)
+AS 'MODULE_PATHNAME', 'test_wal_read_from_buffers'
+LANGUAGE C STRICT;
diff --git a/src/test/modules/test_wal_read_from_buffers/test_wal_read_from_buffers.c b/src/test/modules/test_wal_read_from_buffers/test_wal_read_from_buffers.c
new file mode 100644
index 0000000000..5368f59b16
--- /dev/null
+++ b/src/test/modules/test_wal_read_from_buffers/test_wal_read_from_buffers.c
@@ -0,0 +1,37 @@
+/*--------------------------------------------------------------------------
+ *
+ * test_wal_read_from_buffers.c
+ * Test module to read WAL from WAL buffers.
+ *
+ * Portions Copyright (c) 2023, PostgreSQL Global Development Group
+ *
+ * IDENTIFICATION
+ * src/test/modules/test_wal_read_from_buffers/test_wal_read_from_buffers.c
+ * -------------------------------------------------------------------------
+ */
+
+#include "postgres.h"
+
+#include "access/xlog.h"
+#include "fmgr.h"
+#include "utils/pg_lsn.h"
+
+PG_MODULE_MAGIC;
+
+/*
+ * SQL function for verifying that WAL data at a given LSN can be read from WAL
+ * buffers. Returns true if read from WAL buffers, otherwise false.
+ */
+PG_FUNCTION_INFO_V1(test_wal_read_from_buffers);
+Datum
+test_wal_read_from_buffers(PG_FUNCTION_ARGS)
+{
+ XLogRecPtr startptr = PG_GETARG_LSN(0);
+ char data[XLOG_BLCKSZ] = {0};
+ Size nread = 0;
+
+ if (!XLogRecPtrIsInvalid(startptr))
+ nread = XLogReadFromBuffers(data, startptr, XLOG_BLCKSZ);
+
+ PG_RETURN_BOOL(nread > 0);
+}
diff --git a/src/test/modules/test_wal_read_from_buffers/test_wal_read_from_buffers.control b/src/test/modules/test_wal_read_from_buffers/test_wal_read_from_buffers.control
new file mode 100644
index 0000000000..eda8d47954
--- /dev/null
+++ b/src/test/modules/test_wal_read_from_buffers/test_wal_read_from_buffers.control
@@ -0,0 +1,4 @@
+comment = 'Test module to read WAL from WAL buffers'
+default_version = '1.0'
+module_pathname = '$libdir/test_wal_read_from_buffers'
+relocatable = true
--
2.34.1
^ permalink raw reply [nested|flat] 22+ messages in thread
* Re: Improve WALRead() to suck data directly from WAL buffers when possible
@ 2024-01-25 09:05 Bharath Rupireddy <[email protected]>
parent: Jeff Davis <[email protected]>
0 siblings, 1 reply; 22+ messages in thread
From: Bharath Rupireddy @ 2024-01-25 09:05 UTC (permalink / raw)
To: Jeff Davis <[email protected]>; +Cc: Andres Freund <[email protected]>; Dilip Kumar <[email protected]>; Kyotaro Horiguchi <[email protected]>; [email protected]; Nathan Bossart <[email protected]>; Masahiko Sawada <[email protected]>
On Tue, Jan 23, 2024 at 9:37 AM Jeff Davis <[email protected]> wrote:
>
> On Mon, 2024-01-22 at 12:12 -0800, Andres Freund wrote:
> > I still think that anything that requires such checks shouldn't be
> > merged. It's completely bogus to check page contents for validity
> > when we
> > should have metadata telling us which range of the buffers is valid
> > and which
> > not.
>
> The check seems entirely unnecessary, to me. A leftover from v18?
>
> I have attached a new patch (version "19j") to illustrate some of my
> previous suggestions. I didn't spend a lot of time on it so it's not
> ready for commit, but I believe my suggestions are easier to understand
> in code form.
> Note that, right now, it only works for XLogSendPhysical(). I believe
> it's best to just make it work for 1-3 callers that we understand well,
> and we can generalize later if it makes sense.
+1 to do it for XLogSendPhysical() first. Enabling it for others can
just be done as something like the attached v20-0003.
> I'm still not clear on why some callers are reading XLOG_BLCKSZ
> (expecting zeros at the end), and if it's OK to just change them to use
> the exact byte count.
"expecting zeros at the end" - this can't always be true as the WAL
can get flushed after determining the flush ptr before reading it from
the WAL file. FWIW, here's what I've tried previoulsy -
https://github.com/BRupireddy2/postgres/tree/ensure_extra_read_WAL_page_is_zero_padded_at_the_end_WI...,
the tests hit the Assert(false); added. Which means, the zero-padding
comment around WALRead() call-sites isn't quite right.
/*
* Even though we just determined how much of the page can be validly read
* as 'count', read the whole page anyway. It's guaranteed to be
* zero-padded up to the page boundary if it's incomplete.
*/
if (!WALRead(state, cur_page, targetPagePtr, XLOG_BLCKSZ, tli,
I think this needs to be discussed separately. If okay, I'll start a new thread.
> Also, if we've detected that the first requested buffer has been
> evicted, is there any value in continuing the loop to see if more
> recent buffers are available? For example, if the requested LSNs range
> over buffers 4, 5, and 6, and 4 has already been evicted, should we try
> to return LSN data from 5 and 6 at the proper offset in the dest
> buffer? If so, we'd need to adjust the API so the caller knows what
> parts of the dest buffer were filled in.
I'd second this capability for now to keep the API simple and clear,
but we can consider expanding it as needed.
I reviewed the v19j and attached v20 patch set:
1.
* The caller must ensure that it's reasonable to read from the WAL buffers,
* i.e. that the requested data is from the current timeline, that we're not
* in recovery, etc.
I still think the XLogReadFromBuffers can just return in any of the
above cases instead of comments. I feel we must assume the caller is
going to ask the WAL from a different timeline and/or in recovery and
design the API to deal with it. Done that way in v20 patch.
2. Fixed some typos, reworded a few comments (i.e. used "current
insert/write position" instead of "Insert/Write pointer" like
elsewhere), ran pgindent.
3.
- * XXX probably this should be improved to suck data directly from the
- * WAL buffers when possible.
Removed the above comment before WALRead() since we have that facility
now. Perhaps, we can say the callers can suck data directly from the
WAL buffers using XLogReadFromBuffers. But I have no strong opinion on
this.
4.
+ * Most callers will have already updated LogwrtResult when determining
+ * how far to read, but it's OK if it's out of date. (XXX: is it worth
+ * taking a spinlock to update LogwrtResult and check again before calling
+ * WaitXLogInsertionsToFinish()?)
If the callers use GetFlushRecPtr() to determine how far to read,
LogwrtResult will be *reasonably* latest, otherwise not. If
LogwrtResult is a bit old, XLogReadFromBuffers will call
WaitXLogInsertionsToFinish which will just loop over all insertion
locks and return.
As far as the current WAL readers are concerned, we don't need an
explicit spinlock to determine LogwrtResult because all of them use
GetFlushRecPtr() to determine how far to read. If there's any caller
that's not updating LogwrtResult at all, we can consider reading
LogwrtResult it ourselves in future.
5. I think the two requirements specified at
https://www.postgresql.org/message-id/20231109205836.zjoawdrn4q77yemv%40awork3.anarazel.de
still hold with the v19j.
5.1 Never allow WAL being read that's past
XLogBytePosToRecPtr(XLogCtl->Insert->CurrBytePos) as it does not
exist.
5.2 If the to-be-read LSN is between XLogCtl->LogwrtResult->Write and
XLogBytePosToRecPtr(Insert->CurrBytePos) we need to call
WaitXLogInsertionsToFinish() before copying the data.
+ if (upto > LogwrtResult.Write)
+ {
+ XLogRecPtr writtenUpto = WaitXLogInsertionsToFinish(upto, false);
+
+ upto = Min(upto, writtenUpto);
+ nbytes = upto - startptr;
+ }
XLogReadFromBuffers ensures the above two with adjusting upto based on
Min(upto, writtenUpto) as WaitXLogInsertionsToFinish returns the
oldest insertion that is still in-progress.
For instance, the current write LSN is 100, current insert LSN is 150
and upto is 200 - we only read upto 150 if startptr is < 150; we don't
read anything if startptr is > 150.
6. I've modified the test module in v20-0002 patch as follows:
6.1 Renamed the module to read_wal_from_buffers stripping "test_"
which otherwise is making the name longer. Longer names can cause
failures on some Windows BF members if the PATH/FILE name is too long.
6.2 Tweaked tests to hit WaitXLogInsertionsToFinish() and upto =
Min(upto, writtenUpto); in XLogReadFromBuffers.
PSA v20 patch set.
--
Bharath Rupireddy
PostgreSQL Contributors Team
RDS Open Source Databases
Amazon Web Services: https://aws.amazon.com
Attachments:
[application/octet-stream] v20-0001-Add-XLogReadFromBuffers.patch (9.6K, ../../CALj2ACV=C1GZT9XQRm4iN1NV1T=hLA_hsGWNx2Y5-G+mSwdhNg@mail.gmail.com/2-v20-0001-Add-XLogReadFromBuffers.patch)
download | inline diff:
From 76b71019e9067d96559639c299384222abb1651e Mon Sep 17 00:00:00 2001
From: Bharath Rupireddy <[email protected]>
Date: Thu, 25 Jan 2024 08:19:01 +0000
Subject: [PATCH v20] Add XLogReadFromBuffers().
Allows reading directly from WAL buffers without a lock, avoiding the
need to wait for WAL flushing and read from the filesystem.
For now, the only caller is physical replication, but we can consider
expanding it to other callers as needed.
Author: Bharath Rupireddy
---
src/backend/access/transam/xlog.c | 147 ++++++++++++++++++++++--
src/backend/access/transam/xlogreader.c | 3 -
src/backend/replication/walsender.c | 8 ++
src/include/access/xlog.h | 3 +
4 files changed, 149 insertions(+), 12 deletions(-)
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 478377c4a2..4940e8ca29 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -698,7 +698,7 @@ static void ReserveXLogInsertLocation(int size, XLogRecPtr *StartPos,
XLogRecPtr *EndPos, XLogRecPtr *PrevPtr);
static bool ReserveXLogSwitch(XLogRecPtr *StartPos, XLogRecPtr *EndPos,
XLogRecPtr *PrevPtr);
-static XLogRecPtr WaitXLogInsertionsToFinish(XLogRecPtr upto);
+static XLogRecPtr WaitXLogInsertionsToFinish(XLogRecPtr upto, bool emitLog);
static char *GetXLogBuffer(XLogRecPtr ptr, TimeLineID tli);
static XLogRecPtr XLogBytePosToRecPtr(uint64 bytepos);
static XLogRecPtr XLogBytePosToEndRecPtr(uint64 bytepos);
@@ -1494,7 +1494,7 @@ WALInsertLockUpdateInsertingAt(XLogRecPtr insertingAt)
* to make room for a new one, which in turn requires WALWriteLock.
*/
static XLogRecPtr
-WaitXLogInsertionsToFinish(XLogRecPtr upto)
+WaitXLogInsertionsToFinish(XLogRecPtr upto, bool emitLog)
{
uint64 bytepos;
XLogRecPtr reservedUpto;
@@ -1521,9 +1521,10 @@ WaitXLogInsertionsToFinish(XLogRecPtr upto)
*/
if (upto > reservedUpto)
{
- ereport(LOG,
- (errmsg("request to flush past end of generated WAL; request %X/%X, current position %X/%X",
- LSN_FORMAT_ARGS(upto), LSN_FORMAT_ARGS(reservedUpto))));
+ if (emitLog)
+ ereport(LOG,
+ (errmsg("request to flush past end of generated WAL; request %X/%X, current position %X/%X",
+ LSN_FORMAT_ARGS(upto), LSN_FORMAT_ARGS(reservedUpto))));
upto = reservedUpto;
}
@@ -1705,6 +1706,134 @@ GetXLogBuffer(XLogRecPtr ptr, TimeLineID tli)
return cachedPos + ptr % XLOG_BLCKSZ;
}
+/*
+ * Read WAL directly from WAL buffers, if available.
+ *
+ * This function reads 'count' bytes of WAL from WAL buffers into 'buf'
+ * starting at location 'startptr' and returns total bytes read.
+ *
+ * The bytes read may be fewer than requested if any of the WAL buffers in the
+ * requested range have been evicted, or if the last requested byte is beyond
+ * the current insert position.
+ *
+ * If reading beyond the current write position, this function will wait for
+ * concurrent inserters to finish. Otherwise, it does not wait at all.
+ *
+ * This function returns immediately if the requested data is not from the
+ * current timeline, or if the server is in recovery.
+ */
+Size
+XLogReadFromBuffers(char *buf, XLogRecPtr startptr, Size count, TimeLineID tli)
+{
+ XLogRecPtr ptr = startptr;
+ XLogRecPtr upto = startptr + count;
+ Size nbytes = count;
+ Size ntotal = 0;
+ char *dst = buf;
+
+ if (RecoveryInProgress() ||
+ tli != GetWALInsertionTimeLine())
+ return ntotal;
+
+ Assert(!XLogRecPtrIsInvalid(startptr));
+
+ /*
+ * Caller requested very recent WAL data. Wait for any in-progress WAL
+ * insertions to WAL buffers to finish.
+ *
+ * Most callers will have already updated LogwrtResult when determining
+ * how far to read, but it's OK if it's out of date. XXX: is it worth
+ * taking a spinlock to update LogwrtResult and check again before calling
+ * WaitXLogInsertionsToFinish()?
+ */
+ if (upto > LogwrtResult.Write)
+ {
+ XLogRecPtr writtenUpto = WaitXLogInsertionsToFinish(upto, false);
+
+ upto = Min(upto, writtenUpto);
+ nbytes = upto - startptr;
+ }
+
+ /*
+ * Loop through the buffers without a lock. For each buffer, atomically
+ * read and verify the end pointer, then copy the data out, and finally
+ * re-read and re-verify the end pointer.
+ *
+ * Once a page is evicted, it never returns to the WAL buffers, so if the
+ * end pointer matches the expected end pointer before and after we copy
+ * the data, then the right page must have been present during the data
+ * copy. Read barriers are necessary to ensure that the data copy actually
+ * happens between the two verification steps.
+ *
+ * If the verification fails, we simply terminate the loop and return with
+ * the data that had been already copied out successfully.
+ */
+ while (nbytes > 0)
+ {
+ XLogRecPtr expectedEndPtr;
+ XLogRecPtr endptr;
+ int idx;
+ const char *page;
+ const char *data;
+ Size nread;
+
+ idx = XLogRecPtrToBufIdx(ptr);
+ expectedEndPtr = ptr;
+ expectedEndPtr += XLOG_BLCKSZ - ptr % XLOG_BLCKSZ;
+
+ /*
+ * First verification step: check that the correct page is present in
+ * the WAL buffers.
+ */
+ endptr = pg_atomic_read_u64(&XLogCtl->xlblocks[idx]);
+ if (expectedEndPtr != endptr)
+ break;
+
+ /*
+ * We found WAL buffer page containing given XLogRecPtr. Get starting
+ * address of the page and a pointer to the right location of given
+ * XLogRecPtr in that page.
+ */
+ page = XLogCtl->pages + idx * (Size) XLOG_BLCKSZ;
+ data = page + ptr % XLOG_BLCKSZ;
+
+ /*
+ * Ensure that the data copy and the first verification step are not
+ * reordered.
+ */
+ pg_read_barrier();
+
+ /* how much is available on this page to read? */
+ nread = Min(nbytes, XLOG_BLCKSZ - (data - page));
+
+ /* data copy */
+ memcpy(dst, data, nread);
+
+ /*
+ * Ensure that the data copy and the second verification step are not
+ * reordered.
+ */
+ pg_read_barrier();
+
+ /*
+ * Second verification step: check that the page we read from wasn't
+ * evicted while we were copying the data.
+ */
+ endptr = pg_atomic_read_u64(&XLogCtl->xlblocks[idx]);
+ if (expectedEndPtr != endptr)
+ break;
+
+ dst += nread;
+ ptr += nread;
+ ntotal += nread;
+ nbytes -= nread;
+ }
+
+ Assert(ntotal <= count);
+
+ return ntotal;
+}
+
/*
* Converts a "usable byte position" to XLogRecPtr. A usable byte position
* is the position starting from the beginning of WAL, excluding all WAL
@@ -1895,7 +2024,7 @@ AdvanceXLInsertBuffer(XLogRecPtr upto, TimeLineID tli, bool opportunistic)
*/
LWLockRelease(WALBufMappingLock);
- WaitXLogInsertionsToFinish(OldPageRqstPtr);
+ WaitXLogInsertionsToFinish(OldPageRqstPtr, true);
LWLockAcquire(WALWriteLock, LW_EXCLUSIVE);
@@ -2689,7 +2818,7 @@ XLogFlush(XLogRecPtr record)
* Before actually performing the write, wait for all in-flight
* insertions to the pages we're about to write to finish.
*/
- insertpos = WaitXLogInsertionsToFinish(WriteRqstPtr);
+ insertpos = WaitXLogInsertionsToFinish(WriteRqstPtr, true);
/*
* Try to get the write lock. If we can't get it immediately, wait
@@ -2740,7 +2869,7 @@ XLogFlush(XLogRecPtr record)
* We're only calling it again to allow insertpos to be moved
* further forward, not to actually wait for anyone.
*/
- insertpos = WaitXLogInsertionsToFinish(insertpos);
+ insertpos = WaitXLogInsertionsToFinish(insertpos, true);
}
/* try to write/flush later additions to XLOG as well */
@@ -2919,7 +3048,7 @@ XLogBackgroundFlush(void)
START_CRIT_SECTION();
/* now wait for any in-progress insertions to finish and get write lock */
- WaitXLogInsertionsToFinish(WriteRqst.Write);
+ WaitXLogInsertionsToFinish(WriteRqst.Write, true);
LWLockAcquire(WALWriteLock, LW_EXCLUSIVE);
LogwrtResult = XLogCtl->LogwrtResult;
if (WriteRqst.Write > LogwrtResult.Write ||
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index 7190156f2f..74a6b11866 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -1500,9 +1500,6 @@ err:
*
* Returns true if succeeded, false if an error occurs, in which case
* 'errinfo' receives error details.
- *
- * XXX probably this should be improved to suck data directly from the
- * WAL buffers when possible.
*/
bool
WALRead(XLogReaderState *state,
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index 087031e9dc..95ba656a06 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -2910,6 +2910,7 @@ XLogSendPhysical(void)
Size nbytes;
XLogSegNo segno;
WALReadError errinfo;
+ Size rbytes;
/* If requested switch the WAL sender to the stopping state. */
if (got_STOPPING)
@@ -3125,6 +3126,13 @@ XLogSendPhysical(void)
enlargeStringInfo(&output_message, nbytes);
retry:
+ /* Read from WAL buffers, if available. */
+ rbytes = XLogReadFromBuffers(&output_message.data[output_message.len],
+ startptr, nbytes, xlogreader->seg.ws_tli);
+ output_message.len += rbytes;
+ startptr += rbytes;
+ nbytes -= rbytes;
+
if (!WALRead(xlogreader,
&output_message.data[output_message.len],
startptr,
diff --git a/src/include/access/xlog.h b/src/include/access/xlog.h
index 301c5fa11f..f8c281c799 100644
--- a/src/include/access/xlog.h
+++ b/src/include/access/xlog.h
@@ -252,6 +252,9 @@ extern XLogRecPtr GetLastImportantRecPtr(void);
extern void SetWalWriterSleeping(bool sleeping);
+extern Size XLogReadFromBuffers(char *buf, XLogRecPtr startptr, Size count,
+ TimeLineID tli);
+
/*
* Routines used by xlogrecovery.c to call back into xlog.c during recovery.
*/
--
2.34.1
[application/octet-stream] v20-0002-Add-test-module-for-verifying-WAL-read-from-WAL-.patch (9.2K, ../../CALj2ACV=C1GZT9XQRm4iN1NV1T=hLA_hsGWNx2Y5-G+mSwdhNg@mail.gmail.com/3-v20-0002-Add-test-module-for-verifying-WAL-read-from-WAL-.patch)
download | inline diff:
From d348c029ca3db228753b8db9cea214951eced4ee Mon Sep 17 00:00:00 2001
From: Bharath Rupireddy <[email protected]>
Date: Thu, 25 Jan 2024 06:54:33 +0000
Subject: [PATCH v20] Add test module for verifying WAL read from WAL buffers
This commit adds a test module to verify WAL read from WAL
buffers.
Author: Bharath Rupireddy
Reviewed-by: Dilip Kumar
Discussion: https://www.postgresql.org/message-id/CALj2ACXKKK%3DwbiG5_t6dGao5GoecMwRkhr7GjVBM_jg54%2BNa%3DQ%40mail.gmail.com
---
src/test/modules/Makefile | 1 +
src/test/modules/meson.build | 1 +
.../modules/read_wal_from_buffers/.gitignore | 4 ++
.../modules/read_wal_from_buffers/Makefile | 23 ++++++++
.../modules/read_wal_from_buffers/meson.build | 33 ++++++++++++
.../read_wal_from_buffers--1.0.sql | 14 +++++
.../read_wal_from_buffers.c | 41 +++++++++++++++
.../read_wal_from_buffers.control | 4 ++
.../read_wal_from_buffers/t/001_basic.pl | 52 +++++++++++++++++++
9 files changed, 173 insertions(+)
create mode 100644 src/test/modules/read_wal_from_buffers/.gitignore
create mode 100644 src/test/modules/read_wal_from_buffers/Makefile
create mode 100644 src/test/modules/read_wal_from_buffers/meson.build
create mode 100644 src/test/modules/read_wal_from_buffers/read_wal_from_buffers--1.0.sql
create mode 100644 src/test/modules/read_wal_from_buffers/read_wal_from_buffers.c
create mode 100644 src/test/modules/read_wal_from_buffers/read_wal_from_buffers.control
create mode 100644 src/test/modules/read_wal_from_buffers/t/001_basic.pl
diff --git a/src/test/modules/Makefile b/src/test/modules/Makefile
index e32c8925f6..4eba0fa2e2 100644
--- a/src/test/modules/Makefile
+++ b/src/test/modules/Makefile
@@ -12,6 +12,7 @@ SUBDIRS = \
dummy_seclabel \
libpq_pipeline \
plsample \
+ read_wal_from_buffers \
spgist_name_ops \
test_bloomfilter \
test_copy_callbacks \
diff --git a/src/test/modules/meson.build b/src/test/modules/meson.build
index 397e0906e6..f0b53eced7 100644
--- a/src/test/modules/meson.build
+++ b/src/test/modules/meson.build
@@ -32,6 +32,7 @@ subdir('test_resowner')
subdir('test_rls_hooks')
subdir('test_shm_mq')
subdir('test_slru')
+subdir('read_wal_from_buffers')
subdir('unsafe_tests')
subdir('worker_spi')
subdir('xid_wraparound')
diff --git a/src/test/modules/read_wal_from_buffers/.gitignore b/src/test/modules/read_wal_from_buffers/.gitignore
new file mode 100644
index 0000000000..5dcb3ff972
--- /dev/null
+++ b/src/test/modules/read_wal_from_buffers/.gitignore
@@ -0,0 +1,4 @@
+# Generated subdirectories
+/log/
+/results/
+/tmp_check/
diff --git a/src/test/modules/read_wal_from_buffers/Makefile b/src/test/modules/read_wal_from_buffers/Makefile
new file mode 100644
index 0000000000..9e57a837f9
--- /dev/null
+++ b/src/test/modules/read_wal_from_buffers/Makefile
@@ -0,0 +1,23 @@
+# src/test/modules/read_wal_from_buffers/Makefile
+
+MODULE_big = read_wal_from_buffers
+OBJS = \
+ $(WIN32RES) \
+ read_wal_from_buffers.o
+PGFILEDESC = "read_wal_from_buffers - test module to read WAL from WAL buffers"
+
+EXTENSION = read_wal_from_buffers
+DATA = read_wal_from_buffers--1.0.sql
+
+TAP_TESTS = 1
+
+ifdef USE_PGXS
+PG_CONFIG = pg_config
+PGXS := $(shell $(PG_CONFIG) --pgxs)
+include $(PGXS)
+else
+subdir = src/test/modules/read_wal_from_buffers
+top_builddir = ../../../..
+include $(top_builddir)/src/Makefile.global
+include $(top_srcdir)/contrib/contrib-global.mk
+endif
diff --git a/src/test/modules/read_wal_from_buffers/meson.build b/src/test/modules/read_wal_from_buffers/meson.build
new file mode 100644
index 0000000000..3fac00d616
--- /dev/null
+++ b/src/test/modules/read_wal_from_buffers/meson.build
@@ -0,0 +1,33 @@
+# Copyright (c) 2024, PostgreSQL Global Development Group
+
+read_wal_from_buffers_sources = files(
+ 'read_wal_from_buffers.c',
+)
+
+if host_system == 'windows'
+ read_wal_from_buffers_sources += rc_lib_gen.process(win32ver_rc, extra_args: [
+ '--NAME', 'read_wal_from_buffers',
+ '--FILEDESC', 'read_wal_from_buffers - test module to read WAL from WAL buffers',])
+endif
+
+read_wal_from_buffers = shared_module('read_wal_from_buffers',
+ read_wal_from_buffers_sources,
+ kwargs: pg_test_mod_args,
+)
+test_install_libs += read_wal_from_buffers
+
+test_install_data += files(
+ 'read_wal_from_buffers.control',
+ 'read_wal_from_buffers--1.0.sql',
+)
+
+tests += {
+ 'name': 'read_wal_from_buffers',
+ 'sd': meson.current_source_dir(),
+ 'bd': meson.current_build_dir(),
+ 'tap': {
+ 'tests': [
+ 't/001_basic.pl',
+ ],
+ },
+}
diff --git a/src/test/modules/read_wal_from_buffers/read_wal_from_buffers--1.0.sql b/src/test/modules/read_wal_from_buffers/read_wal_from_buffers--1.0.sql
new file mode 100644
index 0000000000..82fa097d10
--- /dev/null
+++ b/src/test/modules/read_wal_from_buffers/read_wal_from_buffers--1.0.sql
@@ -0,0 +1,14 @@
+/* src/test/modules/read_wal_from_buffers/read_wal_from_buffers--1.0.sql */
+
+-- complain if script is sourced in psql, rather than via CREATE EXTENSION
+\echo Use "CREATE EXTENSION read_wal_from_buffers" to load this file. \quit
+
+--
+-- read_wal_from_buffers()
+--
+-- SQL function to read WAL from WAL buffers. Returns number of bytes read.
+--
+CREATE FUNCTION read_wal_from_buffers(IN lsn pg_lsn, IN bytes_to_read int,
+ bytes_read OUT int)
+AS 'MODULE_PATHNAME', 'read_wal_from_buffers'
+LANGUAGE C STRICT;
diff --git a/src/test/modules/read_wal_from_buffers/read_wal_from_buffers.c b/src/test/modules/read_wal_from_buffers/read_wal_from_buffers.c
new file mode 100644
index 0000000000..da841da3f0
--- /dev/null
+++ b/src/test/modules/read_wal_from_buffers/read_wal_from_buffers.c
@@ -0,0 +1,41 @@
+/*--------------------------------------------------------------------------
+ *
+ * read_wal_from_buffers.c
+ * Test module to read WAL from WAL buffers.
+ *
+ * Portions Copyright (c) 1996-2024, PostgreSQL Global Development Group
+ * Portions Copyright (c) 1994, Regents of the University of California
+ *
+ * IDENTIFICATION
+ * src/test/modules/read_wal_from_buffers/read_wal_from_buffers.c
+ * -------------------------------------------------------------------------
+ */
+
+#include "postgres.h"
+
+#include "access/xlog.h"
+#include "fmgr.h"
+#include "utils/pg_lsn.h"
+
+PG_MODULE_MAGIC;
+
+/*
+ * SQL function to read WAL from WAL buffers. Returns number of bytes read.
+ */
+PG_FUNCTION_INFO_V1(read_wal_from_buffers);
+Datum
+read_wal_from_buffers(PG_FUNCTION_ARGS)
+{
+ XLogRecPtr startptr = PG_GETARG_LSN(0);
+ int32 bytes_to_read = PG_GETARG_INT32(1);
+ Size bytes_read = 0;
+ char *data = palloc0(bytes_to_read);
+
+ bytes_read = XLogReadFromBuffers(data, startptr,
+ (Size) bytes_to_read,
+ GetWALInsertionTimeLine());
+
+ pfree(data);
+
+ PG_RETURN_INT32(bytes_read);
+}
diff --git a/src/test/modules/read_wal_from_buffers/read_wal_from_buffers.control b/src/test/modules/read_wal_from_buffers/read_wal_from_buffers.control
new file mode 100644
index 0000000000..b14d24751c
--- /dev/null
+++ b/src/test/modules/read_wal_from_buffers/read_wal_from_buffers.control
@@ -0,0 +1,4 @@
+comment = 'Test module to read WAL from WAL buffers'
+default_version = '1.0'
+module_pathname = '$libdir/read_wal_from_buffers'
+relocatable = true
diff --git a/src/test/modules/read_wal_from_buffers/t/001_basic.pl b/src/test/modules/read_wal_from_buffers/t/001_basic.pl
new file mode 100644
index 0000000000..e1773da2c8
--- /dev/null
+++ b/src/test/modules/read_wal_from_buffers/t/001_basic.pl
@@ -0,0 +1,52 @@
+# Copyright (c) 2021-2023, PostgreSQL Global Development Group
+
+use strict;
+use warnings;
+
+use PostgreSQL::Test::Cluster;
+use PostgreSQL::Test::Utils;
+use Test::More;
+
+my $node = PostgreSQL::Test::Cluster->new('test');
+
+$node->init;
+
+# Ensure nobody interferes with us so that the WAL in WAL buffers don't get
+# overwritten while running tests.
+$node->append_conf(
+ 'postgresql.conf', qq(
+autovacuum = off
+checkpoint_timeout = 1h
+wal_writer_delay = 10000ms
+wal_writer_flush_after = 1GB
+));
+$node->start;
+
+# Setup.
+$node->safe_psql('postgres', 'CREATE EXTENSION read_wal_from_buffers;');
+
+# Get current insert LSN. After this, we generate some WAL which is guranteed
+# to be in WAL buffers as there is no other WAL generating activity is
+# happening on the server. We then verify if we can read the WAL from WAL
+# buffers using this LSN.
+my $lsn = $node->safe_psql('postgres', 'SELECT pg_current_wal_insert_lsn();');
+
+# Generate minimal WAL so that WAL buffers don't get overwritten.
+$node->safe_psql('postgres',
+ "CREATE TABLE t (c int); INSERT INTO t VALUES (1);");
+
+# Check if WAL is successfully read from WAL buffers.
+my $to_read = 8192;
+my $result = $node->safe_psql('postgres',
+ qq{SELECT read_wal_from_buffers(lsn := '$lsn', bytes_to_read := $to_read) > 0;});
+is($result, 't', "WAL is successfully read from WAL buffers");
+
+# Check with a WAL that doesn't yet exist i.e., 16MB starting from current
+# flush LSN.
+$lsn = $node->safe_psql('postgres', 'SELECT pg_current_wal_flush_lsn()+16777216;');
+$to_read = 8192;
+$result = $node->safe_psql('postgres',
+ qq{SELECT read_wal_from_buffers(lsn := '$lsn', bytes_to_read := $to_read) = 0;});
+is($result, 't', "WAL that doesn't yet exist is not read from WAL buffers");
+
+done_testing();
--
2.34.1
[application/octet-stream] v20-0003-Use-XLogReadFromBuffers-in-more-places.patch (3.9K, ../../CALj2ACV=C1GZT9XQRm4iN1NV1T=hLA_hsGWNx2Y5-G+mSwdhNg@mail.gmail.com/4-v20-0003-Use-XLogReadFromBuffers-in-more-places.patch)
download | inline diff:
From 938d80bdaa6d702cb8e415b582555efa76574e55 Mon Sep 17 00:00:00 2001
From: Bharath Rupireddy <[email protected]>
Date: Thu, 25 Jan 2024 07:13:23 +0000
Subject: [PATCH v20] Use XLogReadFromBuffers in more places
---
src/backend/access/transam/xlogutils.c | 12 +++++++++++-
src/backend/postmaster/walsummarizer.c | 12 +++++++++++-
src/backend/replication/walsender.c | 12 +++++++++++-
3 files changed, 33 insertions(+), 3 deletions(-)
diff --git a/src/backend/access/transam/xlogutils.c b/src/backend/access/transam/xlogutils.c
index aa8667abd1..de526f7da7 100644
--- a/src/backend/access/transam/xlogutils.c
+++ b/src/backend/access/transam/xlogutils.c
@@ -894,6 +894,8 @@ read_local_xlog_page_guts(XLogReaderState *state, XLogRecPtr targetPagePtr,
int count;
WALReadError errinfo;
TimeLineID currTLI;
+ Size nbytes;
+ Size rbytes;
loc = targetPagePtr + reqLen;
@@ -1006,12 +1008,20 @@ read_local_xlog_page_guts(XLogReaderState *state, XLogRecPtr targetPagePtr,
count = read_upto - targetPagePtr;
}
+ /* Read from WAL buffers, if available. */
+ nbytes = XLOG_BLCKSZ;
+ rbytes = XLogReadFromBuffers(cur_page, targetPagePtr,
+ nbytes, currTLI);
+ cur_page += rbytes;
+ targetPagePtr += rbytes;
+ nbytes -= rbytes;
+
/*
* Even though we just determined how much of the page can be validly read
* as 'count', read the whole page anyway. It's guaranteed to be
* zero-padded up to the page boundary if it's incomplete.
*/
- if (!WALRead(state, cur_page, targetPagePtr, XLOG_BLCKSZ, tli,
+ if (!WALRead(state, cur_page, targetPagePtr, nbytes, tli,
&errinfo))
WALReadRaiseError(&errinfo);
diff --git a/src/backend/postmaster/walsummarizer.c b/src/backend/postmaster/walsummarizer.c
index 9b883c21ca..33eb3a4870 100644
--- a/src/backend/postmaster/walsummarizer.c
+++ b/src/backend/postmaster/walsummarizer.c
@@ -1221,6 +1221,8 @@ summarizer_read_local_xlog_page(XLogReaderState *state,
int count;
WALReadError errinfo;
SummarizerReadLocalXLogPrivate *private_data;
+ Size nbytes;
+ Size rbytes;
HandleWalSummarizerInterrupts();
@@ -1318,12 +1320,20 @@ summarizer_read_local_xlog_page(XLogReaderState *state,
}
}
+ /* Read from WAL buffers, if available. */
+ nbytes = XLOG_BLCKSZ;
+ rbytes = XLogReadFromBuffers(cur_page, targetPagePtr,
+ nbytes, private_data->tli);
+ cur_page += rbytes;
+ targetPagePtr += rbytes;
+ nbytes -= rbytes;
+
/*
* Even though we just determined how much of the page can be validly read
* as 'count', read the whole page anyway. It's guaranteed to be
* zero-padded up to the page boundary if it's incomplete.
*/
- if (!WALRead(state, cur_page, targetPagePtr, XLOG_BLCKSZ,
+ if (!WALRead(state, cur_page, targetPagePtr, nbytes,
private_data->tli, &errinfo))
WALReadRaiseError(&errinfo);
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index 95ba656a06..ab119ef29a 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -1059,6 +1059,8 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
WALReadError errinfo;
XLogSegNo segno;
TimeLineID currTLI;
+ Size nbytes;
+ Size rbytes;
/*
* Make sure we have enough WAL available before retrieving the current
@@ -1095,11 +1097,19 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
else
count = flushptr - targetPagePtr; /* part of the page available */
+ /* Read from WAL buffers, if available. */
+ nbytes = XLOG_BLCKSZ;
+ rbytes = XLogReadFromBuffers(cur_page, targetPagePtr,
+ nbytes, currTLI);
+ cur_page += rbytes;
+ targetPagePtr += rbytes;
+ nbytes -= rbytes;
+
/* now actually read the data, we know it's there */
if (!WALRead(state,
cur_page,
targetPagePtr,
- XLOG_BLCKSZ,
+ nbytes,
currTLI, /* Pass the current TLI because only
* WalSndSegmentOpen controls whether new TLI
* is needed. */
--
2.34.1
^ permalink raw reply [nested|flat] 22+ messages in thread
* Re: Improve WALRead() to suck data directly from WAL buffers when possible
@ 2024-01-26 03:01 Jeff Davis <[email protected]>
parent: Bharath Rupireddy <[email protected]>
0 siblings, 1 reply; 22+ messages in thread
From: Jeff Davis @ 2024-01-26 03:01 UTC (permalink / raw)
To: Bharath Rupireddy <[email protected]>; +Cc: Andres Freund <[email protected]>; Dilip Kumar <[email protected]>; Kyotaro Horiguchi <[email protected]>; [email protected]; Nathan Bossart <[email protected]>; Masahiko Sawada <[email protected]>
On Thu, 2024-01-25 at 14:35 +0530, Bharath Rupireddy wrote:
>
> "expecting zeros at the end" - this can't always be true as the WAL
>
...
> I think this needs to be discussed separately. If okay, I'll start a
> new thread.
Thank you for investigating. When the above issue is handled, I'll be
more comfortable expanding the call sites for XLogReadFromBuffers().
> > Also, if we've detected that the first requested buffer has been
> > evicted, is there any value in continuing the loop to see if more
> > recent buffers are available? For example, if the requested LSNs
> > range
> > over buffers 4, 5, and 6, and 4 has already been evicted, should we
> > try
> > to return LSN data from 5 and 6 at the proper offset in the dest
> > buffer? If so, we'd need to adjust the API so the caller knows what
> > parts of the dest buffer were filled in.
>
> I'd second this capability for now to keep the API simple and clear,
> but we can consider expanding it as needed.
Agreed. This case doesn't seem important; I just thought I'd ask about
it.
> If the callers use GetFlushRecPtr() to determine how far to read,
> LogwrtResult will be *reasonably* latest
It will be up-to-date enough that we'd never go through
WaitXLogInsertionsToFinish(), which is all we care about.
> As far as the current WAL readers are concerned, we don't need an
> explicit spinlock to determine LogwrtResult because all of them use
> GetFlushRecPtr() to determine how far to read. If there's any caller
> that's not updating LogwrtResult at all, we can consider reading
> LogwrtResult it ourselves in future.
So we don't actually need that path yet, right?
> 5. I think the two requirements specified at
> https://www.postgresql.org/message-id/20231109205836.zjoawdrn4q77yemv%40awork3.anarazel.de
> still hold with the v19j.
Agreed.
> PSA v20 patch set.
0001 is very close. I have the following suggestions:
* Don't just return zero. If the caller is doing something we don't
expect, we want to fix the caller. I understand you'd like this to be
more like a transparent optimization, and we may do that later, but I
don't think it's a good idea to do that now.
* There's currently no use for reading LSNs between Write and Insert,
so remove the WaitXLogInsertionsToFinish() code path. That also means
we don't need the extra emitLog parameter, so we can remove that. When
we have a use case, we can bring it all back.
If you agree, I can just make those adjustments (and do some final
checking) and commit 0001. Otherwise let me know what you think.
0002: How does the test control whether the data requested is before
the Flush pointer, the Write pointer, or the Insert pointer? What if
the walwriter comes in and moves one of those pointers before the next
statement is executed? Also, do you think a test module is required for
the basic functionality in 0001, or only when we start doing more
complex things like reading past the Flush pointer?
0003: can you explain why this is useful for wal summarizer to read
from the buffers?
Regards,
Jeff Davis
^ permalink raw reply [nested|flat] 22+ messages in thread
* Re: Improve WALRead() to suck data directly from WAL buffers when possible
@ 2024-01-26 14:01 Bharath Rupireddy <[email protected]>
parent: Jeff Davis <[email protected]>
0 siblings, 1 reply; 22+ messages in thread
From: Bharath Rupireddy @ 2024-01-26 14:01 UTC (permalink / raw)
To: Jeff Davis <[email protected]>; +Cc: Andres Freund <[email protected]>; Dilip Kumar <[email protected]>; Kyotaro Horiguchi <[email protected]>; [email protected]; Nathan Bossart <[email protected]>; Masahiko Sawada <[email protected]>
On Fri, Jan 26, 2024 at 8:31 AM Jeff Davis <[email protected]> wrote:
>
> > PSA v20 patch set.
>
> 0001 is very close. I have the following suggestions:
>
> * Don't just return zero. If the caller is doing something we don't
> expect, we want to fix the caller. I understand you'd like this to be
> more like a transparent optimization, and we may do that later, but I
> don't think it's a good idea to do that now.
+ if (RecoveryInProgress() ||
+ tli != GetWALInsertionTimeLine())
+ return ntotal;
+
+ Assert(!XLogRecPtrIsInvalid(startptr));
Are you suggesting to error out instead of returning 0? If yes, I
disagree with it. Because, failure to read due to unmet pre-conditions
doesn't necessarily have to be to error out. If we error out, the
immediate failure we see is in the src/bin/psql TAP test for calling
XLogReadFromBuffers when the server is in recovery. How about
returning a negative value instead of just 0 or returning true/false
just like WALRead?
> * There's currently no use for reading LSNs between Write and Insert,
> so remove the WaitXLogInsertionsToFinish() code path. That also means
> we don't need the extra emitLog parameter, so we can remove that. When
> we have a use case, we can bring it all back.
I disagree with this. I don't see anything wrong with
XLogReadFromBuffers having the capability to wait for in-progress
insertions to finish. In fact, it makes the function near-complete.
Imagine, implementing an extension (may be for fun or learning or
educational or production purposes) to read unflushed WAL directly
from WAL buffers using XLogReadFromBuffers as page_read callback with
xlogreader facility. AFAICT, I don't see a problem with
WaitXLogInsertionsToFinish logic in XLogReadFromBuffers.
FWIW, one important aspect of XLogReadFromBuffers is its ability to
read the unflushed WAL from WAL buffers. Also, see a note from Andres
here https://www.postgresql.org/message-id/20231109205836.zjoawdrn4q77yemv%40awork3.anarazel.de.
> If you agree, I can just make those adjustments (and do some final
> checking) and commit 0001. Otherwise let me know what you think.
Thanks. Please see my responses above.
> 0002: How does the test control whether the data requested is before
> the Flush pointer, the Write pointer, or the Insert pointer? What if
> the walwriter comes in and moves one of those pointers before the next
> statement is executed?
Tried to keep wal_writer quiet with wal_writer_delay=10000ms and
wal_writer_flush_after = 1GB to not to flush WAL in the background.
Also, disabled autovacuum, and set checkpoint_timeout to a higher
value. All of this is done to generate minimal WAL so that WAL buffers
don't get overwritten. Do you see any problems with it?
> Also, do you think a test module is required for
> the basic functionality in 0001, or only when we start doing more
> complex things like reading past the Flush pointer?
With WaitXLogInsertionsToFinish in XLogReadFromBuffers, we have that
capability already in. Having a separate test module ensures the code
is tested properly.
As far as the test is concerned, it verifies 2 cases:
1. Check if WAL is successfully read from WAL buffers. For this, the
test generates minimal WAL and reads from WAL buffers from the start
LSN = current insert LSN captured before the WAL generation.
2. Check with a WAL that doesn't yet exist. For this, the test reads
from WAL buffers from the start LSN = current flush LSN+16MB (a
randomly chosen higher value).
> 0003: can you explain why this is useful for wal summarizer to read
> from the buffers?
Can the WAL summarizer ever read the WAL on current TLI? I'm not so
sure about it, I haven't explored it in detail.
--
Bharath Rupireddy
PostgreSQL Contributors Team
RDS Open Source Databases
Amazon Web Services: https://aws.amazon.com
^ permalink raw reply [nested|flat] 22+ messages in thread
* Re: Improve WALRead() to suck data directly from WAL buffers when possible
@ 2024-01-26 19:34 Jeff Davis <[email protected]>
parent: Bharath Rupireddy <[email protected]>
0 siblings, 1 reply; 22+ messages in thread
From: Jeff Davis @ 2024-01-26 19:34 UTC (permalink / raw)
To: Bharath Rupireddy <[email protected]>; +Cc: Andres Freund <[email protected]>; Dilip Kumar <[email protected]>; Kyotaro Horiguchi <[email protected]>; [email protected]; Nathan Bossart <[email protected]>; Masahiko Sawada <[email protected]>
On Fri, 2024-01-26 at 19:31 +0530, Bharath Rupireddy wrote:
> Are you suggesting to error out instead of returning 0?
We'd do neither of those things, because no caller should actually call
it while RecoveryInProgress() or on a different timeline.
> How about
> returning a negative value instead of just 0 or returning true/false
> just like WALRead?
All of these things are functionally equivalent -- the same thing is
happening at the end. This is just a discussion about API style and how
that will interact with hypothetical callers that don't exist today.
And it can also be easily changed later, so we aren't stuck with
whatever decision happens here.
>
> Imagine, implementing an extension (may be for fun or learning or
> educational or production purposes) to read unflushed WAL directly
> from WAL buffers using XLogReadFromBuffers as page_read callback with
> xlogreader facility.
That makes sense, I didn't realize you intended to use this fron an
extension. I'm fine considering that as a separate patch that could
potentially be committed soon after this one.
I'd like some more details, but can I please just commit the basic
functionality now-ish?
> Tried to keep wal_writer quiet with wal_writer_delay=10000ms and
> wal_writer_flush_after = 1GB to not to flush WAL in the background.
> Also, disabled autovacuum, and set checkpoint_timeout to a higher
> value. All of this is done to generate minimal WAL so that WAL
> buffers
> don't get overwritten. Do you see any problems with it?
Maybe check it against pg_current_wal_lsn(), and see if the Write
pointer moved ahead? Perhaps even have a (limited) loop that tries
again to catch it at the right time?
> Can the WAL summarizer ever read the WAL on current TLI? I'm not so
> sure about it, I haven't explored it in detail.
Let's just not call XLogReadFromBuffers from there.
Regards,
Jeff Davis
^ permalink raw reply [nested|flat] 22+ messages in thread
* Re: Improve WALRead() to suck data directly from WAL buffers when possible
@ 2024-01-27 08:00 Bharath Rupireddy <[email protected]>
parent: Jeff Davis <[email protected]>
0 siblings, 1 reply; 22+ messages in thread
From: Bharath Rupireddy @ 2024-01-27 08:00 UTC (permalink / raw)
To: Jeff Davis <[email protected]>; +Cc: Andres Freund <[email protected]>; Dilip Kumar <[email protected]>; Kyotaro Horiguchi <[email protected]>; [email protected]; Nathan Bossart <[email protected]>; Masahiko Sawada <[email protected]>
On Sat, Jan 27, 2024 at 1:04 AM Jeff Davis <[email protected]> wrote:
>
> All of these things are functionally equivalent -- the same thing is
> happening at the end. This is just a discussion about API style and how
> that will interact with hypothetical callers that don't exist today.
> And it can also be easily changed later, so we aren't stuck with
> whatever decision happens here.
I'll leave that up to you. I'm okay either ways - 1) ensure the caller
doesn't use XLogReadFromBuffers, 2) XLogReadFromBuffers returning
as-if nothing was read when in recovery or on a different timeline.
> > Imagine, implementing an extension (may be for fun or learning or
> > educational or production purposes) to read unflushed WAL directly
> > from WAL buffers using XLogReadFromBuffers as page_read callback with
> > xlogreader facility.
>
> That makes sense, I didn't realize you intended to use this fron an
> extension. I'm fine considering that as a separate patch that could
> potentially be committed soon after this one.
Yes, I've turned that into 0002 patch.
> I'd like some more details, but can I please just commit the basic
> functionality now-ish?
+1.
> > Tried to keep wal_writer quiet with wal_writer_delay=10000ms and
> > wal_writer_flush_after = 1GB to not to flush WAL in the background.
> > Also, disabled autovacuum, and set checkpoint_timeout to a higher
> > value. All of this is done to generate minimal WAL so that WAL
> > buffers
> > don't get overwritten. Do you see any problems with it?
>
> Maybe check it against pg_current_wal_lsn(), and see if the Write
> pointer moved ahead? Perhaps even have a (limited) loop that tries
> again to catch it at the right time?
Adding a loop seems to be reasonable here and done in v21-0003. Also,
I've added wal_level = minimal per
src/test/recovery/t/039_end_of_wal.pl introduced by commit bae868caf22
which also tries to keep WAL activity to minimum.
> > Can the WAL summarizer ever read the WAL on current TLI? I'm not so
> > sure about it, I haven't explored it in detail.
>
> Let's just not call XLogReadFromBuffers from there.
Removed.
PSA v21 patch set.
--
Bharath Rupireddy
PostgreSQL Contributors Team
RDS Open Source Databases
Amazon Web Services: https://aws.amazon.com
Attachments:
[application/octet-stream] v21-0001-Add-XLogReadFromBuffers.patch (5.9K, ../../CALj2ACW4oZzmg6joL0ppQN=cCGOLaVKJknc3OLqmo8Q+cGDRKw@mail.gmail.com/2-v21-0001-Add-XLogReadFromBuffers.patch)
download | inline diff:
From 95ba60dd3afdc134329f3b017588264103f85985 Mon Sep 17 00:00:00 2001
From: Bharath Rupireddy <[email protected]>
Date: Sat, 27 Jan 2024 05:48:30 +0000
Subject: [PATCH v21] Add XLogReadFromBuffers().
Allows reading directly from WAL buffers without a lock, avoiding the
need to wait for WAL flushing and read from the filesystem.
For now, the only caller is physical replication, but we can consider
expanding it to other callers as needed.
---
src/backend/access/transam/xlog.c | 106 ++++++++++++++++++++++++
src/backend/access/transam/xlogreader.c | 3 -
src/backend/replication/walsender.c | 8 ++
src/include/access/xlog.h | 3 +
4 files changed, 117 insertions(+), 3 deletions(-)
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 478377c4a2..eea50bea3c 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -1705,6 +1705,112 @@ GetXLogBuffer(XLogRecPtr ptr, TimeLineID tli)
return cachedPos + ptr % XLOG_BLCKSZ;
}
+/*
+ * Read WAL directly from WAL buffers, if available.
+ *
+ * This function reads 'count' bytes of WAL from WAL buffers into 'buf'
+ * starting at location 'startptr' and returns total bytes read.
+ *
+ * The bytes read may be fewer than requested if any of the WAL buffers in the
+ * requested range have been evicted.
+ *
+ * This function returns immediately if the requested data is not from the
+ * current timeline, or if the server is in recovery.
+ */
+Size
+XLogReadFromBuffers(char *buf, XLogRecPtr startptr, Size count, TimeLineID tli)
+{
+ XLogRecPtr ptr = startptr;
+ Size nbytes = count;
+ Size ntotal = 0;
+ char *dst = buf;
+
+ if (RecoveryInProgress() ||
+ tli != GetWALInsertionTimeLine())
+ return ntotal;
+
+ Assert(!XLogRecPtrIsInvalid(startptr));
+
+ /*
+ * Loop through the buffers without a lock. For each buffer, atomically
+ * read and verify the end pointer, then copy the data out, and finally
+ * re-read and re-verify the end pointer.
+ *
+ * Once a page is evicted, it never returns to the WAL buffers, so if the
+ * end pointer matches the expected end pointer before and after we copy
+ * the data, then the right page must have been present during the data
+ * copy. Read barriers are necessary to ensure that the data copy actually
+ * happens between the two verification steps.
+ *
+ * If the verification fails, we simply terminate the loop and return with
+ * the data that had been already copied out successfully.
+ */
+ while (nbytes > 0)
+ {
+ XLogRecPtr expectedEndPtr;
+ XLogRecPtr endptr;
+ int idx;
+ const char *page;
+ const char *data;
+ Size nread;
+
+ idx = XLogRecPtrToBufIdx(ptr);
+ expectedEndPtr = ptr;
+ expectedEndPtr += XLOG_BLCKSZ - ptr % XLOG_BLCKSZ;
+
+ /*
+ * First verification step: check that the correct page is present in
+ * the WAL buffers.
+ */
+ endptr = pg_atomic_read_u64(&XLogCtl->xlblocks[idx]);
+ if (expectedEndPtr != endptr)
+ break;
+
+ /*
+ * We found WAL buffer page containing given XLogRecPtr. Get starting
+ * address of the page and a pointer to the right location of given
+ * XLogRecPtr in that page.
+ */
+ page = XLogCtl->pages + idx * (Size) XLOG_BLCKSZ;
+ data = page + ptr % XLOG_BLCKSZ;
+
+ /*
+ * Ensure that the data copy and the first verification step are not
+ * reordered.
+ */
+ pg_read_barrier();
+
+ /* how much is available on this page to read? */
+ nread = Min(nbytes, XLOG_BLCKSZ - (data - page));
+
+ /* data copy */
+ memcpy(dst, data, nread);
+
+ /*
+ * Ensure that the data copy and the second verification step are not
+ * reordered.
+ */
+ pg_read_barrier();
+
+ /*
+ * Second verification step: check that the page we read from wasn't
+ * evicted while we were copying the data.
+ */
+ endptr = pg_atomic_read_u64(&XLogCtl->xlblocks[idx]);
+ if (expectedEndPtr != endptr)
+ break;
+
+ dst += nread;
+ ptr += nread;
+ ntotal += nread;
+ nbytes -= nread;
+ }
+
+ Assert(ntotal <= count);
+
+ return ntotal;
+}
+
/*
* Converts a "usable byte position" to XLogRecPtr. A usable byte position
* is the position starting from the beginning of WAL, excluding all WAL
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index 7190156f2f..74a6b11866 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -1500,9 +1500,6 @@ err:
*
* Returns true if succeeded, false if an error occurs, in which case
* 'errinfo' receives error details.
- *
- * XXX probably this should be improved to suck data directly from the
- * WAL buffers when possible.
*/
bool
WALRead(XLogReaderState *state,
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index aa80f3de20..7efe9ad010 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -2910,6 +2910,7 @@ XLogSendPhysical(void)
Size nbytes;
XLogSegNo segno;
WALReadError errinfo;
+ Size rbytes;
/* If requested switch the WAL sender to the stopping state. */
if (got_STOPPING)
@@ -3125,6 +3126,13 @@ XLogSendPhysical(void)
enlargeStringInfo(&output_message, nbytes);
retry:
+ /* Read from WAL buffers, if available. */
+ rbytes = XLogReadFromBuffers(&output_message.data[output_message.len],
+ startptr, nbytes, xlogreader->seg.ws_tli);
+ output_message.len += rbytes;
+ startptr += rbytes;
+ nbytes -= rbytes;
+
if (!WALRead(xlogreader,
&output_message.data[output_message.len],
startptr,
diff --git a/src/include/access/xlog.h b/src/include/access/xlog.h
index 301c5fa11f..f8c281c799 100644
--- a/src/include/access/xlog.h
+++ b/src/include/access/xlog.h
@@ -252,6 +252,9 @@ extern XLogRecPtr GetLastImportantRecPtr(void);
extern void SetWalWriterSleeping(bool sleeping);
+extern Size XLogReadFromBuffers(char *buf, XLogRecPtr startptr, Size count,
+ TimeLineID tli);
+
/*
* Routines used by xlogrecovery.c to call back into xlog.c during recovery.
*/
--
2.34.1
[application/octet-stream] v21-0002-Allow-XLogReadFromBuffers-to-wait-for-in-progres.patch (5.1K, ../../CALj2ACW4oZzmg6joL0ppQN=cCGOLaVKJknc3OLqmo8Q+cGDRKw@mail.gmail.com/3-v21-0002-Allow-XLogReadFromBuffers-to-wait-for-in-progres.patch)
download | inline diff:
From 1e9dcbfb18c26a8e0fbbf90bac6e7890afcf012d Mon Sep 17 00:00:00 2001
From: Bharath Rupireddy <[email protected]>
Date: Sat, 27 Jan 2024 06:23:07 +0000
Subject: [PATCH v21] Allow XLogReadFromBuffers to wait for in-progress
insertions
---
src/backend/access/transam/xlog.c | 43 ++++++++++++++++++++++++-------
1 file changed, 33 insertions(+), 10 deletions(-)
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index eea50bea3c..20fc2c6036 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -698,7 +698,7 @@ static void ReserveXLogInsertLocation(int size, XLogRecPtr *StartPos,
XLogRecPtr *EndPos, XLogRecPtr *PrevPtr);
static bool ReserveXLogSwitch(XLogRecPtr *StartPos, XLogRecPtr *EndPos,
XLogRecPtr *PrevPtr);
-static XLogRecPtr WaitXLogInsertionsToFinish(XLogRecPtr upto);
+static XLogRecPtr WaitXLogInsertionsToFinish(XLogRecPtr upto, bool emitLog);
static char *GetXLogBuffer(XLogRecPtr ptr, TimeLineID tli);
static XLogRecPtr XLogBytePosToRecPtr(uint64 bytepos);
static XLogRecPtr XLogBytePosToEndRecPtr(uint64 bytepos);
@@ -1494,7 +1494,7 @@ WALInsertLockUpdateInsertingAt(XLogRecPtr insertingAt)
* to make room for a new one, which in turn requires WALWriteLock.
*/
static XLogRecPtr
-WaitXLogInsertionsToFinish(XLogRecPtr upto)
+WaitXLogInsertionsToFinish(XLogRecPtr upto, bool emitLog)
{
uint64 bytepos;
XLogRecPtr reservedUpto;
@@ -1521,9 +1521,10 @@ WaitXLogInsertionsToFinish(XLogRecPtr upto)
*/
if (upto > reservedUpto)
{
- ereport(LOG,
- (errmsg("request to flush past end of generated WAL; request %X/%X, current position %X/%X",
- LSN_FORMAT_ARGS(upto), LSN_FORMAT_ARGS(reservedUpto))));
+ if (emitLog)
+ ereport(LOG,
+ (errmsg("request to flush past end of generated WAL; request %X/%X, current position %X/%X",
+ LSN_FORMAT_ARGS(upto), LSN_FORMAT_ARGS(reservedUpto))));
upto = reservedUpto;
}
@@ -1712,7 +1713,11 @@ GetXLogBuffer(XLogRecPtr ptr, TimeLineID tli)
* starting at location 'startptr' and returns total bytes read.
*
* The bytes read may be fewer than requested if any of the WAL buffers in the
- * requested range have been evicted.
+ * requested range have been evicted, or if the last requested byte is beyond
+ * the current insert position.
+ *
+ * If reading beyond the current write position, this function will wait for
+ * concurrent inserters to finish. Otherwise, it does not wait at all.
*
* This function returns immediately if the requested data is not from the
* current timeline, or if the server is in recovery.
@@ -1724,6 +1729,7 @@ XLogReadFromBuffers(char *buf, XLogRecPtr startptr, Size count, TimeLineID tli)
Size nbytes = count;
Size ntotal = 0;
char *dst = buf;
+ XLogRecPtr upto = startptr + count;
if (RecoveryInProgress() ||
tli != GetWALInsertionTimeLine())
@@ -1731,6 +1737,23 @@ XLogReadFromBuffers(char *buf, XLogRecPtr startptr, Size count, TimeLineID tli)
Assert(!XLogRecPtrIsInvalid(startptr));
+ /*
+ * Caller requested very recent WAL data. Wait for any in-progress WAL
+ * insertions to WAL buffers to finish.
+ *
+ * Most callers will have already updated LogwrtResult when determining
+ * how far to read, but it's OK if it's out of date. XXX: is it worth
+ * taking a spinlock to update LogwrtResult and check again before calling
+ * WaitXLogInsertionsToFinish()?
+ */
+ if (upto > LogwrtResult.Write)
+ {
+ XLogRecPtr writtenUpto = WaitXLogInsertionsToFinish(upto, false);
+
+ upto = Min(upto, writtenUpto);
+ nbytes = upto - startptr;
+ }
+
/*
* Loop through the buffers without a lock. For each buffer, atomically
* read and verify the end pointer, then copy the data out, and finally
@@ -2001,7 +2024,7 @@ AdvanceXLInsertBuffer(XLogRecPtr upto, TimeLineID tli, bool opportunistic)
*/
LWLockRelease(WALBufMappingLock);
- WaitXLogInsertionsToFinish(OldPageRqstPtr);
+ WaitXLogInsertionsToFinish(OldPageRqstPtr, true);
LWLockAcquire(WALWriteLock, LW_EXCLUSIVE);
@@ -2795,7 +2818,7 @@ XLogFlush(XLogRecPtr record)
* Before actually performing the write, wait for all in-flight
* insertions to the pages we're about to write to finish.
*/
- insertpos = WaitXLogInsertionsToFinish(WriteRqstPtr);
+ insertpos = WaitXLogInsertionsToFinish(WriteRqstPtr, true);
/*
* Try to get the write lock. If we can't get it immediately, wait
@@ -2846,7 +2869,7 @@ XLogFlush(XLogRecPtr record)
* We're only calling it again to allow insertpos to be moved
* further forward, not to actually wait for anyone.
*/
- insertpos = WaitXLogInsertionsToFinish(insertpos);
+ insertpos = WaitXLogInsertionsToFinish(insertpos, true);
}
/* try to write/flush later additions to XLOG as well */
@@ -3025,7 +3048,7 @@ XLogBackgroundFlush(void)
START_CRIT_SECTION();
/* now wait for any in-progress insertions to finish and get write lock */
- WaitXLogInsertionsToFinish(WriteRqst.Write);
+ WaitXLogInsertionsToFinish(WriteRqst.Write, true);
LWLockAcquire(WALWriteLock, LW_EXCLUSIVE);
LogwrtResult = XLogCtl->LogwrtResult;
if (WriteRqst.Write > LogwrtResult.Write ||
--
2.34.1
[application/octet-stream] v21-0003-Add-test-module-for-verifying-WAL-read-from-WAL-.patch (9.8K, ../../CALj2ACW4oZzmg6joL0ppQN=cCGOLaVKJknc3OLqmo8Q+cGDRKw@mail.gmail.com/4-v21-0003-Add-test-module-for-verifying-WAL-read-from-WAL-.patch)
download | inline diff:
From e78206889c4ffd5a52033b4e35814bdc74560f7b Mon Sep 17 00:00:00 2001
From: Bharath Rupireddy <[email protected]>
Date: Sat, 27 Jan 2024 07:03:48 +0000
Subject: [PATCH v21] Add test module for verifying WAL read from WAL buffers
This commit adds a test module to verify WAL read from WAL
buffers.
Author: Bharath Rupireddy
Reviewed-by: Dilip Kumar
Discussion: https://www.postgresql.org/message-id/CALj2ACXKKK%3DwbiG5_t6dGao5GoecMwRkhr7GjVBM_jg54%2BNa%3DQ%40mail.gmail.com
---
src/test/modules/Makefile | 1 +
src/test/modules/meson.build | 1 +
.../modules/read_wal_from_buffers/.gitignore | 4 ++
.../modules/read_wal_from_buffers/Makefile | 23 ++++++
.../modules/read_wal_from_buffers/meson.build | 33 +++++++++
.../read_wal_from_buffers--1.0.sql | 14 ++++
.../read_wal_from_buffers.c | 41 +++++++++++
.../read_wal_from_buffers.control | 4 ++
.../read_wal_from_buffers/t/001_basic.pl | 71 +++++++++++++++++++
9 files changed, 192 insertions(+)
create mode 100644 src/test/modules/read_wal_from_buffers/.gitignore
create mode 100644 src/test/modules/read_wal_from_buffers/Makefile
create mode 100644 src/test/modules/read_wal_from_buffers/meson.build
create mode 100644 src/test/modules/read_wal_from_buffers/read_wal_from_buffers--1.0.sql
create mode 100644 src/test/modules/read_wal_from_buffers/read_wal_from_buffers.c
create mode 100644 src/test/modules/read_wal_from_buffers/read_wal_from_buffers.control
create mode 100644 src/test/modules/read_wal_from_buffers/t/001_basic.pl
diff --git a/src/test/modules/Makefile b/src/test/modules/Makefile
index e32c8925f6..4eba0fa2e2 100644
--- a/src/test/modules/Makefile
+++ b/src/test/modules/Makefile
@@ -12,6 +12,7 @@ SUBDIRS = \
dummy_seclabel \
libpq_pipeline \
plsample \
+ read_wal_from_buffers \
spgist_name_ops \
test_bloomfilter \
test_copy_callbacks \
diff --git a/src/test/modules/meson.build b/src/test/modules/meson.build
index 397e0906e6..f0b53eced7 100644
--- a/src/test/modules/meson.build
+++ b/src/test/modules/meson.build
@@ -32,6 +32,7 @@ subdir('test_resowner')
subdir('test_rls_hooks')
subdir('test_shm_mq')
subdir('test_slru')
+subdir('read_wal_from_buffers')
subdir('unsafe_tests')
subdir('worker_spi')
subdir('xid_wraparound')
diff --git a/src/test/modules/read_wal_from_buffers/.gitignore b/src/test/modules/read_wal_from_buffers/.gitignore
new file mode 100644
index 0000000000..5dcb3ff972
--- /dev/null
+++ b/src/test/modules/read_wal_from_buffers/.gitignore
@@ -0,0 +1,4 @@
+# Generated subdirectories
+/log/
+/results/
+/tmp_check/
diff --git a/src/test/modules/read_wal_from_buffers/Makefile b/src/test/modules/read_wal_from_buffers/Makefile
new file mode 100644
index 0000000000..9e57a837f9
--- /dev/null
+++ b/src/test/modules/read_wal_from_buffers/Makefile
@@ -0,0 +1,23 @@
+# src/test/modules/read_wal_from_buffers/Makefile
+
+MODULE_big = read_wal_from_buffers
+OBJS = \
+ $(WIN32RES) \
+ read_wal_from_buffers.o
+PGFILEDESC = "read_wal_from_buffers - test module to read WAL from WAL buffers"
+
+EXTENSION = read_wal_from_buffers
+DATA = read_wal_from_buffers--1.0.sql
+
+TAP_TESTS = 1
+
+ifdef USE_PGXS
+PG_CONFIG = pg_config
+PGXS := $(shell $(PG_CONFIG) --pgxs)
+include $(PGXS)
+else
+subdir = src/test/modules/read_wal_from_buffers
+top_builddir = ../../../..
+include $(top_builddir)/src/Makefile.global
+include $(top_srcdir)/contrib/contrib-global.mk
+endif
diff --git a/src/test/modules/read_wal_from_buffers/meson.build b/src/test/modules/read_wal_from_buffers/meson.build
new file mode 100644
index 0000000000..3fac00d616
--- /dev/null
+++ b/src/test/modules/read_wal_from_buffers/meson.build
@@ -0,0 +1,33 @@
+# Copyright (c) 2024, PostgreSQL Global Development Group
+
+read_wal_from_buffers_sources = files(
+ 'read_wal_from_buffers.c',
+)
+
+if host_system == 'windows'
+ read_wal_from_buffers_sources += rc_lib_gen.process(win32ver_rc, extra_args: [
+ '--NAME', 'read_wal_from_buffers',
+ '--FILEDESC', 'read_wal_from_buffers - test module to read WAL from WAL buffers',])
+endif
+
+read_wal_from_buffers = shared_module('read_wal_from_buffers',
+ read_wal_from_buffers_sources,
+ kwargs: pg_test_mod_args,
+)
+test_install_libs += read_wal_from_buffers
+
+test_install_data += files(
+ 'read_wal_from_buffers.control',
+ 'read_wal_from_buffers--1.0.sql',
+)
+
+tests += {
+ 'name': 'read_wal_from_buffers',
+ 'sd': meson.current_source_dir(),
+ 'bd': meson.current_build_dir(),
+ 'tap': {
+ 'tests': [
+ 't/001_basic.pl',
+ ],
+ },
+}
diff --git a/src/test/modules/read_wal_from_buffers/read_wal_from_buffers--1.0.sql b/src/test/modules/read_wal_from_buffers/read_wal_from_buffers--1.0.sql
new file mode 100644
index 0000000000..82fa097d10
--- /dev/null
+++ b/src/test/modules/read_wal_from_buffers/read_wal_from_buffers--1.0.sql
@@ -0,0 +1,14 @@
+/* src/test/modules/read_wal_from_buffers/read_wal_from_buffers--1.0.sql */
+
+-- complain if script is sourced in psql, rather than via CREATE EXTENSION
+\echo Use "CREATE EXTENSION read_wal_from_buffers" to load this file. \quit
+
+--
+-- read_wal_from_buffers()
+--
+-- SQL function to read WAL from WAL buffers. Returns number of bytes read.
+--
+CREATE FUNCTION read_wal_from_buffers(IN lsn pg_lsn, IN bytes_to_read int,
+ bytes_read OUT int)
+AS 'MODULE_PATHNAME', 'read_wal_from_buffers'
+LANGUAGE C STRICT;
diff --git a/src/test/modules/read_wal_from_buffers/read_wal_from_buffers.c b/src/test/modules/read_wal_from_buffers/read_wal_from_buffers.c
new file mode 100644
index 0000000000..da841da3f0
--- /dev/null
+++ b/src/test/modules/read_wal_from_buffers/read_wal_from_buffers.c
@@ -0,0 +1,41 @@
+/*--------------------------------------------------------------------------
+ *
+ * read_wal_from_buffers.c
+ * Test module to read WAL from WAL buffers.
+ *
+ * Portions Copyright (c) 1996-2024, PostgreSQL Global Development Group
+ * Portions Copyright (c) 1994, Regents of the University of California
+ *
+ * IDENTIFICATION
+ * src/test/modules/read_wal_from_buffers/read_wal_from_buffers.c
+ * -------------------------------------------------------------------------
+ */
+
+#include "postgres.h"
+
+#include "access/xlog.h"
+#include "fmgr.h"
+#include "utils/pg_lsn.h"
+
+PG_MODULE_MAGIC;
+
+/*
+ * SQL function to read WAL from WAL buffers. Returns number of bytes read.
+ */
+PG_FUNCTION_INFO_V1(read_wal_from_buffers);
+Datum
+read_wal_from_buffers(PG_FUNCTION_ARGS)
+{
+ XLogRecPtr startptr = PG_GETARG_LSN(0);
+ int32 bytes_to_read = PG_GETARG_INT32(1);
+ Size bytes_read = 0;
+ char *data = palloc0(bytes_to_read);
+
+ bytes_read = XLogReadFromBuffers(data, startptr,
+ (Size) bytes_to_read,
+ GetWALInsertionTimeLine());
+
+ pfree(data);
+
+ PG_RETURN_INT32(bytes_read);
+}
diff --git a/src/test/modules/read_wal_from_buffers/read_wal_from_buffers.control b/src/test/modules/read_wal_from_buffers/read_wal_from_buffers.control
new file mode 100644
index 0000000000..b14d24751c
--- /dev/null
+++ b/src/test/modules/read_wal_from_buffers/read_wal_from_buffers.control
@@ -0,0 +1,4 @@
+comment = 'Test module to read WAL from WAL buffers'
+default_version = '1.0'
+module_pathname = '$libdir/read_wal_from_buffers'
+relocatable = true
diff --git a/src/test/modules/read_wal_from_buffers/t/001_basic.pl b/src/test/modules/read_wal_from_buffers/t/001_basic.pl
new file mode 100644
index 0000000000..62ea21e541
--- /dev/null
+++ b/src/test/modules/read_wal_from_buffers/t/001_basic.pl
@@ -0,0 +1,71 @@
+# Copyright (c) 2021-2023, PostgreSQL Global Development Group
+
+use strict;
+use warnings;
+
+use PostgreSQL::Test::Cluster;
+use PostgreSQL::Test::Utils;
+use Test::More;
+
+# Setup a new node. The configuration chosen here minimizes the number
+# of arbitrary records that could get generated in a cluster. Enlarging
+# checkpoint_timeout avoids noise with checkpoint activity. wal_level
+# set to "minimal" avoids random standby snapshot records. Autovacuum
+# could also trigger randomly, generating random WAL activity of its own.
+# Enlarging wal_writer_delay and wal_writer_flush_after avoid background
+# wal flush by walwriter.
+my $node = PostgreSQL::Test::Cluster->new("node");
+$node->init;
+$node->append_conf(
+ 'postgresql.conf',
+ q[wal_level = minimal
+ autovacuum = off
+ checkpoint_timeout = '30min'
+ wal_writer_delay = 10000ms
+ wal_writer_flush_after = 1GB
+]);
+$node->start;
+
+# Setup.
+$node->safe_psql('postgres', 'CREATE EXTENSION read_wal_from_buffers;');
+
+$node->safe_psql('postgres', 'CREATE TABLE t (c int);');
+
+my $result = 0;
+my $lsn;
+my $to_read;
+
+# Wait until we read from WAL buffers
+for (my $i = 0; $i < 10 * $PostgreSQL::Test::Utils::timeout_default; $i++)
+{
+ # Get current insert LSN. After this, we generate some WAL which is guranteed
+ # to be in WAL buffers as there is no other WAL generating activity is
+ # happening on the server. We then verify if we can read the WAL from WAL
+ # buffers using this LSN.
+ $lsn = $node->safe_psql('postgres', 'SELECT pg_current_wal_insert_lsn();');
+
+ # Generate minimal WAL so that WAL buffers don't get overwritten.
+ $node->safe_psql('postgres', "INSERT INTO t VALUES ($i);");
+
+ $to_read = 8192;
+
+ if ($node->safe_psql('postgres',
+ qq{SELECT read_wal_from_buffers(lsn := '$lsn', bytes_to_read := $to_read) > 0;}))
+ {
+ $result = 1;
+ last;
+ }
+
+ usleep(100_000);
+}
+ok($result, 'waited until WAL is successfully read from WAL buffers');
+
+# Check with a WAL that doesn't yet exist i.e., 16MB starting from current
+# flush LSN.
+$lsn = $node->safe_psql('postgres', 'SELECT pg_current_wal_flush_lsn()+16777216;');
+$to_read = 8192;
+$result = $node->safe_psql('postgres',
+ qq{SELECT read_wal_from_buffers(lsn := '$lsn', bytes_to_read := $to_read) = 0;});
+is($result, 't', "WAL that doesn't yet exist is not read from WAL buffers");
+
+done_testing();
--
2.34.1
[application/octet-stream] v21-0004-Use-XLogReadFromBuffers-in-more-places.patch (2.7K, ../../CALj2ACW4oZzmg6joL0ppQN=cCGOLaVKJknc3OLqmo8Q+cGDRKw@mail.gmail.com/5-v21-0004-Use-XLogReadFromBuffers-in-more-places.patch)
download | inline diff:
From 0a3f3ea0f849e0bef731efe4c66ed17745b44c8d Mon Sep 17 00:00:00 2001
From: Bharath Rupireddy <[email protected]>
Date: Sat, 27 Jan 2024 07:04:51 +0000
Subject: [PATCH v21] Use XLogReadFromBuffers in more places
---
src/backend/access/transam/xlogutils.c | 12 +++++++++++-
src/backend/replication/walsender.c | 12 +++++++++++-
2 files changed, 22 insertions(+), 2 deletions(-)
diff --git a/src/backend/access/transam/xlogutils.c b/src/backend/access/transam/xlogutils.c
index aa8667abd1..de526f7da7 100644
--- a/src/backend/access/transam/xlogutils.c
+++ b/src/backend/access/transam/xlogutils.c
@@ -894,6 +894,8 @@ read_local_xlog_page_guts(XLogReaderState *state, XLogRecPtr targetPagePtr,
int count;
WALReadError errinfo;
TimeLineID currTLI;
+ Size nbytes;
+ Size rbytes;
loc = targetPagePtr + reqLen;
@@ -1006,12 +1008,20 @@ read_local_xlog_page_guts(XLogReaderState *state, XLogRecPtr targetPagePtr,
count = read_upto - targetPagePtr;
}
+ /* Read from WAL buffers, if available. */
+ nbytes = XLOG_BLCKSZ;
+ rbytes = XLogReadFromBuffers(cur_page, targetPagePtr,
+ nbytes, currTLI);
+ cur_page += rbytes;
+ targetPagePtr += rbytes;
+ nbytes -= rbytes;
+
/*
* Even though we just determined how much of the page can be validly read
* as 'count', read the whole page anyway. It's guaranteed to be
* zero-padded up to the page boundary if it's incomplete.
*/
- if (!WALRead(state, cur_page, targetPagePtr, XLOG_BLCKSZ, tli,
+ if (!WALRead(state, cur_page, targetPagePtr, nbytes, tli,
&errinfo))
WALReadRaiseError(&errinfo);
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index 7efe9ad010..4bc8d5e320 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -1059,6 +1059,8 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
WALReadError errinfo;
XLogSegNo segno;
TimeLineID currTLI;
+ Size nbytes;
+ Size rbytes;
/*
* Make sure we have enough WAL available before retrieving the current
@@ -1095,11 +1097,19 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
else
count = flushptr - targetPagePtr; /* part of the page available */
+ /* Read from WAL buffers, if available. */
+ nbytes = XLOG_BLCKSZ;
+ rbytes = XLogReadFromBuffers(cur_page, targetPagePtr,
+ nbytes, currTLI);
+ cur_page += rbytes;
+ targetPagePtr += rbytes;
+ nbytes -= rbytes;
+
/* now actually read the data, we know it's there */
if (!WALRead(state,
cur_page,
targetPagePtr,
- XLOG_BLCKSZ,
+ nbytes,
currTLI, /* Pass the current TLI because only
* WalSndSegmentOpen controls whether new TLI
* is needed. */
--
2.34.1
^ permalink raw reply [nested|flat] 22+ messages in thread
* Re: Improve WALRead() to suck data directly from WAL buffers when possible
@ 2024-01-30 17:31 Alvaro Herrera <[email protected]>
parent: Bharath Rupireddy <[email protected]>
0 siblings, 1 reply; 22+ messages in thread
From: Alvaro Herrera @ 2024-01-30 17:31 UTC (permalink / raw)
To: Bharath Rupireddy <[email protected]>; +Cc: Jeff Davis <[email protected]>; Andres Freund <[email protected]>; Dilip Kumar <[email protected]>; Kyotaro Horiguchi <[email protected]>; [email protected]; Nathan Bossart <[email protected]>; Masahiko Sawada <[email protected]>
Hmm, this looks quite nice and simple. My only comment is that a
sequence like this
/* Read from WAL buffers, if available. */
rbytes = XLogReadFromBuffers(&output_message.data[output_message.len],
startptr, nbytes, xlogreader->seg.ws_tli);
output_message.len += rbytes;
startptr += rbytes;
nbytes -= rbytes;
if (!WALRead(xlogreader,
&output_message.data[output_message.len],
startptr,
leaves you wondering if WALRead() should be called at all or not, in the
case when all bytes were read by XLogReadFromBuffers. I think in many
cases what's going to happen is that nbytes is going to be zero, and
then WALRead is going to return having done nothing in its inner loop.
I think this warrants a comment somewhere. Alternatively, we could
short-circuit the 'if' expression so that WALRead() is not called in
that case (but I'm not sure it's worth the loss of code clarity).
Also, but this is really quite minor, it seems sad to add more functions
with the prefix XLog, when we have renamed things to use the prefix WAL,
and we have kept the old names only to avoid backpatchability issues.
I mean, if we have WALRead() already, wouldn't it make perfect sense to
name the new routine WALReadFromBuffers?
--
Álvaro Herrera 48°01'N 7°57'E — https://www.EnterpriseDB.com/
"Tiene valor aquel que admite que es un cobarde" (Fernandel)
^ permalink raw reply [nested|flat] 22+ messages in thread
* Re: Improve WALRead() to suck data directly from WAL buffers when possible
@ 2024-01-31 09:00 Bharath Rupireddy <[email protected]>
parent: Alvaro Herrera <[email protected]>
0 siblings, 1 reply; 22+ messages in thread
From: Bharath Rupireddy @ 2024-01-31 09:00 UTC (permalink / raw)
To: Alvaro Herrera <[email protected]>; +Cc: Jeff Davis <[email protected]>; Andres Freund <[email protected]>; Dilip Kumar <[email protected]>; Kyotaro Horiguchi <[email protected]>; [email protected]; Nathan Bossart <[email protected]>; Masahiko Sawada <[email protected]>
On Tue, Jan 30, 2024 at 11:01 PM Alvaro Herrera <[email protected]> wrote:
>
> Hmm, this looks quite nice and simple.
Thanks for looking at it.
> My only comment is that a
> sequence like this
>
> /* Read from WAL buffers, if available. */
> rbytes = XLogReadFromBuffers(&output_message.data[output_message.len],
> startptr, nbytes, xlogreader->seg.ws_tli);
> output_message.len += rbytes;
> startptr += rbytes;
> nbytes -= rbytes;
>
> if (!WALRead(xlogreader,
> &output_message.data[output_message.len],
> startptr,
>
> leaves you wondering if WALRead() should be called at all or not, in the
> case when all bytes were read by XLogReadFromBuffers. I think in many
> cases what's going to happen is that nbytes is going to be zero, and
> then WALRead is going to return having done nothing in its inner loop.
> I think this warrants a comment somewhere. Alternatively, we could
> short-circuit the 'if' expression so that WALRead() is not called in
> that case (but I'm not sure it's worth the loss of code clarity).
It might help avoid a function call in case reading from WAL buffers
satisfies the read fully. And, it's not that clumsy with the change,
see following. I've changed it in the attached v22 patch set.
if (nbytes > 0 &&
!WALRead(xlogreader,
> Also, but this is really quite minor, it seems sad to add more functions
> with the prefix XLog, when we have renamed things to use the prefix WAL,
> and we have kept the old names only to avoid backpatchability issues.
> I mean, if we have WALRead() already, wouldn't it make perfect sense to
> name the new routine WALReadFromBuffers?
WALReadFromBuffers looks better. Used that in v22 patch.
Please see the attached v22 patch set.
--
Bharath Rupireddy
PostgreSQL Contributors Team
RDS Open Source Databases
Amazon Web Services: https://aws.amazon.com
Attachments:
[application/octet-stream] v22-0004-Use-WALReadFromBuffers-in-more-places.patch (2.9K, ../../CALj2ACVSR9cgFW=OMxd4mtLceAmucVYHJ2anRAoG3y6OSyLR+g@mail.gmail.com/2-v22-0004-Use-WALReadFromBuffers-in-more-places.patch)
download | inline diff:
From 73d640cbac5d33c151c24c71613fc89f99a78c97 Mon Sep 17 00:00:00 2001
From: Bharath Rupireddy <[email protected]>
Date: Wed, 31 Jan 2024 07:48:57 +0000
Subject: [PATCH v22 4/4] Use WALReadFromBuffers() in more places
---
src/backend/access/transam/xlogutils.c | 14 +++++++++++++-
src/backend/replication/walsender.c | 16 +++++++++++++---
2 files changed, 26 insertions(+), 4 deletions(-)
diff --git a/src/backend/access/transam/xlogutils.c b/src/backend/access/transam/xlogutils.c
index aa8667abd1..1740ac3160 100644
--- a/src/backend/access/transam/xlogutils.c
+++ b/src/backend/access/transam/xlogutils.c
@@ -894,6 +894,8 @@ read_local_xlog_page_guts(XLogReaderState *state, XLogRecPtr targetPagePtr,
int count;
WALReadError errinfo;
TimeLineID currTLI;
+ Size nbytes;
+ Size rbytes;
loc = targetPagePtr + reqLen;
@@ -1006,12 +1008,22 @@ read_local_xlog_page_guts(XLogReaderState *state, XLogRecPtr targetPagePtr,
count = read_upto - targetPagePtr;
}
+ /* Attempt to read WAL from WAL buffers first. */
+ nbytes = XLOG_BLCKSZ;
+ rbytes = WALReadFromBuffers(cur_page, targetPagePtr, nbytes, currTLI);
+ cur_page += rbytes;
+ targetPagePtr += rbytes;
+ nbytes -= rbytes;
+
/*
+ * Now read the remaining WAL from WAL file.
+ *
* Even though we just determined how much of the page can be validly read
* as 'count', read the whole page anyway. It's guaranteed to be
* zero-padded up to the page boundary if it's incomplete.
*/
- if (!WALRead(state, cur_page, targetPagePtr, XLOG_BLCKSZ, tli,
+ if (nbytes > 0 &&
+ !WALRead(state, cur_page, targetPagePtr, nbytes, tli,
&errinfo))
WALReadRaiseError(&errinfo);
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index 0551f0f2d8..3f515bbf18 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -1059,6 +1059,8 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
WALReadError errinfo;
XLogSegNo segno;
TimeLineID currTLI;
+ Size nbytes;
+ Size rbytes;
/*
* Make sure we have enough WAL available before retrieving the current
@@ -1095,11 +1097,19 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
else
count = flushptr - targetPagePtr; /* part of the page available */
- /* now actually read the data, we know it's there */
- if (!WALRead(state,
+ /* Attempt to read WAL from WAL buffers first. */
+ nbytes = XLOG_BLCKSZ;
+ rbytes = WALReadFromBuffers(cur_page, targetPagePtr, nbytes, currTLI);
+ cur_page += rbytes;
+ targetPagePtr += rbytes;
+ nbytes -= rbytes;
+
+ /* Now read the remaining WAL from WAL file. */
+ if (nbytes > 0 &&
+ !WALRead(state,
cur_page,
targetPagePtr,
- XLOG_BLCKSZ,
+ nbytes,
currTLI, /* Pass the current TLI because only
* WalSndSegmentOpen controls whether new TLI
* is needed. */
--
2.34.1
[application/octet-stream] v22-0002-Allow-WALReadFromBuffers-to-wait-for-in-progress.patch (5.1K, ../../CALj2ACVSR9cgFW=OMxd4mtLceAmucVYHJ2anRAoG3y6OSyLR+g@mail.gmail.com/3-v22-0002-Allow-WALReadFromBuffers-to-wait-for-in-progress.patch)
download | inline diff:
From 5b6b2ebc60100d6d062bd837aa30f5943d4212cc Mon Sep 17 00:00:00 2001
From: Bharath Rupireddy <[email protected]>
Date: Wed, 31 Jan 2024 07:27:11 +0000
Subject: [PATCH v22 2/4] Allow WALReadFromBuffers() to wait for in-progress
insertions
---
src/backend/access/transam/xlog.c | 43 ++++++++++++++++++++++++-------
1 file changed, 33 insertions(+), 10 deletions(-)
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 0d87a66c59..d82557886e 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -698,7 +698,7 @@ static void ReserveXLogInsertLocation(int size, XLogRecPtr *StartPos,
XLogRecPtr *EndPos, XLogRecPtr *PrevPtr);
static bool ReserveXLogSwitch(XLogRecPtr *StartPos, XLogRecPtr *EndPos,
XLogRecPtr *PrevPtr);
-static XLogRecPtr WaitXLogInsertionsToFinish(XLogRecPtr upto);
+static XLogRecPtr WaitXLogInsertionsToFinish(XLogRecPtr upto, bool emitLog);
static char *GetXLogBuffer(XLogRecPtr ptr, TimeLineID tli);
static XLogRecPtr XLogBytePosToRecPtr(uint64 bytepos);
static XLogRecPtr XLogBytePosToEndRecPtr(uint64 bytepos);
@@ -1494,7 +1494,7 @@ WALInsertLockUpdateInsertingAt(XLogRecPtr insertingAt)
* to make room for a new one, which in turn requires WALWriteLock.
*/
static XLogRecPtr
-WaitXLogInsertionsToFinish(XLogRecPtr upto)
+WaitXLogInsertionsToFinish(XLogRecPtr upto, bool emitLog)
{
uint64 bytepos;
XLogRecPtr reservedUpto;
@@ -1521,9 +1521,10 @@ WaitXLogInsertionsToFinish(XLogRecPtr upto)
*/
if (upto > reservedUpto)
{
- ereport(LOG,
- (errmsg("request to flush past end of generated WAL; request %X/%X, current position %X/%X",
- LSN_FORMAT_ARGS(upto), LSN_FORMAT_ARGS(reservedUpto))));
+ if (emitLog)
+ ereport(LOG,
+ (errmsg("request to flush past end of generated WAL; request %X/%X, current position %X/%X",
+ LSN_FORMAT_ARGS(upto), LSN_FORMAT_ARGS(reservedUpto))));
upto = reservedUpto;
}
@@ -1712,7 +1713,11 @@ GetXLogBuffer(XLogRecPtr ptr, TimeLineID tli)
* starting at location 'startptr' and returns total bytes read.
*
* The bytes read may be fewer than requested if any of the WAL buffers in the
- * requested range have been evicted.
+ * requested range have been evicted, or if the last requested byte is beyond
+ * the current insert position.
+ *
+ * If reading beyond the current write position, this function will wait for
+ * concurrent inserters to finish. Otherwise, it does not wait at all.
*
* This function returns immediately if the requested data is not from the
* current timeline, or if the server is in recovery.
@@ -1724,6 +1729,7 @@ WALReadFromBuffers(char *buf, XLogRecPtr startptr, Size count, TimeLineID tli)
Size nbytes = count;
Size ntotal = 0;
char *dst = buf;
+ XLogRecPtr upto = startptr + count;
if (RecoveryInProgress() ||
tli != GetWALInsertionTimeLine())
@@ -1731,6 +1737,23 @@ WALReadFromBuffers(char *buf, XLogRecPtr startptr, Size count, TimeLineID tli)
Assert(!XLogRecPtrIsInvalid(startptr));
+ /*
+ * Caller requested very recent WAL data. Wait for any in-progress WAL
+ * insertions to WAL buffers to finish.
+ *
+ * Most callers will have already updated LogwrtResult when determining
+ * how far to read, but it's OK if it's out of date. XXX: is it worth
+ * taking a spinlock to update LogwrtResult and check again before calling
+ * WaitXLogInsertionsToFinish()?
+ */
+ if (upto > LogwrtResult.Write)
+ {
+ XLogRecPtr writtenUpto = WaitXLogInsertionsToFinish(upto, false);
+
+ upto = Min(upto, writtenUpto);
+ nbytes = upto - startptr;
+ }
+
/*
* Loop through the buffers without a lock. For each buffer, atomically
* read and verify the end pointer, then copy the data out, and finally
@@ -2001,7 +2024,7 @@ AdvanceXLInsertBuffer(XLogRecPtr upto, TimeLineID tli, bool opportunistic)
*/
LWLockRelease(WALBufMappingLock);
- WaitXLogInsertionsToFinish(OldPageRqstPtr);
+ WaitXLogInsertionsToFinish(OldPageRqstPtr, true);
LWLockAcquire(WALWriteLock, LW_EXCLUSIVE);
@@ -2795,7 +2818,7 @@ XLogFlush(XLogRecPtr record)
* Before actually performing the write, wait for all in-flight
* insertions to the pages we're about to write to finish.
*/
- insertpos = WaitXLogInsertionsToFinish(WriteRqstPtr);
+ insertpos = WaitXLogInsertionsToFinish(WriteRqstPtr, true);
/*
* Try to get the write lock. If we can't get it immediately, wait
@@ -2846,7 +2869,7 @@ XLogFlush(XLogRecPtr record)
* We're only calling it again to allow insertpos to be moved
* further forward, not to actually wait for anyone.
*/
- insertpos = WaitXLogInsertionsToFinish(insertpos);
+ insertpos = WaitXLogInsertionsToFinish(insertpos, true);
}
/* try to write/flush later additions to XLOG as well */
@@ -3025,7 +3048,7 @@ XLogBackgroundFlush(void)
START_CRIT_SECTION();
/* now wait for any in-progress insertions to finish and get write lock */
- WaitXLogInsertionsToFinish(WriteRqst.Write);
+ WaitXLogInsertionsToFinish(WriteRqst.Write, true);
LWLockAcquire(WALWriteLock, LW_EXCLUSIVE);
LogwrtResult = XLogCtl->LogwrtResult;
if (WriteRqst.Write > LogwrtResult.Write ||
--
2.34.1
[application/octet-stream] v22-0003-Add-test-module-for-verifying-WAL-read-from-WAL-.patch (9.8K, ../../CALj2ACVSR9cgFW=OMxd4mtLceAmucVYHJ2anRAoG3y6OSyLR+g@mail.gmail.com/4-v22-0003-Add-test-module-for-verifying-WAL-read-from-WAL-.patch)
download | inline diff:
From 7d867ca6fb918f9b0ed5145f1c6992ce984bf2eb Mon Sep 17 00:00:00 2001
From: Bharath Rupireddy <[email protected]>
Date: Wed, 31 Jan 2024 07:29:04 +0000
Subject: [PATCH v22 3/4] Add test module for verifying WAL read from WAL
buffers
This commit adds a test module to verify WAL read from WAL
buffers.
Author: Bharath Rupireddy
Reviewed-by: Dilip Kumar
Discussion: https://www.postgresql.org/message-id/CALj2ACXKKK%3DwbiG5_t6dGao5GoecMwRkhr7GjVBM_jg54%2BNa%3DQ%40mail.gmail.com
---
src/test/modules/Makefile | 1 +
src/test/modules/meson.build | 1 +
.../modules/read_wal_from_buffers/.gitignore | 4 ++
.../modules/read_wal_from_buffers/Makefile | 23 ++++++
.../modules/read_wal_from_buffers/meson.build | 33 +++++++++
.../read_wal_from_buffers--1.0.sql | 14 ++++
.../read_wal_from_buffers.c | 41 +++++++++++
.../read_wal_from_buffers.control | 4 ++
.../read_wal_from_buffers/t/001_basic.pl | 71 +++++++++++++++++++
9 files changed, 192 insertions(+)
create mode 100644 src/test/modules/read_wal_from_buffers/.gitignore
create mode 100644 src/test/modules/read_wal_from_buffers/Makefile
create mode 100644 src/test/modules/read_wal_from_buffers/meson.build
create mode 100644 src/test/modules/read_wal_from_buffers/read_wal_from_buffers--1.0.sql
create mode 100644 src/test/modules/read_wal_from_buffers/read_wal_from_buffers.c
create mode 100644 src/test/modules/read_wal_from_buffers/read_wal_from_buffers.control
create mode 100644 src/test/modules/read_wal_from_buffers/t/001_basic.pl
diff --git a/src/test/modules/Makefile b/src/test/modules/Makefile
index 89aa41b5e3..864a3dd72b 100644
--- a/src/test/modules/Makefile
+++ b/src/test/modules/Makefile
@@ -12,6 +12,7 @@ SUBDIRS = \
dummy_seclabel \
libpq_pipeline \
plsample \
+ read_wal_from_buffers \
spgist_name_ops \
test_bloomfilter \
test_copy_callbacks \
diff --git a/src/test/modules/meson.build b/src/test/modules/meson.build
index 8fbe742d38..4f3dd69e58 100644
--- a/src/test/modules/meson.build
+++ b/src/test/modules/meson.build
@@ -33,6 +33,7 @@ subdir('test_resowner')
subdir('test_rls_hooks')
subdir('test_shm_mq')
subdir('test_slru')
+subdir('read_wal_from_buffers')
subdir('unsafe_tests')
subdir('worker_spi')
subdir('xid_wraparound')
diff --git a/src/test/modules/read_wal_from_buffers/.gitignore b/src/test/modules/read_wal_from_buffers/.gitignore
new file mode 100644
index 0000000000..5dcb3ff972
--- /dev/null
+++ b/src/test/modules/read_wal_from_buffers/.gitignore
@@ -0,0 +1,4 @@
+# Generated subdirectories
+/log/
+/results/
+/tmp_check/
diff --git a/src/test/modules/read_wal_from_buffers/Makefile b/src/test/modules/read_wal_from_buffers/Makefile
new file mode 100644
index 0000000000..9e57a837f9
--- /dev/null
+++ b/src/test/modules/read_wal_from_buffers/Makefile
@@ -0,0 +1,23 @@
+# src/test/modules/read_wal_from_buffers/Makefile
+
+MODULE_big = read_wal_from_buffers
+OBJS = \
+ $(WIN32RES) \
+ read_wal_from_buffers.o
+PGFILEDESC = "read_wal_from_buffers - test module to read WAL from WAL buffers"
+
+EXTENSION = read_wal_from_buffers
+DATA = read_wal_from_buffers--1.0.sql
+
+TAP_TESTS = 1
+
+ifdef USE_PGXS
+PG_CONFIG = pg_config
+PGXS := $(shell $(PG_CONFIG) --pgxs)
+include $(PGXS)
+else
+subdir = src/test/modules/read_wal_from_buffers
+top_builddir = ../../../..
+include $(top_builddir)/src/Makefile.global
+include $(top_srcdir)/contrib/contrib-global.mk
+endif
diff --git a/src/test/modules/read_wal_from_buffers/meson.build b/src/test/modules/read_wal_from_buffers/meson.build
new file mode 100644
index 0000000000..3fac00d616
--- /dev/null
+++ b/src/test/modules/read_wal_from_buffers/meson.build
@@ -0,0 +1,33 @@
+# Copyright (c) 2024, PostgreSQL Global Development Group
+
+read_wal_from_buffers_sources = files(
+ 'read_wal_from_buffers.c',
+)
+
+if host_system == 'windows'
+ read_wal_from_buffers_sources += rc_lib_gen.process(win32ver_rc, extra_args: [
+ '--NAME', 'read_wal_from_buffers',
+ '--FILEDESC', 'read_wal_from_buffers - test module to read WAL from WAL buffers',])
+endif
+
+read_wal_from_buffers = shared_module('read_wal_from_buffers',
+ read_wal_from_buffers_sources,
+ kwargs: pg_test_mod_args,
+)
+test_install_libs += read_wal_from_buffers
+
+test_install_data += files(
+ 'read_wal_from_buffers.control',
+ 'read_wal_from_buffers--1.0.sql',
+)
+
+tests += {
+ 'name': 'read_wal_from_buffers',
+ 'sd': meson.current_source_dir(),
+ 'bd': meson.current_build_dir(),
+ 'tap': {
+ 'tests': [
+ 't/001_basic.pl',
+ ],
+ },
+}
diff --git a/src/test/modules/read_wal_from_buffers/read_wal_from_buffers--1.0.sql b/src/test/modules/read_wal_from_buffers/read_wal_from_buffers--1.0.sql
new file mode 100644
index 0000000000..82fa097d10
--- /dev/null
+++ b/src/test/modules/read_wal_from_buffers/read_wal_from_buffers--1.0.sql
@@ -0,0 +1,14 @@
+/* src/test/modules/read_wal_from_buffers/read_wal_from_buffers--1.0.sql */
+
+-- complain if script is sourced in psql, rather than via CREATE EXTENSION
+\echo Use "CREATE EXTENSION read_wal_from_buffers" to load this file. \quit
+
+--
+-- read_wal_from_buffers()
+--
+-- SQL function to read WAL from WAL buffers. Returns number of bytes read.
+--
+CREATE FUNCTION read_wal_from_buffers(IN lsn pg_lsn, IN bytes_to_read int,
+ bytes_read OUT int)
+AS 'MODULE_PATHNAME', 'read_wal_from_buffers'
+LANGUAGE C STRICT;
diff --git a/src/test/modules/read_wal_from_buffers/read_wal_from_buffers.c b/src/test/modules/read_wal_from_buffers/read_wal_from_buffers.c
new file mode 100644
index 0000000000..9fad86a962
--- /dev/null
+++ b/src/test/modules/read_wal_from_buffers/read_wal_from_buffers.c
@@ -0,0 +1,41 @@
+/*--------------------------------------------------------------------------
+ *
+ * read_wal_from_buffers.c
+ * Test module to read WAL from WAL buffers.
+ *
+ * Portions Copyright (c) 1996-2024, PostgreSQL Global Development Group
+ * Portions Copyright (c) 1994, Regents of the University of California
+ *
+ * IDENTIFICATION
+ * src/test/modules/read_wal_from_buffers/read_wal_from_buffers.c
+ * -------------------------------------------------------------------------
+ */
+
+#include "postgres.h"
+
+#include "access/xlog.h"
+#include "fmgr.h"
+#include "utils/pg_lsn.h"
+
+PG_MODULE_MAGIC;
+
+/*
+ * SQL function to read WAL from WAL buffers. Returns number of bytes read.
+ */
+PG_FUNCTION_INFO_V1(read_wal_from_buffers);
+Datum
+read_wal_from_buffers(PG_FUNCTION_ARGS)
+{
+ XLogRecPtr startptr = PG_GETARG_LSN(0);
+ int32 bytes_to_read = PG_GETARG_INT32(1);
+ Size bytes_read = 0;
+ char *data = palloc0(bytes_to_read);
+
+ bytes_read = WALReadFromBuffers(data, startptr,
+ (Size) bytes_to_read,
+ GetWALInsertionTimeLine());
+
+ pfree(data);
+
+ PG_RETURN_INT32(bytes_read);
+}
diff --git a/src/test/modules/read_wal_from_buffers/read_wal_from_buffers.control b/src/test/modules/read_wal_from_buffers/read_wal_from_buffers.control
new file mode 100644
index 0000000000..b14d24751c
--- /dev/null
+++ b/src/test/modules/read_wal_from_buffers/read_wal_from_buffers.control
@@ -0,0 +1,4 @@
+comment = 'Test module to read WAL from WAL buffers'
+default_version = '1.0'
+module_pathname = '$libdir/read_wal_from_buffers'
+relocatable = true
diff --git a/src/test/modules/read_wal_from_buffers/t/001_basic.pl b/src/test/modules/read_wal_from_buffers/t/001_basic.pl
new file mode 100644
index 0000000000..62ea21e541
--- /dev/null
+++ b/src/test/modules/read_wal_from_buffers/t/001_basic.pl
@@ -0,0 +1,71 @@
+# Copyright (c) 2021-2023, PostgreSQL Global Development Group
+
+use strict;
+use warnings;
+
+use PostgreSQL::Test::Cluster;
+use PostgreSQL::Test::Utils;
+use Test::More;
+
+# Setup a new node. The configuration chosen here minimizes the number
+# of arbitrary records that could get generated in a cluster. Enlarging
+# checkpoint_timeout avoids noise with checkpoint activity. wal_level
+# set to "minimal" avoids random standby snapshot records. Autovacuum
+# could also trigger randomly, generating random WAL activity of its own.
+# Enlarging wal_writer_delay and wal_writer_flush_after avoid background
+# wal flush by walwriter.
+my $node = PostgreSQL::Test::Cluster->new("node");
+$node->init;
+$node->append_conf(
+ 'postgresql.conf',
+ q[wal_level = minimal
+ autovacuum = off
+ checkpoint_timeout = '30min'
+ wal_writer_delay = 10000ms
+ wal_writer_flush_after = 1GB
+]);
+$node->start;
+
+# Setup.
+$node->safe_psql('postgres', 'CREATE EXTENSION read_wal_from_buffers;');
+
+$node->safe_psql('postgres', 'CREATE TABLE t (c int);');
+
+my $result = 0;
+my $lsn;
+my $to_read;
+
+# Wait until we read from WAL buffers
+for (my $i = 0; $i < 10 * $PostgreSQL::Test::Utils::timeout_default; $i++)
+{
+ # Get current insert LSN. After this, we generate some WAL which is guranteed
+ # to be in WAL buffers as there is no other WAL generating activity is
+ # happening on the server. We then verify if we can read the WAL from WAL
+ # buffers using this LSN.
+ $lsn = $node->safe_psql('postgres', 'SELECT pg_current_wal_insert_lsn();');
+
+ # Generate minimal WAL so that WAL buffers don't get overwritten.
+ $node->safe_psql('postgres', "INSERT INTO t VALUES ($i);");
+
+ $to_read = 8192;
+
+ if ($node->safe_psql('postgres',
+ qq{SELECT read_wal_from_buffers(lsn := '$lsn', bytes_to_read := $to_read) > 0;}))
+ {
+ $result = 1;
+ last;
+ }
+
+ usleep(100_000);
+}
+ok($result, 'waited until WAL is successfully read from WAL buffers');
+
+# Check with a WAL that doesn't yet exist i.e., 16MB starting from current
+# flush LSN.
+$lsn = $node->safe_psql('postgres', 'SELECT pg_current_wal_flush_lsn()+16777216;');
+$to_read = 8192;
+$result = $node->safe_psql('postgres',
+ qq{SELECT read_wal_from_buffers(lsn := '$lsn', bytes_to_read := $to_read) = 0;});
+is($result, 't', "WAL that doesn't yet exist is not read from WAL buffers");
+
+done_testing();
--
2.34.1
[application/octet-stream] v22-0001-Add-WALReadFromBuffers.patch (6.1K, ../../CALj2ACVSR9cgFW=OMxd4mtLceAmucVYHJ2anRAoG3y6OSyLR+g@mail.gmail.com/5-v22-0001-Add-WALReadFromBuffers.patch)
download | inline diff:
From 5826ad95c980f4543517cb7014af21bd9a1aa917 Mon Sep 17 00:00:00 2001
From: Bharath Rupireddy <[email protected]>
Date: Wed, 31 Jan 2024 07:25:22 +0000
Subject: [PATCH v22 1/4] Add WALReadFromBuffers().
Allows reading directly from WAL buffers without a lock, avoiding the
need to wait for WAL flushing and read from the filesystem.
For now, the only caller is physical replication, but we can consider
expanding it to other callers as needed.
---
src/backend/access/transam/xlog.c | 106 ++++++++++++++++++++++++
src/backend/access/transam/xlogreader.c | 3 -
src/backend/replication/walsender.c | 12 ++-
src/include/access/xlog.h | 3 +
4 files changed, 120 insertions(+), 4 deletions(-)
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 478377c4a2..0d87a66c59 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -1705,6 +1705,112 @@ GetXLogBuffer(XLogRecPtr ptr, TimeLineID tli)
return cachedPos + ptr % XLOG_BLCKSZ;
}
+/*
+ * Read WAL directly from WAL buffers, if available.
+ *
+ * This function reads 'count' bytes of WAL from WAL buffers into 'buf'
+ * starting at location 'startptr' and returns total bytes read.
+ *
+ * The bytes read may be fewer than requested if any of the WAL buffers in the
+ * requested range have been evicted.
+ *
+ * This function returns immediately if the requested data is not from the
+ * current timeline, or if the server is in recovery.
+ */
+Size
+WALReadFromBuffers(char *buf, XLogRecPtr startptr, Size count, TimeLineID tli)
+{
+ XLogRecPtr ptr = startptr;
+ Size nbytes = count;
+ Size ntotal = 0;
+ char *dst = buf;
+
+ if (RecoveryInProgress() ||
+ tli != GetWALInsertionTimeLine())
+ return ntotal;
+
+ Assert(!XLogRecPtrIsInvalid(startptr));
+
+ /*
+ * Loop through the buffers without a lock. For each buffer, atomically
+ * read and verify the end pointer, then copy the data out, and finally
+ * re-read and re-verify the end pointer.
+ *
+ * Once a page is evicted, it never returns to the WAL buffers, so if the
+ * end pointer matches the expected end pointer before and after we copy
+ * the data, then the right page must have been present during the data
+ * copy. Read barriers are necessary to ensure that the data copy actually
+ * happens between the two verification steps.
+ *
+ * If the verification fails, we simply terminate the loop and return with
+ * the data that had been already copied out successfully.
+ */
+ while (nbytes > 0)
+ {
+ XLogRecPtr expectedEndPtr;
+ XLogRecPtr endptr;
+ int idx;
+ const char *page;
+ const char *data;
+ Size nread;
+
+ idx = XLogRecPtrToBufIdx(ptr);
+ expectedEndPtr = ptr;
+ expectedEndPtr += XLOG_BLCKSZ - ptr % XLOG_BLCKSZ;
+
+ /*
+ * First verification step: check that the correct page is present in
+ * the WAL buffers.
+ */
+ endptr = pg_atomic_read_u64(&XLogCtl->xlblocks[idx]);
+ if (expectedEndPtr != endptr)
+ break;
+
+ /*
+ * We found WAL buffer page containing given XLogRecPtr. Get starting
+ * address of the page and a pointer to the right location of given
+ * XLogRecPtr in that page.
+ */
+ page = XLogCtl->pages + idx * (Size) XLOG_BLCKSZ;
+ data = page + ptr % XLOG_BLCKSZ;
+
+ /*
+ * Ensure that the data copy and the first verification step are not
+ * reordered.
+ */
+ pg_read_barrier();
+
+ /* how much is available on this page to read? */
+ nread = Min(nbytes, XLOG_BLCKSZ - (data - page));
+
+ /* data copy */
+ memcpy(dst, data, nread);
+
+ /*
+ * Ensure that the data copy and the second verification step are not
+ * reordered.
+ */
+ pg_read_barrier();
+
+ /*
+ * Second verification step: check that the page we read from wasn't
+ * evicted while we were copying the data.
+ */
+ endptr = pg_atomic_read_u64(&XLogCtl->xlblocks[idx]);
+ if (expectedEndPtr != endptr)
+ break;
+
+ dst += nread;
+ ptr += nread;
+ ntotal += nread;
+ nbytes -= nread;
+ }
+
+ Assert(ntotal <= count);
+
+ return ntotal;
+}
+
/*
* Converts a "usable byte position" to XLogRecPtr. A usable byte position
* is the position starting from the beginning of WAL, excluding all WAL
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index 7190156f2f..74a6b11866 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -1500,9 +1500,6 @@ err:
*
* Returns true if succeeded, false if an error occurs, in which case
* 'errinfo' receives error details.
- *
- * XXX probably this should be improved to suck data directly from the
- * WAL buffers when possible.
*/
bool
WALRead(XLogReaderState *state,
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index 77c8baa32a..0551f0f2d8 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -2966,6 +2966,7 @@ XLogSendPhysical(void)
Size nbytes;
XLogSegNo segno;
WALReadError errinfo;
+ Size rbytes;
/* If requested switch the WAL sender to the stopping state. */
if (got_STOPPING)
@@ -3181,7 +3182,16 @@ XLogSendPhysical(void)
enlargeStringInfo(&output_message, nbytes);
retry:
- if (!WALRead(xlogreader,
+ /* Attempt to read WAL from WAL buffers first. */
+ rbytes = WALReadFromBuffers(&output_message.data[output_message.len],
+ startptr, nbytes, xlogreader->seg.ws_tli);
+ output_message.len += rbytes;
+ startptr += rbytes;
+ nbytes -= rbytes;
+
+ /* Now read the remaining WAL from WAL file. */
+ if (nbytes > 0 &&
+ !WALRead(xlogreader,
&output_message.data[output_message.len],
startptr,
nbytes,
diff --git a/src/include/access/xlog.h b/src/include/access/xlog.h
index 301c5fa11f..6d5de9812c 100644
--- a/src/include/access/xlog.h
+++ b/src/include/access/xlog.h
@@ -252,6 +252,9 @@ extern XLogRecPtr GetLastImportantRecPtr(void);
extern void SetWalWriterSleeping(bool sleeping);
+extern Size WALReadFromBuffers(char *buf, XLogRecPtr startptr, Size count,
+ TimeLineID tli);
+
/*
* Routines used by xlogrecovery.c to call back into xlog.c during recovery.
*/
--
2.34.1
^ permalink raw reply [nested|flat] 22+ messages in thread
* Re: Improve WALRead() to suck data directly from WAL buffers when possible
@ 2024-01-31 09:31 Alvaro Herrera <[email protected]>
parent: Bharath Rupireddy <[email protected]>
0 siblings, 1 reply; 22+ messages in thread
From: Alvaro Herrera @ 2024-01-31 09:31 UTC (permalink / raw)
To: Bharath Rupireddy <[email protected]>; +Cc: Jeff Davis <[email protected]>; Andres Freund <[email protected]>; Dilip Kumar <[email protected]>; Kyotaro Horiguchi <[email protected]>; [email protected]; Nathan Bossart <[email protected]>; Masahiko Sawada <[email protected]>
Looking at 0003, where an XXX comment is added about taking a spinlock
to read LogwrtResult, I suspect the answer is probably not, because it
is likely to slow down the other uses of LogwrtResult. But I wonder if
a better path forward would be to base further work on my older
uncommitted patch to make LogwrtResult use atomics. With that, you
wouldn't have to block others in order to read the value. I last posted
that patch in [1] in case you're curious.
[1] https://postgr.es/m/[email protected]
The reason I abandoned that patch is that the performance problem that I
was fixing no longer existed -- it was fixed in a different way.
--
Álvaro Herrera Breisgau, Deutschland — https://www.EnterpriseDB.com/
"In fact, the basic problem with Perl 5's subroutines is that they're not
crufty enough, so the cruft leaks out into user-defined code instead, by
the Conservation of Cruft Principle." (Larry Wall, Apocalypse 6)
^ permalink raw reply [nested|flat] 22+ messages in thread
* Re: Improve WALRead() to suck data directly from WAL buffers when possible
@ 2024-01-31 11:36 Bharath Rupireddy <[email protected]>
parent: Alvaro Herrera <[email protected]>
0 siblings, 0 replies; 22+ messages in thread
From: Bharath Rupireddy @ 2024-01-31 11:36 UTC (permalink / raw)
To: Alvaro Herrera <[email protected]>; +Cc: Jeff Davis <[email protected]>; Andres Freund <[email protected]>; Dilip Kumar <[email protected]>; Kyotaro Horiguchi <[email protected]>; [email protected]; Nathan Bossart <[email protected]>; Masahiko Sawada <[email protected]>
On Wed, Jan 31, 2024 at 3:01 PM Alvaro Herrera <[email protected]> wrote:
>
> Looking at 0003, where an XXX comment is added about taking a spinlock
> to read LogwrtResult, I suspect the answer is probably not, because it
> is likely to slow down the other uses of LogwrtResult.
We avoided keeping LogwrtResult latest as the current callers for
WALReadFromBuffers() all determine the flush LSN using
GetFlushRecPtr(), see comment #4 from
https://www.postgresql.org/message-id/CALj2ACV%3DC1GZT9XQRm4iN1NV1T%3DhLA_hsGWNx2Y5-G%2BmSwdhNg%40ma....
> But I wonder if
> a better path forward would be to base further work on my older
> uncommitted patch to make LogwrtResult use atomics. With that, you
> wouldn't have to block others in order to read the value. I last posted
> that patch in [1] in case you're curious.
>
> [1] https://postgr.es/m/[email protected]
>
> The reason I abandoned that patch is that the performance problem that I
> was fixing no longer existed -- it was fixed in a different way.
Nice. I'll respond in that thread. FWIW, there's been a recent
attempt at turning unloggedLSN to 64-bit atomic -
https://commitfest.postgresql.org/46/4330/ and that might need
pg_atomic_monotonic_advance_u64. I guess we would have to bring your
patch and the unloggedLSN into a single thread to have a better
discussion.
--
Bharath Rupireddy
PostgreSQL Contributors Team
RDS Open Source Databases
Amazon Web Services: https://aws.amazon.com
^ permalink raw reply [nested|flat] 22+ messages in thread
end of thread, other threads:[~2024-01-31 11:36 UTC | newest]
Thread overview: 22+ messages (download: mbox mbox.gz follow: Atom feed)
-- links below jump to the message on this page --
2019-09-26 11:51 [PATCH 2/2] Remove the old implemenations of XLogRead(). Antonin Houska <[email protected]>
2022-12-09 09:03 Improve WALRead() to suck data directly from WAL buffers when possible Bharath Rupireddy <[email protected]>
2022-12-12 02:57 ` Re: Improve WALRead() to suck data directly from WAL buffers when possible Kyotaro Horiguchi <[email protected]>
2022-12-12 03:06 ` Re: Improve WALRead() to suck data directly from WAL buffers when possible Kyotaro Horiguchi <[email protected]>
2022-12-12 03:08 ` Re: Improve WALRead() to suck data directly from WAL buffers when possible Kyotaro Horiguchi <[email protected]>
2022-12-23 10:15 ` Re: Improve WALRead() to suck data directly from WAL buffers when possible Bharath Rupireddy <[email protected]>
2022-12-25 11:25 ` Re: Improve WALRead() to suck data directly from WAL buffers when possible Dilip Kumar <[email protected]>
2022-12-26 08:50 ` Re: Improve WALRead() to suck data directly from WAL buffers when possible Bharath Rupireddy <[email protected]>
2024-01-10 14:29 Re: Improve WALRead() to suck data directly from WAL buffers when possible Bharath Rupireddy <[email protected]>
2024-01-20 01:47 ` Re: Improve WALRead() to suck data directly from WAL buffers when possible Jeff Davis <[email protected]>
2024-01-22 14:03 ` Re: Improve WALRead() to suck data directly from WAL buffers when possible Melih Mutlu <[email protected]>
2024-01-22 20:12 ` Re: Improve WALRead() to suck data directly from WAL buffers when possible Andres Freund <[email protected]>
2024-01-23 04:07 ` Re: Improve WALRead() to suck data directly from WAL buffers when possible Jeff Davis <[email protected]>
2024-01-25 09:05 ` Re: Improve WALRead() to suck data directly from WAL buffers when possible Bharath Rupireddy <[email protected]>
2024-01-26 03:01 ` Re: Improve WALRead() to suck data directly from WAL buffers when possible Jeff Davis <[email protected]>
2024-01-26 14:01 ` Re: Improve WALRead() to suck data directly from WAL buffers when possible Bharath Rupireddy <[email protected]>
2024-01-26 19:34 ` Re: Improve WALRead() to suck data directly from WAL buffers when possible Jeff Davis <[email protected]>
2024-01-27 08:00 ` Re: Improve WALRead() to suck data directly from WAL buffers when possible Bharath Rupireddy <[email protected]>
2024-01-30 17:31 ` Re: Improve WALRead() to suck data directly from WAL buffers when possible Alvaro Herrera <[email protected]>
2024-01-31 09:00 ` Re: Improve WALRead() to suck data directly from WAL buffers when possible Bharath Rupireddy <[email protected]>
2024-01-31 09:31 ` Re: Improve WALRead() to suck data directly from WAL buffers when possible Alvaro Herrera <[email protected]>
2024-01-31 11:36 ` Re: Improve WALRead() to suck data directly from WAL buffers when possible Bharath Rupireddy <[email protected]>
This inbox is served by agora; see mirroring instructions
for how to clone and mirror all data and code used for this inbox